@alteran/astro 0.3.9 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (150) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +19 -30
  3. package/index.js +34 -28
  4. package/migrations/0007_bored_spitfire.sql +26 -0
  5. package/migrations/0008_furry_ozymandias.sql +2 -0
  6. package/migrations/meta/0007_snapshot.json +534 -0
  7. package/migrations/meta/0008_snapshot.json +548 -0
  8. package/migrations/meta/_journal.json +14 -0
  9. package/package.json +10 -9
  10. package/src/app.ts +8 -4
  11. package/src/db/account.ts +25 -6
  12. package/src/db/client.ts +1 -1
  13. package/src/db/dal.ts +34 -23
  14. package/src/db/repo.ts +38 -38
  15. package/src/db/schema.ts +5 -1
  16. package/src/db/seed.ts +5 -13
  17. package/src/entrypoints/server.ts +2 -22
  18. package/src/handlers/debug.ts +1 -1
  19. package/src/handlers/ready.ts +1 -1
  20. package/src/handlers/root.ts +4 -4
  21. package/src/handlers/xrpc.server.refreshSession.ts +6 -6
  22. package/src/lib/account-state.ts +156 -0
  23. package/src/lib/actor.ts +29 -13
  24. package/src/lib/appview/auth-policy.ts +66 -0
  25. package/src/lib/appview/did-resolver.ts +233 -0
  26. package/src/lib/appview/proxy.ts +221 -0
  27. package/src/lib/appview/service-config.ts +61 -0
  28. package/src/lib/appview/service-jwt.ts +93 -0
  29. package/src/lib/appview/types.ts +25 -0
  30. package/src/lib/appview.ts +5 -532
  31. package/src/lib/auth-errors.ts +24 -0
  32. package/src/lib/auth.ts +63 -15
  33. package/src/lib/blockstore-gc.ts +6 -5
  34. package/src/lib/cache.ts +30 -4
  35. package/src/lib/chat.ts +20 -14
  36. package/src/lib/commit-log-pruning.ts +2 -2
  37. package/src/lib/commit.ts +26 -36
  38. package/src/lib/config.ts +26 -15
  39. package/src/lib/did-document.ts +32 -0
  40. package/src/lib/errors.ts +54 -0
  41. package/src/lib/feed.ts +18 -19
  42. package/src/lib/firehose/frames.ts +87 -47
  43. package/src/lib/firehose/validation.ts +3 -3
  44. package/src/lib/jwt.ts +85 -177
  45. package/src/lib/labeler.ts +43 -30
  46. package/src/lib/logger.ts +4 -0
  47. package/src/lib/mst/block-map.ts +172 -0
  48. package/src/lib/mst/blockstore.ts +56 -93
  49. package/src/lib/mst/index.ts +1 -0
  50. package/src/lib/mst/leaf.ts +25 -0
  51. package/src/lib/mst/mst.ts +81 -237
  52. package/src/lib/mst/serialize.ts +97 -0
  53. package/src/lib/mst/types.ts +21 -0
  54. package/src/lib/oauth/clients.ts +67 -0
  55. package/src/lib/oauth/dpop-errors.ts +15 -0
  56. package/src/lib/oauth/dpop.ts +150 -0
  57. package/src/lib/oauth/resource.ts +199 -0
  58. package/src/lib/oauth/store.ts +77 -0
  59. package/src/lib/preferences.ts +12 -37
  60. package/src/lib/ratelimit.ts +4 -4
  61. package/src/lib/refresh-session.ts +161 -0
  62. package/src/lib/relay.ts +10 -8
  63. package/src/lib/secrets.ts +6 -7
  64. package/src/lib/sequencer.ts +14 -5
  65. package/src/lib/service-auth.ts +184 -0
  66. package/src/lib/session-tokens.ts +28 -76
  67. package/src/lib/streaming-car.ts +3 -0
  68. package/src/lib/tracing.ts +4 -3
  69. package/src/lib/util.ts +65 -15
  70. package/src/middleware.ts +1 -1
  71. package/src/pages/.well-known/did.json.ts +27 -30
  72. package/src/pages/.well-known/oauth-authorization-server.ts +31 -0
  73. package/src/pages/.well-known/oauth-protected-resource.ts +22 -0
  74. package/src/pages/debug/blob/[...key].ts +2 -2
  75. package/src/pages/debug/db/bootstrap.ts +1 -1
  76. package/src/pages/debug/db/commits.ts +1 -1
  77. package/src/pages/debug/gc/blobs.ts +1 -1
  78. package/src/pages/debug/record.ts +1 -1
  79. package/src/pages/debug/sequencer.ts +28 -0
  80. package/src/pages/health.ts +4 -4
  81. package/src/pages/oauth/authorize.ts +78 -0
  82. package/src/pages/oauth/consent.ts +80 -0
  83. package/src/pages/oauth/par.ts +121 -0
  84. package/src/pages/oauth/token.ts +158 -0
  85. package/src/pages/ready.ts +2 -2
  86. package/src/pages/xrpc/[...nsid].ts +61 -0
  87. package/src/pages/xrpc/app.bsky.actor.getPreferences.ts +12 -13
  88. package/src/pages/xrpc/app.bsky.actor.putPreferences.ts +23 -23
  89. package/src/pages/xrpc/app.bsky.unspecced.getAgeAssuranceState.ts +9 -2
  90. package/src/pages/xrpc/chat.bsky.convo.getLog.ts +9 -2
  91. package/src/pages/xrpc/chat.bsky.convo.listConvos.ts +9 -2
  92. package/src/pages/xrpc/com.atproto.identity.getRecommendedDidCredentials.ts +43 -41
  93. package/src/pages/xrpc/com.atproto.identity.requestPlcOperationSignature.ts +10 -3
  94. package/src/pages/xrpc/com.atproto.identity.resolveHandle.ts +40 -9
  95. package/src/pages/xrpc/com.atproto.identity.signPlcOperation.ts +41 -29
  96. package/src/pages/xrpc/com.atproto.identity.submitPlcOperation.ts +20 -6
  97. package/src/pages/xrpc/com.atproto.identity.updateHandle.ts +1 -1
  98. package/src/pages/xrpc/com.atproto.repo.applyWrites.ts +101 -11
  99. package/src/pages/xrpc/com.atproto.repo.createRecord.ts +44 -14
  100. package/src/pages/xrpc/com.atproto.repo.deleteRecord.ts +41 -13
  101. package/src/pages/xrpc/com.atproto.repo.describeRepo.ts +2 -2
  102. package/src/pages/xrpc/com.atproto.repo.getRecord.ts +14 -1
  103. package/src/pages/xrpc/com.atproto.repo.listMissingBlobs.ts +14 -6
  104. package/src/pages/xrpc/com.atproto.repo.listRecords.ts +1 -1
  105. package/src/pages/xrpc/com.atproto.repo.putRecord.ts +42 -14
  106. package/src/pages/xrpc/com.atproto.repo.uploadBlob.ts +76 -15
  107. package/src/pages/xrpc/com.atproto.server.checkAccountStatus.ts +20 -8
  108. package/src/pages/xrpc/com.atproto.server.createSession.ts +32 -12
  109. package/src/pages/xrpc/com.atproto.server.describeServer.ts +1 -1
  110. package/src/pages/xrpc/com.atproto.server.getServiceAuth.ts +12 -5
  111. package/src/pages/xrpc/com.atproto.server.getSession.ts +22 -8
  112. package/src/pages/xrpc/com.atproto.server.refreshSession.ts +30 -72
  113. package/src/pages/xrpc/com.atproto.sync.getBlob.ts +72 -23
  114. package/src/pages/xrpc/com.atproto.sync.getCheckout.json.ts +1 -1
  115. package/src/pages/xrpc/com.atproto.sync.getCheckout.ts +1 -1
  116. package/src/pages/xrpc/com.atproto.sync.getHead.ts +7 -2
  117. package/src/pages/xrpc/com.atproto.sync.getLatestCommit.ts +1 -1
  118. package/src/pages/xrpc/com.atproto.sync.getRecord.ts +5 -27
  119. package/src/pages/xrpc/com.atproto.sync.getRepo.json.ts +1 -1
  120. package/src/pages/xrpc/com.atproto.sync.getRepo.ts +50 -5
  121. package/src/pages/xrpc/com.atproto.sync.getRepoStatus.ts +58 -0
  122. package/src/pages/xrpc/com.atproto.sync.listBlobs.ts +2 -2
  123. package/src/pages/xrpc/com.atproto.sync.listRepos.ts +5 -3
  124. package/src/services/car.ts +209 -57
  125. package/src/services/r2-blob-store.ts +4 -4
  126. package/src/services/repo/blockstore-ops.ts +29 -0
  127. package/src/services/repo/operations.ts +133 -0
  128. package/src/services/repo-manager.ts +203 -254
  129. package/src/worker/runtime.ts +56 -11
  130. package/src/worker/sequencer/broadcast.ts +91 -0
  131. package/src/worker/sequencer/cid-helpers.ts +39 -0
  132. package/src/worker/sequencer/payload.ts +84 -0
  133. package/src/worker/sequencer/types.ts +36 -0
  134. package/src/worker/sequencer/upgrade.ts +141 -0
  135. package/src/worker/sequencer.ts +264 -406
  136. package/types/env.d.ts +18 -6
  137. package/src/pages/xrpc/app.bsky.actor.getProfile.ts +0 -49
  138. package/src/pages/xrpc/app.bsky.actor.getProfiles.ts +0 -51
  139. package/src/pages/xrpc/app.bsky.feed.getActorFeeds.ts +0 -25
  140. package/src/pages/xrpc/app.bsky.feed.getAuthorFeed.ts +0 -42
  141. package/src/pages/xrpc/app.bsky.feed.getFeedGenerators.ts +0 -25
  142. package/src/pages/xrpc/app.bsky.feed.getPostThread.ts +0 -37
  143. package/src/pages/xrpc/app.bsky.feed.getPosts.ts +0 -26
  144. package/src/pages/xrpc/app.bsky.feed.getSuggestedFeeds.ts +0 -23
  145. package/src/pages/xrpc/app.bsky.feed.getTimeline.ts +0 -47
  146. package/src/pages/xrpc/app.bsky.graph.getFollowers.ts +0 -29
  147. package/src/pages/xrpc/app.bsky.graph.getFollows.ts +0 -29
  148. package/src/pages/xrpc/app.bsky.notification.getUnreadCount.ts +0 -20
  149. package/src/pages/xrpc/app.bsky.notification.listNotifications.ts +0 -27
  150. package/src/pages/xrpc/app.bsky.unspecced.getSuggestedFeeds.ts +0 -23
@@ -1,65 +1,41 @@
1
- // Types via tsconfig
2
-
3
1
  import type { DurableObjectState, D1Database } from '@cloudflare/workers-types';
4
2
  import { drizzle } from 'drizzle-orm/d1';
5
3
  import { commit_log } from '../db/schema';
6
4
  import { gt, eq, desc } from 'drizzle-orm';
7
- import {
8
- createInfoFrame,
9
- createCommitFrame,
10
- createIdentityFrame,
11
- createAccountFrame,
12
- createErrorFrame,
13
- type CommitMessage,
14
- type RepoOp,
15
- } from '../lib/firehose/frames';
16
- import { checkCursor } from '../lib/firehose/validation';
17
- import { CID } from 'multiformats/cid';
18
- import { encodeBlocksForCommit } from '../services/car';
5
+ import { encodeInfoFrame, encodeCommitFrame } from '../lib/firehose/frames';
19
6
  import type { Env } from '../env';
7
+ import { fromWireStatus } from '../lib/account-state';
8
+ import type {
9
+ AccountEvent,
10
+ Client,
11
+ CommitEvent,
12
+ IdentityEvent,
13
+ } from './sequencer/types';
14
+ import { reviveOps, base64ToBytes } from './sequencer/cid-helpers';
15
+ import { createCommitPayload } from './sequencer/payload';
16
+ import {
17
+ handleUpgrade,
18
+ type HibernatableSocket,
19
+ type HibernatableState,
20
+ type WebSocketAttachment,
21
+ } from './sequencer/upgrade';
22
+ import {
23
+ broadcastAccount,
24
+ broadcastCommit,
25
+ broadcastIdentity,
26
+ } from './sequencer/broadcast';
27
+
28
+ export type {
29
+ AccountEvent,
30
+ Client,
31
+ CommitEvent,
32
+ IdentityEvent,
33
+ SequencerEvent,
34
+ } from './sequencer/types';
20
35
 
21
- interface Client {
22
- webSocket: WebSocket;
23
- id: string;
24
- cursor: number;
25
- }
26
-
27
- interface CommitEvent {
28
- seq: number;
29
- did: string;
30
- commitCid: string;
31
- rev: string;
32
- data: string; // JSON-encoded commit data
33
- sig: string; // base64 signature
34
- ts: number;
35
- ops?: RepoOp[];
36
- blocks?: Uint8Array;
37
- }
38
-
39
- interface IdentityEvent {
40
- seq: number;
41
- did: string;
42
- handle?: string;
43
- ts: number;
44
- }
45
-
46
- interface AccountEvent {
47
- seq: number;
48
- did: string;
49
- active: boolean;
50
- status?: string;
51
- ts: number;
52
- }
53
-
54
- type SequencerEvent = CommitEvent | IdentityEvent | AccountEvent;
55
-
56
- /**
57
- * Sequencer Durable Object
58
- * Manages the firehose event stream for repository updates
59
- */
60
36
  export class Sequencer {
61
- private readonly state: DurableObjectState;
62
- private readonly env: Env & { PDS_SEQ_WINDOW?: string };
37
+ private readonly state: HibernatableState;
38
+ private readonly env: Env;
63
39
  private readonly clients = new Map<string, Client>();
64
40
  private buffer: CommitEvent[] = [];
65
41
  private readonly db: D1Database;
@@ -67,17 +43,39 @@ export class Sequencer {
67
43
  private nextSeq = 1;
68
44
  private droppedFrameCount = 0;
69
45
 
70
- constructor(state: DurableObjectState, env: Env & { PDS_SEQ_WINDOW?: string }) {
71
- this.state = state;
46
+ constructor(state: DurableObjectState, env: Env) {
47
+ this.state = state as HibernatableState;
72
48
  this.env = env;
73
- this.db = env.DB;
49
+ this.db = env.ALTERAN_DB;
74
50
  this.maxWindow = parseInt(env.PDS_SEQ_WINDOW || '512', 10);
75
51
 
76
- // Initialize from storage
52
+ // Reconcile nextSeq with DB on construction so replay and append agree
53
+ // after worker restarts or DO migrations.
77
54
  this.state.blockConcurrencyWhile(async () => {
78
- const stored = await this.state.storage.get<number>('nextSeq');
79
- if (stored) {
80
- this.nextSeq = stored;
55
+ let base = 0;
56
+ try {
57
+ base = (await this.state.storage.get<number>('nextSeq')) || 0;
58
+ } catch (storageError) {
59
+ console.warn('Sequencer: storage.get(nextSeq) failed:', storageError);
60
+ }
61
+ try {
62
+ const db = drizzle(this.db);
63
+ const last = await db
64
+ .select({ seq: commit_log.seq })
65
+ .from(commit_log)
66
+ .orderBy(desc(commit_log.seq))
67
+ .limit(1)
68
+ .get();
69
+ const dbNext = last?.seq ? Number(last.seq) + 1 : 1;
70
+ if (!base || dbNext > base) base = dbNext;
71
+ } catch (dbError) {
72
+ console.warn('Sequencer: commit_log max(seq) failed:', dbError);
73
+ }
74
+ this.nextSeq = base > 0 ? base : 1;
75
+ try {
76
+ await this.state.storage.put('nextSeq', this.nextSeq);
77
+ } catch (storageError) {
78
+ console.warn('Sequencer: storage.put(nextSeq) failed:', storageError);
81
79
  }
82
80
  });
83
81
  }
@@ -85,18 +83,16 @@ export class Sequencer {
85
83
  async fetch(request: Request): Promise<Response> {
86
84
  const url = new URL(request.url);
87
85
 
88
- // Handle event notifications from PDS
89
86
  if (request.method === 'POST') {
90
- if (url.pathname === '/commit') {
91
- return this.handleCommitNotification(request);
92
- } else if (url.pathname === '/identity') {
93
- return this.handleIdentityNotification(request);
94
- } else if (url.pathname === '/account') {
95
- return this.handleAccountNotification(request);
96
- }
87
+ if (url.pathname === '/commit') return this.handleCommitNotification(request);
88
+ if (url.pathname === '/identity') return this.handleIdentityNotification(request);
89
+ if (url.pathname === '/account') return this.handleAccountNotification(request);
90
+ }
91
+
92
+ if (request.method === 'GET' && url.pathname === '/metrics') {
93
+ return this.handleMetrics();
97
94
  }
98
95
 
99
- // Handle WebSocket upgrade for firehose subscription
100
96
  const upgradeHeader = request.headers.get('Upgrade');
101
97
  if (upgradeHeader !== 'websocket') {
102
98
  return new Response('Expected websocket', { status: 426 });
@@ -105,9 +101,6 @@ export class Sequencer {
105
101
  return this.handleWebSocketUpgrade(request, url);
106
102
  }
107
103
 
108
- /**
109
- * Handle commit notification from PDS
110
- */
111
104
  private async handleCommitNotification(request: Request): Promise<Response> {
112
105
  try {
113
106
  const body = (await request.json()) as {
@@ -116,50 +109,79 @@ export class Sequencer {
116
109
  rev: string;
117
110
  data: string;
118
111
  sig: string;
119
- ops?: RepoOp[];
120
- blocks?: string; // base64-encoded CAR
112
+ ops?: unknown;
113
+ blocks?: string;
121
114
  };
122
115
 
123
- // Helper: base64 to Uint8Array (workers-safe)
124
- const b64ToBytes = (b64: string): Uint8Array => {
125
- const bin = atob(b64);
126
- const out = new Uint8Array(bin.length);
127
- for (let i = 0; i < bin.length; i++) out[i] = bin.charCodeAt(i);
128
- return out;
129
- };
116
+ const ops = reviveOps(body.ops);
117
+
118
+ const db = drizzle(this.db);
119
+ let seqForEvent: number | null = null;
120
+ let tsForEvent = Date.now();
121
+ try {
122
+ const row = await db
123
+ .select({
124
+ seq: commit_log.seq,
125
+ rev: commit_log.rev,
126
+ data: commit_log.data,
127
+ sig: commit_log.sig,
128
+ ts: commit_log.ts,
129
+ })
130
+ .from(commit_log)
131
+ .where(eq(commit_log.cid, body.commitCid))
132
+ .limit(1)
133
+ .get();
134
+ if (row && typeof row.seq === 'number') {
135
+ seqForEvent = row.seq;
136
+ body.rev = row.rev;
137
+ body.data = row.data;
138
+ body.sig = row.sig;
139
+ tsForEvent = row.ts ?? tsForEvent;
140
+ }
141
+ } catch (lookupError) {
142
+ console.warn('commit_log lookup failed:', lookupError);
143
+ }
144
+
145
+ if (seqForEvent == null) {
146
+ seqForEvent = this.nextSeq++;
147
+ await this.state.storage.put('nextSeq', this.nextSeq);
148
+ try {
149
+ await db
150
+ .insert(commit_log)
151
+ .values({
152
+ seq: seqForEvent,
153
+ cid: body.commitCid,
154
+ rev: body.rev,
155
+ data: body.data,
156
+ sig: body.sig,
157
+ ts: tsForEvent,
158
+ })
159
+ .run();
160
+ } catch (insertError) {
161
+ console.warn('commit_log insert failed:', insertError);
162
+ }
163
+ } else if (seqForEvent >= this.nextSeq) {
164
+ this.nextSeq = seqForEvent + 1;
165
+ try {
166
+ await this.state.storage.put('nextSeq', this.nextSeq);
167
+ } catch (putError) {
168
+ console.warn('Sequencer: storage.put(nextSeq) failed:', putError);
169
+ }
170
+ }
130
171
 
131
172
  const event: CommitEvent = {
132
- seq: this.nextSeq++,
173
+ seq: seqForEvent,
133
174
  did: body.did,
134
175
  commitCid: body.commitCid,
135
176
  rev: body.rev,
136
177
  data: body.data,
137
178
  sig: body.sig,
138
- ts: Date.now(),
139
- ops: body.ops,
140
- blocks: body.blocks ? b64ToBytes(body.blocks) : undefined,
179
+ ts: tsForEvent,
180
+ ops,
181
+ blocks: body.blocks ? base64ToBytes(body.blocks) : undefined,
141
182
  };
142
183
 
143
- // Persist sequence number
144
- await this.state.storage.put('nextSeq', this.nextSeq);
145
-
146
- // Update commit_log with assigned sequence for this commit (if row exists)
147
- try {
148
- const db = drizzle(this.db);
149
- const res = await db.update(commit_log).set({ seq: event.seq }).where(eq(commit_log.cid, event.commitCid)).run();
150
- // If the row didn't exist (unexpected), insert a minimal row so replay works
151
- // Note: drizzle's run() returns a driver-specific result; we just best-effort insert
152
- if ((res as any)?.success === false) {
153
- await db.insert(commit_log).values({ seq: event.seq, cid: event.commitCid, rev: event.rev, data: event.data, sig: event.sig, ts: event.ts }).run();
154
- }
155
- } catch (e) {
156
- console.warn('commit_log seq update failed:', e);
157
- }
158
-
159
- // Add to buffer
160
184
  this.appendCommit(event);
161
-
162
- // Broadcast to all connected clients
163
185
  await this.broadcastCommit(event);
164
186
 
165
187
  return new Response('ok');
@@ -169,29 +191,21 @@ export class Sequencer {
169
191
  }
170
192
  }
171
193
 
172
- /**
173
- * Handle identity notification from PDS (handle changes)
174
- */
175
194
  private async handleIdentityNotification(request: Request): Promise<Response> {
176
195
  try {
177
- const body = (await request.json()) as {
178
- did: string;
179
- handle?: string;
180
- };
181
-
196
+ const body = (await request.json()) as { did: string; handle?: string };
182
197
  const event: IdentityEvent = {
183
198
  seq: this.nextSeq++,
184
199
  did: body.did,
185
200
  handle: body.handle,
186
201
  ts: Date.now(),
187
202
  };
188
-
189
- // Persist sequence number
190
- await this.state.storage.put('nextSeq', this.nextSeq);
191
-
192
- // Broadcast to all connected clients
203
+ try {
204
+ await this.state.storage.put('nextSeq', this.nextSeq);
205
+ } catch (putError) {
206
+ console.warn('Sequencer: storage.put(nextSeq) failed:', putError);
207
+ }
193
208
  await this.broadcastIdentity(event);
194
-
195
209
  return new Response('ok');
196
210
  } catch (error) {
197
211
  console.error('Failed to handle identity notification:', error);
@@ -199,31 +213,21 @@ export class Sequencer {
199
213
  }
200
214
  }
201
215
 
202
- /**
203
- * Handle account notification from PDS (account status changes)
204
- */
205
216
  private async handleAccountNotification(request: Request): Promise<Response> {
206
217
  try {
207
- const body = (await request.json()) as {
208
- did: string;
209
- active: boolean;
210
- status?: string;
211
- };
212
-
218
+ const body = (await request.json()) as { did: string; active: boolean; status?: string };
213
219
  const event: AccountEvent = {
214
220
  seq: this.nextSeq++,
215
221
  did: body.did,
216
- active: body.active,
217
- status: body.status,
222
+ state: fromWireStatus({ active: body.active, status: body.status }),
218
223
  ts: Date.now(),
219
224
  };
220
-
221
- // Persist sequence number
222
- await this.state.storage.put('nextSeq', this.nextSeq);
223
-
224
- // Broadcast to all connected clients
225
+ try {
226
+ await this.state.storage.put('nextSeq', this.nextSeq);
227
+ } catch (putError) {
228
+ console.warn('Sequencer: storage.put(nextSeq) failed:', putError);
229
+ }
225
230
  await this.broadcastAccount(event);
226
-
227
231
  return new Response('ok');
228
232
  } catch (error) {
229
233
  console.error('Failed to handle account notification:', error);
@@ -231,291 +235,110 @@ export class Sequencer {
231
235
  }
232
236
  }
233
237
 
234
- /**
235
- * Handle WebSocket upgrade for firehose subscription
236
- */
237
- private async handleWebSocketUpgrade(request: Request, url: URL): Promise<Response> {
238
- const pair = new WebSocketPair();
239
- const [client, server] = Object.values(pair);
240
- const id = crypto.randomUUID();
241
-
242
- // Parse cursor parameter
243
- const cursorParam = url.searchParams.get('cursor');
244
- const cursor = cursorParam ? parseInt(cursorParam, 10) : 0;
245
-
246
- // Validate cursor
247
- if (cursor > this.nextSeq - 1) {
248
- // Future cursor error
249
- const err = checkCursor(cursor, this.nextSeq - 1) ?? createErrorFrame('FutureCursor', 'Cursor is ahead of current sequence').toFramedBytes();
250
- server.send(err);
251
- server.close(1008, 'FutureCursor');
252
- return new Response(null, { status: 101, webSocket: client });
253
- }
254
-
255
- // CRITICAL: Use Cloudflare's hibernatable WebSocket API
256
- // This keeps the connection alive even after the response is returned
257
- // Without this, the WebSocket closes immediately when the worker context ends
258
- this.state.acceptWebSocket(server as any, [id, cursor.toString()]);
259
-
260
- const clientObj: Client = { webSocket: server as unknown as WebSocket, id, cursor };
261
- this.clients.set(id, clientObj);
262
-
263
- // Send #info frame on connection
264
- const infoFrame = createInfoFrame('com.atproto.sync.subscribeRepos', 'Connected to PDS firehose');
265
- try {
266
- server.send(infoFrame.toFramedBytes());
267
- } catch (error) {
268
- console.error('Failed to send info frame:', error);
269
- }
270
-
271
- // Replay buffered events if cursor provided
272
- if (cursor > 0) {
273
- await this.replayFromCursor(server as unknown as WebSocket, cursor);
274
- }
275
-
276
- return new Response(null, { status: 101, webSocket: client });
238
+ private handleWebSocketUpgrade(request: Request, url: URL): Response {
239
+ const hibernate = String(this.env.PDS_WS_HIBERNATE ?? 'true').toLowerCase() !== 'false';
240
+ return handleUpgrade(request, url, {
241
+ state: this.state,
242
+ nextSeq: this.nextSeq,
243
+ hibernate,
244
+ onClient: (id, cursor, server) => {
245
+ this.clients.set(id, { webSocket: server, id, cursor });
246
+ },
247
+ });
277
248
  }
278
249
 
279
- /**
280
- * Replay events from cursor
281
- */
282
250
  private async replayFromCursor(ws: WebSocket, cursor: number): Promise<void> {
283
- // First try from buffer
284
251
  const bufferedEvents = this.buffer.filter((e) => e.seq > cursor);
285
252
 
286
253
  if (bufferedEvents.length > 0) {
287
254
  for (const event of bufferedEvents) {
288
255
  try {
289
- const frame = await this.createCommitFrame(event);
290
- ws.send(frame.toFramedBytes());
256
+ const message = await createCommitPayload(this.env, this.db, event);
257
+ ws.send(encodeCommitFrame(message));
291
258
  } catch (error) {
292
259
  console.error('Failed to send buffered event:', error);
293
260
  }
294
261
  }
295
- } else {
296
- // Fetch from database if not in buffer
297
- try {
298
- const db = drizzle(this.db);
299
- const events = await db
300
- .select()
301
- .from(commit_log)
302
- .where(gt(commit_log.seq, cursor))
303
- .orderBy(commit_log.seq)
304
- .limit(100)
305
- .all();
306
-
307
- for (const event of events) {
308
- try {
309
- const commitEvent: CommitEvent = {
310
- seq: event.seq!,
311
- did: JSON.parse(event.data).did,
312
- commitCid: event.cid,
313
- rev: event.rev,
314
- data: event.data,
315
- sig: event.sig,
316
- ts: event.ts,
317
- };
318
- const frame = await this.createCommitFrame(commitEvent);
319
- ws.send(frame.toFramedBytes());
320
- } catch (error) {
321
- console.error('Failed to send database event:', error);
322
- }
323
- }
324
- } catch (error) {
325
- console.error('Failed to fetch events from database:', error);
326
- }
262
+ return;
327
263
  }
328
- }
329
-
330
- /**
331
- * Broadcast commit event to all connected clients
332
- */
333
- private async broadcastCommit(event: CommitEvent): Promise<void> {
334
- const frame = await this.createCommitFrame(event);
335
- const bytes = frame.toFramedBytes();
336
264
 
337
- const disconnected: string[] = [];
338
-
339
- for (const [id, client] of Array.from(this.clients.entries())) {
340
- try {
341
- // Check if client's cursor is caught up
342
- if (event.seq > client.cursor) {
343
- client.webSocket.send(bytes);
344
- client.cursor = event.seq;
265
+ try {
266
+ const db = drizzle(this.db);
267
+ const events = await db
268
+ .select()
269
+ .from(commit_log)
270
+ .where(gt(commit_log.seq, cursor))
271
+ .orderBy(commit_log.seq)
272
+ .limit(100)
273
+ .all();
274
+
275
+ for (const event of events) {
276
+ try {
277
+ if (event.seq == null) continue;
278
+ const commitEvent: CommitEvent = {
279
+ seq: event.seq,
280
+ did: JSON.parse(event.data).did,
281
+ commitCid: event.cid,
282
+ rev: event.rev,
283
+ data: event.data,
284
+ sig: event.sig,
285
+ ts: event.ts,
286
+ };
287
+ const message = await createCommitPayload(this.env, this.db, commitEvent);
288
+ ws.send(encodeCommitFrame(message));
289
+ } catch (error) {
290
+ console.error('Failed to send database event:', error);
345
291
  }
346
- } catch (error) {
347
- console.error(`Failed to send to client ${id}:`, error);
348
- disconnected.push(id);
349
292
  }
350
- }
351
-
352
- // Clean up disconnected clients
353
- for (const id of disconnected) {
354
- this.clients.delete(id);
293
+ } catch (error) {
294
+ console.error('Failed to fetch events from database:', error);
355
295
  }
356
296
  }
357
297
 
358
- /**
359
- * Broadcast identity event to all connected clients
360
- */
361
- private async broadcastIdentity(event: IdentityEvent): Promise<void> {
362
- const frame = createIdentityFrame({
363
- seq: event.seq,
364
- did: event.did,
365
- time: new Date(event.ts).toISOString(),
366
- handle: event.handle,
367
- });
368
- const bytes = frame.toFramedBytes();
369
-
370
- const disconnected: string[] = [];
371
-
372
- for (const [id, client] of Array.from(this.clients.entries())) {
373
- try {
374
- if (event.seq > client.cursor) {
375
- client.webSocket.send(bytes);
376
- client.cursor = event.seq;
377
- }
378
- } catch (error) {
379
- console.error(`Failed to send to client ${id}:`, error);
380
- disconnected.push(id);
381
- }
382
- }
383
-
384
- for (const id of disconnected) {
385
- this.clients.delete(id);
298
+ private getSocketTargets(): WebSocket[] {
299
+ let sockets: WebSocket[] = [];
300
+ try {
301
+ // workers-types WebSocket misses a few DOM-types members; the values
302
+ // are wire-compatible at runtime, so widen through unknown.
303
+ sockets = (this.state.getWebSockets?.() || []) as unknown as WebSocket[];
304
+ } catch (error) {
305
+ console.warn('Sequencer: getWebSockets failed:', error);
386
306
  }
307
+ return sockets.length > 0
308
+ ? sockets
309
+ : Array.from(this.clients.values()).map((c) => c.webSocket);
387
310
  }
388
311
 
389
- /**
390
- * Broadcast account event to all connected clients
391
- */
392
- private async broadcastAccount(event: AccountEvent): Promise<void> {
393
- const accountFrame = createAccountFrame({
394
- seq: event.seq,
395
- did: event.did,
396
- time: new Date(event.ts).toISOString(),
397
- active: event.active,
398
- status: event.status,
399
- });
400
- // Emit compatibility #sync frame as well
401
- const { createSyncFrame } = await import('../lib/firehose/frames');
402
- const syncLike = createSyncFrame({
403
- seq: event.seq,
404
- did: event.did,
405
- time: new Date(event.ts).toISOString(),
406
- active: event.active,
407
- status: event.status,
408
- });
409
- const bytesAccount = accountFrame.toFramedBytes();
410
- const bytesSync = syncLike.toFramedBytes();
411
-
412
- const disconnected: string[] = [];
413
-
414
- for (const [id, client] of Array.from(this.clients.entries())) {
415
- try {
416
- if (event.seq > client.cursor) {
417
- client.webSocket.send(bytesAccount);
418
- client.webSocket.send(bytesSync);
419
- client.cursor = event.seq;
420
- }
421
- } catch (error) {
422
- console.error(`Failed to send to client ${id}:`, error);
423
- disconnected.push(id);
424
- }
425
- }
426
-
427
- for (const id of disconnected) {
428
- this.clients.delete(id);
429
- }
312
+ private broadcastCommit(event: CommitEvent): Promise<void> {
313
+ return broadcastCommit(this.env, this.db, this.getSocketTargets(), event);
430
314
  }
431
315
 
432
- /**
433
- * Create a #commit frame from event
434
- */
435
- private async createCommitFrame(event: CommitEvent): Promise<ReturnType<typeof createCommitFrame>> {
436
- const commitData = JSON.parse(event.data);
437
-
438
- // If blocks weren't provided, encode them now
439
- let blocks = event.blocks;
440
- if (!blocks && event.ops && event.ops.length > 0) {
441
- try {
442
- const commitCid = CID.parse(event.commitCid);
443
- // Extract MST root from commit data
444
- const mstRoot = commitData.data ? CID.parse(commitData.data) : commitCid;
445
- blocks = await encodeBlocksForCommit(
446
- this.env as Env,
447
- commitCid,
448
- mstRoot,
449
- event.ops,
450
- );
451
- } catch (error) {
452
- console.error('Failed to encode blocks for commit:', error);
453
- blocks = new Uint8Array();
454
- }
455
- }
456
-
457
- // Resolve prev commit and since (previous rev) when available
458
- let prevCid: CID | null = null;
459
- try {
460
- if (commitData.prev) prevCid = CID.parse(String(commitData.prev));
461
- } catch {}
462
-
463
- let since: string | null = null;
464
- try {
465
- const db = drizzle(this.db);
466
- if (prevCid) {
467
- const prev = await db.select().from(commit_log).where(eq(commit_log.cid, prevCid.toString())).get();
468
- since = prev?.rev ?? null;
469
- } else {
470
- const row = await db.select().from(commit_log).where(gt(commit_log.seq, 0 as any)).orderBy(desc(commit_log.seq)).limit(1).get();
471
- since = row?.rev ?? null;
472
- }
473
- } catch {}
474
-
475
- const message: CommitMessage = {
476
- seq: event.seq,
477
- rebase: false,
478
- tooBig: false,
479
- repo: event.did,
480
- commit: CID.parse(event.commitCid),
481
- prev: prevCid,
482
- rev: event.rev,
483
- since,
484
- blocks: blocks || new Uint8Array(),
485
- ops: event.ops || [],
486
- blobs: [],
487
- time: new Date(event.ts).toISOString(),
488
- };
316
+ private broadcastIdentity(event: IdentityEvent): void {
317
+ broadcastIdentity(this.getSocketTargets(), event);
318
+ }
489
319
 
490
- return createCommitFrame(message);
320
+ private broadcastAccount(event: AccountEvent): void {
321
+ broadcastAccount(this.getSocketTargets(), event);
491
322
  }
492
323
 
493
- /**
494
- * Append commit event to buffer with backpressure
495
- */
496
324
  private appendCommit(event: CommitEvent): void {
497
325
  this.buffer.push(event);
498
326
 
499
- // Implement backpressure: drop oldest events if buffer is full
500
327
  if (this.buffer.length > this.maxWindow) {
501
328
  const dropped = this.buffer.shift();
502
329
  this.droppedFrameCount++;
503
- console.warn(`Dropped event seq=${dropped?.seq} due to backpressure (total dropped: ${this.droppedFrameCount})`);
504
-
505
- // Send #info frame to all clients about dropped frames
330
+ console.warn(
331
+ `Dropped event seq=${dropped?.seq} due to backpressure (total dropped: ${this.droppedFrameCount})`,
332
+ );
506
333
  this.notifyFramesDropped();
507
334
  }
508
335
  }
509
336
 
510
- /**
511
- * Notify all clients that frames were dropped
512
- */
513
337
  private notifyFramesDropped(): void {
514
- const infoFrame = createInfoFrame(
338
+ const bytes = encodeInfoFrame(
515
339
  'FramesDropped',
516
340
  `${this.droppedFrameCount} frame(s) dropped due to backpressure`,
517
341
  );
518
- const bytes = infoFrame.toFramedBytes();
519
342
 
520
343
  for (const [id, client] of Array.from(this.clients.entries())) {
521
344
  try {
@@ -526,9 +349,6 @@ export class Sequencer {
526
349
  }
527
350
  }
528
351
 
529
- /**
530
- * Get metrics
531
- */
532
352
  getMetrics(): {
533
353
  connectedClients: number;
534
354
  bufferSize: number;
@@ -543,12 +363,7 @@ export class Sequencer {
543
363
  };
544
364
  }
545
365
 
546
- /**
547
- * WebSocket hibernation handler: called when a message is received
548
- * This is required for Cloudflare's hibernatable WebSocket API
549
- */
550
366
  async webSocketMessage(ws: WebSocket, message: string | ArrayBuffer): Promise<void> {
551
- // Find client by WebSocket instance
552
367
  const client = Array.from(this.clients.values()).find((c) => c.webSocket === ws);
553
368
  if (!client) {
554
369
  console.warn('Received message from unknown WebSocket');
@@ -557,34 +372,77 @@ export class Sequencer {
557
372
 
558
373
  try {
559
374
  const data = typeof message === 'string' ? message : new TextDecoder().decode(message);
560
- if (data === 'ping') {
561
- ws.send('pong');
562
- }
375
+ if (data === 'ping') ws.send('pong');
563
376
  } catch (error) {
564
377
  console.error('WebSocket message error:', error);
565
378
  }
566
379
  }
567
380
 
568
- /**
569
- * WebSocket hibernation handler: called when connection closes
570
- * This is required for Cloudflare's hibernatable WebSocket API
571
- */
572
- async webSocketClose(ws: WebSocket, code: number, reason: string, wasClean: boolean): Promise<void> {
573
- // Find and remove client
574
- const entry = Array.from(this.clients.entries()).find(([_, c]) => c.webSocket === ws);
381
+ private async handleMetrics(): Promise<Response> {
382
+ const base = this.getMetrics();
383
+ let sockets: WebSocket[] = [];
384
+ try {
385
+ // workers-types WebSocket misses a few DOM-types members; the values
386
+ // are wire-compatible at runtime, so widen through unknown.
387
+ sockets = (this.state.getWebSockets?.() || []) as unknown as WebSocket[];
388
+ } catch (error) {
389
+ console.warn('Sequencer: getWebSockets failed in metrics:', error);
390
+ }
391
+ const clients = sockets.map((ws) => {
392
+ let attachment: WebSocketAttachment | undefined;
393
+ try {
394
+ attachment = (ws as HibernatableSocket).deserializeAttachment?.();
395
+ } catch (attachError) {
396
+ console.warn('Sequencer: deserializeAttachment failed in metrics:', attachError);
397
+ }
398
+ return { attachment: attachment ?? null };
399
+ });
400
+ const body = {
401
+ ...base,
402
+ hibernatedSockets: sockets.length,
403
+ clients,
404
+ };
405
+ return new Response(JSON.stringify(body), {
406
+ headers: { 'Content-Type': 'application/json' },
407
+ });
408
+ }
409
+
410
+ async webSocketOpen(ws: WebSocket): Promise<void> {
411
+ console.log(JSON.stringify({ level: 'info', type: 'ws_open', ts: new Date().toISOString() }));
412
+
413
+ let cursor: number | undefined;
414
+ try {
415
+ const attachment = (ws as HibernatableSocket).deserializeAttachment?.();
416
+ if (attachment && typeof attachment.cursor === 'number') cursor = attachment.cursor;
417
+ } catch (attachError) {
418
+ console.warn('Sequencer: deserializeAttachment failed on open:', attachError);
419
+ }
420
+ if (cursor == null) {
421
+ const client = Array.from(this.clients.values()).find((c) => c.webSocket === ws);
422
+ if (client) cursor = client.cursor;
423
+ }
424
+ if (cursor && cursor > 0) {
425
+ await this.replayFromCursor(ws, cursor);
426
+ }
427
+ }
428
+
429
+ async webSocketClose(
430
+ ws: WebSocket,
431
+ code: number,
432
+ reason: string,
433
+ wasClean: boolean,
434
+ ): Promise<void> {
435
+ const entry = Array.from(this.clients.entries()).find(([, c]) => c.webSocket === ws);
575
436
  if (entry) {
576
437
  this.clients.delete(entry[0]);
577
- console.log(`Client ${entry[0]} disconnected: code=${code} reason="${reason}" clean=${wasClean}`);
438
+ console.log(
439
+ `Client ${entry[0]} disconnected: code=${code} reason="${reason}" clean=${wasClean}`,
440
+ );
578
441
  }
579
442
  }
580
443
 
581
- /**
582
- * WebSocket hibernation handler: called when an error occurs
583
- * This is required for Cloudflare's hibernatable WebSocket API
584
- */
585
444
  async webSocketError(ws: WebSocket, error: unknown): Promise<void> {
586
- // Find and remove client
587
- const entry = Array.from(this.clients.entries()).find(([_, c]) => c.webSocket === ws);
445
+ const entry = Array.from(this.clients.entries()).find(([, c]) => c.webSocket === ws);
588
446
  if (entry) {
589
447
  this.clients.delete(entry[0]);
590
448
  console.error(`Client ${entry[0]} error:`, error);