@syncular/server-workers 0.78.3 → 0.79.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -88,13 +88,27 @@
88
88
  * sessions, and re-arms while sockets remain. The host owns the alarm: a
89
89
  * subclass that needs another alarm schedule cannot share it.
90
90
  *
91
- * ## The wake path (external-command fan-out)
91
+ * ## Fanout between objects (§8.9)
92
92
  *
93
93
  * Ordinary HTTP `/sync` and `/operations` (with `RealtimeDOConfig.operations`)
94
- * are forwarded into this DO and fan out in-process. An
95
- * external authoritative command host that already provides equivalent D1
96
- * partition serialization may call `/__wake` after its own commit so sockets
97
- * re-pull. See `durableObjectRealtimeNotifier` for that caller side.
94
+ * are forwarded into this DO and fan out in-process. A commit applied
95
+ * elsewhere reaches the partition's object through
96
+ * `durableObjectRealtimeNotifier`: a POST to the fanout path carrying the
97
+ * commit, which the object delivers to its hub as a delta.
98
+ *
99
+ * On PostgreSQL the Worker may shard sockets over several objects
100
+ * (`WorkersRealtimeOptions.shard`) and run every round itself. The
101
+ * partition's object (`idFromName(partition)`) then holds no socket: it keeps
102
+ * the names of the shard objects that registered with it in its storage and
103
+ * relays every fanout message to each of them. A shard registers when it
104
+ * accepts a socket, before its hub connects it; a shard that answers a relay with 404 (no socket
105
+ * left) is removed unless it registered again meanwhile. A shard's hub
106
+ * (`RealtimeHubConfig.fanout`) sends the commits of its own rounds through the
107
+ * partition's object too, so every shard, the shard itself included, receives
108
+ * each commit once. A host scope refresh (`refreshScopes` on the notifier)
109
+ * takes the same path. A relay that fails leaves a record of what the shard
110
+ * owes in the partition object's storage, and the alarm retries it every
111
+ * second until the shard answers or is removed.
98
112
  */
99
113
  import {
100
114
  adapterSyncError,
@@ -118,9 +132,13 @@ import {
118
132
  type RemoteOperationRegistry,
119
133
  type RemoteOperationResolver,
120
134
  RingBufferEvents,
135
+ type RealtimeNotifier,
121
136
  type ServerStorage,
122
137
  SSP2_CONTENT_TYPE,
138
+ type StoredCommit,
123
139
  SyncError,
140
+ type SyncularServerEvents,
141
+ emitEvent,
124
142
  SyncularAdmin,
125
143
  type SyncularErrorRoute,
126
144
  type SyncRequestContext,
@@ -135,15 +153,36 @@ import { decodeRequest } from '@syncular/core';
135
153
 
136
154
  export interface DurableObjectStateLike {
137
155
  acceptWebSocket(ws: WebSocketLike, tags?: string[]): void;
156
+ /** Keeps work alive after a response: the notifier's retries (§8.9). */
157
+ waitUntil(promise: Promise<unknown>): void;
138
158
  getWebSockets(tag?: string): WebSocketLike[];
139
- /** The alarm half of the DO storage API; the host owns the alarm and
140
- * drives §8.5 heartbeats with it. */
159
+ /** The DO storage API the host uses: the alarm drives §8.5 heartbeats, and
160
+ * the partition's object keeps its registered shards as keys. */
141
161
  readonly storage: {
142
162
  getAlarm(): Promise<number | null>;
143
163
  setAlarm(scheduledTime: number): Promise<void>;
164
+ get(key: string): Promise<unknown>;
165
+ put(key: string, value: string): Promise<void>;
166
+ delete(key: string): Promise<boolean>;
167
+ list(options: { prefix: string }): Promise<Map<string, unknown>>;
144
168
  };
145
169
  }
146
170
 
171
+ /** A DO stub: the callable handle to one Durable Object instance. */
172
+ export interface DurableObjectStubLike {
173
+ fetch(request: Request): Promise<Response>;
174
+ }
175
+
176
+ /** A DO namespace binding: `idFromName` → `get(id)` → a stub. */
177
+ export interface DurableObjectNamespaceLike {
178
+ idFromName(name: string): DurableObjectIdLike;
179
+ get(id: DurableObjectIdLike): DurableObjectStubLike;
180
+ }
181
+
182
+ export interface DurableObjectIdLike {
183
+ toString(): string;
184
+ }
185
+
147
186
  export interface WebSocketLike {
148
187
  accept?(): void;
149
188
  send(data: string | ArrayBuffer | ArrayBufferView): void;
@@ -211,6 +250,15 @@ export type RealtimeDOConfig<S extends RealtimeHostStorage = D1ServerStorage> =
211
250
  * the scope resolver per request (REMOTE.md §5.1).
212
251
  */
213
252
  readonly operations?: RemoteOperationRegistry | RemoteOperationResolver;
253
+ /**
254
+ * The namespace this class is bound to. Required when the Worker shards
255
+ * sockets (`WorkersRealtimeOptions.shard`, §8.9): a shard registers with
256
+ * the partition's object through it, the partition's object relays every
257
+ * commit to the registered shards, and a shard's own rounds send their
258
+ * commits through the partition's object. The storage must lock
259
+ * partitions in the database (PostgreSQL); a D1 host refuses it.
260
+ */
261
+ readonly namespace?: DurableObjectNamespaceLike;
214
262
  };
215
263
 
216
264
  /**
@@ -271,6 +319,199 @@ function encodeAttachment(
271
319
  return Uint8Array.from(bytes);
272
320
  }
273
321
 
322
+ /**
323
+ * §8.9: what a shard missed when a relay to it failed, delivered from the
324
+ * partition object's alarm: the highest commit sequence (sessions behind it get
325
+ * a `catchup-required` wake), a fail-closed refresh of every session's scopes,
326
+ * and changed registrations (a wake, whose round reloads them).
327
+ */
328
+ interface Owed {
329
+ readonly commitSeq?: number;
330
+ readonly refresh?: true;
331
+ readonly registrations?: true;
332
+ }
333
+
334
+ /** A message between objects (§8.9): a commit, a client's changed
335
+ * registrations, a host refresh of scopes, or what a failed relay owes. */
336
+ type FanoutMessage =
337
+ | { readonly partition: string; readonly commit: StoredCommit }
338
+ | { readonly partition: string; readonly clientId: string }
339
+ | {
340
+ readonly partition: string;
341
+ readonly refresh: { readonly actorId?: string };
342
+ }
343
+ | { readonly partition: string; readonly owed: Owed };
344
+
345
+ /**
346
+ * Fanout body: a u32 length, the message as UTF-8 JSON with each change
347
+ * payload replaced by its byte length, then the payloads in change order.
348
+ */
349
+ function encodeFanout(message: FanoutMessage): Uint8Array<ArrayBuffer> {
350
+ const payloads: Uint8Array[] = [];
351
+ const head = new TextEncoder().encode(
352
+ JSON.stringify(
353
+ 'commit' in message
354
+ ? {
355
+ ...message,
356
+ commit: {
357
+ ...message.commit,
358
+ changes: message.commit.changes.map(({ payload, ...change }) => {
359
+ if (payload === undefined) return change;
360
+ payloads.push(payload);
361
+ return { ...change, payloadBytes: payload.byteLength };
362
+ }),
363
+ },
364
+ }
365
+ : message,
366
+ ),
367
+ );
368
+ const bytes = new Uint8Array(
369
+ payloads.reduce((total, p) => total + p.byteLength, 4 + head.byteLength),
370
+ );
371
+ new DataView(bytes.buffer).setUint32(0, head.byteLength);
372
+ bytes.set(head, 4);
373
+ let at = 4 + head.byteLength;
374
+ for (const payload of payloads) {
375
+ bytes.set(payload, at);
376
+ at += payload.byteLength;
377
+ }
378
+ return bytes;
379
+ }
380
+
381
+ function decodeFanout(bytes: Uint8Array): FanoutMessage {
382
+ const length = new DataView(bytes.buffer, bytes.byteOffset).getUint32(0);
383
+ const message = JSON.parse(
384
+ new TextDecoder().decode(bytes.subarray(4, 4 + length)),
385
+ ) as
386
+ | {
387
+ readonly partition: string;
388
+ readonly commit: Omit<StoredCommit, 'changes'> & {
389
+ readonly changes: readonly (Omit<
390
+ StoredCommit['changes'][number],
391
+ 'payload'
392
+ > & { readonly payloadBytes?: number })[];
393
+ };
394
+ }
395
+ | Exclude<FanoutMessage, { readonly commit: StoredCommit }>;
396
+ if (!('commit' in message)) return message;
397
+ let at = 4 + length;
398
+ return {
399
+ partition: message.partition,
400
+ commit: {
401
+ ...message.commit,
402
+ changes: message.commit.changes.map(({ payloadBytes, ...change }) =>
403
+ payloadBytes === undefined
404
+ ? change
405
+ : { ...change, payload: bytes.slice(at, (at += payloadBytes)) },
406
+ ),
407
+ },
408
+ };
409
+ }
410
+
411
+ /** Options of `durableObjectRealtimeNotifier`. */
412
+ export interface DurableObjectRealtimeNotifierOptions {
413
+ /**
414
+ * Keeps the retries of a failed request alive after the response: the
415
+ * Worker's `ctx.waitUntil`, or the Durable Object state's.
416
+ */
417
+ readonly waitUntil: (promise: Promise<unknown>) => void;
418
+ /** Receives `realtime.relay_failed` when the retries give up. */
419
+ readonly events?: SyncularServerEvents;
420
+ /**
421
+ * Resolves when retry `attempt` (1 to 3) may start. Default: after 250 ms,
422
+ * 1 s and 4 s.
423
+ */
424
+ readonly retryDelay?: (attempt: number) => Promise<void>;
425
+ }
426
+
427
+ /**
428
+ * The `RealtimeNotifier` (§8.2, §8.9) that sends a commit, or a client's
429
+ * changed registrations, to the partition's Durable Object, which delivers it
430
+ * to its own sockets and relays it to every shard registered with it, retrying
431
+ * a shard whose relay failed. Pass it as the Worker config's `realtime`
432
+ * whenever the Worker applies commits itself: sharded sockets, or an external
433
+ * command host. A failed request to the partition's object never fails the
434
+ * push, whose commit is durable: the notifier retries it three times in
435
+ * `waitUntil` and emits `realtime.relay_failed` when the last one fails.
436
+ *
437
+ * `refreshScopes(partition, actorId?)` is `RealtimeHub.refreshScopes` for
438
+ * every object of the partition: call it after a grant changes. It throws when
439
+ * the partition's object did not take the refresh.
440
+ */
441
+ export function durableObjectRealtimeNotifier(
442
+ namespace: DurableObjectNamespaceLike,
443
+ options: DurableObjectRealtimeNotifierOptions,
444
+ ): RealtimeNotifier & {
445
+ refreshScopes(partition: string, actorId?: string): Promise<void>;
446
+ } {
447
+ // A synthetic origin: the DO reads only the path and the body. The partition
448
+ // object answers 204, or 404 when no object of the partition holds a socket.
449
+ const send = async (
450
+ message: FanoutMessage,
451
+ body = encodeFanout(message),
452
+ ): Promise<number> => {
453
+ try {
454
+ const response = await namespace
455
+ .get(namespace.idFromName(message.partition))
456
+ .fetch(
457
+ new Request(`https://do${REALTIME_DO_FANOUT_PATH}`, {
458
+ method: 'POST',
459
+ body,
460
+ }),
461
+ );
462
+ return response.status;
463
+ } catch {
464
+ return 0;
465
+ }
466
+ };
467
+ const deliver = async (message: FanoutMessage): Promise<void> => {
468
+ const body = encodeFanout(message);
469
+ const status = await send(message, body);
470
+ if (status === 204 || status === 404) return;
471
+ options.waitUntil(
472
+ (async () => {
473
+ for (let attempt = 1; attempt <= 3; attempt++) {
474
+ await (options.retryDelay?.(attempt) ??
475
+ new Promise((resolve) =>
476
+ setTimeout(resolve, 250 * 4 ** (attempt - 1)),
477
+ ));
478
+ const retried = await send(message, body);
479
+ if (retried === 204 || retried === 404) return;
480
+ }
481
+ if (options.events !== undefined) {
482
+ emitEvent(options.events, {
483
+ type: 'realtime.relay_failed',
484
+ atMs: Date.now(),
485
+ partition: message.partition,
486
+ ...('commit' in message
487
+ ? { commitSeq: message.commit.commitSeq }
488
+ : {}),
489
+ attempts: 4,
490
+ });
491
+ }
492
+ })(),
493
+ );
494
+ };
495
+ return {
496
+ notifyCommit: (partition, commit) => deliver({ partition, commit }),
497
+ notifySubscriptions: (partition, clientId) =>
498
+ deliver({ partition, clientId }),
499
+ async refreshScopes(partition, actorId) {
500
+ const status = await send({
501
+ partition,
502
+ refresh: actorId === undefined ? {} : { actorId },
503
+ });
504
+ if (status !== 204 && status !== 404) {
505
+ throw new SyncError(
506
+ 'sync.internal_error',
507
+ 'realtime scope refresh failed',
508
+ JSON.stringify({ status }),
509
+ );
510
+ }
511
+ },
512
+ };
513
+ }
514
+
274
515
  /** The identity the upgrade request must carry (resolved by the Worker). */
275
516
  export interface RealtimeUpgradeIdentity {
276
517
  readonly partition: string;
@@ -281,7 +522,8 @@ export interface RealtimeUpgradeIdentity {
281
522
  }
282
523
 
283
524
  /** Internal control-request paths on the DO stub (never client-facing). */
284
- export const REALTIME_DO_WAKE_PATH = '/__syncular_realtime/wake';
525
+ export const REALTIME_DO_FANOUT_PATH = '/__syncular_realtime/fanout';
526
+ export const REALTIME_DO_SHARDS_PATH = '/__syncular_realtime/shards';
285
527
  export const REALTIME_DO_UPGRADE_PATH = '/__syncular_realtime/upgrade';
286
528
  export const SYNC_DO_REQUEST_PATH = '/__syncular_realtime/sync';
287
529
  export const OPERATIONS_DO_REQUEST_PATH = '/__syncular_realtime/operations';
@@ -348,6 +590,8 @@ export class SyncularRealtimeHost<
348
590
  #rehydration: Promise<void> | undefined;
349
591
  /** Explicit FIFO: Durable Object events may interleave at `await`. */
350
592
  #partitionTail: Promise<void> = Promise.resolve();
593
+ /** The shard name this instance registered with the partition's object. */
594
+ #registeredShard: string | undefined;
351
595
 
352
596
  /**
353
597
  * Host a partition over a D1 binding (the host builds the coordinated
@@ -372,6 +616,11 @@ export class SyncularRealtimeHost<
372
616
  config: RealtimeDOConfig<D1ServerStorage> | RealtimeDOConfig<S>,
373
617
  ) {
374
618
  this.#state = state;
619
+ if (config.namespace !== undefined && 'prepare' in storage) {
620
+ throw new Error(
621
+ 'RealtimeDOConfig.namespace needs a storage that locks partitions in the database; D1 pushes need the partition FIFO',
622
+ );
623
+ }
375
624
  this.#storage =
376
625
  'prepare' in storage
377
626
  ? // One DO is selected per partition, and every round enters
@@ -407,6 +656,15 @@ export class SyncularRealtimeHost<
407
656
  ? composeEvents(ring, config.events)
408
657
  : ring,
409
658
  };
659
+ if (this.#config.namespace !== undefined) {
660
+ withRing.fanout = durableObjectRealtimeNotifier(
661
+ this.#config.namespace,
662
+ {
663
+ waitUntil: (promise) => this.#state.waitUntil(promise),
664
+ events: withRing.events,
665
+ },
666
+ );
667
+ }
410
668
  const hub = createRealtimeHub(withRing);
411
669
  const cap = hub.maxPresenceBytes;
412
670
  this.#presenceReserve =
@@ -434,10 +692,10 @@ export class SyncularRealtimeHost<
434
692
  }
435
693
 
436
694
  /**
437
- * The DO `fetch` handler: routes the internal upgrade + wake control paths.
438
- * The Worker forwards `GET <mount>/realtime` here as an upgrade with the
439
- * resolved identity in headers (see `forwardRealtimeUpgrade` in `index.ts`),
440
- * and accepts external-command wakes at `/__syncular_realtime/wake`.
695
+ * The DO `fetch` handler: routes the internal control paths. The Worker
696
+ * forwards `GET <mount>/realtime` here as an upgrade with the resolved
697
+ * identity in headers (see `forwardRealtimeUpgrade` in `index.ts`), and
698
+ * commits applied elsewhere arrive at the fanout path (§8.9).
441
699
  */
442
700
  async fetch(request: Request): Promise<Response> {
443
701
  await this.#rehydrateSockets();
@@ -470,8 +728,18 @@ export class SyncularRealtimeHost<
470
728
  if (url.pathname === OPERATIONS_DO_REQUEST_PATH) {
471
729
  return this.#serializePartition(() => this.#handleOperation(request));
472
730
  }
473
- if (url.pathname === REALTIME_DO_WAKE_PATH) {
474
- return this.#serializePartition(() => this.#handleWake(request));
731
+ if (url.pathname === REALTIME_DO_FANOUT_PATH) {
732
+ // Beside the FIFO: a round that applied this commit may be waiting on
733
+ // the relay that delivers it here.
734
+ return this.#handleFanout(request);
735
+ }
736
+ if (url.pathname === REALTIME_DO_SHARDS_PATH) {
737
+ const { shard, token } = (await request.json()) as {
738
+ shard: string;
739
+ token: string;
740
+ };
741
+ await this.#state.storage.put(`shard:${shard}`, token);
742
+ return new Response(null, { status: 204 });
475
743
  }
476
744
  if (url.pathname === REALTIME_DO_UPGRADE_PATH) {
477
745
  return this.#handleUpgrade(request);
@@ -721,20 +989,113 @@ export class SyncularRealtimeHost<
721
989
  });
722
990
  }
723
991
 
724
- /** §8.3 wake: an external coordinated command landed; re-pull the delta. */
725
- async #handleWake(request: Request): Promise<Response> {
726
- const body = (await request.json().catch(() => null)) as {
727
- partition?: unknown;
728
- } | null;
729
- const partition = body?.partition;
730
- if (typeof partition !== 'string') {
731
- return new Response('bad wake', { status: 400 });
992
+ /**
993
+ * §8.9: deliver a message to this object's sockets, then relay it to every
994
+ * shard registered here. A relay that fails (a thrown request, any status
995
+ * but 2xx and 404) leaves the shard owing what the message carried, and the
996
+ * alarm retries it. This object answers 404 when it holds no socket and no
997
+ * shard.
998
+ */
999
+ async #handleFanout(request: Request): Promise<Response> {
1000
+ const bytes = new Uint8Array(await request.arrayBuffer());
1001
+ const message = decodeFanout(bytes);
1002
+ const hub = this.#getHub();
1003
+ if ('commit' in message) {
1004
+ await hub.notifyCommit(message.partition, message.commit);
1005
+ } else if ('clientId' in message) {
1006
+ await hub.notifySubscriptions(message.partition, message.clientId);
1007
+ } else if ('refresh' in message) {
1008
+ await hub.refreshScopes(message.partition, message.refresh.actorId);
1009
+ } else {
1010
+ if (message.owed.refresh) await hub.refreshScopes(message.partition);
1011
+ if (message.owed.registrations) {
1012
+ hub.wake(message.partition, 'catchup-required');
1013
+ }
1014
+ if (message.owed.commitSeq !== undefined) {
1015
+ hub.wakeBehind(message.partition, message.owed.commitSeq);
1016
+ }
732
1017
  }
733
- // Wake only if this DO actually holds sockets (a cold DO has none — the
734
- // socket that would care is on this DO by construction, so a wake to a
735
- // DO with no sockets is simply a no-op, not an error).
736
- this.#getHub().wake(partition, 'catchup-required');
737
- return new Response(null, { status: 204 });
1018
+ const shards = await this.#state.storage.list({ prefix: 'shard:' });
1019
+ const owes: Owed =
1020
+ 'commit' in message
1021
+ ? { commitSeq: message.commit.commitSeq }
1022
+ : 'clientId' in message
1023
+ ? { registrations: true }
1024
+ : 'refresh' in message
1025
+ ? { refresh: true }
1026
+ : message.owed;
1027
+ await Promise.all(
1028
+ [...shards].map(async ([key, token]) => {
1029
+ const shard = key.slice('shard:'.length);
1030
+ const status = await this.#relay(shard, token, bytes);
1031
+ if (status !== 'failed') return;
1032
+ const stored = await this.#state.storage.get(`owed:${shard}`);
1033
+ const before =
1034
+ typeof stored === 'string'
1035
+ ? (JSON.parse(stored) as { readonly owed: Owed }).owed
1036
+ : {};
1037
+ const commitSeq = Math.max(
1038
+ before.commitSeq ?? -1,
1039
+ owes.commitSeq ?? -1,
1040
+ );
1041
+ const owed: Owed = {
1042
+ ...(commitSeq >= 0 ? { commitSeq } : {}),
1043
+ ...(before.refresh || owes.refresh ? { refresh: true } : {}),
1044
+ ...(before.registrations || owes.registrations
1045
+ ? { registrations: true }
1046
+ : {}),
1047
+ };
1048
+ await this.#state.storage.put(
1049
+ `owed:${shard}`,
1050
+ JSON.stringify({ partition: message.partition, owed }),
1051
+ );
1052
+ const at = await this.#state.storage.getAlarm();
1053
+ if (at === null || at > Date.now() + 1_000) {
1054
+ await this.#state.storage.setAlarm(Date.now() + 1_000);
1055
+ }
1056
+ }),
1057
+ );
1058
+ if (this.#state.getWebSockets().length > 0 || shards.size > 0) {
1059
+ return new Response(null, { status: 204 });
1060
+ }
1061
+ this.#registeredShard = undefined;
1062
+ return new Response(null, { status: 404 });
1063
+ }
1064
+
1065
+ /**
1066
+ * Send a fanout body to a registered shard. A shard that answers 404 holds
1067
+ * no socket: it is removed with what it owes, unless it registered again
1068
+ * with a new token meanwhile.
1069
+ */
1070
+ async #relay(
1071
+ shard: string,
1072
+ token: unknown,
1073
+ body: Uint8Array<ArrayBuffer>,
1074
+ ): Promise<'delivered' | 'gone' | 'failed'> {
1075
+ const namespace = this.#config.namespace;
1076
+ if (namespace === undefined) {
1077
+ throw new Error('a registered shard needs RealtimeDOConfig.namespace');
1078
+ }
1079
+ let status = 0;
1080
+ try {
1081
+ const response = await namespace.get(namespace.idFromName(shard)).fetch(
1082
+ new Request(`https://do${REALTIME_DO_FANOUT_PATH}`, {
1083
+ method: 'POST',
1084
+ body,
1085
+ }),
1086
+ );
1087
+ status = response.status;
1088
+ } catch {
1089
+ return 'failed';
1090
+ }
1091
+ if (status === 404) {
1092
+ if ((await this.#state.storage.get(`shard:${shard}`)) === token) {
1093
+ await this.#state.storage.delete(`shard:${shard}`);
1094
+ await this.#state.storage.delete(`owed:${shard}`);
1095
+ }
1096
+ return 'gone';
1097
+ }
1098
+ return status >= 200 && status < 300 ? 'delivered' : 'failed';
738
1099
  }
739
1100
 
740
1101
  /** The WebSocket upgrade: accept a hibernatable socket, connect the hub. */
@@ -770,6 +1131,31 @@ export class SyncularRealtimeHost<
770
1131
  }),
771
1132
  );
772
1133
  }
1134
+ // §8.9: a shard registers with the partition's object after it accepts
1135
+ // the socket (so a relay meanwhile does not find it empty and drop it)
1136
+ // and before the hub reads the latest sequence, so a commit relayed
1137
+ // after that read reaches the session.
1138
+ const shard = request.headers.get(REALTIME_SHARD_HEADER);
1139
+ if (shard !== null && this.#registeredShard !== shard) {
1140
+ const namespace = this.#config.namespace;
1141
+ if (namespace === undefined) {
1142
+ throw new Error(
1143
+ 'a sharded realtime upgrade needs RealtimeDOConfig.namespace',
1144
+ );
1145
+ }
1146
+ const registered = await namespace
1147
+ .get(namespace.idFromName(identity.partition))
1148
+ .fetch(
1149
+ new Request(new URL(REALTIME_DO_SHARDS_PATH, request.url), {
1150
+ method: 'POST',
1151
+ body: JSON.stringify({ shard, token: crypto.randomUUID() }),
1152
+ }),
1153
+ );
1154
+ if (registered.status !== 204) {
1155
+ throw new Error('shard registration failed');
1156
+ }
1157
+ this.#registeredShard = shard;
1158
+ }
773
1159
  const session = await hub.connect(this.#connectOptions(server, identity));
774
1160
  this.#sessions.set(server, session);
775
1161
  if ((await this.#state.storage.getAlarm()) === null) {
@@ -809,9 +1195,11 @@ export class SyncularRealtimeHost<
809
1195
  }
810
1196
 
811
1197
  /**
812
- * The Durable Object alarm: send a §8.5 heartbeat to every socket and re-arm
813
- * while sockets remain. The frame goes straight to the hibernatable sockets,
814
- * so a heartbeat rebuilds no session.
1198
+ * The Durable Object alarm: send a §8.5 heartbeat to every socket, retry
1199
+ * every shard that owes a failed relay (§8.9), and re-arm: after the
1200
+ * heartbeat interval while sockets remain, after one second while a shard
1201
+ * still owes. The heartbeat goes straight to the hibernatable sockets, so it
1202
+ * rebuilds no session.
815
1203
  */
816
1204
  async alarm(): Promise<void> {
817
1205
  const sockets = this.#state.getWebSockets();
@@ -823,9 +1211,28 @@ export class SyncularRealtimeHost<
823
1211
  // A closing socket; its close callback cleans up.
824
1212
  }
825
1213
  }
826
- if (sockets.length > 0) {
827
- await this.#state.storage.setAlarm(
828
- Date.now() + (this.#config.heartbeatIntervalMs ?? 30_000),
1214
+ const storage = this.#state.storage;
1215
+ for (const [key, value] of await storage.list({ prefix: 'owed:' })) {
1216
+ const shard = key.slice('owed:'.length);
1217
+ const token = await storage.get(`shard:${shard}`);
1218
+ const message = JSON.parse(String(value)) as {
1219
+ readonly partition: string;
1220
+ readonly owed: Owed;
1221
+ };
1222
+ // A newer failure may have merged into the record during the retry.
1223
+ if (
1224
+ (await this.#relay(shard, token, encodeFanout(message))) ===
1225
+ 'delivered' &&
1226
+ (await storage.get(key)) === value
1227
+ ) {
1228
+ await storage.delete(key);
1229
+ }
1230
+ }
1231
+ const owing = (await storage.list({ prefix: 'owed:' })).size > 0;
1232
+ if (sockets.length > 0 || owing) {
1233
+ await storage.setAlarm(
1234
+ Date.now() +
1235
+ (owing ? 1_000 : (this.#config.heartbeatIntervalMs ?? 30_000)),
829
1236
  );
830
1237
  }
831
1238
  }
@@ -935,6 +1342,8 @@ const REALTIME_ID_HEADER = {
935
1342
  } as const;
936
1343
 
937
1344
  const ADMIN_ACTIONS_HEADER = 'x-syncular-admin-actions';
1345
+ /** The object name of the shard a sharded upgrade goes to (§8.9). */
1346
+ const REALTIME_SHARD_HEADER = 'x-syncular-shard';
938
1347
 
939
1348
  /** Address a Worker-authorized admin request to the DO (Worker side). */
940
1349
  export function adminRequestForDurableObject(
@@ -978,13 +1387,20 @@ export function writeRequestIdentityHeaders(
978
1387
  }
979
1388
  }
980
1389
 
981
- /** Write the resolved identity onto an upgrade request's headers (Worker side). */
1390
+ /**
1391
+ * Write the resolved identity, and the object name of a sharded upgrade's
1392
+ * shard (§8.9), onto an upgrade request's headers (Worker side). A shard
1393
+ * header the client sent is removed when the upgrade is not sharded.
1394
+ */
982
1395
  export function writeIdentityHeaders(
983
1396
  headers: Headers,
984
1397
  identity: RealtimeUpgradeIdentity,
1398
+ shard?: string,
985
1399
  ): void {
986
1400
  writeRequestIdentityHeaders(headers, identity);
987
1401
  headers.set(REALTIME_ID_HEADER.clientId, identity.clientId);
1402
+ if (shard === undefined) headers.delete(REALTIME_SHARD_HEADER);
1403
+ else headers.set(REALTIME_SHARD_HEADER, shard);
988
1404
  }
989
1405
 
990
1406
  function readIdentityHeaders(