@syncular/server-workers 0.78.3 → 0.79.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -4
- package/dist/index.d.ts +31 -40
- package/dist/index.js +31 -44
- package/dist/realtime-do.d.ts +100 -18
- package/dist/realtime-do.js +313 -32
- package/package.json +4 -4
- package/src/index.ts +61 -69
- package/src/realtime-do.ts +450 -34
package/src/realtime-do.ts
CHANGED
|
@@ -88,13 +88,27 @@
|
|
|
88
88
|
* sessions, and re-arms while sockets remain. The host owns the alarm: a
|
|
89
89
|
* subclass that needs another alarm schedule cannot share it.
|
|
90
90
|
*
|
|
91
|
-
* ##
|
|
91
|
+
* ## Fanout between objects (§8.9)
|
|
92
92
|
*
|
|
93
93
|
* Ordinary HTTP `/sync` and `/operations` (with `RealtimeDOConfig.operations`)
|
|
94
|
-
* are forwarded into this DO and fan out in-process.
|
|
95
|
-
*
|
|
96
|
-
*
|
|
97
|
-
*
|
|
94
|
+
* are forwarded into this DO and fan out in-process. A commit applied
|
|
95
|
+
* elsewhere reaches the partition's object through
|
|
96
|
+
* `durableObjectRealtimeNotifier`: a POST to the fanout path carrying the
|
|
97
|
+
* commit, which the object delivers to its hub as a delta.
|
|
98
|
+
*
|
|
99
|
+
* On PostgreSQL the Worker may shard sockets over several objects
|
|
100
|
+
* (`WorkersRealtimeOptions.shard`) and run every round itself. The
|
|
101
|
+
* partition's object (`idFromName(partition)`) then holds no socket: it keeps
|
|
102
|
+
* the names of the shard objects that registered with it in its storage and
|
|
103
|
+
* relays every fanout message to each of them. A shard registers when it
|
|
104
|
+
* accepts a socket, before its hub connects it; a shard that answers a relay with 404 (no socket
|
|
105
|
+
* left) is removed unless it registered again meanwhile. A shard's hub
|
|
106
|
+
* (`RealtimeHubConfig.fanout`) sends the commits of its own rounds through the
|
|
107
|
+
* partition's object too, so every shard, the shard itself included, receives
|
|
108
|
+
* each commit once. A host scope refresh (`refreshScopes` on the notifier)
|
|
109
|
+
* takes the same path. A relay that fails leaves a record of what the shard
|
|
110
|
+
* owes in the partition object's storage, and the alarm retries it every
|
|
111
|
+
* second until the shard answers or is removed.
|
|
98
112
|
*/
|
|
99
113
|
import {
|
|
100
114
|
adapterSyncError,
|
|
@@ -118,9 +132,13 @@ import {
|
|
|
118
132
|
type RemoteOperationRegistry,
|
|
119
133
|
type RemoteOperationResolver,
|
|
120
134
|
RingBufferEvents,
|
|
135
|
+
type RealtimeNotifier,
|
|
121
136
|
type ServerStorage,
|
|
122
137
|
SSP2_CONTENT_TYPE,
|
|
138
|
+
type StoredCommit,
|
|
123
139
|
SyncError,
|
|
140
|
+
type SyncularServerEvents,
|
|
141
|
+
emitEvent,
|
|
124
142
|
SyncularAdmin,
|
|
125
143
|
type SyncularErrorRoute,
|
|
126
144
|
type SyncRequestContext,
|
|
@@ -135,15 +153,36 @@ import { decodeRequest } from '@syncular/core';
|
|
|
135
153
|
|
|
136
154
|
export interface DurableObjectStateLike {
|
|
137
155
|
acceptWebSocket(ws: WebSocketLike, tags?: string[]): void;
|
|
156
|
+
/** Keeps work alive after a response: the notifier's retries (§8.9). */
|
|
157
|
+
waitUntil(promise: Promise<unknown>): void;
|
|
138
158
|
getWebSockets(tag?: string): WebSocketLike[];
|
|
139
|
-
/** The
|
|
140
|
-
*
|
|
159
|
+
/** The DO storage API the host uses: the alarm drives §8.5 heartbeats, and
|
|
160
|
+
* the partition's object keeps its registered shards as keys. */
|
|
141
161
|
readonly storage: {
|
|
142
162
|
getAlarm(): Promise<number | null>;
|
|
143
163
|
setAlarm(scheduledTime: number): Promise<void>;
|
|
164
|
+
get(key: string): Promise<unknown>;
|
|
165
|
+
put(key: string, value: string): Promise<void>;
|
|
166
|
+
delete(key: string): Promise<boolean>;
|
|
167
|
+
list(options: { prefix: string }): Promise<Map<string, unknown>>;
|
|
144
168
|
};
|
|
145
169
|
}
|
|
146
170
|
|
|
171
|
+
/** A DO stub: the callable handle to one Durable Object instance. */
|
|
172
|
+
export interface DurableObjectStubLike {
|
|
173
|
+
fetch(request: Request): Promise<Response>;
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/** A DO namespace binding: `idFromName` → `get(id)` → a stub. */
|
|
177
|
+
export interface DurableObjectNamespaceLike {
|
|
178
|
+
idFromName(name: string): DurableObjectIdLike;
|
|
179
|
+
get(id: DurableObjectIdLike): DurableObjectStubLike;
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
export interface DurableObjectIdLike {
|
|
183
|
+
toString(): string;
|
|
184
|
+
}
|
|
185
|
+
|
|
147
186
|
export interface WebSocketLike {
|
|
148
187
|
accept?(): void;
|
|
149
188
|
send(data: string | ArrayBuffer | ArrayBufferView): void;
|
|
@@ -211,6 +250,15 @@ export type RealtimeDOConfig<S extends RealtimeHostStorage = D1ServerStorage> =
|
|
|
211
250
|
* the scope resolver per request (REMOTE.md §5.1).
|
|
212
251
|
*/
|
|
213
252
|
readonly operations?: RemoteOperationRegistry | RemoteOperationResolver;
|
|
253
|
+
/**
|
|
254
|
+
* The namespace this class is bound to. Required when the Worker shards
|
|
255
|
+
* sockets (`WorkersRealtimeOptions.shard`, §8.9): a shard registers with
|
|
256
|
+
* the partition's object through it, the partition's object relays every
|
|
257
|
+
* commit to the registered shards, and a shard's own rounds send their
|
|
258
|
+
* commits through the partition's object. The storage must lock
|
|
259
|
+
* partitions in the database (PostgreSQL); a D1 host refuses it.
|
|
260
|
+
*/
|
|
261
|
+
readonly namespace?: DurableObjectNamespaceLike;
|
|
214
262
|
};
|
|
215
263
|
|
|
216
264
|
/**
|
|
@@ -271,6 +319,199 @@ function encodeAttachment(
|
|
|
271
319
|
return Uint8Array.from(bytes);
|
|
272
320
|
}
|
|
273
321
|
|
|
322
|
+
/**
|
|
323
|
+
* §8.9: what a shard missed when a relay to it failed, delivered from the
|
|
324
|
+
* partition object's alarm: the highest commit sequence (sessions behind it get
|
|
325
|
+
* a `catchup-required` wake), a fail-closed refresh of every session's scopes,
|
|
326
|
+
* and changed registrations (a wake, whose round reloads them).
|
|
327
|
+
*/
|
|
328
|
+
interface Owed {
|
|
329
|
+
readonly commitSeq?: number;
|
|
330
|
+
readonly refresh?: true;
|
|
331
|
+
readonly registrations?: true;
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
/** A message between objects (§8.9): a commit, a client's changed
|
|
335
|
+
* registrations, a host refresh of scopes, or what a failed relay owes. */
|
|
336
|
+
type FanoutMessage =
|
|
337
|
+
| { readonly partition: string; readonly commit: StoredCommit }
|
|
338
|
+
| { readonly partition: string; readonly clientId: string }
|
|
339
|
+
| {
|
|
340
|
+
readonly partition: string;
|
|
341
|
+
readonly refresh: { readonly actorId?: string };
|
|
342
|
+
}
|
|
343
|
+
| { readonly partition: string; readonly owed: Owed };
|
|
344
|
+
|
|
345
|
+
/**
|
|
346
|
+
* Fanout body: a u32 length, the message as UTF-8 JSON with each change
|
|
347
|
+
* payload replaced by its byte length, then the payloads in change order.
|
|
348
|
+
*/
|
|
349
|
+
function encodeFanout(message: FanoutMessage): Uint8Array<ArrayBuffer> {
|
|
350
|
+
const payloads: Uint8Array[] = [];
|
|
351
|
+
const head = new TextEncoder().encode(
|
|
352
|
+
JSON.stringify(
|
|
353
|
+
'commit' in message
|
|
354
|
+
? {
|
|
355
|
+
...message,
|
|
356
|
+
commit: {
|
|
357
|
+
...message.commit,
|
|
358
|
+
changes: message.commit.changes.map(({ payload, ...change }) => {
|
|
359
|
+
if (payload === undefined) return change;
|
|
360
|
+
payloads.push(payload);
|
|
361
|
+
return { ...change, payloadBytes: payload.byteLength };
|
|
362
|
+
}),
|
|
363
|
+
},
|
|
364
|
+
}
|
|
365
|
+
: message,
|
|
366
|
+
),
|
|
367
|
+
);
|
|
368
|
+
const bytes = new Uint8Array(
|
|
369
|
+
payloads.reduce((total, p) => total + p.byteLength, 4 + head.byteLength),
|
|
370
|
+
);
|
|
371
|
+
new DataView(bytes.buffer).setUint32(0, head.byteLength);
|
|
372
|
+
bytes.set(head, 4);
|
|
373
|
+
let at = 4 + head.byteLength;
|
|
374
|
+
for (const payload of payloads) {
|
|
375
|
+
bytes.set(payload, at);
|
|
376
|
+
at += payload.byteLength;
|
|
377
|
+
}
|
|
378
|
+
return bytes;
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
function decodeFanout(bytes: Uint8Array): FanoutMessage {
|
|
382
|
+
const length = new DataView(bytes.buffer, bytes.byteOffset).getUint32(0);
|
|
383
|
+
const message = JSON.parse(
|
|
384
|
+
new TextDecoder().decode(bytes.subarray(4, 4 + length)),
|
|
385
|
+
) as
|
|
386
|
+
| {
|
|
387
|
+
readonly partition: string;
|
|
388
|
+
readonly commit: Omit<StoredCommit, 'changes'> & {
|
|
389
|
+
readonly changes: readonly (Omit<
|
|
390
|
+
StoredCommit['changes'][number],
|
|
391
|
+
'payload'
|
|
392
|
+
> & { readonly payloadBytes?: number })[];
|
|
393
|
+
};
|
|
394
|
+
}
|
|
395
|
+
| Exclude<FanoutMessage, { readonly commit: StoredCommit }>;
|
|
396
|
+
if (!('commit' in message)) return message;
|
|
397
|
+
let at = 4 + length;
|
|
398
|
+
return {
|
|
399
|
+
partition: message.partition,
|
|
400
|
+
commit: {
|
|
401
|
+
...message.commit,
|
|
402
|
+
changes: message.commit.changes.map(({ payloadBytes, ...change }) =>
|
|
403
|
+
payloadBytes === undefined
|
|
404
|
+
? change
|
|
405
|
+
: { ...change, payload: bytes.slice(at, (at += payloadBytes)) },
|
|
406
|
+
),
|
|
407
|
+
},
|
|
408
|
+
};
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
/** Options of `durableObjectRealtimeNotifier`. */
|
|
412
|
+
export interface DurableObjectRealtimeNotifierOptions {
|
|
413
|
+
/**
|
|
414
|
+
* Keeps the retries of a failed request alive after the response: the
|
|
415
|
+
* Worker's `ctx.waitUntil`, or the Durable Object state's.
|
|
416
|
+
*/
|
|
417
|
+
readonly waitUntil: (promise: Promise<unknown>) => void;
|
|
418
|
+
/** Receives `realtime.relay_failed` when the retries give up. */
|
|
419
|
+
readonly events?: SyncularServerEvents;
|
|
420
|
+
/**
|
|
421
|
+
* Resolves when retry `attempt` (1 to 3) may start. Default: after 250 ms,
|
|
422
|
+
* 1 s and 4 s.
|
|
423
|
+
*/
|
|
424
|
+
readonly retryDelay?: (attempt: number) => Promise<void>;
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
/**
|
|
428
|
+
* The `RealtimeNotifier` (§8.2, §8.9) that sends a commit, or a client's
|
|
429
|
+
* changed registrations, to the partition's Durable Object, which delivers it
|
|
430
|
+
* to its own sockets and relays it to every shard registered with it, retrying
|
|
431
|
+
* a shard whose relay failed. Pass it as the Worker config's `realtime`
|
|
432
|
+
* whenever the Worker applies commits itself: sharded sockets, or an external
|
|
433
|
+
* command host. A failed request to the partition's object never fails the
|
|
434
|
+
* push, whose commit is durable: the notifier retries it three times in
|
|
435
|
+
* `waitUntil` and emits `realtime.relay_failed` when the last one fails.
|
|
436
|
+
*
|
|
437
|
+
* `refreshScopes(partition, actorId?)` is `RealtimeHub.refreshScopes` for
|
|
438
|
+
* every object of the partition: call it after a grant changes. It throws when
|
|
439
|
+
* the partition's object did not take the refresh.
|
|
440
|
+
*/
|
|
441
|
+
export function durableObjectRealtimeNotifier(
|
|
442
|
+
namespace: DurableObjectNamespaceLike,
|
|
443
|
+
options: DurableObjectRealtimeNotifierOptions,
|
|
444
|
+
): RealtimeNotifier & {
|
|
445
|
+
refreshScopes(partition: string, actorId?: string): Promise<void>;
|
|
446
|
+
} {
|
|
447
|
+
// A synthetic origin: the DO reads only the path and the body. The partition
|
|
448
|
+
// object answers 204, or 404 when no object of the partition holds a socket.
|
|
449
|
+
const send = async (
|
|
450
|
+
message: FanoutMessage,
|
|
451
|
+
body = encodeFanout(message),
|
|
452
|
+
): Promise<number> => {
|
|
453
|
+
try {
|
|
454
|
+
const response = await namespace
|
|
455
|
+
.get(namespace.idFromName(message.partition))
|
|
456
|
+
.fetch(
|
|
457
|
+
new Request(`https://do${REALTIME_DO_FANOUT_PATH}`, {
|
|
458
|
+
method: 'POST',
|
|
459
|
+
body,
|
|
460
|
+
}),
|
|
461
|
+
);
|
|
462
|
+
return response.status;
|
|
463
|
+
} catch {
|
|
464
|
+
return 0;
|
|
465
|
+
}
|
|
466
|
+
};
|
|
467
|
+
const deliver = async (message: FanoutMessage): Promise<void> => {
|
|
468
|
+
const body = encodeFanout(message);
|
|
469
|
+
const status = await send(message, body);
|
|
470
|
+
if (status === 204 || status === 404) return;
|
|
471
|
+
options.waitUntil(
|
|
472
|
+
(async () => {
|
|
473
|
+
for (let attempt = 1; attempt <= 3; attempt++) {
|
|
474
|
+
await (options.retryDelay?.(attempt) ??
|
|
475
|
+
new Promise((resolve) =>
|
|
476
|
+
setTimeout(resolve, 250 * 4 ** (attempt - 1)),
|
|
477
|
+
));
|
|
478
|
+
const retried = await send(message, body);
|
|
479
|
+
if (retried === 204 || retried === 404) return;
|
|
480
|
+
}
|
|
481
|
+
if (options.events !== undefined) {
|
|
482
|
+
emitEvent(options.events, {
|
|
483
|
+
type: 'realtime.relay_failed',
|
|
484
|
+
atMs: Date.now(),
|
|
485
|
+
partition: message.partition,
|
|
486
|
+
...('commit' in message
|
|
487
|
+
? { commitSeq: message.commit.commitSeq }
|
|
488
|
+
: {}),
|
|
489
|
+
attempts: 4,
|
|
490
|
+
});
|
|
491
|
+
}
|
|
492
|
+
})(),
|
|
493
|
+
);
|
|
494
|
+
};
|
|
495
|
+
return {
|
|
496
|
+
notifyCommit: (partition, commit) => deliver({ partition, commit }),
|
|
497
|
+
notifySubscriptions: (partition, clientId) =>
|
|
498
|
+
deliver({ partition, clientId }),
|
|
499
|
+
async refreshScopes(partition, actorId) {
|
|
500
|
+
const status = await send({
|
|
501
|
+
partition,
|
|
502
|
+
refresh: actorId === undefined ? {} : { actorId },
|
|
503
|
+
});
|
|
504
|
+
if (status !== 204 && status !== 404) {
|
|
505
|
+
throw new SyncError(
|
|
506
|
+
'sync.internal_error',
|
|
507
|
+
'realtime scope refresh failed',
|
|
508
|
+
JSON.stringify({ status }),
|
|
509
|
+
);
|
|
510
|
+
}
|
|
511
|
+
},
|
|
512
|
+
};
|
|
513
|
+
}
|
|
514
|
+
|
|
274
515
|
/** The identity the upgrade request must carry (resolved by the Worker). */
|
|
275
516
|
export interface RealtimeUpgradeIdentity {
|
|
276
517
|
readonly partition: string;
|
|
@@ -281,7 +522,8 @@ export interface RealtimeUpgradeIdentity {
|
|
|
281
522
|
}
|
|
282
523
|
|
|
283
524
|
/** Internal control-request paths on the DO stub (never client-facing). */
|
|
284
|
-
export const
|
|
525
|
+
export const REALTIME_DO_FANOUT_PATH = '/__syncular_realtime/fanout';
|
|
526
|
+
export const REALTIME_DO_SHARDS_PATH = '/__syncular_realtime/shards';
|
|
285
527
|
export const REALTIME_DO_UPGRADE_PATH = '/__syncular_realtime/upgrade';
|
|
286
528
|
export const SYNC_DO_REQUEST_PATH = '/__syncular_realtime/sync';
|
|
287
529
|
export const OPERATIONS_DO_REQUEST_PATH = '/__syncular_realtime/operations';
|
|
@@ -348,6 +590,8 @@ export class SyncularRealtimeHost<
|
|
|
348
590
|
#rehydration: Promise<void> | undefined;
|
|
349
591
|
/** Explicit FIFO: Durable Object events may interleave at `await`. */
|
|
350
592
|
#partitionTail: Promise<void> = Promise.resolve();
|
|
593
|
+
/** The shard name this instance registered with the partition's object. */
|
|
594
|
+
#registeredShard: string | undefined;
|
|
351
595
|
|
|
352
596
|
/**
|
|
353
597
|
* Host a partition over a D1 binding (the host builds the coordinated
|
|
@@ -372,6 +616,11 @@ export class SyncularRealtimeHost<
|
|
|
372
616
|
config: RealtimeDOConfig<D1ServerStorage> | RealtimeDOConfig<S>,
|
|
373
617
|
) {
|
|
374
618
|
this.#state = state;
|
|
619
|
+
if (config.namespace !== undefined && 'prepare' in storage) {
|
|
620
|
+
throw new Error(
|
|
621
|
+
'RealtimeDOConfig.namespace needs a storage that locks partitions in the database; D1 pushes need the partition FIFO',
|
|
622
|
+
);
|
|
623
|
+
}
|
|
375
624
|
this.#storage =
|
|
376
625
|
'prepare' in storage
|
|
377
626
|
? // One DO is selected per partition, and every round enters
|
|
@@ -407,6 +656,15 @@ export class SyncularRealtimeHost<
|
|
|
407
656
|
? composeEvents(ring, config.events)
|
|
408
657
|
: ring,
|
|
409
658
|
};
|
|
659
|
+
if (this.#config.namespace !== undefined) {
|
|
660
|
+
withRing.fanout = durableObjectRealtimeNotifier(
|
|
661
|
+
this.#config.namespace,
|
|
662
|
+
{
|
|
663
|
+
waitUntil: (promise) => this.#state.waitUntil(promise),
|
|
664
|
+
events: withRing.events,
|
|
665
|
+
},
|
|
666
|
+
);
|
|
667
|
+
}
|
|
410
668
|
const hub = createRealtimeHub(withRing);
|
|
411
669
|
const cap = hub.maxPresenceBytes;
|
|
412
670
|
this.#presenceReserve =
|
|
@@ -434,10 +692,10 @@ export class SyncularRealtimeHost<
|
|
|
434
692
|
}
|
|
435
693
|
|
|
436
694
|
/**
|
|
437
|
-
* The DO `fetch` handler: routes the internal
|
|
438
|
-
*
|
|
439
|
-
*
|
|
440
|
-
*
|
|
695
|
+
* The DO `fetch` handler: routes the internal control paths. The Worker
|
|
696
|
+
* forwards `GET <mount>/realtime` here as an upgrade with the resolved
|
|
697
|
+
* identity in headers (see `forwardRealtimeUpgrade` in `index.ts`), and
|
|
698
|
+
* commits applied elsewhere arrive at the fanout path (§8.9).
|
|
441
699
|
*/
|
|
442
700
|
async fetch(request: Request): Promise<Response> {
|
|
443
701
|
await this.#rehydrateSockets();
|
|
@@ -470,8 +728,18 @@ export class SyncularRealtimeHost<
|
|
|
470
728
|
if (url.pathname === OPERATIONS_DO_REQUEST_PATH) {
|
|
471
729
|
return this.#serializePartition(() => this.#handleOperation(request));
|
|
472
730
|
}
|
|
473
|
-
if (url.pathname ===
|
|
474
|
-
|
|
731
|
+
if (url.pathname === REALTIME_DO_FANOUT_PATH) {
|
|
732
|
+
// Beside the FIFO: a round that applied this commit may be waiting on
|
|
733
|
+
// the relay that delivers it here.
|
|
734
|
+
return this.#handleFanout(request);
|
|
735
|
+
}
|
|
736
|
+
if (url.pathname === REALTIME_DO_SHARDS_PATH) {
|
|
737
|
+
const { shard, token } = (await request.json()) as {
|
|
738
|
+
shard: string;
|
|
739
|
+
token: string;
|
|
740
|
+
};
|
|
741
|
+
await this.#state.storage.put(`shard:${shard}`, token);
|
|
742
|
+
return new Response(null, { status: 204 });
|
|
475
743
|
}
|
|
476
744
|
if (url.pathname === REALTIME_DO_UPGRADE_PATH) {
|
|
477
745
|
return this.#handleUpgrade(request);
|
|
@@ -721,20 +989,113 @@ export class SyncularRealtimeHost<
|
|
|
721
989
|
});
|
|
722
990
|
}
|
|
723
991
|
|
|
724
|
-
/**
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
992
|
+
/**
|
|
993
|
+
* §8.9: deliver a message to this object's sockets, then relay it to every
|
|
994
|
+
* shard registered here. A relay that fails (a thrown request, any status
|
|
995
|
+
* but 2xx and 404) leaves the shard owing what the message carried, and the
|
|
996
|
+
* alarm retries it. This object answers 404 when it holds no socket and no
|
|
997
|
+
* shard.
|
|
998
|
+
*/
|
|
999
|
+
async #handleFanout(request: Request): Promise<Response> {
|
|
1000
|
+
const bytes = new Uint8Array(await request.arrayBuffer());
|
|
1001
|
+
const message = decodeFanout(bytes);
|
|
1002
|
+
const hub = this.#getHub();
|
|
1003
|
+
if ('commit' in message) {
|
|
1004
|
+
await hub.notifyCommit(message.partition, message.commit);
|
|
1005
|
+
} else if ('clientId' in message) {
|
|
1006
|
+
await hub.notifySubscriptions(message.partition, message.clientId);
|
|
1007
|
+
} else if ('refresh' in message) {
|
|
1008
|
+
await hub.refreshScopes(message.partition, message.refresh.actorId);
|
|
1009
|
+
} else {
|
|
1010
|
+
if (message.owed.refresh) await hub.refreshScopes(message.partition);
|
|
1011
|
+
if (message.owed.registrations) {
|
|
1012
|
+
hub.wake(message.partition, 'catchup-required');
|
|
1013
|
+
}
|
|
1014
|
+
if (message.owed.commitSeq !== undefined) {
|
|
1015
|
+
hub.wakeBehind(message.partition, message.owed.commitSeq);
|
|
1016
|
+
}
|
|
732
1017
|
}
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
1018
|
+
const shards = await this.#state.storage.list({ prefix: 'shard:' });
|
|
1019
|
+
const owes: Owed =
|
|
1020
|
+
'commit' in message
|
|
1021
|
+
? { commitSeq: message.commit.commitSeq }
|
|
1022
|
+
: 'clientId' in message
|
|
1023
|
+
? { registrations: true }
|
|
1024
|
+
: 'refresh' in message
|
|
1025
|
+
? { refresh: true }
|
|
1026
|
+
: message.owed;
|
|
1027
|
+
await Promise.all(
|
|
1028
|
+
[...shards].map(async ([key, token]) => {
|
|
1029
|
+
const shard = key.slice('shard:'.length);
|
|
1030
|
+
const status = await this.#relay(shard, token, bytes);
|
|
1031
|
+
if (status !== 'failed') return;
|
|
1032
|
+
const stored = await this.#state.storage.get(`owed:${shard}`);
|
|
1033
|
+
const before =
|
|
1034
|
+
typeof stored === 'string'
|
|
1035
|
+
? (JSON.parse(stored) as { readonly owed: Owed }).owed
|
|
1036
|
+
: {};
|
|
1037
|
+
const commitSeq = Math.max(
|
|
1038
|
+
before.commitSeq ?? -1,
|
|
1039
|
+
owes.commitSeq ?? -1,
|
|
1040
|
+
);
|
|
1041
|
+
const owed: Owed = {
|
|
1042
|
+
...(commitSeq >= 0 ? { commitSeq } : {}),
|
|
1043
|
+
...(before.refresh || owes.refresh ? { refresh: true } : {}),
|
|
1044
|
+
...(before.registrations || owes.registrations
|
|
1045
|
+
? { registrations: true }
|
|
1046
|
+
: {}),
|
|
1047
|
+
};
|
|
1048
|
+
await this.#state.storage.put(
|
|
1049
|
+
`owed:${shard}`,
|
|
1050
|
+
JSON.stringify({ partition: message.partition, owed }),
|
|
1051
|
+
);
|
|
1052
|
+
const at = await this.#state.storage.getAlarm();
|
|
1053
|
+
if (at === null || at > Date.now() + 1_000) {
|
|
1054
|
+
await this.#state.storage.setAlarm(Date.now() + 1_000);
|
|
1055
|
+
}
|
|
1056
|
+
}),
|
|
1057
|
+
);
|
|
1058
|
+
if (this.#state.getWebSockets().length > 0 || shards.size > 0) {
|
|
1059
|
+
return new Response(null, { status: 204 });
|
|
1060
|
+
}
|
|
1061
|
+
this.#registeredShard = undefined;
|
|
1062
|
+
return new Response(null, { status: 404 });
|
|
1063
|
+
}
|
|
1064
|
+
|
|
1065
|
+
/**
|
|
1066
|
+
* Send a fanout body to a registered shard. A shard that answers 404 holds
|
|
1067
|
+
* no socket: it is removed with what it owes, unless it registered again
|
|
1068
|
+
* with a new token meanwhile.
|
|
1069
|
+
*/
|
|
1070
|
+
async #relay(
|
|
1071
|
+
shard: string,
|
|
1072
|
+
token: unknown,
|
|
1073
|
+
body: Uint8Array<ArrayBuffer>,
|
|
1074
|
+
): Promise<'delivered' | 'gone' | 'failed'> {
|
|
1075
|
+
const namespace = this.#config.namespace;
|
|
1076
|
+
if (namespace === undefined) {
|
|
1077
|
+
throw new Error('a registered shard needs RealtimeDOConfig.namespace');
|
|
1078
|
+
}
|
|
1079
|
+
let status = 0;
|
|
1080
|
+
try {
|
|
1081
|
+
const response = await namespace.get(namespace.idFromName(shard)).fetch(
|
|
1082
|
+
new Request(`https://do${REALTIME_DO_FANOUT_PATH}`, {
|
|
1083
|
+
method: 'POST',
|
|
1084
|
+
body,
|
|
1085
|
+
}),
|
|
1086
|
+
);
|
|
1087
|
+
status = response.status;
|
|
1088
|
+
} catch {
|
|
1089
|
+
return 'failed';
|
|
1090
|
+
}
|
|
1091
|
+
if (status === 404) {
|
|
1092
|
+
if ((await this.#state.storage.get(`shard:${shard}`)) === token) {
|
|
1093
|
+
await this.#state.storage.delete(`shard:${shard}`);
|
|
1094
|
+
await this.#state.storage.delete(`owed:${shard}`);
|
|
1095
|
+
}
|
|
1096
|
+
return 'gone';
|
|
1097
|
+
}
|
|
1098
|
+
return status >= 200 && status < 300 ? 'delivered' : 'failed';
|
|
738
1099
|
}
|
|
739
1100
|
|
|
740
1101
|
/** The WebSocket upgrade: accept a hibernatable socket, connect the hub. */
|
|
@@ -770,6 +1131,31 @@ export class SyncularRealtimeHost<
|
|
|
770
1131
|
}),
|
|
771
1132
|
);
|
|
772
1133
|
}
|
|
1134
|
+
// §8.9: a shard registers with the partition's object after it accepts
|
|
1135
|
+
// the socket (so a relay meanwhile does not find it empty and drop it)
|
|
1136
|
+
// and before the hub reads the latest sequence, so a commit relayed
|
|
1137
|
+
// after that read reaches the session.
|
|
1138
|
+
const shard = request.headers.get(REALTIME_SHARD_HEADER);
|
|
1139
|
+
if (shard !== null && this.#registeredShard !== shard) {
|
|
1140
|
+
const namespace = this.#config.namespace;
|
|
1141
|
+
if (namespace === undefined) {
|
|
1142
|
+
throw new Error(
|
|
1143
|
+
'a sharded realtime upgrade needs RealtimeDOConfig.namespace',
|
|
1144
|
+
);
|
|
1145
|
+
}
|
|
1146
|
+
const registered = await namespace
|
|
1147
|
+
.get(namespace.idFromName(identity.partition))
|
|
1148
|
+
.fetch(
|
|
1149
|
+
new Request(new URL(REALTIME_DO_SHARDS_PATH, request.url), {
|
|
1150
|
+
method: 'POST',
|
|
1151
|
+
body: JSON.stringify({ shard, token: crypto.randomUUID() }),
|
|
1152
|
+
}),
|
|
1153
|
+
);
|
|
1154
|
+
if (registered.status !== 204) {
|
|
1155
|
+
throw new Error('shard registration failed');
|
|
1156
|
+
}
|
|
1157
|
+
this.#registeredShard = shard;
|
|
1158
|
+
}
|
|
773
1159
|
const session = await hub.connect(this.#connectOptions(server, identity));
|
|
774
1160
|
this.#sessions.set(server, session);
|
|
775
1161
|
if ((await this.#state.storage.getAlarm()) === null) {
|
|
@@ -809,9 +1195,11 @@ export class SyncularRealtimeHost<
|
|
|
809
1195
|
}
|
|
810
1196
|
|
|
811
1197
|
/**
|
|
812
|
-
* The Durable Object alarm: send a §8.5 heartbeat to every socket
|
|
813
|
-
*
|
|
814
|
-
*
|
|
1198
|
+
* The Durable Object alarm: send a §8.5 heartbeat to every socket, retry
|
|
1199
|
+
* every shard that owes a failed relay (§8.9), and re-arm: after the
|
|
1200
|
+
* heartbeat interval while sockets remain, after one second while a shard
|
|
1201
|
+
* still owes. The heartbeat goes straight to the hibernatable sockets, so it
|
|
1202
|
+
* rebuilds no session.
|
|
815
1203
|
*/
|
|
816
1204
|
async alarm(): Promise<void> {
|
|
817
1205
|
const sockets = this.#state.getWebSockets();
|
|
@@ -823,9 +1211,28 @@ export class SyncularRealtimeHost<
|
|
|
823
1211
|
// A closing socket; its close callback cleans up.
|
|
824
1212
|
}
|
|
825
1213
|
}
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
1214
|
+
const storage = this.#state.storage;
|
|
1215
|
+
for (const [key, value] of await storage.list({ prefix: 'owed:' })) {
|
|
1216
|
+
const shard = key.slice('owed:'.length);
|
|
1217
|
+
const token = await storage.get(`shard:${shard}`);
|
|
1218
|
+
const message = JSON.parse(String(value)) as {
|
|
1219
|
+
readonly partition: string;
|
|
1220
|
+
readonly owed: Owed;
|
|
1221
|
+
};
|
|
1222
|
+
// A newer failure may have merged into the record during the retry.
|
|
1223
|
+
if (
|
|
1224
|
+
(await this.#relay(shard, token, encodeFanout(message))) ===
|
|
1225
|
+
'delivered' &&
|
|
1226
|
+
(await storage.get(key)) === value
|
|
1227
|
+
) {
|
|
1228
|
+
await storage.delete(key);
|
|
1229
|
+
}
|
|
1230
|
+
}
|
|
1231
|
+
const owing = (await storage.list({ prefix: 'owed:' })).size > 0;
|
|
1232
|
+
if (sockets.length > 0 || owing) {
|
|
1233
|
+
await storage.setAlarm(
|
|
1234
|
+
Date.now() +
|
|
1235
|
+
(owing ? 1_000 : (this.#config.heartbeatIntervalMs ?? 30_000)),
|
|
829
1236
|
);
|
|
830
1237
|
}
|
|
831
1238
|
}
|
|
@@ -935,6 +1342,8 @@ const REALTIME_ID_HEADER = {
|
|
|
935
1342
|
} as const;
|
|
936
1343
|
|
|
937
1344
|
const ADMIN_ACTIONS_HEADER = 'x-syncular-admin-actions';
|
|
1345
|
+
/** The object name of the shard a sharded upgrade goes to (§8.9). */
|
|
1346
|
+
const REALTIME_SHARD_HEADER = 'x-syncular-shard';
|
|
938
1347
|
|
|
939
1348
|
/** Address a Worker-authorized admin request to the DO (Worker side). */
|
|
940
1349
|
export function adminRequestForDurableObject(
|
|
@@ -978,13 +1387,20 @@ export function writeRequestIdentityHeaders(
|
|
|
978
1387
|
}
|
|
979
1388
|
}
|
|
980
1389
|
|
|
981
|
-
/**
|
|
1390
|
+
/**
|
|
1391
|
+
* Write the resolved identity, and the object name of a sharded upgrade's
|
|
1392
|
+
* shard (§8.9), onto an upgrade request's headers (Worker side). A shard
|
|
1393
|
+
* header the client sent is removed when the upgrade is not sharded.
|
|
1394
|
+
*/
|
|
982
1395
|
export function writeIdentityHeaders(
|
|
983
1396
|
headers: Headers,
|
|
984
1397
|
identity: RealtimeUpgradeIdentity,
|
|
1398
|
+
shard?: string,
|
|
985
1399
|
): void {
|
|
986
1400
|
writeRequestIdentityHeaders(headers, identity);
|
|
987
1401
|
headers.set(REALTIME_ID_HEADER.clientId, identity.clientId);
|
|
1402
|
+
if (shard === undefined) headers.delete(REALTIME_SHARD_HEADER);
|
|
1403
|
+
else headers.set(REALTIME_SHARD_HEADER, shard);
|
|
988
1404
|
}
|
|
989
1405
|
|
|
990
1406
|
function readIdentityHeaders(
|