@evolu/common 8.17.0 → 8.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. package/dist/src/Bytes.d.ts +31 -14
  2. package/dist/src/Bytes.d.ts.map +1 -1
  3. package/dist/src/Bytes.js +48 -8
  4. package/dist/src/Error.d.ts.map +1 -1
  5. package/dist/src/Error.js +5 -1
  6. package/dist/src/Polyfills.d.ts.map +1 -1
  7. package/dist/src/Polyfills.js +3 -2
  8. package/dist/src/Task.d.ts +4 -3
  9. package/dist/src/Task.d.ts.map +1 -1
  10. package/dist/src/Task.js +4 -3
  11. package/dist/src/local-first/Db.d.ts.map +1 -1
  12. package/dist/src/local-first/Db.js +18 -7
  13. package/dist/src/local-first/Evolu.d.ts +10 -1
  14. package/dist/src/local-first/Evolu.d.ts.map +1 -1
  15. package/dist/src/local-first/Evolu.js +5 -1
  16. package/dist/src/local-first/Protocol.d.ts +215 -93
  17. package/dist/src/local-first/Protocol.d.ts.map +1 -1
  18. package/dist/src/local-first/Protocol.js +799 -489
  19. package/dist/src/local-first/Relay.d.ts +9 -1
  20. package/dist/src/local-first/Relay.d.ts.map +1 -1
  21. package/dist/src/local-first/Relay.js +41 -36
  22. package/dist/src/local-first/Shared.d.ts +75 -49
  23. package/dist/src/local-first/Shared.d.ts.map +1 -1
  24. package/dist/src/local-first/Shared.js +62 -42
  25. package/dist/src/local-first/Storage.d.ts +80 -18
  26. package/dist/src/local-first/Storage.d.ts.map +1 -1
  27. package/dist/src/local-first/Storage.js +20 -4
  28. package/package.json +1 -1
  29. package/src/Bytes.test.ts +212 -0
  30. package/src/Bytes.ts +67 -17
  31. package/src/Error.ts +5 -1
  32. package/src/Polyfills.ts +3 -2
  33. package/src/Task.ts +4 -3
  34. package/src/local-first/Db.ts +29 -14
  35. package/src/local-first/Evolu.test.ts +43 -12
  36. package/src/local-first/Evolu.ts +16 -2
  37. package/src/local-first/Protocol.test.ts +1541 -44
  38. package/src/local-first/Protocol.ts +1070 -605
  39. package/src/local-first/Relay.ts +80 -69
  40. package/src/local-first/Shared.ts +106 -67
  41. package/src/local-first/Storage.ts +100 -27
@@ -43,14 +43,24 @@
43
43
  * | - {@link NonNegativeInt} | Number of ranges. |
44
44
  * | - {@link Range} | |
45
45
  *
46
+ * Range upper bounds must not decrease. The last range always has
47
+ * {@link InfiniteUpperBound}, which is not encoded. A message with decreasing
48
+ * bounds, or with a hasWriteKey or subscriptionFlag value not listed above, is
49
+ * rejected as {@link ProtocolInvalidDataError}. So is a request with a change
50
+ * shorter than 41 bytes, the smallest {@link EncryptedDbChange}. A client passes
51
+ * such a change to storage, which skips a change it cannot read.
52
+ *
46
53
  * ## WriteKey validation
47
54
  *
48
55
  * The initiator sends a hasWriteKey flag and optionally a WriteKey. The
49
56
  * WriteKey is required when sending messages as a secure token proving the
50
57
  * initiator can write changes. It's ok to not send a WriteKey if the initiator
51
58
  * is only syncing (read-only) and not sending messages. The non-initiator
52
- * validates the WriteKey immediately after parsing the initiator header, before
53
- * processing any messages or ranges.
59
+ * decodes the whole request first and then validates the WriteKey, before
60
+ * storing any messages or reconciling ranges. Only the subscriptionFlag takes
61
+ * effect before that, because subscribing needs no WriteKey: broadcasts carry
62
+ * only encrypted changes. So a request rejected with
63
+ * {@link ProtocolWriteKeyError} still changes its subscription.
54
64
  *
55
65
  * ## Synchronization
56
66
  *
@@ -64,9 +74,14 @@
64
74
  * if further sync is needed or possible, continuing until both sides are
65
75
  * synchronized.
66
76
  *
67
- * The **non-initiator always responds** to provide sync completion feedback,
68
- * even with empty messages containing only the header and no error. This allows
69
- * the initiator to detect when synchronization is complete.
77
+ * The **non-initiator answers every request it can decode** to provide sync
78
+ * completion feedback, even with empty messages containing only the header and
79
+ * no error. This allows the initiator to detect when synchronization is
80
+ * complete. A request it cannot decode, or one larger than the relay's
81
+ * `totalMaxSize`, gets no answer and has no effect.
82
+ *
83
+ * Ranges are compared by their {@link Fingerprint}, which anyone who can write
84
+ * for an owner can make collide. Its documentation describes the consequences.
70
85
  *
71
86
  * Both **Messages** and **Ranges** are optional, allowing each side to send,
72
87
  * sync, or only subscribe data as needed.
@@ -81,18 +96,35 @@
81
96
  *
82
97
  * ## Protocol errors
83
98
  *
84
- * The protocol uses error codes in the header to signal issues:
99
+ * A Response carries a {@link ProtocolErrorCode} in its header. The initiator
100
+ * reports every code except `NoError` as an error with the `OwnerId`:
85
101
  *
86
- * - {@link ProtocolWriteKeyError}: The provided WriteKey is invalid or missing.
87
- * - {@link ProtocolWriteError}: A serious relay-side write failure occurred.
102
+ * - {@link ProtocolWriteKeyError}: The WriteKey is invalid, or missing from a
103
+ * request with messages.
104
+ * - {@link ProtocolWriteError}: The relay failed to store the messages or to
105
+ * validate the WriteKey.
88
106
  * - {@link ProtocolQuotaError}: Storage or billing quota exceeded.
89
- * - {@link ProtocolSyncError}: A serious relay-side synchronization failure
90
- * occurred.
91
- * - {@link ProtocolVersionError}: Protocol version mismatch.
92
- * - {@link ProtocolInvalidDataError}: The message is malformed or corrupted.
107
+ * - {@link ProtocolSyncError}: The relay's storage failed while it reconciled the
108
+ * ranges.
109
+ *
110
+ * The initiator also reports {@link ProtocolVersionError}, with the `OwnerId`,
111
+ * for a reply of another version, and {@link ProtocolInvalidDataError}, without
112
+ * it, for a message it cannot decode, including one with an unknown code.
93
113
  *
94
- * All protocol errors except `ProtocolInvalidDataError` include the `OwnerId`
95
- * to allow clients to associate errors with the correct owner.
114
+ * A relay answers every request it can decode within its `totalMaxSize`. It
115
+ * decodes a whole request before acting on it, so for a malformed one it
116
+ * returns `ProtocolInvalidDataError` without subscribing, storing, or
117
+ * broadcasting anything, and sends no reply. When its storage throws while
118
+ * validating the WriteKey, for example because SQLite cannot store a new
119
+ * owner's key on a full disk, the relay logs the error and answers with
120
+ * `ProtocolWriteError`, like a failed write. Likewise, a client applies nothing
121
+ * from a message it cannot decode.
122
+ *
123
+ * {@link decryptAndDecodeDbChange} returns `ProtocolInvalidDataError`,
124
+ * {@link ProtocolTimestampMismatchError}, or
125
+ * {@link DecryptWithXChaCha20Poly1305Error} for a change it cannot read. These
126
+ * describe a change rather than a protocol message, and client storage skips
127
+ * such a change.
96
128
  *
97
129
  * ## Message size limit
98
130
  *
@@ -105,7 +137,10 @@
105
137
  *
106
138
  * Each mutation is limited to {@link maxMutationSize}, so every change fits one
107
139
  * message of {@link defaultProtocolMessageMaxSize} next to the largest ranges
108
- * section.
140
+ * section. Changes saved before that limit existed, and crafted changes a relay
141
+ * stores, can be larger. Sync skips a stored change that cannot fit an empty
142
+ * message after a pending Skip range, which is the message a later round is
143
+ * guaranteed to reach, and reports it as a {@link ProtocolChangeTooLargeError}.
109
144
  *
110
145
  * ## Why Binary?
111
146
  *
@@ -129,24 +164,18 @@
129
164
  *
130
165
  * ## Versioning
131
166
  *
132
- * Evolu Protocol uses explicit versioning to ensure compatibility between
133
- * clients and relays (or peers). Each protocol message begins with a version
134
- * number and an `ownerId` in its header.
135
- *
136
- * **How version negotiation works:**
137
- *
138
- * - The initiator (usually a client) sends a `ProtocolMessage` that includes its
139
- * protocol version and the `ownerId`.
140
- * - The non-initiator (usually a relay or peer) checks the version.
167
+ * Every message of every protocol version begins with the version and the
168
+ * `OwnerId`, so a peer can route and report a message of any version. Nothing
169
+ * negotiates a version, and no side falls back to the other's.
141
170
  *
142
- * - If the versions match, synchronization proceeds as normal.
143
- * - If the versions do not match, the non-initiator responds with a message
144
- * containing **its own protocol version and the same `ownerId`**.
145
- * - The initiator can then detect the version mismatch for that specific owner
146
- * and handle it appropriately (e.g., prompt for an update or halt sync).
147
- *
148
- * Version negotiation is per-owner, allowing Evolu Protocol to evolve safely
149
- * over time and provide clear feedback about version mismatches.
171
+ * A non-initiator answers a request of another version with only its own
172
+ * version and that `OwnerId`. The initiator reports it as
173
+ * {@link ProtocolVersionError} for that owner, whose `isInitiator` tells which
174
+ * side is older, and stops syncing the owner through that relay. Clients from
175
+ * `@evolu/common` 8.0.0 before 8.11.0 drop that reply and stop syncing the
176
+ * owner through that relay without reporting anything, while 7.x clients report
177
+ * it as a `ProtocolVersionError`. So a relay that moves to another version must
178
+ * keep answering version 1 while such clients remain.
150
179
  *
151
180
  * ## Credible exit
152
181
  *
@@ -180,13 +209,7 @@ import {
180
209
  isNonEmptyArray,
181
210
  type NonEmptyReadonlyArray,
182
211
  } from "../Array.ts";
183
- import {
184
- assert,
185
- assertNonEmptyArray,
186
- assertNonNullable,
187
- assertNotUndefined,
188
- assertSame,
189
- } from "../Assert.ts";
212
+ import { assert, assertNonEmptyArray, assertNonNullable } from "../Assert.ts";
190
213
  import type { Brand } from "../Brand.ts";
191
214
  import {
192
215
  type Buffer,
@@ -214,12 +237,13 @@ import {
214
237
  type DecryptWithXChaCha20Poly1305Error,
215
238
  EncryptionKey,
216
239
  encryptWithXChaCha20Poly1305,
217
- Entropy24,
240
+ type Entropy24,
218
241
  type RandomBytesDep,
219
- XChaCha20Poly1305Ciphertext,
242
+ type XChaCha20Poly1305Ciphertext,
220
243
  xChaCha20Poly1305NonceLength,
221
244
  } from "../Crypto.ts";
222
245
  import { eqArrayNumber } from "../Eq.ts";
246
+ import { exhaustiveCheck } from "../Function.ts";
223
247
  import { computeBalancedBuckets } from "../Number.ts";
224
248
  import { createMutableRecord, objectToEntries } from "../Object.ts";
225
249
  import { err, ok, type Result } from "../Result.ts";
@@ -232,7 +256,7 @@ import {
232
256
  base64UrlToUint8Array,
233
257
  between,
234
258
  DateIso,
235
- FiniteNumber,
259
+ type FiniteNumber,
236
260
  Id,
237
261
  IdBytes,
238
262
  idBytesToId,
@@ -243,7 +267,7 @@ import {
243
267
  jsonToJsonValue,
244
268
  NonNegativeInt,
245
269
  onePositiveInt,
246
- PositiveInt,
270
+ type PositiveInt,
247
271
  type Typed,
248
272
  uint8ArrayToBase64Url,
249
273
  zeroNonNegativeInt,
@@ -285,8 +309,9 @@ import {
285
309
  nodeIdBytesLength,
286
310
  nodeIdBytesToNodeId,
287
311
  nodeIdToNodeIdBytes,
312
+ orderTimestamp,
288
313
  Timestamp,
289
- TimestampBytes,
314
+ type TimestampBytes,
290
315
  timestampBytesLength,
291
316
  timestampBytesToTimestamp,
292
317
  timestampToTimestampBytes,
@@ -294,22 +319,27 @@ import {
294
319
 
295
320
  const minProtocolMessageMaxSize = 1_000_000;
296
321
  const maxProtocolMessageMaxSize = 100_000_000;
322
+ const maxProtocolMessageRangesMaxSize = 100_000;
297
323
 
298
324
  /**
299
- * Protocol message maximum size.
325
+ * Protocol message maximum size, from 1MB to 100MB.
300
326
  *
301
- * Defines the upper limit for how large a single protocol message can be.
302
- * Implementations must enforce a maximum size between 1MB and 100MB to ensure
303
- * compatibility across all Evolu implementations (the maximum size of mutation
304
- * change is hardcoded and enforced hence the maximum size can't be smaller).
327
+ * A message never exceeds its maximum size, inclusive. Builders measure each
328
+ * write exactly and keep room to close the message (see
329
+ * {@link ProtocolMessageBuffer.tryWrite}), and sync sends what does not fit in
330
+ * later rounds.
305
331
  *
306
- * Larger maximum sizes can be configured by relays to reduce roundtrips. For
307
- * example, a dedicated relay with ample resources could configure a 100MB
308
- * maximum to minimize roundtrips for large syncs.
332
+ * Clients send at most {@link defaultProtocolMessageMaxSize} and enforce no
333
+ * receive limit. A relay can answer with larger messages, configured with the
334
+ * `totalMaxSize` option of {@link applyProtocolMessageAsRelay}, to reduce
335
+ * roundtrips for large syncs. Clients must not send larger messages, because
336
+ * relays accept at most the default: the Node.js relay closes the connection on
337
+ * a larger one, and relays up to `@evolu/nodejs` 4.0.0 crash on it.
309
338
  *
310
- * Only relays can safely configure larger sizes, as clients will handle them.
311
- * Increasing this value on the client side would break compatibility with
312
- * relays that enforce smaller limits.
339
+ * The default cannot be smaller either. Deployed clients send messages of that
340
+ * size, a change within {@link maxMutationSize} fits one message next to the
341
+ * largest ranges section, and changes saved before that limit existed, which
342
+ * can be nearly as large as a message, must stay servable.
313
343
  */
314
344
  export const ProtocolMessageMaxSize = /*#__PURE__*/ between(
315
345
  minProtocolMessageMaxSize,
@@ -321,8 +351,9 @@ export type ProtocolMessageMaxSize = typeof ProtocolMessageMaxSize.Output;
321
351
  /**
322
352
  * Default {@link ProtocolMessageMaxSize} (1MB).
323
353
  *
324
- * The standard size used across Evolu implementations. Relays with more
325
- * resources can configure larger sizes to reduce roundtrips.
354
+ * The standard size used across Evolu implementations. Clients send at most
355
+ * this size, and relays accept it. Relays with more resources can answer with
356
+ * larger messages to reduce roundtrips.
326
357
  */
327
358
  export const defaultProtocolMessageMaxSize =
328
359
  minProtocolMessageMaxSize as ProtocolMessageMaxSize;
@@ -336,11 +367,12 @@ export const defaultProtocolMessageMaxSize =
336
367
  *
337
368
  * The upper bound is set to ensure ranges fit within the default 1MB
338
369
  * {@link defaultProtocolMessageMaxSize}, maintaining compatibility between all
339
- * clients and relays.
370
+ * clients and relays. A received message whose ranges section exceeds twice the
371
+ * upper bound is rejected as {@link ProtocolInvalidDataError}.
340
372
  */
341
373
  export const ProtocolMessageRangesMaxSize = /*#__PURE__*/ between(
342
374
  3_000,
343
- 100_000,
375
+ maxProtocolMessageRangesMaxSize,
344
376
  )(Int);
345
377
  export type ProtocolMessageRangesMaxSize =
346
378
  typeof ProtocolMessageRangesMaxSize.Output;
@@ -419,6 +451,14 @@ export const SubscriptionFlags = {
419
451
  export type SubscriptionFlag =
420
452
  (typeof SubscriptionFlags)[keyof typeof SubscriptionFlags];
421
453
 
454
+ /**
455
+ * The error code in the header of a Response.
456
+ *
457
+ * A client reports a code it does not know as {@link ProtocolInvalidDataError},
458
+ * which carries no `OwnerId`, and that relay's round for the owner ends. Every
459
+ * released client does this, so a relay can add a code under the same
460
+ * {@link protocolVersion} only where that outcome is acceptable.
461
+ */
422
462
  export const ProtocolErrorCode = {
423
463
  NoError: 0,
424
464
  /** A code for {@link ProtocolWriteKeyError}. */
@@ -431,7 +471,7 @@ export const ProtocolErrorCode = {
431
471
  SyncError: 4,
432
472
  } as const;
433
473
 
434
- type ProtocolErrorCode =
474
+ export type ProtocolErrorCode =
435
475
  (typeof ProtocolErrorCode)[keyof typeof ProtocolErrorCode];
436
476
 
437
477
  export type ProtocolError =
@@ -454,7 +494,16 @@ export interface ProtocolVersionError
454
494
  readonly isInitiator: boolean;
455
495
  }
456
496
 
457
- /** Error for invalid or corrupted protocol message data. */
497
+ /**
498
+ * Error for a malformed {@link ProtocolMessage} or {@link EncryptedDbChange},
499
+ * with its bytes as `data` and the decoding error as `error`.
500
+ *
501
+ * A protocol message is decoded whole before any of it is applied, so a
502
+ * malformed one has no effect, and a relay does not reply to it. A message of a
503
+ * type its receiver does not accept, such as a Request sent to a client, is
504
+ * malformed too. A throw while applying a decoded message is a defect, not this
505
+ * error.
506
+ */
458
507
  export interface ProtocolInvalidDataError extends Typed<"ProtocolInvalidDataError"> {
459
508
  readonly data: Uint8Array;
460
509
  readonly error: unknown;
@@ -507,12 +556,40 @@ export interface ProtocolTimestampMismatchError extends Typed<"ProtocolTimestamp
507
556
  readonly timestamp: Timestamp;
508
557
  }
509
558
 
559
+ /**
560
+ * Error for a stored change that sync skipped because it cannot fit an empty
561
+ * {@link ProtocolMessage} after a pending Skip range, which is the message a
562
+ * later round is guaranteed to reach.
563
+ *
564
+ * Every change within {@link maxMutationSize} fits. Larger ones are changes
565
+ * saved before that limit existed and crafted changes a relay stores. A change
566
+ * up to 22 bytes too large for that message can still fit one without a pending
567
+ * skip, so sync may send it, sometimes after reporting it, but usually reports
568
+ * it in every sync like a larger one. With
569
+ * {@link defaultProtocolMessageMaxSize}, PADMÉ padding leaves no honest change
570
+ * size in that 22-byte window.
571
+ *
572
+ * An answer to a Timestamps range neither sends nor lists a skipped change, so
573
+ * range fingerprints keep disagreeing about it. A request or a split still
574
+ * lists its timestamp, so a peer that lacks it may ask for it once per sync and
575
+ * gets an answer without it. Every sync narrows the fingerprints to it, skips
576
+ * it again, and ends.
577
+ */
578
+ export interface ProtocolChangeTooLargeError extends Typed<"ProtocolChangeTooLargeError"> {
579
+ readonly timestamp: Timestamp;
580
+ /** The length of the encrypted change in bytes. */
581
+ readonly size: PositiveInt;
582
+ }
583
+
510
584
  /**
511
585
  * Creates a {@link ProtocolMessage} from CRDT messages.
512
586
  *
513
- * If the message size would exceed {@link defaultProtocolMessageMaxSize}, the
514
- * protocol ensures all messages will be sent in the next round(s) even over
515
- * unidirectional and stateless transports.
587
+ * The message holds the leading CRDT messages that fit `maxSize`, by default
588
+ * {@link defaultProtocolMessageMaxSize}, measured exactly. When one does not
589
+ * fit, it and the rest are left out, and the message ends with a range that
590
+ * makes the non-initiator answer with ranges. Sync then sends them in later
591
+ * rounds, even over unidirectional and stateless transports, because every
592
+ * change within {@link maxMutationSize} fits an empty message.
516
593
  */
517
594
  export const createProtocolMessageFromCrdtMessages =
518
595
  (deps: RandomBytesDep) =>
@@ -527,50 +604,46 @@ export const createProtocolMessageFromCrdtMessages =
527
604
  writeKey: owner.writeKey,
528
605
  });
529
606
 
530
- let notAllMessagesSent = false;
531
-
532
607
  for (const message of messages) {
533
608
  const change = encodeAndEncryptDbChange(deps)(
534
609
  message,
535
610
  owner.encryptionKey,
536
611
  );
537
612
  const encryptedCrdtMessage = { timestamp: message.timestamp, change };
538
- if (buffer.canAddMessage(encryptedCrdtMessage)) {
539
- buffer.addMessage(encryptedCrdtMessage);
540
- } else {
541
- notAllMessagesSent = true;
613
+ if (
614
+ !buffer.tryWrite(() => {
615
+ buffer.addMessage(encryptedCrdtMessage);
616
+ })
617
+ ) {
618
+ /**
619
+ * DEV: If not all messages fit due to size limits, we trigger a sync
620
+ * continuation by appending a Range with a random fingerprint. This
621
+ * ensures the receiver always responds with ranges, prompting another
622
+ * sync round.
623
+ *
624
+ * The ideal approach would be to send three ranges (skip, fingerprint,
625
+ * skip) where the fingerprint of unsent messages would act as narrow
626
+ * sync probe. I think we can send `zeroFingerprint` which can be
627
+ * interpreted as an indication that the other side should reply with
628
+ * {@link TimestampsRange}, so no need to restart syncing.
629
+ *
630
+ * For now, using a random fingerprint avoids extra complexity and is
631
+ * good enough for this case.
632
+ */
633
+ const randomFingerprint = deps.randomBytes.create(
634
+ fingerprintSize,
635
+ ) as unknown as Fingerprint;
636
+
637
+ // Every kept trial reserved space for this closing range.
638
+ buffer.addRange({
639
+ type: RangeType.Fingerprint,
640
+ upperBound: InfiniteUpperBound,
641
+ fingerprint: randomFingerprint,
642
+ });
542
643
  break;
543
644
  }
544
645
  }
545
646
 
546
- if (notAllMessagesSent) {
547
- /**
548
- * DEV: If not all messages fit due to size limits, we trigger a sync
549
- * continuation by appending a Range with a random fingerprint. This
550
- * ensures the receiver always responds with ranges, prompting another
551
- * sync round.
552
- *
553
- * The ideal approach would be to send three ranges (skip, fingerprint,
554
- * skip) where the fingerprint of unsent messages would act as narrow sync
555
- * probe. I think we can send `zeroFingerprint` which can be interpreted
556
- * as an indication that the other side should reply with
557
- * {@link TimestampsRange}, so no need to restart syncing.
558
- *
559
- * For now, using a random fingerprint avoids extra complexity and is good
560
- * enough for this case.
561
- */
562
- const randomFingerprint = deps.randomBytes.create(
563
- fingerprintSize,
564
- ) as unknown as Fingerprint;
565
-
566
- // There is always a space for Fingerprint with InfiniteUpperBound.
567
- buffer.addRange({
568
- type: RangeType.Fingerprint,
569
- upperBound: InfiniteUpperBound,
570
- fingerprint: randomFingerprint,
571
- });
572
- }
573
-
574
647
  return buffer.unwrap();
575
648
  };
576
649
 
@@ -631,20 +704,22 @@ export const createProtocolBroadcastMessagesFromCrdtMessages =
631
704
  change: encodeAndEncryptDbChange(deps)(message, owner.encryptionKey),
632
705
  };
633
706
 
634
- if (!buffer.canAddMessage(encryptedMessage)) {
635
- const nextBuffer = createProtocolMessageBuffer(owner.id, {
636
- messageType: MessageType.Broadcast,
637
- totalMaxSize: maxSize,
638
- });
639
- assert(
640
- nextBuffer.canAddMessage(encryptedMessage),
641
- "the message is too big",
642
- );
643
- broadcasts.push(buffer.unwrap());
644
- buffer = nextBuffer;
707
+ const frame = buffer;
708
+ if (
709
+ frame.tryWrite(() => {
710
+ frame.addMessage(encryptedMessage);
711
+ })
712
+ ) {
713
+ continue;
645
714
  }
646
715
 
716
+ buffer = createProtocolMessageBuffer(owner.id, {
717
+ messageType: MessageType.Broadcast,
718
+ totalMaxSize: maxSize,
719
+ });
720
+ // Outside a trial, it asserts that the message fits an empty frame.
647
721
  buffer.addMessage(encryptedMessage);
722
+ broadcasts.push(frame.unwrap());
648
723
  }
649
724
 
650
725
  broadcasts.push(buffer.unwrap());
@@ -664,13 +739,13 @@ export const createProtocolMessageForSync =
664
739
 
665
740
  const size = deps.storage.getSize(ownerIdBytes);
666
741
 
667
- splitRange(deps)(
742
+ const ranges = readSplitRanges(deps)(
668
743
  ownerIdBytes,
669
744
  zeroNonNegativeInt,
670
745
  size,
671
746
  InfiniteUpperBound,
672
- buffer,
673
747
  );
748
+ for (const range of ranges) buffer.addRange(range);
674
749
 
675
750
  return buffer.unwrap();
676
751
  };
@@ -686,24 +761,43 @@ export const createProtocolMessageForUnsubscribe = (
686
761
  /**
687
762
  * Mutable builder for constructing {@link ProtocolMessage} respecting size
688
763
  * limits.
764
+ *
765
+ * A frame never exceeds `totalMaxSize`. Make ordinary writes inside `tryWrite`,
766
+ * which keeps them only if the frame can still be closed. A write outside a
767
+ * trial is a closing write: `addMessage` and `addRange` assert that the frame
768
+ * fits `totalMaxSize`.
769
+ *
770
+ * The builder references each change, which `unwrap` copies into the frame it
771
+ * returns, so a change must not be modified while the builder is in use.
772
+ * `unwrap` leaves the builder unchanged, so it can be called again.
689
773
  */
690
774
  export interface ProtocolMessageBuffer {
691
- readonly canAddMessage: (message: EncryptedCrdtMessage) => boolean;
692
-
693
775
  readonly addMessage: (message: EncryptedCrdtMessage) => void;
694
776
 
695
- readonly canSplitRange: () => boolean;
696
-
697
- readonly canAddTimestampsRangeAndMessage: (
698
- timestamps: TimestampsBuffer,
699
- message: EncryptedCrdtMessage | null,
700
- ) => boolean;
701
-
702
777
  readonly addRange: (
703
778
  range: SkipRange | FingerprintRange | TimestampsRangeWithTimestampsBuffer,
704
779
  ) => void;
705
780
 
781
+ /**
782
+ * Runs `write` and keeps what it wrote only if the frame can still be closed,
783
+ * returning whether it was kept.
784
+ *
785
+ * The frame can be closed when its exact size plus the bytes needed to append
786
+ * one Fingerprint range with {@link InfiniteUpperBound} plus `reserve` is at
787
+ * most `totalMaxSize`, and its ranges section plus the same bytes is at most
788
+ * `rangesMaxSize`. Nothing is needed to close a broadcast, which holds no
789
+ * ranges, or a frame whose last range has InfiniteUpperBound. Use `reserve`
790
+ * for bytes the caller adds later, such as a range it is still collecting.
791
+ *
792
+ * When the frame cannot be closed, or `write` throws, everything `write`
793
+ * wrote is undone, so the frame is as if `write` never ran. Inside `write`,
794
+ * `addMessage` and `addRange` do not assert the size limit.
795
+ */
796
+ readonly tryWrite: (write: () => void, reserve?: NonNegativeInt) => boolean;
797
+
706
798
  readonly unwrap: () => ProtocolMessage;
799
+
800
+ /** Returns the exact encoded size of the frame. */
707
801
  readonly getSize: () => PositiveInt;
708
802
  }
709
803
 
@@ -738,7 +832,7 @@ export const createProtocolMessageBuffer = (
738
832
  header: createBuffer(),
739
833
  messages: {
740
834
  timestamps: createTimestampsBuffer(),
741
- dbChanges: createBuffer(),
835
+ dbChangeLengths: createBuffer(),
742
836
  },
743
837
  ranges: {
744
838
  timestamps: createTimestampsBuffer(),
@@ -764,86 +858,44 @@ export const createProtocolMessageBuffer = (
764
858
  buffers.header.extend([options.errorCode]);
765
859
  }
766
860
 
861
+ // Changes are referenced, not copied, so a trial undoes messages by
862
+ // shortening this array, and unwrap copies each change once.
863
+ const dbChanges: Array<EncryptedDbChange> = [];
864
+ let dbChangesLength = 0;
767
865
  let isLastRangeInfinite = false;
768
-
769
- const isWithinSizeLimits = () => getSize() <= totalMaxSize;
770
-
771
- const getSize = () =>
772
- PositiveInt.orThrow(getHeaderAndMessagesSize() + getRangesSize());
773
-
774
- const getHeaderAndMessagesSize = () =>
775
- buffers.header.getLength() +
776
- buffers.messages.timestamps.getLength() +
777
- buffers.messages.dbChanges.getLength();
778
-
866
+ let isInTrial = false;
867
+
868
+ // Writes outside a trial assert this, so an overflow fails where it happens.
869
+ // A frame must never exceed totalMaxSize: relays at @evolu/nodejs 4.0.0 or
870
+ // older crash on a frame one byte larger than their 1,000,000-byte limit.
871
+ const isWithinSizeLimits = () => getFrameSize() <= totalMaxSize;
872
+
873
+ // Lengths after a header that is never empty.
874
+ const getFrameSize = () =>
875
+ (buffers.header.getLength() +
876
+ buffers.messages.timestamps.getLength() +
877
+ buffers.messages.dbChangeLengths.getLength() +
878
+ dbChangesLength +
879
+ getRangesSize()) as PositiveInt;
880
+
881
+ // Without ranges, the ranges section is omitted, including its count.
779
882
  const getRangesSize = () =>
780
883
  buffers.ranges.timestamps.getCount() > 0
781
884
  ? buffers.ranges.timestamps.getLength() +
782
885
  buffers.ranges.types.getLength() +
783
- buffers.ranges.payloads.getLength() +
784
- safeMargins.remainingRange
886
+ buffers.ranges.payloads.getLength()
785
887
  : 0;
786
888
 
787
- /**
788
- * We calculated worst-case sizes as closely as possible and added a small
789
- * safety margin, since computing exact worst cases is difficult due to
790
- * variable-length, run-length, and delta encoding.
791
- *
792
- * Runtime assertions (`assert`) are used to guarantee that size limits are
793
- * never exceeded. If a limit is exceeded, the assertion will fail at the
794
- * precise location, making it easy to identify and fix the issue.
795
- *
796
- * While it would be possible to avoid the safety margin by snapshotting
797
- * buffer states and rolling back changes, this would likely impact
798
- * performance. If someone has time and wants to experiment with this
799
- * approach, contributions are welcome.
800
- */
801
- const safeMargins = {
802
- // bytes: range type + possible increased count varint
803
- remainingRange: fingerprintSize + 10,
804
- // bytes: max millis + max count + NodeId
805
- timestamp: 30,
806
- // bytes: maximum encoded DbChange length varint
807
- dbChangeLength: 8,
808
- // bytes: worst case is around 650 bytes
809
- splitRange: 800,
810
- // bytes: range type + its upperBound + possible increased count varint
811
- timestampsRange: 50,
812
- };
813
-
814
- const addMessageSafeMargin =
815
- safeMargins.timestamp +
816
- safeMargins.dbChangeLength +
817
- safeMargins.remainingRange;
818
-
889
+ // A trial makes the write, measures the exact frame, and undoes the write in
890
+ // constant time when the frame no longer fits, so it needs no worst-case
891
+ // margins and costs about as much as the write itself.
819
892
  return {
820
- canAddMessage: (message) =>
821
- getSize() + addMessageSafeMargin + message.change.length <= totalMaxSize,
822
-
823
893
  addMessage: (message) => {
824
894
  buffers.messages.timestamps.add(message.timestamp);
825
- encodeLength(buffers.messages.dbChanges, message.change);
826
- buffers.messages.dbChanges.extend(message.change);
827
- assert(isWithinSizeLimits(), "the message is too big");
828
- },
829
-
830
- canSplitRange: () =>
831
- getRangesSize() + safeMargins.splitRange <= rangesMaxSize,
832
-
833
- canAddTimestampsRangeAndMessage: (timestamps, message) => {
834
- const rangesNewSize =
835
- getRangesSize() + timestamps.getLength() + safeMargins.timestampsRange;
836
-
837
- return (
838
- rangesNewSize <= rangesMaxSize &&
839
- (message
840
- ? getHeaderAndMessagesSize() +
841
- rangesNewSize +
842
- addMessageSafeMargin +
843
- message.change.length <=
844
- totalMaxSize
845
- : true)
846
- );
895
+ encodeLength(buffers.messages.dbChangeLengths, message.change);
896
+ dbChanges.push(message.change);
897
+ dbChangesLength += message.change.length;
898
+ assert(isInTrial || isWithinSizeLimits(), "the message is too big");
847
899
  },
848
900
 
849
901
  addRange: (range) => {
@@ -871,10 +923,7 @@ export const createProtocolMessageBuffer = (
871
923
  buffers.ranges.timestamps.addInfinite();
872
924
  }
873
925
 
874
- encodeNonNegativeInt(
875
- buffers.ranges.types,
876
- NonNegativeInt.orThrow(range.type),
877
- );
926
+ encodeNonNegativeInt(buffers.ranges.types, range.type as NonNegativeInt);
878
927
 
879
928
  switch (range.type) {
880
929
  case RangeType.Skip:
@@ -888,7 +937,57 @@ export const createProtocolMessageBuffer = (
888
937
  }
889
938
  }
890
939
 
891
- assert(isWithinSizeLimits(), `the range ${range.type} is too big`);
940
+ assert(
941
+ isInTrial || isWithinSizeLimits(),
942
+ `the range ${range.type} is too big`,
943
+ );
944
+ },
945
+
946
+ tryWrite: (write, reserve = zeroNonNegativeInt) => {
947
+ const restoreMessageTimestamps = buffers.messages.timestamps.checkpoint();
948
+ const dbChangeLengthsLength =
949
+ buffers.messages.dbChangeLengths.getLength();
950
+ const dbChangesCount = dbChanges.length;
951
+ const checkpointDbChangesLength = dbChangesLength;
952
+ const restoreRangeTimestamps = buffers.ranges.timestamps.checkpoint();
953
+ const typesLength = buffers.ranges.types.getLength();
954
+ const payloadsLength = buffers.ranges.payloads.getLength();
955
+ const wasLastRangeInfinite = isLastRangeInfinite;
956
+ const wasInTrial = isInTrial;
957
+
958
+ let isKept = false;
959
+ isInTrial = true;
960
+ try {
961
+ write();
962
+ // Fingerprint(InfiniteUpperBound) encodes no upper bound, only its
963
+ // type (1 byte) and fingerprint (12 bytes). The ranges count varint
964
+ // gains a byte when the new count is a power of 128. That includes
965
+ // 128^0 = 1, since the first range adds the count itself. So closing
966
+ // takes 13 or 14 bytes, and nothing for a Broadcast or a closed frame.
967
+ let newCount = buffers.ranges.timestamps.getCount() + 1;
968
+ while (newCount % 128 === 0) newCount /= 128;
969
+ const closingReserve =
970
+ options.messageType === MessageType.Broadcast || isLastRangeInfinite
971
+ ? 0
972
+ : 1 + fingerprintSize + (newCount === 1 ? 1 : 0);
973
+ const reserves = closingReserve + reserve;
974
+ isKept =
975
+ getFrameSize() + reserves <= totalMaxSize &&
976
+ getRangesSize() + reserves <= rangesMaxSize;
977
+ } finally {
978
+ isInTrial = wasInTrial;
979
+ if (!isKept) {
980
+ restoreMessageTimestamps();
981
+ buffers.messages.dbChangeLengths.truncate(dbChangeLengthsLength);
982
+ dbChanges.length = dbChangesCount;
983
+ dbChangesLength = checkpointDbChangesLength;
984
+ restoreRangeTimestamps();
985
+ buffers.ranges.types.truncate(typesLength);
986
+ buffers.ranges.payloads.truncate(payloadsLength);
987
+ isLastRangeInfinite = wasLastRangeInfinite;
988
+ }
989
+ }
990
+ return isKept;
892
991
  },
893
992
 
894
993
  unwrap: () => {
@@ -899,19 +998,44 @@ export const createProtocolMessageBuffer = (
899
998
  );
900
999
  }
901
1000
 
902
- buffers.messages.timestamps.append(buffers.header);
903
- buffers.header.extend(buffers.messages.dbChanges.unwrap());
1001
+ const frame = new Uint8Array(getFrameSize());
1002
+ frame.set(buffers.header.unwrap());
1003
+ let offset: number = buffers.header.getLength();
1004
+
1005
+ // Written to a new buffer, so the builder is unchanged and unwrap can be
1006
+ // called again.
1007
+ const messageTimestamps = createBuffer();
1008
+ buffers.messages.timestamps.append(messageTimestamps);
1009
+ frame.set(messageTimestamps.unwrap(), offset);
1010
+ offset += messageTimestamps.getLength();
1011
+
1012
+ // Each change follows its length, whose last varint byte is below 128.
1013
+ const lengths = buffers.messages.dbChangeLengths.unwrap();
1014
+ let lengthsOffset = 0;
1015
+ for (const change of dbChanges) {
1016
+ do {
1017
+ frame[offset++] = lengths[lengthsOffset];
1018
+ } while (lengths[lengthsOffset++] >= 128);
1019
+ frame.set(change, offset);
1020
+ offset += change.length;
1021
+ }
904
1022
 
905
1023
  if (buffers.ranges.timestamps.getCount() > 0) {
906
- buffers.ranges.timestamps.append(buffers.header);
907
- buffers.header.extend(buffers.ranges.types.unwrap());
908
- buffers.header.extend(buffers.ranges.payloads.unwrap());
1024
+ const ranges = createBuffer();
1025
+ buffers.ranges.timestamps.append(ranges);
1026
+ ranges.extend(buffers.ranges.types.unwrap());
1027
+ ranges.extend(buffers.ranges.payloads.unwrap());
1028
+ frame.set(ranges.unwrap(), offset);
1029
+ offset += ranges.getLength();
909
1030
  }
910
1031
 
911
- return buffers.header.unwrap() as ProtocolMessage;
1032
+ // An overcounted size would end the frame with zero bytes peers accept.
1033
+ assert(offset === frame.length, "the frame size is exact");
1034
+
1035
+ return frame as ProtocolMessage;
912
1036
  },
913
1037
 
914
- getSize,
1038
+ getSize: getFrameSize,
915
1039
  };
916
1040
  };
917
1041
 
@@ -926,6 +1050,15 @@ export interface TimestampsBuffer {
926
1050
  readonly getCount: () => NonNegativeInt;
927
1051
  readonly getLength: () => number;
928
1052
  readonly append: (buffer: Buffer) => void;
1053
+
1054
+ /**
1055
+ * Returns a function that restores the buffer, in constant time, to the state
1056
+ * it had when this was called, discarding every timestamp added since.
1057
+ *
1058
+ * A restore function can be called repeatedly. It is valid until the buffer
1059
+ * is restored to an earlier checkpoint.
1060
+ */
1061
+ readonly checkpoint: () => () => void;
929
1062
  }
930
1063
 
931
1064
  export const createTimestampsBuffer = (): TimestampsBuffer => {
@@ -983,16 +1116,41 @@ export const createTimestampsBuffer = (): TimestampsBuffer => {
983
1116
  buffer.extend(counterEncoder.unwrap());
984
1117
  buffer.extend(nodeIdEncoder.unwrap());
985
1118
  },
1119
+
1120
+ checkpoint: () => {
1121
+ const checkpointCount = count;
1122
+ const millisLength = millisBuffer.getLength();
1123
+ const checkpointPreviousMillis = previousMillis;
1124
+ const restoreCounters = counterEncoder.checkpoint();
1125
+ const restoreNodeIds = nodeIdEncoder.checkpoint();
1126
+
1127
+ return () => {
1128
+ count = checkpointCount;
1129
+ syncCount();
1130
+ millisBuffer.truncate(millisLength);
1131
+ previousMillis = checkpointPreviousMillis;
1132
+ restoreCounters();
1133
+ restoreNodeIds();
1134
+ };
1135
+ },
986
1136
  };
987
1137
  };
988
1138
 
989
1139
  export interface ApplyProtocolMessageAsClientOptions {
990
- writeKey?: OwnerWriteKey;
1140
+ readonly writeKey?: OwnerWriteKey;
991
1141
 
992
- rangesMaxSize?: ProtocolMessageRangesMaxSize;
1142
+ readonly rangesMaxSize?: ProtocolMessageRangesMaxSize;
1143
+
1144
+ /**
1145
+ * Called for each stored change that sync skipped as a
1146
+ * {@link ProtocolChangeTooLargeError}. Without it, the error is logged with
1147
+ * `console.warn`. It must not throw, because a throw is a defect that aborts
1148
+ * the Run.
1149
+ */
1150
+ readonly onChangeTooLarge?: (error: ProtocolChangeTooLargeError) => void;
993
1151
 
994
1152
  /** For tests only. */
995
- version?: NonNegativeInt;
1153
+ readonly version?: NonNegativeInt;
996
1154
  }
997
1155
 
998
1156
  /**
@@ -1027,11 +1185,11 @@ export interface ApplyProtocolMessageAsClientReadonly extends Typed<"Readonly">
1027
1185
  * Result of {@link applyProtocolMessageAsClient}: the protocol logged an
1028
1186
  * exception thrown by calling the storage's `writeMessages` (`Write`) or a
1029
1187
  * failed range reconciliation (`Sync`) to the console. An exception while the
1030
- * returned Task runs, as the built-in storages throw, is a defect that aborts
1031
- * the Run instead. Expected write rejections return the original
1032
- * {@link StorageWriteMessagesError} through {@link Result}. Reconciliation can
1033
- * fail after messages have been committed; that failure does not roll back the
1034
- * write.
1188
+ * returned Task runs is a defect that aborts the Run instead. A rejected or
1189
+ * failed write, which the built-in storages return when SQLite fails it, gives
1190
+ * the original {@link StorageWriteMessagesError} through {@link Result}.
1191
+ * Reconciliation can fail after messages have been committed; that failure does
1192
+ * not roll back the write.
1035
1193
  */
1036
1194
  export interface ApplyProtocolMessageAsClientFailed extends Typed<"Failed"> {
1037
1195
  readonly cause: "Write" | "Sync";
@@ -1055,149 +1213,157 @@ export const applyProtocolMessageAsClient =
1055
1213
  > =>
1056
1214
  async (run) => {
1057
1215
  const { storage } = run.deps;
1058
- try {
1059
- const input = createBuffer(inputMessage);
1060
- const [requestedVersion, ownerId] = decodeVersionAndOwner(input);
1061
- const version = options.version ?? protocolVersion;
1062
-
1063
- if (requestedVersion !== version) {
1064
- return err<ProtocolVersionError>({
1065
- type: "ProtocolVersionError",
1066
- version: requestedVersion,
1067
- isInitiator: version < requestedVersion,
1068
- ownerId,
1069
- });
1070
- }
1216
+ const version = options.version ?? protocolVersion;
1217
+ const decoded = decodeProtocolMessage(inputMessage, version);
1218
+ if (!decoded.ok) return decoded;
1219
+ const message = decoded.value;
1220
+
1221
+ if (message.type === "OtherVersion") {
1222
+ return err<ProtocolVersionError>({
1223
+ type: "ProtocolVersionError",
1224
+ version: message.version,
1225
+ isInitiator: version < message.version,
1226
+ ownerId: message.ownerId,
1227
+ });
1228
+ }
1071
1229
 
1072
- const messageType = input.shift() as MessageType;
1073
- assert(
1074
- messageType === MessageType.Response ||
1075
- messageType === MessageType.Broadcast,
1076
- "Invalid MessageType",
1077
- );
1230
+ if (message.type === "Request") {
1231
+ return err<ProtocolInvalidDataError>({
1232
+ type: "ProtocolInvalidDataError",
1233
+ data: inputMessage,
1234
+ error: new ProtocolDecodeError("Expected a Response or a Broadcast"),
1235
+ });
1236
+ }
1078
1237
 
1079
- if (messageType === MessageType.Response) {
1080
- const errorCode = input.shift();
1081
- if (errorCode !== ProtocolErrorCode.NoError) {
1082
- switch (errorCode) {
1083
- case ProtocolErrorCode.WriteKeyError:
1084
- return err<ProtocolWriteKeyError>({
1085
- type: "ProtocolWriteKeyError",
1086
- ownerId,
1087
- });
1088
- case ProtocolErrorCode.WriteError:
1089
- return err<ProtocolWriteError>({
1090
- type: "ProtocolWriteError",
1091
- ownerId,
1092
- });
1093
- case ProtocolErrorCode.QuotaError:
1094
- return err<ProtocolQuotaError>({
1095
- type: "ProtocolQuotaError",
1096
- ownerId,
1097
- });
1098
- case ProtocolErrorCode.SyncError:
1099
- return err<ProtocolSyncError>({
1100
- type: "ProtocolSyncError",
1101
- ownerId,
1102
- });
1103
- default:
1104
- throw new ProtocolDecodeError(
1105
- `Invalid ProtocolErrorCode: ${errorCode}`,
1106
- );
1107
- }
1108
- }
1238
+ const { ownerId } = message;
1239
+
1240
+ if (message.type === "ErrorResponse") {
1241
+ switch (message.errorCode) {
1242
+ case ProtocolErrorCode.WriteKeyError:
1243
+ return err<ProtocolWriteKeyError>({
1244
+ type: "ProtocolWriteKeyError",
1245
+ ownerId,
1246
+ });
1247
+ case ProtocolErrorCode.WriteError:
1248
+ return err<ProtocolWriteError>({
1249
+ type: "ProtocolWriteError",
1250
+ ownerId,
1251
+ });
1252
+ case ProtocolErrorCode.QuotaError:
1253
+ return err<ProtocolQuotaError>({
1254
+ type: "ProtocolQuotaError",
1255
+ ownerId,
1256
+ });
1257
+ case ProtocolErrorCode.SyncError:
1258
+ return err<ProtocolSyncError>({
1259
+ type: "ProtocolSyncError",
1260
+ ownerId,
1261
+ });
1109
1262
  }
1263
+ }
1110
1264
 
1111
- const messages = decodeMessages(input);
1112
- const ownerIdBytes = ownerIdToOwnerIdBytes(ownerId);
1265
+ const { messages } = message;
1266
+ const ownerIdBytes = ownerIdToOwnerIdBytes(ownerId);
1113
1267
 
1114
- if (isNonEmptyArray(messages)) {
1115
- try {
1116
- const result = await run(
1117
- storage.writeMessages(ownerIdBytes, messages),
1118
- );
1119
- if (!result.ok) return result;
1120
- } catch (error) {
1121
- if (AbortError.is(error)) throw error;
1122
- run.deps.console.error(error);
1123
- return ok({ type: "Failed", cause: "Write" });
1124
- }
1268
+ if (isNonEmptyArray(messages)) {
1269
+ try {
1270
+ const result = await run(storage.writeMessages(ownerIdBytes, messages));
1271
+ if (!result.ok) return result;
1272
+ } catch (error) {
1273
+ if (AbortError.is(error)) throw error;
1274
+ run.deps.console.error(error);
1275
+ return ok({ type: "Failed", cause: "Write" });
1125
1276
  }
1277
+ }
1126
1278
 
1127
- if (messageType === MessageType.Broadcast) {
1128
- return ok({ type: "Broadcast" });
1129
- }
1279
+ if (message.type === "Broadcast") {
1280
+ return ok({ type: "Broadcast" });
1281
+ }
1130
1282
 
1131
- // Now: No writeKey, no sync.
1132
- // TODO: Allow to sync SharedReadonlyOwner
1133
- // Without local changes, writeKey will not be required.
1134
- // With local changes, writeKey will be required and if not provided,
1135
- // the sync will stop.
1136
- const writeKey = options.writeKey;
1137
- if (writeKey == null) {
1138
- return ok({ type: "Readonly" });
1139
- }
1283
+ // Now: No writeKey, no sync.
1284
+ // TODO: Allow to sync SharedReadonlyOwner
1285
+ // Without local changes, writeKey will not be required.
1286
+ // With local changes, writeKey will be required and if not provided,
1287
+ // the sync will stop.
1288
+ const writeKey = options.writeKey;
1289
+ if (writeKey == null) {
1290
+ return ok({ type: "Readonly" });
1291
+ }
1140
1292
 
1141
- const ranges = decodeRanges(input);
1293
+ const { ranges } = message;
1142
1294
 
1143
- if (!isNonEmptyArray(ranges)) {
1144
- return ok({ type: "Converged" });
1145
- }
1295
+ if (!isNonEmptyArray(ranges)) {
1296
+ return ok({ type: "Converged" });
1297
+ }
1146
1298
 
1147
- const output = createProtocolMessageBuffer(ownerId, {
1299
+ const createOutput = () =>
1300
+ createProtocolMessageBuffer(ownerId, {
1148
1301
  messageType: MessageType.Request,
1149
1302
  writeKey,
1150
1303
  rangesMaxSize: options.rangesMaxSize,
1151
1304
  });
1305
+ const output = createOutput();
1152
1306
 
1153
- let broadcast: ProtocolMessageBuffer | undefined;
1154
- const result = sync(run.deps)(ranges, output, ownerIdBytes, (message) => {
1307
+ let broadcast: ProtocolMessageBuffer | undefined;
1308
+ const result = sync(run.deps)(ranges, output, ownerIdBytes, {
1309
+ createEmptyOutput: createOutput,
1310
+ onChangeTooLarge: options.onChangeTooLarge ?? run.deps.console.warn,
1311
+ onMessage: (message) => {
1155
1312
  broadcast ??= createProtocolMessageBuffer(ownerId, {
1156
1313
  messageType: MessageType.Broadcast,
1157
1314
  });
1315
+ // The request holds the same messages after a larger header.
1158
1316
  broadcast.addMessage(message);
1159
- });
1317
+ },
1318
+ });
1160
1319
 
1161
- // A failure was logged by sync.
1162
- if (!result.ok) return ok({ type: "Failed", cause: "Sync" });
1163
- if (!result.value) return ok({ type: "Converged" });
1320
+ // A failure was logged by sync.
1321
+ if (!result.ok) return ok({ type: "Failed", cause: "Sync" });
1322
+ if (!result.value) return ok({ type: "Converged" });
1164
1323
 
1165
- return ok({
1166
- type: "Response",
1167
- message: output.unwrap(),
1168
- ...(broadcast && { broadcast: broadcast.unwrap() }),
1169
- });
1170
- } catch (error) {
1171
- if (AbortError.is(error)) throw error;
1172
- return err<ProtocolInvalidDataError>({
1173
- type: "ProtocolInvalidDataError",
1174
- data: inputMessage,
1175
- error,
1176
- });
1177
- }
1324
+ return ok({
1325
+ type: "Response",
1326
+ message: output.unwrap(),
1327
+ ...(broadcast && { broadcast: broadcast.unwrap() }),
1328
+ });
1178
1329
  };
1179
1330
 
1331
+ /**
1332
+ * Options for {@link applyProtocolMessageAsRelay}. The callbacks must not throw,
1333
+ * because a throw is a defect that aborts the Run.
1334
+ */
1180
1335
  export interface ApplyProtocolMessageAsRelayOptions {
1181
1336
  /** To subscribe an owner for broadcasting. */
1182
- subscribe?: (ownerId: OwnerId) => void;
1337
+ readonly subscribe?: (ownerId: OwnerId) => void;
1183
1338
 
1184
1339
  /** To unsubscribe an owner from broadcasting. */
1185
- unsubscribe?: (ownerId: OwnerId) => void;
1340
+ readonly unsubscribe?: (ownerId: OwnerId) => void;
1186
1341
 
1187
1342
  /** To broadcast a protocol message to all subscribers. */
1188
- broadcast?: (ownerId: OwnerId, message: ProtocolMessage) => void;
1343
+ readonly broadcast?: (ownerId: OwnerId, message: ProtocolMessage) => void;
1189
1344
 
1190
- totalMaxSize?: ProtocolMessageMaxSize;
1191
- rangesMaxSize?: ProtocolMessageRangesMaxSize;
1345
+ /**
1346
+ * The maximum size of the relay's responses and broadcasts, and of the
1347
+ * requests it accepts. A larger request is a
1348
+ * {@link ProtocolInvalidDataError}.
1349
+ */
1350
+ readonly totalMaxSize?: ProtocolMessageMaxSize;
1351
+
1352
+ readonly rangesMaxSize?: ProtocolMessageRangesMaxSize;
1192
1353
  }
1193
1354
 
1194
1355
  /**
1195
1356
  * Result type for {@link applyProtocolMessageAsRelay}.
1196
1357
  *
1197
- * Unlike {@link ApplyProtocolMessageAsClientResult}, relays always respond with
1198
- * a message to provide sync completion feedback. This ensures the initiator can
1199
- * reliably detect when synchronization is complete, even when there's nothing
1200
- * to sync. Clients may choose not to respond in certain cases (like when they
1358
+ * Unlike {@link ApplyProtocolMessageAsClientResult}, a relay answers every
1359
+ * request it can decode within its `totalMaxSize` to provide sync completion
1360
+ * feedback. This ensures the initiator can reliably detect when synchronization
1361
+ * is complete, even when there's nothing to sync. A storage that throws while
1362
+ * validating the write key is answered with a `WriteError` code, like a failed
1363
+ * write. For a larger request or one it cannot decode,
1364
+ * {@link applyProtocolMessageAsRelay} returns {@link ProtocolInvalidDataError}
1365
+ * before subscribing, storing, or broadcasting anything, and the relay sends
1366
+ * nothing. Clients may choose not to respond in certain cases (like when they
1201
1367
  * receive broadcast messages or when they lack a write key for syncing).
1202
1368
  */
1203
1369
  export interface ApplyProtocolMessageAsRelayResult extends Typed<"Response"> {
@@ -1217,165 +1383,313 @@ export const applyProtocolMessageAsRelay =
1217
1383
  > =>
1218
1384
  async (run) => {
1219
1385
  const { storage } = run.deps;
1220
- try {
1221
- const input = createBuffer(inputMessage);
1222
- const [requestedVersion, ownerId] = decodeVersionAndOwner(input);
1223
- const ownerIdBytes = ownerIdToOwnerIdBytes(ownerId);
1224
-
1225
- if (requestedVersion !== version) {
1226
- // Non-initiator responds with its version and ownerId.
1227
- const output = createBuffer();
1228
- encodeNonNegativeInt(output, version);
1229
- output.extend(ownerIdBytes);
1230
- return ok({
1231
- type: "Response",
1232
- message: output.unwrap() as ProtocolMessage,
1233
- });
1234
- }
1235
1386
 
1236
- const messageType = input.shift() as MessageType;
1237
- assertSame(messageType, MessageType.Request);
1387
+ // The relay broadcasts a request's messages in a frame of totalMaxSize.
1388
+ // That frame omits the request's write key, subscription flag, and ranges,
1389
+ // so it holds the messages of any request up to that size.
1390
+ if (
1391
+ inputMessage.length >
1392
+ (options.totalMaxSize ?? defaultProtocolMessageMaxSize)
1393
+ )
1394
+ return err<ProtocolInvalidDataError>({
1395
+ type: "ProtocolInvalidDataError",
1396
+ data: inputMessage,
1397
+ error: new ProtocolDecodeError("Request is too large"),
1398
+ });
1238
1399
 
1239
- const hasWriteKey = input.shift();
1240
- let writeKey: OwnerWriteKey | undefined;
1400
+ const decoded = decodeProtocolMessage(inputMessage, version);
1401
+ if (!decoded.ok) return decoded;
1402
+ const request = decoded.value;
1241
1403
 
1242
- if (hasWriteKey === 1) {
1243
- writeKey = input.shiftN(ownerWriteKeyLength) as OwnerWriteKey;
1244
- }
1404
+ if (request.type === "OtherVersion") {
1405
+ // Non-initiator responds with its version and ownerId.
1406
+ const output = createBuffer();
1407
+ encodeNonNegativeInt(output, version);
1408
+ output.extend(ownerIdToOwnerIdBytes(request.ownerId));
1409
+ return ok({
1410
+ type: "Response",
1411
+ message: output.unwrap() as ProtocolMessage,
1412
+ });
1413
+ }
1245
1414
 
1246
- const subscriptionFlag = input.shift() as SubscriptionFlag;
1415
+ if (request.type !== "Request") {
1416
+ return err<ProtocolInvalidDataError>({
1417
+ type: "ProtocolInvalidDataError",
1418
+ data: inputMessage,
1419
+ error: new ProtocolDecodeError("Expected a Request"),
1420
+ });
1421
+ }
1247
1422
 
1248
- switch (subscriptionFlag) {
1249
- case SubscriptionFlags.Subscribe:
1250
- options.subscribe?.(ownerId);
1251
- break;
1252
- case SubscriptionFlags.Unsubscribe:
1253
- options.unsubscribe?.(ownerId);
1254
- break;
1255
- case SubscriptionFlags.None:
1256
- break;
1257
- }
1423
+ const { ownerId, writeKey, messages, ranges } = request;
1424
+ const ownerIdBytes = ownerIdToOwnerIdBytes(ownerId);
1258
1425
 
1259
- if (writeKey) {
1260
- const isValid = storage.validateWriteKey(ownerIdBytes, writeKey);
1261
- if (!isValid) {
1262
- return ok({
1263
- type: "Response",
1264
- message: createProtocolMessageBuffer(ownerId, {
1265
- messageType: MessageType.Response,
1266
- errorCode: ProtocolErrorCode.WriteKeyError,
1267
- }).unwrap(),
1268
- });
1269
- }
1270
- }
1426
+ const createErrorResponse = (
1427
+ errorCode: ProtocolErrorCode,
1428
+ ): ApplyProtocolMessageAsRelayResult => ({
1429
+ type: "Response",
1430
+ message: createProtocolMessageBuffer(ownerId, {
1431
+ messageType: MessageType.Response,
1432
+ errorCode,
1433
+ }).unwrap(),
1434
+ });
1271
1435
 
1272
- const messages = decodeMessages(input);
1436
+ switch (request.subscriptionFlag) {
1437
+ case SubscriptionFlags.Subscribe:
1438
+ options.subscribe?.(ownerId);
1439
+ break;
1440
+ case SubscriptionFlags.Unsubscribe:
1441
+ options.unsubscribe?.(ownerId);
1442
+ break;
1443
+ case SubscriptionFlags.None:
1444
+ break;
1445
+ default:
1446
+ exhaustiveCheck(request.subscriptionFlag);
1447
+ }
1273
1448
 
1274
- if (isNonEmptyArray(messages)) {
1275
- if (!writeKey) {
1276
- return ok({
1277
- type: "Response",
1278
- message: createProtocolMessageBuffer(ownerId, {
1279
- messageType: MessageType.Response,
1280
- errorCode: ProtocolErrorCode.WriteKeyError,
1281
- }).unwrap(),
1282
- });
1283
- }
1449
+ if (writeKey) {
1450
+ let isValid: boolean;
1451
+ try {
1452
+ isValid = storage.validateWriteKey(ownerIdBytes, writeKey);
1453
+ } catch (error) {
1454
+ // A relay storage stores the write key of a new owner, which SQLite
1455
+ // can fail, for example on a full disk. A boolean has no room for the
1456
+ // error, so it is answered like a failed write.
1457
+ run.deps.console.error(error);
1458
+ return ok(createErrorResponse(ProtocolErrorCode.WriteError));
1459
+ }
1460
+ if (!isValid) {
1461
+ return ok(createErrorResponse(ProtocolErrorCode.WriteKeyError));
1462
+ }
1463
+ }
1284
1464
 
1285
- try {
1286
- const result = await run(
1287
- storage.writeMessages(ownerIdBytes, messages),
1288
- );
1465
+ if (isNonEmptyArray(messages)) {
1466
+ if (!writeKey) {
1467
+ return ok(createErrorResponse(ProtocolErrorCode.WriteKeyError));
1468
+ }
1289
1469
 
1290
- if (!result.ok) {
1291
- const message = createProtocolMessageBuffer(ownerId, {
1292
- messageType: MessageType.Response,
1293
- errorCode: ProtocolErrorCode.QuotaError,
1294
- }).unwrap();
1295
- return ok({ type: "Response", message });
1296
- }
1297
- } catch (error) {
1298
- if (AbortError.is(error)) throw error;
1299
- run.deps.console.error(error);
1300
- const message = createProtocolMessageBuffer(ownerId, {
1301
- messageType: MessageType.Response,
1302
- errorCode: ProtocolErrorCode.WriteError,
1303
- }).unwrap();
1304
- return ok({ type: "Response", message });
1470
+ try {
1471
+ const result = await run(storage.writeMessages(ownerIdBytes, messages));
1472
+
1473
+ if (!result.ok) {
1474
+ // A storage returns a failed write without reporting it.
1475
+ if (result.error.type === "UnknownError")
1476
+ run.deps.console.error(result.error);
1477
+ return ok(
1478
+ createErrorResponse(
1479
+ result.error.type === "StorageQuotaError"
1480
+ ? ProtocolErrorCode.QuotaError
1481
+ : ProtocolErrorCode.WriteError,
1482
+ ),
1483
+ );
1305
1484
  }
1485
+ } catch (error) {
1486
+ if (AbortError.is(error)) throw error;
1487
+ run.deps.console.error(error);
1488
+ return ok(createErrorResponse(ProtocolErrorCode.WriteError));
1489
+ }
1306
1490
 
1307
- /**
1308
- * Broadcast messages to all subscribed owners for real-time
1309
- * synchronization between clients.
1310
- *
1311
- * Messages are only broadcasted after successful write to ensure
1312
- * devices that can still sync aren't affected by quota errors, and to
1313
- * prevent using a half-working relay service (broadcasting without
1314
- * persistence).
1315
- *
1316
- * When a relay's database is deleted or clients migrate to a new relay
1317
- * (without data migration), clients will sync their data to the relay,
1318
- * and the relay will broadcast those messages to other connected
1319
- * clients. Those clients may receive messages they already have, but
1320
- * this is safe because Evolu sync is idempotent. As the relay becomes
1321
- * more synchronized with clients over time, fewer duplicate messages
1322
- * will be broadcasted.
1323
- */
1324
- if (options.broadcast) {
1325
- const broadcastBuffer = createProtocolMessageBuffer(ownerId, {
1326
- messageType: MessageType.Broadcast,
1327
- totalMaxSize: options.totalMaxSize,
1328
- rangesMaxSize: options.rangesMaxSize,
1329
- version,
1330
- });
1331
- for (const message of messages) {
1332
- broadcastBuffer.addMessage(message);
1333
- }
1334
- options.broadcast(ownerId, broadcastBuffer.unwrap());
1491
+ /**
1492
+ * Broadcast messages to all subscribed owners for real-time
1493
+ * synchronization between clients.
1494
+ *
1495
+ * Messages are only broadcasted after successful write to ensure devices
1496
+ * that can still sync aren't affected by quota errors, and to prevent
1497
+ * using a half-working relay service (broadcasting without persistence).
1498
+ *
1499
+ * When a relay's database is deleted or clients migrate to a new relay
1500
+ * (without data migration), clients will sync their data to the relay,
1501
+ * and the relay will broadcast those messages to other connected clients.
1502
+ * Those clients may receive messages they already have, but this is safe
1503
+ * because Evolu sync is idempotent. As the relay becomes more
1504
+ * synchronized with clients over time, fewer duplicate messages will be
1505
+ * broadcasted.
1506
+ */
1507
+ if (options.broadcast) {
1508
+ const broadcastBuffer = createProtocolMessageBuffer(ownerId, {
1509
+ messageType: MessageType.Broadcast,
1510
+ totalMaxSize: options.totalMaxSize,
1511
+ rangesMaxSize: options.rangesMaxSize,
1512
+ version,
1513
+ });
1514
+ for (const message of messages) {
1515
+ broadcastBuffer.addMessage(message);
1335
1516
  }
1517
+ options.broadcast(ownerId, broadcastBuffer.unwrap());
1336
1518
  }
1519
+ }
1337
1520
 
1338
- const ranges = decodeRanges(input);
1339
-
1340
- const output = createProtocolMessageBuffer(ownerId, {
1521
+ const createOutput = () =>
1522
+ createProtocolMessageBuffer(ownerId, {
1341
1523
  messageType: MessageType.Response,
1342
1524
  errorCode: ProtocolErrorCode.NoError,
1343
1525
  totalMaxSize: options.totalMaxSize,
1344
1526
  rangesMaxSize: options.rangesMaxSize,
1345
1527
  });
1528
+ const output = createOutput();
1346
1529
 
1347
- // Non-initiators always respond to provide sync completion feedback,
1348
- // even when there's nothing to sync.
1349
- if (!isNonEmptyArray(ranges)) {
1350
- return ok({ type: "Response", message: output.unwrap() });
1351
- }
1530
+ // A relay answers every request it decodes, even with nothing to sync, so
1531
+ // the initiator knows the sync is complete.
1532
+ if (!isNonEmptyArray(ranges)) {
1533
+ return ok({ type: "Response", message: output.unwrap() });
1534
+ }
1535
+
1536
+ const result = sync(run.deps)(ranges, output, ownerIdBytes, {
1537
+ createEmptyOutput: createOutput,
1538
+ onChangeTooLarge: run.deps.console.warn,
1539
+ });
1352
1540
 
1353
- const result = sync(run.deps)(ranges, output, ownerIdBytes);
1541
+ // A relay answers every request it decodes, a failed reconciliation with
1542
+ // its error code.
1543
+ return ok(
1544
+ result.ok
1545
+ ? { type: "Response", message: output.unwrap() }
1546
+ : createErrorResponse(result.error),
1547
+ );
1548
+ };
1354
1549
 
1355
- const message = result.ok
1356
- ? output.unwrap()
1357
- : createProtocolMessageBuffer(ownerId, {
1358
- messageType: MessageType.Response,
1359
- errorCode: result.error,
1360
- }).unwrap();
1550
+ /**
1551
+ * Decodes a whole {@link ProtocolMessage}, so a malformed one is rejected before
1552
+ * anything it carries is applied.
1553
+ *
1554
+ * Applying a message turns only a throw from here into
1555
+ * {@link ProtocolInvalidDataError}. A throw after decoding is a defect.
1556
+ */
1557
+ const decodeProtocolMessage = (
1558
+ inputMessage: Uint8Array,
1559
+ version: NonNegativeInt,
1560
+ ): Result<DecodedProtocolMessage, ProtocolInvalidDataError> => {
1561
+ try {
1562
+ const input = createBuffer(inputMessage);
1563
+ const [messageVersion, ownerId] = decodeVersionAndOwner(input);
1564
+
1565
+ if (messageVersion !== version)
1566
+ return ok({ type: "OtherVersion", version: messageVersion, ownerId });
1567
+
1568
+ switch (decodeMessageType(input)) {
1569
+ case MessageType.Request: {
1570
+ const hasWriteKey = input.shift();
1571
+ if (hasWriteKey > 1)
1572
+ throw new ProtocolDecodeError(`Invalid hasWriteKey: ${hasWriteKey}`);
1573
+ const writeKey =
1574
+ hasWriteKey === 1
1575
+ ? (input.shiftN(ownerWriteKeyLength) as OwnerWriteKey)
1576
+ : null;
1577
+
1578
+ const subscriptionFlag: number = input.shift();
1579
+ switch (subscriptionFlag) {
1580
+ case SubscriptionFlags.None:
1581
+ case SubscriptionFlags.Subscribe:
1582
+ case SubscriptionFlags.Unsubscribe:
1583
+ break;
1584
+ default:
1585
+ throw new ProtocolDecodeError(
1586
+ `Invalid SubscriptionFlag: ${subscriptionFlag}`,
1587
+ );
1588
+ }
1361
1589
 
1362
- // Non-initiators always respond to provide sync completion feedback,
1363
- return ok({ type: "Response", message });
1364
- } catch (error) {
1365
- if (AbortError.is(error)) throw error;
1366
- return err<ProtocolInvalidDataError>({
1367
- type: "ProtocolInvalidDataError",
1368
- data: inputMessage,
1369
- error,
1370
- });
1590
+ const messages = decodeMessages(input);
1591
+
1592
+ // Only a relay accepts a Request, so only the relay checks this.
1593
+ // Deployed relays already store shorter changes, and a client skips a
1594
+ // change it cannot read, whereas a check in decodeMessages would make
1595
+ // it reject every response holding one.
1596
+ for (const { change } of messages)
1597
+ if (change.length < minEncryptedDbChangeLength)
1598
+ throw new ProtocolDecodeError("EncryptedDbChange is too short");
1599
+
1600
+ const ranges = decodeRanges(input);
1601
+
1602
+ return ok({
1603
+ type: "Request",
1604
+ ownerId,
1605
+ writeKey,
1606
+ subscriptionFlag,
1607
+ messages,
1608
+ ranges,
1609
+ });
1610
+ }
1611
+
1612
+ case MessageType.Response: {
1613
+ const errorCode: number = input.shift();
1614
+ switch (errorCode) {
1615
+ case ProtocolErrorCode.NoError: {
1616
+ const messages = decodeMessages(input);
1617
+ const ranges = decodeRanges(input);
1618
+ return ok({ type: "Response", ownerId, messages, ranges });
1619
+ }
1620
+ case ProtocolErrorCode.WriteKeyError:
1621
+ case ProtocolErrorCode.WriteError:
1622
+ case ProtocolErrorCode.QuotaError:
1623
+ case ProtocolErrorCode.SyncError:
1624
+ return ok({ type: "ErrorResponse", ownerId, errorCode });
1625
+ default:
1626
+ throw new ProtocolDecodeError(
1627
+ `Invalid ProtocolErrorCode: ${errorCode}`,
1628
+ );
1629
+ }
1630
+ }
1631
+
1632
+ case MessageType.Broadcast:
1633
+ return ok({
1634
+ type: "Broadcast",
1635
+ ownerId,
1636
+ messages: decodeMessages(input),
1637
+ });
1371
1638
  }
1372
- };
1639
+ } catch (error) {
1640
+ return err<ProtocolInvalidDataError>({
1641
+ type: "ProtocolInvalidDataError",
1642
+ data: inputMessage,
1643
+ error,
1644
+ });
1645
+ }
1646
+ };
1647
+
1648
+ type DecodedProtocolMessage =
1649
+ | DecodedOtherVersionMessage
1650
+ | DecodedRequest
1651
+ | DecodedErrorResponse
1652
+ | DecodedResponse
1653
+ | DecodedBroadcast;
1654
+
1655
+ /** A message of another version, whose layout after the owner is unknown. */
1656
+ interface DecodedOtherVersionMessage extends Typed<"OtherVersion"> {
1657
+ readonly version: NonNegativeInt;
1658
+ readonly ownerId: OwnerId;
1659
+ }
1660
+
1661
+ interface DecodedRequest extends Typed<"Request"> {
1662
+ readonly ownerId: OwnerId;
1663
+ readonly writeKey: OwnerWriteKey | null;
1664
+ readonly subscriptionFlag: SubscriptionFlag;
1665
+ readonly messages: ReadonlyArray<EncryptedCrdtMessage>;
1666
+ readonly ranges: ReadonlyArray<Range>;
1667
+ }
1668
+
1669
+ /** A Response with an error code, after which nothing is decoded. */
1670
+ interface DecodedErrorResponse extends Typed<"ErrorResponse"> {
1671
+ readonly ownerId: OwnerId;
1672
+ readonly errorCode: Exclude<
1673
+ ProtocolErrorCode,
1674
+ typeof ProtocolErrorCode.NoError
1675
+ >;
1676
+ }
1677
+
1678
+ interface DecodedResponse extends Typed<"Response"> {
1679
+ readonly ownerId: OwnerId;
1680
+ readonly messages: ReadonlyArray<EncryptedCrdtMessage>;
1681
+ readonly ranges: ReadonlyArray<Range>;
1682
+ }
1683
+
1684
+ interface DecodedBroadcast extends Typed<"Broadcast"> {
1685
+ readonly ownerId: OwnerId;
1686
+ readonly messages: ReadonlyArray<EncryptedCrdtMessage>;
1687
+ }
1373
1688
 
1374
1689
  const decodeVersionAndOwner = (input: Buffer): [NonNegativeInt, OwnerId] => {
1375
1690
  // This structure must never change across protocol versions. The version
1376
1691
  // and owner ID must always be the first two fields in every protocol message
1377
- // to enable version negotiation and owner identification before any other
1378
- // processing occurs.
1692
+ // to route and report a version mismatch before any other processing occurs.
1379
1693
  const version = decodeNonNegativeInt(input);
1380
1694
  const ownerId = decodeId(input) as OwnerId;
1381
1695
  return [version, ownerId];
@@ -1388,24 +1702,20 @@ const parseProtocolHeaderFromBuffer = (input: Buffer): ProtocolHeader => {
1388
1702
  return { type: "ProtocolHeader", version, ownerId };
1389
1703
  }
1390
1704
 
1391
- const messageTypeValue = input.shift();
1392
- let messageType: MessageType;
1705
+ const messageType = decodeMessageType(input);
1706
+ return { type: "ProtocolHeader", version, ownerId, messageType };
1707
+ };
1393
1708
 
1394
- switch (messageTypeValue) {
1709
+ const decodeMessageType = (input: Buffer): MessageType => {
1710
+ const messageType: number = input.shift();
1711
+ switch (messageType) {
1395
1712
  case MessageType.Request:
1396
- messageType = MessageType.Request;
1397
- break;
1398
1713
  case MessageType.Response:
1399
- messageType = MessageType.Response;
1400
- break;
1401
1714
  case MessageType.Broadcast:
1402
- messageType = MessageType.Broadcast;
1403
- break;
1715
+ return messageType;
1404
1716
  default:
1405
1717
  throw new ProtocolDecodeError("Invalid MessageType");
1406
1718
  }
1407
-
1408
- return { type: "ProtocolHeader", version, ownerId, messageType };
1409
1719
  };
1410
1720
 
1411
1721
  /**
@@ -1416,8 +1726,6 @@ class ProtocolDecodeError extends Error {
1416
1726
  constructor(message: string) {
1417
1727
  super(message);
1418
1728
  this.name = this.constructor.name;
1419
-
1420
- Error.captureStackTrace(this, this.constructor);
1421
1729
  }
1422
1730
  }
1423
1731
 
@@ -1437,13 +1745,56 @@ const decodeMessages = (
1437
1745
  return messages;
1438
1746
  };
1439
1747
 
1748
+ // The smallest envelope encodeAndEncryptDbChange can produce: the nonce, a
1749
+ // 1-byte ciphertext length, and the 16-byte Poly1305 tag. Every v1 client sends
1750
+ // at least 77 bytes. The relay quota counts change bytes, so a relay storing
1751
+ // shorter changes, zero-length ones above all, would store rows the quota does
1752
+ // not count.
1753
+ const minEncryptedDbChangeLength = xChaCha20Poly1305NonceLength + 1 + 16;
1754
+
1755
+ /**
1756
+ * Answers the ranges into `output`, returning whether it has anything to send.
1757
+ *
1758
+ * Every write except a closing one is a trial (see
1759
+ * {@link ProtocolMessageBuffer.tryWrite}), so the frame always has room to close
1760
+ * with one Fingerprint range with {@link InfiniteUpperBound}. When a write does
1761
+ * not fit, that range closes the frame. Its fingerprint covers everything from
1762
+ * the last upper bound the frame states, which every v1 peer assumes, and the
1763
+ * peer reconciles it in the next round.
1764
+ *
1765
+ * A stored change that cannot fit an empty frame with this header, with only
1766
+ * its own timestamp listed after a pending skip, is skipped, and
1767
+ * `onChangeTooLarge` reports it as a {@link ProtocolChangeTooLargeError}. A
1768
+ * later round is guaranteed to reach exactly that frame, so a change that fits
1769
+ * it is sent eventually.
1770
+ *
1771
+ * An answer to a Timestamps range neither sends nor lists a skipped change, but
1772
+ * a request or a split lists its timestamp, so a peer that lacks it may ask for
1773
+ * it once per sync and gets an answer without it. Fingerprints keep disagreeing
1774
+ * about it, so every sync narrows them to it, skips it again, and ends.
1775
+ *
1776
+ * A throw from storage, or from a split's checks of what storage returned, is
1777
+ * logged and returns `SyncError`, except an AbortError from a split's reads,
1778
+ * which is rethrown. Any other throw, such as from writing the frame,
1779
+ * `onChangeTooLarge`, or `onMessage`, is a defect.
1780
+ */
1440
1781
  const sync =
1441
1782
  (deps: StorageDep & ConsoleDep) =>
1442
1783
  (
1443
1784
  ranges: NonEmptyReadonlyArray<Range>,
1444
1785
  output: ProtocolMessageBuffer,
1445
1786
  ownerIdBytes: OwnerIdBytes,
1446
- onMessage?: (message: EncryptedCrdtMessage) => void,
1787
+ {
1788
+ createEmptyOutput,
1789
+ onChangeTooLarge,
1790
+ onMessage,
1791
+ }: {
1792
+ /** Creates an empty frame with the header of `output`. */
1793
+ createEmptyOutput: () => ProtocolMessageBuffer;
1794
+ onChangeTooLarge: (error: ProtocolChangeTooLargeError) => void;
1795
+ /** Called with each message `output` keeps. */
1796
+ onMessage?: (message: EncryptedCrdtMessage) => void;
1797
+ },
1447
1798
  ): Result<boolean, typeof ProtocolErrorCode.SyncError> => {
1448
1799
  const outputInitialSize = output.getSize();
1449
1800
  let storageSize: NonNegativeInt;
@@ -1456,7 +1807,11 @@ const sync =
1456
1807
 
1457
1808
  let prevUpperBound: RangeUpperBound | null = null;
1458
1809
  let prevIndex = zeroNonNegativeInt;
1810
+ // The index of the last upper bound the frame states, 0 before any.
1811
+ let statedIndex = zeroNonNegativeInt;
1459
1812
 
1813
+ // Consecutive skipped ranges become one Skip range, written only before
1814
+ // the next non-skip range.
1460
1815
  let skip = false;
1461
1816
  let nonSkipRangeAdded = false;
1462
1817
 
@@ -1465,6 +1820,9 @@ const sync =
1465
1820
  ) => {
1466
1821
  // The last range, if any non skip was added, must have InfiniteUpperBound.
1467
1822
  if (nonSkipRangeAdded && range.upperBound === InfiniteUpperBound) {
1823
+ // A closing write. Like the closing Fingerprint range, it encodes no
1824
+ // upper bound, and its type takes 1 byte of the 13 that every kept
1825
+ // trial reserved beyond the ranges count.
1468
1826
  output.addRange({
1469
1827
  type: RangeType.Skip,
1470
1828
  upperBound: InfiniteUpperBound,
@@ -1474,42 +1832,60 @@ const sync =
1474
1832
  }
1475
1833
  };
1476
1834
 
1477
- const coalesceSkipsBeforeAdd = () => {
1478
- // Set to true because we are going to add a non skip range.
1479
- nonSkipRangeAdded = true;
1480
- if (skip) {
1835
+ /**
1836
+ * Tries to write a non-skip range ending at the index `upper` after the
1837
+ * pending skip, if any.
1838
+ */
1839
+ const tryWriteRange = (
1840
+ upper: NonNegativeInt,
1841
+ write: () => void,
1842
+ ): boolean => {
1843
+ const isKept = output.tryWrite(() => {
1844
+ if (skip) {
1845
+ assertNonNullable(prevUpperBound, "prevUpperBound is null");
1846
+ output.addRange({
1847
+ type: RangeType.Skip,
1848
+ upperBound: prevUpperBound,
1849
+ });
1850
+ }
1851
+ write();
1852
+ });
1853
+ if (isKept) {
1481
1854
  skip = false;
1482
- assertNonNullable(prevUpperBound, "prevUpperBound is null");
1483
- // There is always a space for a skip range before adding.
1484
- output.addRange({
1485
- type: RangeType.Skip,
1486
- upperBound: prevUpperBound,
1487
- });
1855
+ nonSkipRangeAdded = true;
1856
+ statedIndex = upper;
1488
1857
  }
1858
+ return isKept;
1489
1859
  };
1490
1860
 
1491
- // When we don't have a space...
1492
- const addFingerprintForRemainingRange = (
1493
- begin: NonNegativeInt,
1494
- ): boolean => {
1861
+ /**
1862
+ * Closes the frame with a Fingerprint range with InfiniteUpperBound over
1863
+ * the items from the last upper bound the frame states. A frame that cannot
1864
+ * fit a range closes without writing the pending Skip range, because only
1865
+ * this range has room reserved, so it covers the skipped items too.
1866
+ */
1867
+ const closeWithFingerprint = (): Result<
1868
+ true,
1869
+ typeof ProtocolErrorCode.SyncError
1870
+ > => {
1495
1871
  let fingerprint: Fingerprint;
1496
1872
  try {
1497
1873
  fingerprint = deps.storage.fingerprint(
1498
1874
  ownerIdBytes,
1499
- begin,
1875
+ statedIndex,
1500
1876
  storageSize,
1501
1877
  );
1502
1878
  } catch (error) {
1503
1879
  deps.console.error(error);
1504
- return false;
1880
+ return err(ProtocolErrorCode.SyncError);
1505
1881
  }
1506
- // There is always a space for a ramaining range.
1882
+ // A closing write. Every kept trial reserved room for it.
1507
1883
  output.addRange({
1508
1884
  type: RangeType.Fingerprint,
1509
1885
  upperBound: InfiniteUpperBound,
1510
1886
  fingerprint,
1511
1887
  });
1512
- return true;
1888
+ return ok(true);
1513
1889
  };
1514
1890
 
1515
1891
  for (const range of ranges) {
@@ -1550,26 +1926,28 @@ const sync =
1550
1926
 
1551
1927
  if (eqArrayNumber(range.fingerprint, ourFingerprint)) {
1552
1928
  skipRange(range);
1553
- } else if (output.canSplitRange()) {
1554
- coalesceSkipsBeforeAdd();
1555
- try {
1556
- splitRange(deps)(
1557
- ownerIdBytes,
1558
- lower,
1559
- upper,
1560
- currentUpperBound,
1561
- output,
1562
- );
1563
- } catch (error) {
1564
- if (AbortError.is(error)) throw error;
1565
- deps.console.error(error);
1566
- return err(ProtocolErrorCode.SyncError);
1567
- }
1568
- } else {
1569
- return addFingerprintForRemainingRange(upper)
1570
- ? ok(true)
1571
- : err(ProtocolErrorCode.SyncError);
1929
+ break;
1930
+ }
1931
+
1932
+ let splitRanges: ReadonlyArray<
1933
+ FingerprintRange | TimestampsRangeWithTimestampsBuffer
1934
+ >;
1935
+ try {
1936
+ splitRanges = readSplitRanges(deps)(
1937
+ ownerIdBytes,
1938
+ lower,
1939
+ upper,
1940
+ currentUpperBound,
1941
+ );
1942
+ } catch (error) {
1943
+ if (AbortError.is(error)) throw error;
1944
+ deps.console.error(error);
1945
+ return err(ProtocolErrorCode.SyncError);
1572
1946
  }
1947
+ const isSplit = tryWriteRange(upper, () => {
1948
+ for (const splitRange of splitRanges) output.addRange(splitRange);
1949
+ });
1950
+ if (!isSplit) return closeWithFingerprint();
1573
1951
  break;
1574
1952
  }
1575
1953
 
@@ -1583,6 +1961,12 @@ const sync =
1583
1961
 
1584
1962
  let exceeded = false as boolean;
1585
1963
  let iterateFailed = false as boolean;
1964
+ // A pending skip stays pending until the range is written.
1965
+ const isSkipPending = skip;
1966
+ // Only a throw from the storage itself is a storage failure. A throw
1967
+ // from the callback, such as from onChangeTooLarge or onMessage, is
1968
+ // a defect, rethrown after iterating.
1969
+ let callbackError = null as { readonly error: unknown } | null;
1586
1970
 
1587
1971
  try {
1588
1972
  deps.storage.iterate(
@@ -1590,78 +1974,116 @@ const sync =
1590
1974
  lower,
1591
1975
  upper,
1592
1976
  (timestamp, index) => {
1593
- const timestampString = timestamp.join();
1594
- const timestampBinary = timestampBytesToTimestamp(timestamp);
1595
-
1596
- let message: EncryptedCrdtMessage | null = null;
1597
-
1598
- if (timestampsWeNeed.has(timestampString)) {
1599
- timestampsWeNeed.delete(timestampString);
1600
- } else {
1601
- try {
1602
- message = {
1603
- timestamp: timestampBinary,
1604
- change: deps.storage.readDbChange(
1605
- ownerIdBytes,
1606
- timestamp,
1607
- ),
1608
- };
1609
- } catch (error) {
1610
- deps.console.error(error);
1611
- iterateFailed = true;
1612
- return false;
1977
+ try {
1978
+ const timestampString = timestamp.join();
1979
+ const timestampBinary = timestampBytesToTimestamp(timestamp);
1980
+
1981
+ let message: EncryptedCrdtMessage | null = null;
1982
+
1983
+ if (timestampsWeNeed.has(timestampString)) {
1984
+ timestampsWeNeed.delete(timestampString);
1985
+ } else {
1986
+ try {
1987
+ message = {
1988
+ timestamp: timestampBinary,
1989
+ change: deps.storage.readDbChange(
1990
+ ownerIdBytes,
1991
+ timestamp,
1992
+ ),
1993
+ };
1994
+ } catch (error) {
1995
+ deps.console.error(error);
1996
+ iterateFailed = true;
1997
+ return false;
1998
+ }
1999
+ }
2000
+
2001
+ // One trial per timestamp: its entry in ourTimestamps and its
2002
+ // message if the peer lacks it, keeping room to write
2003
+ // ourTimestamps as a range after the pending skip.
2004
+ const restoreOurTimestamps = ourTimestamps.checkpoint();
2005
+ ourTimestamps.add(timestampBinary);
2006
+ if (
2007
+ output.tryWrite(
2008
+ () => {
2009
+ if (message) output.addMessage(message);
2010
+ },
2011
+ getTimestampsRangeReserve(ourTimestamps, isSkipPending),
2012
+ )
2013
+ ) {
2014
+ if (message) onMessage?.(message);
2015
+ return true;
2016
+ }
2017
+ restoreOurTimestamps();
2018
+
2019
+ if (message) {
2020
+ // Whether an empty frame can hold the message in the trial
2021
+ // a Timestamps range makes for it, with only its own
2022
+ // timestamp listed and a skip pending. A later round is
2023
+ // guaranteed to reach exactly that frame, so a message that
2024
+ // fits is sent eventually. A trial without a pending skip
2025
+ // reserves 22 bytes less, so a message up to 22 bytes too
2026
+ // large for this check may still be sent in such a frame.
2027
+ const emptyOutput = createEmptyOutput();
2028
+ const emptyOutputTimestamps = createTimestampsBuffer();
2029
+ emptyOutputTimestamps.add(message.timestamp);
2030
+ const fitsEmptyOutput = emptyOutput.tryWrite(
2031
+ () => {
2032
+ emptyOutput.addMessage(message);
2033
+ },
2034
+ getTimestampsRangeReserve(emptyOutputTimestamps, true),
2035
+ );
2036
+ if (!fitsEmptyOutput) {
2037
+ onChangeTooLarge({
2038
+ type: "ProtocolChangeTooLargeError",
2039
+ timestamp: timestampBinary,
2040
+ size: message.change.length as PositiveInt,
2041
+ });
2042
+ return true;
2043
+ }
1613
2044
  }
1614
- }
1615
2045
 
1616
- if (
1617
- !output.canAddTimestampsRangeAndMessage(
1618
- ourTimestamps,
1619
- message,
1620
- )
1621
- ) {
1622
2046
  exceeded = true;
1623
2047
  endBound = timestamp;
1624
2048
  upper = index;
1625
2049
  return false;
2050
+ } catch (error) {
2051
+ callbackError = { error };
2052
+ return false;
1626
2053
  }
1627
-
1628
- ourTimestamps.add(timestampBinary);
1629
- if (message) {
1630
- output.addMessage(message);
1631
- onMessage?.(message);
1632
- }
1633
- return true;
1634
2054
  },
1635
2055
  );
1636
2056
  } catch (error) {
1637
2057
  deps.console.error(error);
1638
- return err(ProtocolErrorCode.SyncError);
2058
+ iterateFailed = true;
1639
2059
  }
1640
2060
 
2061
+ if (callbackError) throw callbackError.error;
2062
+
1641
2063
  if (iterateFailed) {
1642
2064
  return err(ProtocolErrorCode.SyncError);
1643
2065
  }
1644
2066
 
1645
- const addRange = () => {
1646
- coalesceSkipsBeforeAdd();
1647
- output.addRange({
1648
- type: RangeType.Timestamps,
1649
- upperBound: endBound,
1650
- timestamps: ourTimestamps,
2067
+ // When any timestamp was kept, its trial reserved room for this
2068
+ // write. Otherwise the range may not fit, and the frame closes
2069
+ // without it.
2070
+ const tryWriteTimestampsRange = () =>
2071
+ tryWriteRange(upper, () => {
2072
+ output.addRange({
2073
+ type: RangeType.Timestamps,
2074
+ upperBound: endBound,
2075
+ timestamps: ourTimestamps,
2076
+ });
1651
2077
  });
1652
- };
1653
2078
 
1654
2079
  if (exceeded) {
1655
- addRange();
1656
- if (!addFingerprintForRemainingRange(upper)) {
1657
- return err(ProtocolErrorCode.SyncError);
1658
- }
1659
- return ok(true);
2080
+ tryWriteTimestampsRange();
2081
+ return closeWithFingerprint();
1660
2082
  }
1661
2083
 
1662
2084
  // If we need something, we have to respond with our timestamps.
1663
2085
  if (timestampsWeNeed.size > 0) {
1664
- addRange();
2086
+ if (!tryWriteTimestampsRange()) return closeWithFingerprint();
1665
2087
  } else {
1666
2088
  skipRange(range);
1667
2089
  }
@@ -1680,15 +2102,42 @@ const sync =
1680
2102
  return ok(hasChange);
1681
2103
  };
1682
2104
 
1683
- const splitRange =
2105
+ // The most a range adds to a frame beyond its payload. Its upper bound adds at
2106
+ // most 20 bytes to the ranges' timestamps: a millis delta varint of up to 7
2107
+ // bytes, as maxMillis has 48 bits; up to 4 for the counter, as a new run is a
2108
+ // varint of up to 3 bytes, Counter being at most 65,535, and a 1-byte run
2109
+ // length, while extending a run adds at most 1 byte; and up to 9 for the
2110
+ // NodeId, as a new run is 8 bytes and a 1-byte run length. Its type takes 1
2111
+ // byte, and the ranges count gains at most 1 byte.
2112
+ const maxRangeOverhead = 22;
2113
+
2114
+ /**
2115
+ * Returns the bytes a frame must keep to write `timestamps` as a Timestamps
2116
+ * range, after a Skip range when one is pending. A Skip range has no payload,
2117
+ * and the length of `timestamps` is exact.
2118
+ */
2119
+ const getTimestampsRangeReserve = (
2120
+ timestamps: TimestampsBuffer,
2121
+ isSkipPending: boolean,
2122
+ ): NonNegativeInt =>
2123
+ (timestamps.getLength() +
2124
+ maxRangeOverhead +
2125
+ (isSkipPending ? maxRangeOverhead : 0)) as NonNegativeInt;
2126
+
2127
+ /**
2128
+ * Reads the ranges that split the items from `lower` to `upper`: one Timestamps
2129
+ * range listing them when they are too few for buckets, otherwise Fingerprint
2130
+ * ranges over the buckets. It only reads storage, so a throw is a storage
2131
+ * failure.
2132
+ */
2133
+ const readSplitRanges =
1684
2134
  (deps: StorageDep) =>
1685
2135
  (
1686
2136
  ownerId: OwnerIdBytes,
1687
2137
  lower: NonNegativeInt,
1688
2138
  upper: NonNegativeInt,
1689
2139
  upperBound: RangeUpperBound,
1690
- buffer: ProtocolMessageBuffer,
1691
- ): void => {
2140
+ ): ReadonlyArray<FingerprintRange | TimestampsRangeWithTimestampsBuffer> => {
1692
2141
  const itemCount = NonNegativeInt.orThrow(upper - lower);
1693
2142
  const buckets = computeBalancedBuckets(itemCount);
1694
2143
 
@@ -1704,18 +2153,14 @@ const splitRange =
1704
2153
  return true;
1705
2154
  });
1706
2155
 
1707
- buffer.addRange(range);
1708
- return;
2156
+ return [range];
1709
2157
  }
1710
2158
 
1711
2159
  // Check Storage.ts `fingerprint` and `fingerprintRanges` docs.
1712
2160
  const fingerprintRangesBuckets =
1713
2161
  lower === 0
1714
2162
  ? buckets.value
1715
- : [
1716
- lower,
1717
- ...buckets.value.map((b) => NonNegativeInt.orThrow(b + lower)),
1718
- ];
2163
+ : [lower, ...buckets.value.map((b) => (b + lower) as NonNegativeInt)];
1719
2164
 
1720
2165
  const fingerprintRanges = deps.storage.fingerprintRanges(
1721
2166
  ownerId,
@@ -1723,23 +2168,45 @@ const splitRange =
1723
2168
  upperBound,
1724
2169
  );
1725
2170
 
1726
- const rangesToUse =
1727
- lower > 0 ? fingerprintRanges.slice(1) : fingerprintRanges;
1728
-
1729
- for (const range of rangesToUse) {
1730
- buffer.addRange(range);
1731
- }
2171
+ return lower > 0 ? fingerprintRanges.slice(1) : fingerprintRanges;
1732
2172
  };
1733
2173
 
2174
+ // Twice the largest rangesMaxSize rather than the receiver's own, because
2175
+ // deployed senders exceed theirs. Up to @evolu/common 8.17, sync answered each
2176
+ // Timestamps range that listed timestamps it lacked, while it held none in that
2177
+ // range, with an empty Timestamps range, without checking the size. Such an
2178
+ // echo reuses the peer's bounds with a 1-byte payload, so the echoes take less
2179
+ // than the peer's ranges that list timestamps, which the peer's size checks
2180
+ // kept within its rangesMaxSize. An empty list asks for nothing, so an echo is
2181
+ // never echoed again.
2182
+ const maxRangesSectionSize = 2 * maxProtocolMessageRangesMaxSize;
2183
+
1734
2184
  const decodeRanges = (buffer: Buffer): ReadonlyArray<Range> => {
2185
+ // The ranges section ends the frame, so it is checked before any allocation.
2186
+ if (buffer.getLength() > maxRangesSectionSize)
2187
+ throw new ProtocolDecodeError(
2188
+ `Ranges section exceeds ${maxRangesSectionSize} bytes`,
2189
+ );
2190
+
1735
2191
  if (buffer.getLength() === 0) return [];
1736
2192
 
1737
2193
  const rangesCount = decodeNonNegativeInt(buffer);
1738
2194
  if (rangesCount === 0) return [];
1739
2195
 
1740
- const timestampsCount = NonNegativeInt.orThrow(rangesCount - 1);
2196
+ const timestampsCount = (rangesCount - 1) as NonNegativeInt;
1741
2197
  const timestamps = decodeTimestamps(buffer, timestampsCount);
1742
2198
 
2199
+ // Storage resolves each bound from the owner's first timestamp, so a lower
2200
+ // bound would move back over ranges already answered. Equal bounds are
2201
+ // valid: sync ends a range at the change that did not fit, which can be
2202
+ // where the previous range ended. Timestamps listed in a range are not
2203
+ // checked, because peers before @evolu/common 8.11 list some outside it.
2204
+ for (let i = 1; i < timestamps.length; i++)
2205
+ if (orderTimestamp(timestamps[i - 1], timestamps[i]) > 0)
2206
+ throw new ProtocolDecodeError(
2207
+ "Range upper bounds must be non-decreasing",
2208
+ );
2209
+
1743
2210
  const rangeTypes = createMutableArray<RangeType>(rangesCount);
1744
2211
 
1745
2212
  for (let i = 0; i < rangesCount; i++) {
@@ -1802,6 +2269,9 @@ const decodeTimestamps = (
1802
2269
  length?: NonNegativeInt,
1803
2270
  ): ReadonlyArray<Timestamp> => {
1804
2271
  length ??= decodeNonNegativeInt(buffer);
2272
+ // Every timestamp takes at least its 1-byte millis delta.
2273
+ if (length > buffer.getLength())
2274
+ throw new ProtocolDecodeError("Invalid timestamps count");
1805
2275
 
1806
2276
  let previousMillis = 0 as Millis;
1807
2277
 
@@ -1841,13 +2311,28 @@ const decodeId = (buffer: Buffer): Id => {
1841
2311
  return idBytesToId(bytes as IdBytes);
1842
2312
  };
1843
2313
 
2314
+ /**
2315
+ * The format version that starts every {@link EncryptedDbChange} plaintext.
2316
+ *
2317
+ * It is independent of {@link protocolVersion}, so a new protocol version that
2318
+ * keeps this layout does not make decoders reject changes. 6.0.1-preview.35
2319
+ * wrote 0 with this layout, so decoding accepts every version up to this one.
2320
+ *
2321
+ * Decoders in `@evolu/common` 8.17 and earlier ignore the version. A future
2322
+ * layout must still make them fail, for example with a value they cannot
2323
+ * decode, rather than let them decode wrong values.
2324
+ */
2325
+ const encryptedDbChangeVersion = onePositiveInt;
2326
+
1844
2327
  /**
1845
2328
  * Encodes and encrypts a {@link DbChange} using the provided owner's encryption
1846
2329
  * key. Returns an encrypted binary representation as {@link EncryptedDbChange}.
1847
2330
  *
1848
- * The format includes the protocol version for backward compatibility and the
1849
- * timestamp for tamper-proof verification that the timestamp matches the change
1850
- * data.
2331
+ * The plaintext starts with the format version of the change, which is
2332
+ * independent of {@link protocolVersion}, and the timestamp, which proves that
2333
+ * the change belongs to the timestamp it is sent with.
2334
+ * {@link decryptAndDecodeDbChange} rejects a newer format version and accepts
2335
+ * older ones.
1851
2336
  */
1852
2337
  export const encodeAndEncryptDbChange =
1853
2338
  (deps: RandomBytesDep) =>
@@ -1881,7 +2366,7 @@ export const encodeAndEncryptDbChange =
1881
2366
  * within it fits one protocol message.
1882
2367
  */
1883
2368
  export const encodeDbChange = (buffer: Buffer, message: CrdtMessage): void => {
1884
- encodeNonNegativeInt(buffer, protocolVersion);
2369
+ encodeNonNegativeInt(buffer, encryptedDbChangeVersion);
1885
2370
 
1886
2371
  // Encode the timestamp to prevent tampering (e.g., a malicious relay
1887
2372
  // assigning this EncryptedDbChange to a different EncryptedCrdtMessage)
@@ -1901,9 +2386,10 @@ export const encodeDbChange = (buffer: Buffer, message: CrdtMessage): void => {
1901
2386
 
1902
2387
  encodeLength(buffer, entries);
1903
2388
  for (const [column, value] of entries) {
1904
- assertNotUndefined(value);
1905
2389
  encodeString(buffer, column);
1906
- encodeSqliteValue(buffer, value);
2390
+ // DbChange validated every value as a SqliteValue; only its Partial record
2391
+ // type admits undefined.
2392
+ encodeSqliteValue(buffer, value as SqliteValue);
1907
2393
  }
1908
2394
  };
1909
2395
 
@@ -1911,6 +2397,9 @@ export const encodeDbChange = (buffer: Buffer, message: CrdtMessage): void => {
1911
2397
  * Decrypts and decodes an {@link EncryptedCrdtMessage} using the provided
1912
2398
  * owner's encryption key. Verifies that the embedded timestamp matches the
1913
2399
  * expected timestamp to ensure message integrity.
2400
+ *
2401
+ * A change with a newer format version than {@link encodeAndEncryptDbChange}
2402
+ * writes is a {@link ProtocolInvalidDataError}.
1914
2403
  */
1915
2404
  export const decryptAndDecodeDbChange = (
1916
2405
  message: EncryptedCrdtMessage,
@@ -1928,8 +2417,8 @@ export const decryptAndDecodeDbChange = (
1928
2417
  const ciphertext = buffer.shiftN(decodeLength(buffer));
1929
2418
 
1930
2419
  const plaintextBytes = decryptWithXChaCha20Poly1305(
1931
- XChaCha20Poly1305Ciphertext.orThrow(ciphertext),
1932
- Entropy24.orThrow(nonce),
2420
+ ciphertext as XChaCha20Poly1305Ciphertext,
2421
+ nonce as Entropy24,
1933
2422
  key,
1934
2423
  );
1935
2424
  if (!plaintextBytes.ok) return plaintextBytes;
@@ -1937,11 +2426,11 @@ export const decryptAndDecodeDbChange = (
1937
2426
  buffer.reset();
1938
2427
  buffer.extend(plaintextBytes.value);
1939
2428
 
1940
- // Decode version (for future compatibility, not need yet)
1941
- decodeNonNegativeInt(buffer);
2429
+ if (decodeNonNegativeInt(buffer) > encryptedDbChangeVersion)
2430
+ throw new ProtocolDecodeError("Unsupported EncryptedDbChange version");
1942
2431
 
1943
2432
  const timestamp = timestampBytesToTimestamp(
1944
- TimestampBytes.orThrow(buffer.shiftN(timestampBytesLength)),
2433
+ buffer.shiftN(timestampBytesLength) as TimestampBytes,
1945
2434
  );
1946
2435
 
1947
2436
  if (!eqTimestamp(timestamp, message.timestamp)) {
@@ -1952,7 +2441,7 @@ export const decryptAndDecodeDbChange = (
1952
2441
  });
1953
2442
  }
1954
2443
 
1955
- const flags = decodeFlags(buffer, PositiveInt.orThrow(3));
2444
+ const flags = decodeFlags(buffer, 3 as PositiveInt);
1956
2445
  const table = decodeString(buffer);
1957
2446
  const id = decodeId(buffer);
1958
2447
 
@@ -1983,30 +2472,6 @@ export const decryptAndDecodeDbChange = (
1983
2472
  }
1984
2473
  };
1985
2474
 
1986
- /**
1987
- * Decodes a ProtocolMessage into a readable JSON object for debugging.
1988
- *
1989
- * Note: This is a stub for future implementation. It should use:
1990
- *
1991
- * - DecodeVersionAndOwner
1992
- * - DecodeError or decodeWriteKeys (depending on context)
1993
- * - DecodeMessages
1994
- * - DecodeRanges
1995
- *
1996
- * If you want to help, please contribute to this function.
1997
- */
1998
- export const decodeProtocolMessageToJson = (
1999
- _protocolMessage: ProtocolMessage,
2000
- _isInitiator: boolean,
2001
- ): unknown => {
2002
- // TODO: Implement using
2003
- // - decodeVersionAndOwner
2004
- // -- decodeError or decodeWriteKeys (should be refactored out),
2005
- // -- decodeMessages, and decodeRanges.
2006
- // This is a stub for PRs and community contributions.
2007
- throw new Error("decodeProtocolMessageToJson is not implemented yet.");
2008
- };
2009
-
2010
2475
  // Small ints are encoded into ProtocolValueType, saving one byte per int.
2011
2476
  export const ProtocolValueType = {
2012
2477
  // 0-19 small ints
@@ -2067,7 +2532,7 @@ export const encodeSqliteValue = (buffer: Buffer, value: SqliteValue): void => {
2067
2532
  buffer,
2068
2533
  ProtocolValueType.DateIsoWithNegativeTime,
2069
2534
  );
2070
- encodeNumber(buffer, FiniteNumber.orThrow(time));
2535
+ encodeNumber(buffer, time as FiniteNumber);
2071
2536
  }
2072
2537
  return;
2073
2538
  }
@@ -2082,7 +2547,7 @@ export const encodeSqliteValue = (buffer: Buffer, value: SqliteValue): void => {
2082
2547
  const json = Json.from.parent(value);
2083
2548
  if (json.ok) {
2084
2549
  const jsonValue = jsonToJsonValue(json.value);
2085
- jsonBuffer.reset();
2550
+ const jsonBuffer = createBuffer();
2086
2551
  try {
2087
2552
  // Encoding first rejects nesting deeper than decoding allows, before
2088
2553
  // the recursive JSON.stringify below could overflow the stack. Such
@@ -2101,8 +2566,6 @@ export const encodeSqliteValue = (buffer: Buffer, value: SqliteValue): void => {
2101
2566
  }
2102
2567
  } catch (error) {
2103
2568
  if (!(error instanceof BufferError)) throw error;
2104
- } finally {
2105
- jsonBuffer.reset();
2106
2569
  }
2107
2570
  }
2108
2571
 
@@ -2115,6 +2578,10 @@ export const encodeSqliteValue = (buffer: Buffer, value: SqliteValue): void => {
2115
2578
  return;
2116
2579
  }
2117
2580
 
2581
+ // encodeString replaces a lone surrogate with U+FFFD, so the bytes are
2582
+ // always valid UTF-8. Encoding WTF-8 instead would make peers disagree:
2583
+ // deployed decoders read a lone surrogate in WTF-8 as three U+FFFD,
2584
+ // while a decoder that kept it would read the surrogate.
2118
2585
  encodeNonNegativeInt(buffer, ProtocolValueType.String);
2119
2586
  encodeString(buffer, value);
2120
2587
  return;
@@ -2206,7 +2673,5 @@ export const decodeSqliteValue = (buffer: Buffer): SqliteValue => {
2206
2673
  }
2207
2674
  };
2208
2675
 
2209
- const jsonBuffer = createBuffer();
2210
-
2211
2676
  const isSmallInt: Predicate<number> = (value: number) =>
2212
2677
  value >= 0 && value < 20;