@evolu/common 8.17.0 → 8.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/dist/src/Bytes.d.ts +31 -14
  2. package/dist/src/Bytes.d.ts.map +1 -1
  3. package/dist/src/Bytes.js +48 -8
  4. package/dist/src/Error.d.ts.map +1 -1
  5. package/dist/src/Error.js +5 -1
  6. package/dist/src/Polyfills.d.ts.map +1 -1
  7. package/dist/src/Polyfills.js +3 -2
  8. package/dist/src/Sqlite.d.ts +1 -1
  9. package/dist/src/Task.d.ts +4 -3
  10. package/dist/src/Task.d.ts.map +1 -1
  11. package/dist/src/Task.js +4 -3
  12. package/dist/src/index.d.ts +1 -1
  13. package/dist/src/index.d.ts.map +1 -1
  14. package/dist/src/local-first/Db.d.ts +32 -3
  15. package/dist/src/local-first/Db.d.ts.map +1 -1
  16. package/dist/src/local-first/Db.js +30 -9
  17. package/dist/src/local-first/Evolu.d.ts +21 -6
  18. package/dist/src/local-first/Evolu.d.ts.map +1 -1
  19. package/dist/src/local-first/Evolu.js +5 -1
  20. package/dist/src/local-first/Protocol.d.ts +215 -93
  21. package/dist/src/local-first/Protocol.d.ts.map +1 -1
  22. package/dist/src/local-first/Protocol.js +799 -489
  23. package/dist/src/local-first/Relay.d.ts +9 -1
  24. package/dist/src/local-first/Relay.d.ts.map +1 -1
  25. package/dist/src/local-first/Relay.js +41 -36
  26. package/dist/src/local-first/Shared.d.ts +79 -53
  27. package/dist/src/local-first/Shared.d.ts.map +1 -1
  28. package/dist/src/local-first/Shared.js +67 -42
  29. package/dist/src/local-first/Storage.d.ts +80 -18
  30. package/dist/src/local-first/Storage.d.ts.map +1 -1
  31. package/dist/src/local-first/Storage.js +20 -4
  32. package/package.json +1 -1
  33. package/src/Bytes.test.ts +212 -0
  34. package/src/Bytes.ts +67 -17
  35. package/src/Error.ts +5 -1
  36. package/src/Polyfills.ts +3 -2
  37. package/src/Sqlite.ts +1 -1
  38. package/src/Task.ts +4 -3
  39. package/src/index.ts +4 -1
  40. package/src/local-first/Db.ts +76 -17
  41. package/src/local-first/Evolu.test.ts +43 -12
  42. package/src/local-first/Evolu.ts +34 -7
  43. package/src/local-first/Protocol.test.ts +1541 -44
  44. package/src/local-first/Protocol.ts +1070 -605
  45. package/src/local-first/Relay.ts +80 -69
  46. package/src/local-first/Shared.test.ts +37 -1
  47. package/src/local-first/Shared.ts +124 -73
  48. package/src/local-first/Storage.ts +100 -27
@@ -43,14 +43,24 @@
43
43
  * | - {@link NonNegativeInt} | Number of ranges. |
44
44
  * | - {@link Range} | |
45
45
  *
46
+ * Range upper bounds must not decrease. The last range always has
47
+ * {@link InfiniteUpperBound}, which is not encoded. A message with decreasing
48
+ * bounds, or with a hasWriteKey or subscriptionFlag value not listed above, is
49
+ * rejected as {@link ProtocolInvalidDataError}. So is a request with a change
50
+ * shorter than 41 bytes, the smallest {@link EncryptedDbChange}. A client passes
51
+ * such a change to storage, which skips a change it cannot read.
52
+ *
46
53
  * ## WriteKey validation
47
54
  *
48
55
  * The initiator sends a hasWriteKey flag and optionally a WriteKey. The
49
56
  * WriteKey is required when sending messages as a secure token proving the
50
57
  * initiator can write changes. It's ok to not send a WriteKey if the initiator
51
58
  * is only syncing (read-only) and not sending messages. The non-initiator
52
- * validates the WriteKey immediately after parsing the initiator header, before
53
- * processing any messages or ranges.
59
+ * decodes the whole request first and then validates the WriteKey, before
60
+ * storing any messages or reconciling ranges. Only the subscriptionFlag takes
61
+ * effect before that, because subscribing needs no WriteKey: broadcasts carry
62
+ * only encrypted changes. So a request rejected with
63
+ * {@link ProtocolWriteKeyError} still changes its subscription.
54
64
  *
55
65
  * ## Synchronization
56
66
  *
@@ -64,9 +74,14 @@
64
74
  * if further sync is needed or possible, continuing until both sides are
65
75
  * synchronized.
66
76
  *
67
- * The **non-initiator always responds** to provide sync completion feedback,
68
- * even with empty messages containing only the header and no error. This allows
69
- * the initiator to detect when synchronization is complete.
77
+ * The **non-initiator answers every request it can decode** to provide sync
78
+ * completion feedback, even with empty messages containing only the header and
79
+ * no error. This allows the initiator to detect when synchronization is
80
+ * complete. A request it cannot decode, or one larger than the relay's
81
+ * `totalMaxSize`, gets no answer and has no effect.
82
+ *
83
+ * Ranges are compared by their {@link Fingerprint}, which anyone who can write
84
+ * for an owner can make collide. Its documentation describes the consequences.
70
85
  *
71
86
  * Both **Messages** and **Ranges** are optional, allowing each side to send,
72
87
  * sync, or only subscribe data as needed.
@@ -81,18 +96,35 @@
81
96
  *
82
97
  * ## Protocol errors
83
98
  *
84
- * The protocol uses error codes in the header to signal issues:
99
+ * A Response carries a {@link ProtocolErrorCode} in its header. The initiator
100
+ * reports every code except `NoError` as an error with the `OwnerId`:
85
101
  *
86
- * - {@link ProtocolWriteKeyError}: The provided WriteKey is invalid or missing.
87
- * - {@link ProtocolWriteError}: A serious relay-side write failure occurred.
102
+ * - {@link ProtocolWriteKeyError}: The WriteKey is invalid, or missing from a
103
+ * request with messages.
104
+ * - {@link ProtocolWriteError}: The relay failed to store the messages or to
105
+ * validate the WriteKey.
88
106
  * - {@link ProtocolQuotaError}: Storage or billing quota exceeded.
89
- * - {@link ProtocolSyncError}: A serious relay-side synchronization failure
90
- * occurred.
91
- * - {@link ProtocolVersionError}: Protocol version mismatch.
92
- * - {@link ProtocolInvalidDataError}: The message is malformed or corrupted.
107
+ * - {@link ProtocolSyncError}: The relay's storage failed while it reconciled the
108
+ * ranges.
109
+ *
110
+ * The initiator also reports {@link ProtocolVersionError}, with the `OwnerId`,
111
+ * for a reply of another version, and {@link ProtocolInvalidDataError}, without
112
+ * it, for a message it cannot decode, including one with an unknown code.
113
+ *
114
+ * A relay answers every request it can decode within its `totalMaxSize`. It
115
+ * decodes a whole request before acting on it, so for a malformed one it
116
+ * returns `ProtocolInvalidDataError` without subscribing, storing, or
117
+ * broadcasting anything, and sends no reply. When its storage throws while
118
+ * validating the WriteKey, for example because SQLite cannot store a new
119
+ * owner's key on a full disk, the relay logs the error and answers with
120
+ * `ProtocolWriteError`, like a failed write. Likewise, a client applies nothing
121
+ * from a message it cannot decode.
93
122
  *
94
- * All protocol errors except `ProtocolInvalidDataError` include the `OwnerId`
95
- * to allow clients to associate errors with the correct owner.
123
+ * {@link decryptAndDecodeDbChange} returns `ProtocolInvalidDataError`,
124
+ * {@link ProtocolTimestampMismatchError}, or
125
+ * {@link DecryptWithXChaCha20Poly1305Error} for a change it cannot read. These
126
+ * describe a change rather than a protocol message, and client storage skips
127
+ * such a change.
96
128
  *
97
129
  * ## Message size limit
98
130
  *
@@ -105,7 +137,10 @@
105
137
  *
106
138
  * Each mutation is limited to {@link maxMutationSize}, so every change fits one
107
139
  * message of {@link defaultProtocolMessageMaxSize} next to the largest ranges
108
- * section.
140
+ * section. Changes saved before that limit existed, and crafted changes a relay
141
+ * stores, can be larger. Sync skips a stored change that cannot fit an empty
142
+ * message after a pending Skip range, which is the message a later round is
143
+ * guaranteed to reach, and reports it as a {@link ProtocolChangeTooLargeError}.
109
144
  *
110
145
  * ## Why Binary?
111
146
  *
@@ -129,24 +164,18 @@
129
164
  *
130
165
  * ## Versioning
131
166
  *
132
- * Evolu Protocol uses explicit versioning to ensure compatibility between
133
- * clients and relays (or peers). Each protocol message begins with a version
134
- * number and an `ownerId` in its header.
167
+ * Every message of every protocol version begins with the version and the
168
+ * `OwnerId`, so a peer can route and report a message of any version. Nothing
169
+ * negotiates a version, and no side falls back to the other's.
135
170
  *
136
- * **How version negotiation works:**
137
- *
138
- * - The initiator (usually a client) sends a `ProtocolMessage` that includes its
139
- * protocol version and the `ownerId`.
140
- * - The non-initiator (usually a relay or peer) checks the version.
141
- *
142
- * - If the versions match, synchronization proceeds as normal.
143
- * - If the versions do not match, the non-initiator responds with a message
144
- * containing **its own protocol version and the same `ownerId`**.
145
- * - The initiator can then detect the version mismatch for that specific owner
146
- * and handle it appropriately (e.g., prompt for an update or halt sync).
147
- *
148
- * Version negotiation is per-owner, allowing Evolu Protocol to evolve safely
149
- * over time and provide clear feedback about version mismatches.
171
+ * A non-initiator answers a request of another version with only its own
172
+ * version and that `OwnerId`. The initiator reports it as
173
+ * {@link ProtocolVersionError} for that owner, whose `isInitiator` tells which
174
+ * side is older, and stops syncing the owner through that relay. Clients from
175
+ * `@evolu/common` 8.0.0 before 8.11.0 drop that reply and stop syncing the
176
+ * owner through that relay without reporting anything, while 7.x clients report
177
+ * it as a `ProtocolVersionError`. So a relay that moves to another version must
178
+ * keep answering version 1 while such clients remain.
150
179
  *
151
180
  * ## Credible exit
152
181
  *
@@ -174,43 +203,50 @@
174
203
  * indicators) to be forwarded by relays without storage overhead.
175
204
  */
176
205
  import { createMutableArray, isNonEmptyArray, } from "../Array.js";
177
- import { assert, assertNonEmptyArray, assertNonNullable, assertNotUndefined, assertSame, } from "../Assert.js";
206
+ import { assert, assertNonEmptyArray, assertNonNullable } from "../Assert.js";
178
207
  import { BufferError, createBuffer, createRunLengthEncoder, decodeFlags, decodeJsonValue, decodeLength, decodeNonNegativeInt, decodeNumber, decodeRle, decodeString, encodeFlags, encodeJsonValue, encodeLength, encodeNonNegativeInt, encodeNumber, encodeString, } from "../Bytes.js";
179
- import { createPadmePadding, decryptWithXChaCha20Poly1305, EncryptionKey, encryptWithXChaCha20Poly1305, Entropy24, XChaCha20Poly1305Ciphertext, xChaCha20Poly1305NonceLength, } from "../Crypto.js";
208
+ import { createPadmePadding, decryptWithXChaCha20Poly1305, EncryptionKey, encryptWithXChaCha20Poly1305, xChaCha20Poly1305NonceLength, } from "../Crypto.js";
180
209
  import { eqArrayNumber } from "../Eq.js";
210
+ import { exhaustiveCheck } from "../Function.js";
181
211
  import { computeBalancedBuckets } from "../Number.js";
182
212
  import { createMutableRecord, objectToEntries } from "../Object.js";
183
213
  import { err, ok } from "../Result.js";
184
214
  import { AbortError } from "../Task.js";
185
215
  import { Millis } from "../Time.js";
186
- import { assertType, Base64Url, base64UrlToUint8Array, between, DateIso, FiniteNumber, Id, IdBytes, idBytesToId, idBytesTypeValueLength, idToIdBytes, Int, Json, jsonToJsonValue, NonNegativeInt, onePositiveInt, PositiveInt, uint8ArrayToBase64Url, zeroNonNegativeInt, } from "../Type.js";
216
+ import { assertType, Base64Url, base64UrlToUint8Array, between, DateIso, Id, IdBytes, idBytesToId, idBytesTypeValueLength, idToIdBytes, Int, Json, jsonToJsonValue, NonNegativeInt, onePositiveInt, uint8ArrayToBase64Url, zeroNonNegativeInt, } from "../Type.js";
187
217
  import { OwnerId, OwnerIdBytes, ownerIdToOwnerIdBytes, OwnerWriteKey, ownerWriteKeyLength, } from "./Owner.js";
188
218
  import { DbChange, fingerprintSize, InfiniteUpperBound, RangeType, } from "./Storage.js";
189
- import { Counter, eqTimestamp, NodeId, nodeIdBytesLength, nodeIdBytesToNodeId, nodeIdToNodeIdBytes, Timestamp, TimestampBytes, timestampBytesLength, timestampBytesToTimestamp, timestampToTimestampBytes, } from "./Timestamp.js";
219
+ import { Counter, eqTimestamp, NodeId, nodeIdBytesLength, nodeIdBytesToNodeId, nodeIdToNodeIdBytes, orderTimestamp, Timestamp, timestampBytesLength, timestampBytesToTimestamp, timestampToTimestampBytes, } from "./Timestamp.js";
190
220
  const minProtocolMessageMaxSize = 1_000_000;
191
221
  const maxProtocolMessageMaxSize = 100_000_000;
222
+ const maxProtocolMessageRangesMaxSize = 100_000;
192
223
  /**
193
- * Protocol message maximum size.
224
+ * Protocol message maximum size, from 1MB to 100MB.
194
225
  *
195
- * Defines the upper limit for how large a single protocol message can be.
196
- * Implementations must enforce a maximum size between 1MB and 100MB to ensure
197
- * compatibility across all Evolu implementations (the maximum size of mutation
198
- * change is hardcoded and enforced hence the maximum size can't be smaller).
226
+ * A message never exceeds its maximum size, inclusive. Builders measure each
227
+ * write exactly and keep room to close the message (see
228
+ * {@link ProtocolMessageBuffer.tryWrite}), and sync sends what does not fit in
229
+ * later rounds.
199
230
  *
200
- * Larger maximum sizes can be configured by relays to reduce roundtrips. For
201
- * example, a dedicated relay with ample resources could configure a 100MB
202
- * maximum to minimize roundtrips for large syncs.
231
+ * Clients send at most {@link defaultProtocolMessageMaxSize} and enforce no
232
+ * receive limit. A relay can answer with larger messages, configured with the
233
+ * `totalMaxSize` option of {@link applyProtocolMessageAsRelay}, to reduce
234
+ * roundtrips for large syncs. Clients must not send larger messages, because
235
+ * relays accept at most the default: the Node.js relay closes the connection on
236
+ * a larger one, and relays up to `@evolu/nodejs` 4.0.0 crash on it.
203
237
  *
204
- * Only relays can safely configure larger sizes, as clients will handle them.
205
- * Increasing this value on the client side would break compatibility with
206
- * relays that enforce smaller limits.
238
+ * The default cannot be smaller either. Deployed clients send messages of that
239
+ * size, a change within {@link maxMutationSize} fits one message next to the
240
+ * largest ranges section, and changes saved before that limit existed, which
241
+ * can be nearly as large as a message, must stay servable.
207
242
  */
208
243
  export const ProtocolMessageMaxSize = /*#__PURE__*/ between(minProtocolMessageMaxSize, maxProtocolMessageMaxSize)(Int);
209
244
  /**
210
245
  * Default {@link ProtocolMessageMaxSize} (1MB).
211
246
  *
212
- * The standard size used across Evolu implementations. Relays with more
213
- * resources can configure larger sizes to reduce roundtrips.
247
+ * The standard size used across Evolu implementations. Clients send at most
248
+ * this size, and relays accept it. Relays with more resources can answer with
249
+ * larger messages to reduce roundtrips.
214
250
  */
215
251
  export const defaultProtocolMessageMaxSize = minProtocolMessageMaxSize;
216
252
  /**
@@ -222,9 +258,10 @@ export const defaultProtocolMessageMaxSize = minProtocolMessageMaxSize;
222
258
  *
223
259
  * The upper bound is set to ensure ranges fit within the default 1MB
224
260
  * {@link defaultProtocolMessageMaxSize}, maintaining compatibility between all
225
- * clients and relays.
261
+ * clients and relays. A received message whose ranges section exceeds twice the
262
+ * upper bound is rejected as {@link ProtocolInvalidDataError}.
226
263
  */
227
- export const ProtocolMessageRangesMaxSize = /*#__PURE__*/ between(3_000, 100_000)(Int);
264
+ export const ProtocolMessageRangesMaxSize = /*#__PURE__*/ between(3_000, maxProtocolMessageRangesMaxSize)(Int);
228
265
  /**
229
266
  * Default {@link ProtocolMessageRangesMaxSize} (30KB).
230
267
  *
@@ -270,6 +307,14 @@ export const SubscriptionFlags = {
270
307
  /** Unsubscribe from updates for this owner. */
271
308
  Unsubscribe: 2,
272
309
  };
310
+ /**
311
+ * The error code in the header of a Response.
312
+ *
313
+ * A client reports a code it does not know as {@link ProtocolInvalidDataError},
314
+ * which carries no `OwnerId`, and that relay's round for the owner ends. Every
315
+ * released client does this, so a relay can add a code under the same
316
+ * {@link protocolVersion} only where that outcome is acceptable.
317
+ */
273
318
  export const ProtocolErrorCode = {
274
319
  NoError: 0,
275
320
  /** A code for {@link ProtocolWriteKeyError}. */
@@ -284,9 +329,12 @@ export const ProtocolErrorCode = {
284
329
  /**
285
330
  * Creates a {@link ProtocolMessage} from CRDT messages.
286
331
  *
287
- * If the message size would exceed {@link defaultProtocolMessageMaxSize}, the
288
- * protocol ensures all messages will be sent in the next round(s) even over
289
- * unidirectional and stateless transports.
332
+ * The message holds the leading CRDT messages that fit `maxSize`, by default
333
+ * {@link defaultProtocolMessageMaxSize}, measured exactly. When one does not
334
+ * fit, it and the rest are left out, and the message ends with a range that
335
+ * makes the non-initiator answer with ranges. Sync then sends them in later
336
+ * rounds, even over unidirectional and stateless transports, because every
337
+ * change within {@link maxMutationSize} fits an empty message.
290
338
  */
291
339
  export const createProtocolMessageFromCrdtMessages = (deps) => (owner, messages, maxSize) => {
292
340
  const buffer = createProtocolMessageBuffer(owner.id, {
@@ -294,42 +342,37 @@ export const createProtocolMessageFromCrdtMessages = (deps) => (owner, messages,
294
342
  totalMaxSize: maxSize ?? defaultProtocolMessageMaxSize,
295
343
  writeKey: owner.writeKey,
296
344
  });
297
- let notAllMessagesSent = false;
298
345
  for (const message of messages) {
299
346
  const change = encodeAndEncryptDbChange(deps)(message, owner.encryptionKey);
300
347
  const encryptedCrdtMessage = { timestamp: message.timestamp, change };
301
- if (buffer.canAddMessage(encryptedCrdtMessage)) {
348
+ if (!buffer.tryWrite(() => {
302
349
  buffer.addMessage(encryptedCrdtMessage);
303
- }
304
- else {
305
- notAllMessagesSent = true;
350
+ })) {
351
+ /**
352
+ * DEV: If not all messages fit due to size limits, we trigger a sync
353
+ * continuation by appending a Range with a random fingerprint. This
354
+ * ensures the receiver always responds with ranges, prompting another
355
+ * sync round.
356
+ *
357
+ * The ideal approach would be to send three ranges (skip, fingerprint,
358
+ * skip) where the fingerprint of unsent messages would act as narrow
359
+ * sync probe. I think we can send `zeroFingerprint` which can be
360
+ * interpreted as an indication that the other side should reply with
361
+ * {@link TimestampsRange}, so no need to restart syncing.
362
+ *
363
+ * For now, using a random fingerprint avoids extra complexity and is
364
+ * good enough for this case.
365
+ */
366
+ const randomFingerprint = deps.randomBytes.create(fingerprintSize);
367
+ // Every kept trial reserved space for this closing range.
368
+ buffer.addRange({
369
+ type: RangeType.Fingerprint,
370
+ upperBound: InfiniteUpperBound,
371
+ fingerprint: randomFingerprint,
372
+ });
306
373
  break;
307
374
  }
308
375
  }
309
- if (notAllMessagesSent) {
310
- /**
311
- * DEV: If not all messages fit due to size limits, we trigger a sync
312
- * continuation by appending a Range with a random fingerprint. This
313
- * ensures the receiver always responds with ranges, prompting another
314
- * sync round.
315
- *
316
- * The ideal approach would be to send three ranges (skip, fingerprint,
317
- * skip) where the fingerprint of unsent messages would act as narrow sync
318
- * probe. I think we can send `zeroFingerprint` which can be interpreted
319
- * as an indication that the other side should reply with
320
- * {@link TimestampsRange}, so no need to restart syncing.
321
- *
322
- * For now, using a random fingerprint avoids extra complexity and is good
323
- * enough for this case.
324
- */
325
- const randomFingerprint = deps.randomBytes.create(fingerprintSize);
326
- // There is always a space for Fingerprint with InfiniteUpperBound.
327
- buffer.addRange({
328
- type: RangeType.Fingerprint,
329
- upperBound: InfiniteUpperBound,
330
- fingerprint: randomFingerprint,
331
- });
332
- }
333
376
  return buffer.unwrap();
334
377
  };
335
378
  /**
@@ -382,16 +425,19 @@ export const createProtocolBroadcastMessagesFromCrdtMessages = (deps) => (owner,
382
425
  timestamp: message.timestamp,
383
426
  change: encodeAndEncryptDbChange(deps)(message, owner.encryptionKey),
384
427
  };
385
- if (!buffer.canAddMessage(encryptedMessage)) {
386
- const nextBuffer = createProtocolMessageBuffer(owner.id, {
387
- messageType: MessageType.Broadcast,
388
- totalMaxSize: maxSize,
389
- });
390
- assert(nextBuffer.canAddMessage(encryptedMessage), "the message is too big");
391
- broadcasts.push(buffer.unwrap());
392
- buffer = nextBuffer;
428
+ const frame = buffer;
429
+ if (frame.tryWrite(() => {
430
+ frame.addMessage(encryptedMessage);
431
+ })) {
432
+ continue;
393
433
  }
434
+ buffer = createProtocolMessageBuffer(owner.id, {
435
+ messageType: MessageType.Broadcast,
436
+ totalMaxSize: maxSize,
437
+ });
438
+ // Outside a trial, it asserts that the message fits an empty frame.
394
439
  buffer.addMessage(encryptedMessage);
440
+ broadcasts.push(frame.unwrap());
395
441
  }
396
442
  broadcasts.push(buffer.unwrap());
397
443
  assertNonEmptyArray(broadcasts);
@@ -405,7 +451,9 @@ export const createProtocolMessageForSync = (deps) => (ownerId, subscriptionFlag
405
451
  });
406
452
  const ownerIdBytes = ownerIdToOwnerIdBytes(ownerId);
407
453
  const size = deps.storage.getSize(ownerIdBytes);
408
- splitRange(deps)(ownerIdBytes, zeroNonNegativeInt, size, InfiniteUpperBound, buffer);
454
+ const ranges = readSplitRanges(deps)(ownerIdBytes, zeroNonNegativeInt, size, InfiniteUpperBound);
455
+ for (const range of ranges)
456
+ buffer.addRange(range);
409
457
  return buffer.unwrap();
410
458
  };
411
459
  export const createProtocolMessageForUnsubscribe = (ownerId) => createProtocolMessageBuffer(ownerId, {
@@ -418,7 +466,7 @@ export const createProtocolMessageBuffer = (ownerId, options) => {
418
466
  header: createBuffer(),
419
467
  messages: {
420
468
  timestamps: createTimestampsBuffer(),
421
- dbChanges: createBuffer(),
469
+ dbChangeLengths: createBuffer(),
422
470
  },
423
471
  ranges: {
424
472
  timestamps: createTimestampsBuffer(),
@@ -443,66 +491,38 @@ export const createProtocolMessageBuffer = (ownerId, options) => {
443
491
  else if (options.messageType === MessageType.Response) {
444
492
  buffers.header.extend([options.errorCode]);
445
493
  }
494
+ // Changes are referenced, not copied, so a trial undoes messages by
495
+ // shortening this array, and unwrap copies each change once.
496
+ const dbChanges = [];
497
+ let dbChangesLength = 0;
446
498
  let isLastRangeInfinite = false;
447
- const isWithinSizeLimits = () => getSize() <= totalMaxSize;
448
- const getSize = () => PositiveInt.orThrow(getHeaderAndMessagesSize() + getRangesSize());
449
- const getHeaderAndMessagesSize = () => buffers.header.getLength() +
499
+ let isInTrial = false;
500
+ // Writes outside a trial assert this, so an overflow fails where it happens.
501
+ // A frame must never exceed totalMaxSize: relays at @evolu/nodejs 4.0.0 or
502
+ // older crash on a frame one byte larger than their 1,000,000-byte limit.
503
+ const isWithinSizeLimits = () => getFrameSize() <= totalMaxSize;
504
+ // Lengths after a header that is never empty.
505
+ const getFrameSize = () => (buffers.header.getLength() +
450
506
  buffers.messages.timestamps.getLength() +
451
- buffers.messages.dbChanges.getLength();
507
+ buffers.messages.dbChangeLengths.getLength() +
508
+ dbChangesLength +
509
+ getRangesSize());
510
+ // Without ranges, the ranges section is omitted, including its count.
452
511
  const getRangesSize = () => buffers.ranges.timestamps.getCount() > 0
453
512
  ? buffers.ranges.timestamps.getLength() +
454
513
  buffers.ranges.types.getLength() +
455
- buffers.ranges.payloads.getLength() +
456
- safeMargins.remainingRange
514
+ buffers.ranges.payloads.getLength()
457
515
  : 0;
458
- /**
459
- * We calculated worst-case sizes as closely as possible and added a small
460
- * safety margin, since computing exact worst cases is difficult due to
461
- * variable-length, run-length, and delta encoding.
462
- *
463
- * Runtime assertions (`assert`) are used to guarantee that size limits are
464
- * never exceeded. If a limit is exceeded, the assertion will fail at the
465
- * precise location, making it easy to identify and fix the issue.
466
- *
467
- * While it would be possible to avoid the safety margin by snapshotting
468
- * buffer states and rolling back changes, this would likely impact
469
- * performance. If someone has time and wants to experiment with this
470
- * approach, contributions are welcome.
471
- */
472
- const safeMargins = {
473
- // bytes: range type + possible increased count varint
474
- remainingRange: fingerprintSize + 10,
475
- // bytes: max millis + max count + NodeId
476
- timestamp: 30,
477
- // bytes: maximum encoded DbChange length varint
478
- dbChangeLength: 8,
479
- // bytes: worst case is around 650 bytes
480
- splitRange: 800,
481
- // bytes: range type + its upperBound + possible increased count varint
482
- timestampsRange: 50,
483
- };
484
- const addMessageSafeMargin = safeMargins.timestamp +
485
- safeMargins.dbChangeLength +
486
- safeMargins.remainingRange;
516
+ // A trial makes the write, measures the exact frame, and undoes the write in
517
+ // constant time when the frame no longer fits, so it needs no worst-case
518
+ // margins and costs about as much as the write itself.
487
519
  return {
488
- canAddMessage: (message) => getSize() + addMessageSafeMargin + message.change.length <= totalMaxSize,
489
520
  addMessage: (message) => {
490
521
  buffers.messages.timestamps.add(message.timestamp);
491
- encodeLength(buffers.messages.dbChanges, message.change);
492
- buffers.messages.dbChanges.extend(message.change);
493
- assert(isWithinSizeLimits(), "the message is too big");
494
- },
495
- canSplitRange: () => getRangesSize() + safeMargins.splitRange <= rangesMaxSize,
496
- canAddTimestampsRangeAndMessage: (timestamps, message) => {
497
- const rangesNewSize = getRangesSize() + timestamps.getLength() + safeMargins.timestampsRange;
498
- return (rangesNewSize <= rangesMaxSize &&
499
- (message
500
- ? getHeaderAndMessagesSize() +
501
- rangesNewSize +
502
- addMessageSafeMargin +
503
- message.change.length <=
504
- totalMaxSize
505
- : true));
522
+ encodeLength(buffers.messages.dbChangeLengths, message.change);
523
+ dbChanges.push(message.change);
524
+ dbChangesLength += message.change.length;
525
+ assert(isInTrial || isWithinSizeLimits(), "the message is too big");
506
526
  },
507
527
  addRange: (range) => {
508
528
  assert(options.messageType !== MessageType.Broadcast, "Cannot add a range into broadcast message");
@@ -518,7 +538,7 @@ export const createProtocolMessageBuffer = (ownerId, options) => {
518
538
  else {
519
539
  buffers.ranges.timestamps.addInfinite();
520
540
  }
521
- encodeNonNegativeInt(buffers.ranges.types, NonNegativeInt.orThrow(range.type));
541
+ encodeNonNegativeInt(buffers.ranges.types, range.type);
522
542
  switch (range.type) {
523
543
  case RangeType.Skip:
524
544
  break;
@@ -530,22 +550,89 @@ export const createProtocolMessageBuffer = (ownerId, options) => {
530
550
  break;
531
551
  }
532
552
  }
533
- assert(isWithinSizeLimits(), `the range ${range.type} is too big`);
553
+ assert(isInTrial || isWithinSizeLimits(), `the range ${range.type} is too big`);
554
+ },
555
+ tryWrite: (write, reserve = zeroNonNegativeInt) => {
556
+ const restoreMessageTimestamps = buffers.messages.timestamps.checkpoint();
557
+ const dbChangeLengthsLength = buffers.messages.dbChangeLengths.getLength();
558
+ const dbChangesCount = dbChanges.length;
559
+ const checkpointDbChangesLength = dbChangesLength;
560
+ const restoreRangeTimestamps = buffers.ranges.timestamps.checkpoint();
561
+ const typesLength = buffers.ranges.types.getLength();
562
+ const payloadsLength = buffers.ranges.payloads.getLength();
563
+ const wasLastRangeInfinite = isLastRangeInfinite;
564
+ const wasInTrial = isInTrial;
565
+ let isKept = false;
566
+ isInTrial = true;
567
+ try {
568
+ write();
569
+ // Fingerprint(InfiniteUpperBound) encodes no upper bound, only its
570
+ // type (1 byte) and fingerprint (12 bytes). The ranges count varint
571
+ // gains a byte when the new count is a power of 128. That includes
572
+ // 128^0 = 1, since the first range adds the count itself. So closing
573
+ // takes 13 or 14 bytes, and nothing for a Broadcast or a closed frame.
574
+ let newCount = buffers.ranges.timestamps.getCount() + 1;
575
+ while (newCount % 128 === 0)
576
+ newCount /= 128;
577
+ const closingReserve = options.messageType === MessageType.Broadcast || isLastRangeInfinite
578
+ ? 0
579
+ : 1 + fingerprintSize + (newCount === 1 ? 1 : 0);
580
+ const reserves = closingReserve + reserve;
581
+ isKept =
582
+ getFrameSize() + reserves <= totalMaxSize &&
583
+ getRangesSize() + reserves <= rangesMaxSize;
584
+ }
585
+ finally {
586
+ isInTrial = wasInTrial;
587
+ if (!isKept) {
588
+ restoreMessageTimestamps();
589
+ buffers.messages.dbChangeLengths.truncate(dbChangeLengthsLength);
590
+ dbChanges.length = dbChangesCount;
591
+ dbChangesLength = checkpointDbChangesLength;
592
+ restoreRangeTimestamps();
593
+ buffers.ranges.types.truncate(typesLength);
594
+ buffers.ranges.payloads.truncate(payloadsLength);
595
+ isLastRangeInfinite = wasLastRangeInfinite;
596
+ }
597
+ }
598
+ return isKept;
534
599
  },
535
600
  unwrap: () => {
536
601
  if (buffers.ranges.timestamps.getCount() > 0) {
537
602
  assert(isLastRangeInfinite, "The last range's upperBound must be InfiniteUpperBound");
538
603
  }
539
- buffers.messages.timestamps.append(buffers.header);
540
- buffers.header.extend(buffers.messages.dbChanges.unwrap());
604
+ const frame = new Uint8Array(getFrameSize());
605
+ frame.set(buffers.header.unwrap());
606
+ let offset = buffers.header.getLength();
607
+ // Written to a new buffer, so the builder is unchanged and unwrap can be
608
+ // called again.
609
+ const messageTimestamps = createBuffer();
610
+ buffers.messages.timestamps.append(messageTimestamps);
611
+ frame.set(messageTimestamps.unwrap(), offset);
612
+ offset += messageTimestamps.getLength();
613
+ // Each change follows its length, whose last varint byte is below 128.
614
+ const lengths = buffers.messages.dbChangeLengths.unwrap();
615
+ let lengthsOffset = 0;
616
+ for (const change of dbChanges) {
617
+ do {
618
+ frame[offset++] = lengths[lengthsOffset];
619
+ } while (lengths[lengthsOffset++] >= 128);
620
+ frame.set(change, offset);
621
+ offset += change.length;
622
+ }
541
623
  if (buffers.ranges.timestamps.getCount() > 0) {
542
- buffers.ranges.timestamps.append(buffers.header);
543
- buffers.header.extend(buffers.ranges.types.unwrap());
544
- buffers.header.extend(buffers.ranges.payloads.unwrap());
624
+ const ranges = createBuffer();
625
+ buffers.ranges.timestamps.append(ranges);
626
+ ranges.extend(buffers.ranges.types.unwrap());
627
+ ranges.extend(buffers.ranges.payloads.unwrap());
628
+ frame.set(ranges.unwrap(), offset);
629
+ offset += ranges.getLength();
545
630
  }
546
- return buffers.header.unwrap();
631
+ // An overcounted size would end the frame with zero bytes peers accept.
632
+ assert(offset === frame.length, "the frame size is exact");
633
+ return frame;
547
634
  },
548
- getSize,
635
+ getSize: getFrameSize,
549
636
  };
550
637
  };
551
638
  export const createTimestampsBuffer = () => {
@@ -590,252 +677,348 @@ export const createTimestampsBuffer = () => {
590
677
  buffer.extend(counterEncoder.unwrap());
591
678
  buffer.extend(nodeIdEncoder.unwrap());
592
679
  },
680
+ checkpoint: () => {
681
+ const checkpointCount = count;
682
+ const millisLength = millisBuffer.getLength();
683
+ const checkpointPreviousMillis = previousMillis;
684
+ const restoreCounters = counterEncoder.checkpoint();
685
+ const restoreNodeIds = nodeIdEncoder.checkpoint();
686
+ return () => {
687
+ count = checkpointCount;
688
+ syncCount();
689
+ millisBuffer.truncate(millisLength);
690
+ previousMillis = checkpointPreviousMillis;
691
+ restoreCounters();
692
+ restoreNodeIds();
693
+ };
694
+ },
593
695
  };
594
696
  };
595
697
  export const applyProtocolMessageAsClient = (inputMessage, options = {}) => async (run) => {
596
698
  const { storage } = run.deps;
597
- try {
598
- const input = createBuffer(inputMessage);
599
- const [requestedVersion, ownerId] = decodeVersionAndOwner(input);
600
- const version = options.version ?? protocolVersion;
601
- if (requestedVersion !== version) {
602
- return err({
603
- type: "ProtocolVersionError",
604
- version: requestedVersion,
605
- isInitiator: version < requestedVersion,
606
- ownerId,
607
- });
608
- }
609
- const messageType = input.shift();
610
- assert(messageType === MessageType.Response ||
611
- messageType === MessageType.Broadcast, "Invalid MessageType");
612
- if (messageType === MessageType.Response) {
613
- const errorCode = input.shift();
614
- if (errorCode !== ProtocolErrorCode.NoError) {
615
- switch (errorCode) {
616
- case ProtocolErrorCode.WriteKeyError:
617
- return err({
618
- type: "ProtocolWriteKeyError",
619
- ownerId,
620
- });
621
- case ProtocolErrorCode.WriteError:
622
- return err({
623
- type: "ProtocolWriteError",
624
- ownerId,
625
- });
626
- case ProtocolErrorCode.QuotaError:
627
- return err({
628
- type: "ProtocolQuotaError",
629
- ownerId,
630
- });
631
- case ProtocolErrorCode.SyncError:
632
- return err({
633
- type: "ProtocolSyncError",
634
- ownerId,
635
- });
636
- default:
637
- throw new ProtocolDecodeError(`Invalid ProtocolErrorCode: ${errorCode}`);
638
- }
639
- }
640
- }
641
- const messages = decodeMessages(input);
642
- const ownerIdBytes = ownerIdToOwnerIdBytes(ownerId);
643
- if (isNonEmptyArray(messages)) {
644
- try {
645
- const result = await run(storage.writeMessages(ownerIdBytes, messages));
646
- if (!result.ok)
647
- return result;
648
- }
649
- catch (error) {
650
- if (AbortError.is(error))
651
- throw error;
652
- run.deps.console.error(error);
653
- return ok({ type: "Failed", cause: "Write" });
654
- }
655
- }
656
- if (messageType === MessageType.Broadcast) {
657
- return ok({ type: "Broadcast" });
699
+ const version = options.version ?? protocolVersion;
700
+ const decoded = decodeProtocolMessage(inputMessage, version);
701
+ if (!decoded.ok)
702
+ return decoded;
703
+ const message = decoded.value;
704
+ if (message.type === "OtherVersion") {
705
+ return err({
706
+ type: "ProtocolVersionError",
707
+ version: message.version,
708
+ isInitiator: version < message.version,
709
+ ownerId: message.ownerId,
710
+ });
711
+ }
712
+ if (message.type === "Request") {
713
+ return err({
714
+ type: "ProtocolInvalidDataError",
715
+ data: inputMessage,
716
+ error: new ProtocolDecodeError("Expected a Response or a Broadcast"),
717
+ });
718
+ }
719
+ const { ownerId } = message;
720
+ if (message.type === "ErrorResponse") {
721
+ switch (message.errorCode) {
722
+ case ProtocolErrorCode.WriteKeyError:
723
+ return err({
724
+ type: "ProtocolWriteKeyError",
725
+ ownerId,
726
+ });
727
+ case ProtocolErrorCode.WriteError:
728
+ return err({
729
+ type: "ProtocolWriteError",
730
+ ownerId,
731
+ });
732
+ case ProtocolErrorCode.QuotaError:
733
+ return err({
734
+ type: "ProtocolQuotaError",
735
+ ownerId,
736
+ });
737
+ case ProtocolErrorCode.SyncError:
738
+ return err({
739
+ type: "ProtocolSyncError",
740
+ ownerId,
741
+ });
658
742
  }
659
- // Now: No writeKey, no sync.
660
- // TODO: Allow to sync SharedReadonlyOwner
661
- // Without local changes, writeKey will not be required.
662
- // With local changes, writeKey will be required and if not provided,
663
- // the sync will stop.
664
- const writeKey = options.writeKey;
665
- if (writeKey == null) {
666
- return ok({ type: "Readonly" });
743
+ }
744
+ const { messages } = message;
745
+ const ownerIdBytes = ownerIdToOwnerIdBytes(ownerId);
746
+ if (isNonEmptyArray(messages)) {
747
+ try {
748
+ const result = await run(storage.writeMessages(ownerIdBytes, messages));
749
+ if (!result.ok)
750
+ return result;
667
751
  }
668
- const ranges = decodeRanges(input);
669
- if (!isNonEmptyArray(ranges)) {
670
- return ok({ type: "Converged" });
752
+ catch (error) {
753
+ if (AbortError.is(error))
754
+ throw error;
755
+ run.deps.console.error(error);
756
+ return ok({ type: "Failed", cause: "Write" });
671
757
  }
672
- const output = createProtocolMessageBuffer(ownerId, {
673
- messageType: MessageType.Request,
674
- writeKey,
675
- rangesMaxSize: options.rangesMaxSize,
676
- });
677
- let broadcast;
678
- const result = sync(run.deps)(ranges, output, ownerIdBytes, (message) => {
758
+ }
759
+ if (message.type === "Broadcast") {
760
+ return ok({ type: "Broadcast" });
761
+ }
762
+ // Now: No writeKey, no sync.
763
+ // TODO: Allow to sync SharedReadonlyOwner
764
+ // Without local changes, writeKey will not be required.
765
+ // With local changes, writeKey will be required and if not provided,
766
+ // the sync will stop.
767
+ const writeKey = options.writeKey;
768
+ if (writeKey == null) {
769
+ return ok({ type: "Readonly" });
770
+ }
771
+ const { ranges } = message;
772
+ if (!isNonEmptyArray(ranges)) {
773
+ return ok({ type: "Converged" });
774
+ }
775
+ const createOutput = () => createProtocolMessageBuffer(ownerId, {
776
+ messageType: MessageType.Request,
777
+ writeKey,
778
+ rangesMaxSize: options.rangesMaxSize,
779
+ });
780
+ const output = createOutput();
781
+ let broadcast;
782
+ const result = sync(run.deps)(ranges, output, ownerIdBytes, {
783
+ createEmptyOutput: createOutput,
784
+ onChangeTooLarge: options.onChangeTooLarge ?? run.deps.console.warn,
785
+ onMessage: (message) => {
679
786
  broadcast ??= createProtocolMessageBuffer(ownerId, {
680
787
  messageType: MessageType.Broadcast,
681
788
  });
789
+ // The request holds the same messages after a larger header.
682
790
  broadcast.addMessage(message);
791
+ },
792
+ });
793
+ // A failure was logged by sync.
794
+ if (!result.ok)
795
+ return ok({ type: "Failed", cause: "Sync" });
796
+ if (!result.value)
797
+ return ok({ type: "Converged" });
798
+ return ok({
799
+ type: "Response",
800
+ message: output.unwrap(),
801
+ ...(broadcast && { broadcast: broadcast.unwrap() }),
802
+ });
803
+ };
804
+ export const applyProtocolMessageAsRelay = (inputMessage, options = {},
805
+ /** For tests only. */
806
+ version = protocolVersion) => async (run) => {
807
+ const { storage } = run.deps;
808
+ // The relay broadcasts a request's messages in a frame of totalMaxSize.
809
+ // That frame omits the request's write key, subscription flag, and ranges,
810
+ // so it holds the messages of any request up to that size.
811
+ if (inputMessage.length >
812
+ (options.totalMaxSize ?? defaultProtocolMessageMaxSize))
813
+ return err({
814
+ type: "ProtocolInvalidDataError",
815
+ data: inputMessage,
816
+ error: new ProtocolDecodeError("Request is too large"),
683
817
  });
684
- // A failure was logged by sync.
685
- if (!result.ok)
686
- return ok({ type: "Failed", cause: "Sync" });
687
- if (!result.value)
688
- return ok({ type: "Converged" });
818
+ const decoded = decodeProtocolMessage(inputMessage, version);
819
+ if (!decoded.ok)
820
+ return decoded;
821
+ const request = decoded.value;
822
+ if (request.type === "OtherVersion") {
823
+ // Non-initiator responds with its version and ownerId.
824
+ const output = createBuffer();
825
+ encodeNonNegativeInt(output, version);
826
+ output.extend(ownerIdToOwnerIdBytes(request.ownerId));
689
827
  return ok({
690
828
  type: "Response",
691
829
  message: output.unwrap(),
692
- ...(broadcast && { broadcast: broadcast.unwrap() }),
693
830
  });
694
831
  }
695
- catch (error) {
696
- if (AbortError.is(error))
697
- throw error;
832
+ if (request.type !== "Request") {
698
833
  return err({
699
834
  type: "ProtocolInvalidDataError",
700
835
  data: inputMessage,
701
- error,
836
+ error: new ProtocolDecodeError("Expected a Request"),
702
837
  });
703
838
  }
704
- };
705
- export const applyProtocolMessageAsRelay = (inputMessage, options = {},
706
- /** For tests only. */
707
- version = protocolVersion) => async (run) => {
708
- const { storage } = run.deps;
709
- try {
710
- const input = createBuffer(inputMessage);
711
- const [requestedVersion, ownerId] = decodeVersionAndOwner(input);
712
- const ownerIdBytes = ownerIdToOwnerIdBytes(ownerId);
713
- if (requestedVersion !== version) {
714
- // Non-initiator responds with its version and ownerId.
715
- const output = createBuffer();
716
- encodeNonNegativeInt(output, version);
717
- output.extend(ownerIdBytes);
718
- return ok({
719
- type: "Response",
720
- message: output.unwrap(),
721
- });
839
+ const { ownerId, writeKey, messages, ranges } = request;
840
+ const ownerIdBytes = ownerIdToOwnerIdBytes(ownerId);
841
+ const createErrorResponse = (errorCode) => ({
842
+ type: "Response",
843
+ message: createProtocolMessageBuffer(ownerId, {
844
+ messageType: MessageType.Response,
845
+ errorCode,
846
+ }).unwrap(),
847
+ });
848
+ switch (request.subscriptionFlag) {
849
+ case SubscriptionFlags.Subscribe:
850
+ options.subscribe?.(ownerId);
851
+ break;
852
+ case SubscriptionFlags.Unsubscribe:
853
+ options.unsubscribe?.(ownerId);
854
+ break;
855
+ case SubscriptionFlags.None:
856
+ break;
857
+ default:
858
+ exhaustiveCheck(request.subscriptionFlag);
859
+ }
860
+ if (writeKey) {
861
+ let isValid;
862
+ try {
863
+ isValid = storage.validateWriteKey(ownerIdBytes, writeKey);
722
864
  }
723
- const messageType = input.shift();
724
- assertSame(messageType, MessageType.Request);
725
- const hasWriteKey = input.shift();
726
- let writeKey;
727
- if (hasWriteKey === 1) {
728
- writeKey = input.shiftN(ownerWriteKeyLength);
865
+ catch (error) {
866
+ // A relay storage stores the write key of a new owner, which SQLite
867
+ // can fail, for example on a full disk. A boolean has no room for the
868
+ // error, so it is answered like a failed write.
869
+ run.deps.console.error(error);
870
+ return ok(createErrorResponse(ProtocolErrorCode.WriteError));
729
871
  }
730
- const subscriptionFlag = input.shift();
731
- switch (subscriptionFlag) {
732
- case SubscriptionFlags.Subscribe:
733
- options.subscribe?.(ownerId);
734
- break;
735
- case SubscriptionFlags.Unsubscribe:
736
- options.unsubscribe?.(ownerId);
737
- break;
738
- case SubscriptionFlags.None:
739
- break;
872
+ if (!isValid) {
873
+ return ok(createErrorResponse(ProtocolErrorCode.WriteKeyError));
740
874
  }
741
- if (writeKey) {
742
- const isValid = storage.validateWriteKey(ownerIdBytes, writeKey);
743
- if (!isValid) {
744
- return ok({
745
- type: "Response",
746
- message: createProtocolMessageBuffer(ownerId, {
747
- messageType: MessageType.Response,
748
- errorCode: ProtocolErrorCode.WriteKeyError,
749
- }).unwrap(),
750
- });
875
+ }
876
+ if (isNonEmptyArray(messages)) {
877
+ if (!writeKey) {
878
+ return ok(createErrorResponse(ProtocolErrorCode.WriteKeyError));
879
+ }
880
+ try {
881
+ const result = await run(storage.writeMessages(ownerIdBytes, messages));
882
+ if (!result.ok) {
883
+ // A storage returns a failed write without reporting it.
884
+ if (result.error.type === "UnknownError")
885
+ run.deps.console.error(result.error);
886
+ return ok(createErrorResponse(result.error.type === "StorageQuotaError"
887
+ ? ProtocolErrorCode.QuotaError
888
+ : ProtocolErrorCode.WriteError));
751
889
  }
752
890
  }
753
- const messages = decodeMessages(input);
754
- if (isNonEmptyArray(messages)) {
755
- if (!writeKey) {
891
+ catch (error) {
892
+ if (AbortError.is(error))
893
+ throw error;
894
+ run.deps.console.error(error);
895
+ return ok(createErrorResponse(ProtocolErrorCode.WriteError));
896
+ }
897
+ /**
898
+ * Broadcast messages to all subscribed owners for real-time
899
+ * synchronization between clients.
900
+ *
901
+ * Messages are only broadcasted after successful write to ensure devices
902
+ * that can still sync aren't affected by quota errors, and to prevent
903
+ * using a half-working relay service (broadcasting without persistence).
904
+ *
905
+ * When a relay's database is deleted or clients migrate to a new relay
906
+ * (without data migration), clients will sync their data to the relay,
907
+ * and the relay will broadcast those messages to other connected clients.
908
+ * Those clients may receive messages they already have, but this is safe
909
+ * because Evolu sync is idempotent. As the relay becomes more
910
+ * synchronized with clients over time, fewer duplicate messages will be
911
+ * broadcasted.
912
+ */
913
+ if (options.broadcast) {
914
+ const broadcastBuffer = createProtocolMessageBuffer(ownerId, {
915
+ messageType: MessageType.Broadcast,
916
+ totalMaxSize: options.totalMaxSize,
917
+ rangesMaxSize: options.rangesMaxSize,
918
+ version,
919
+ });
920
+ for (const message of messages) {
921
+ broadcastBuffer.addMessage(message);
922
+ }
923
+ options.broadcast(ownerId, broadcastBuffer.unwrap());
924
+ }
925
+ }
926
+ const createOutput = () => createProtocolMessageBuffer(ownerId, {
927
+ messageType: MessageType.Response,
928
+ errorCode: ProtocolErrorCode.NoError,
929
+ totalMaxSize: options.totalMaxSize,
930
+ rangesMaxSize: options.rangesMaxSize,
931
+ });
932
+ const output = createOutput();
933
+ // A relay answers every request it decodes, even with nothing to sync, so
934
+ // the initiator knows the sync is complete.
935
+ if (!isNonEmptyArray(ranges)) {
936
+ return ok({ type: "Response", message: output.unwrap() });
937
+ }
938
+ const result = sync(run.deps)(ranges, output, ownerIdBytes, {
939
+ createEmptyOutput: createOutput,
940
+ onChangeTooLarge: run.deps.console.warn,
941
+ });
942
+ // A relay answers every request it decodes, a failed reconciliation with
943
+ // its error code.
944
+ return ok(result.ok
945
+ ? { type: "Response", message: output.unwrap() }
946
+ : createErrorResponse(result.error));
947
+ };
948
+ /**
949
+ * Decodes a whole {@link ProtocolMessage}, so a malformed one is rejected before
950
+ * anything it carries is applied.
951
+ *
952
+ * Applying a message turns only a throw from here into
953
+ * {@link ProtocolInvalidDataError}. A throw after decoding is a defect.
954
+ */
955
+ const decodeProtocolMessage = (inputMessage, version) => {
956
+ try {
957
+ const input = createBuffer(inputMessage);
958
+ const [messageVersion, ownerId] = decodeVersionAndOwner(input);
959
+ if (messageVersion !== version)
960
+ return ok({ type: "OtherVersion", version: messageVersion, ownerId });
961
+ switch (decodeMessageType(input)) {
962
+ case MessageType.Request: {
963
+ const hasWriteKey = input.shift();
964
+ if (hasWriteKey > 1)
965
+ throw new ProtocolDecodeError(`Invalid hasWriteKey: ${hasWriteKey}`);
966
+ const writeKey = hasWriteKey === 1
967
+ ? input.shiftN(ownerWriteKeyLength)
968
+ : null;
969
+ const subscriptionFlag = input.shift();
970
+ switch (subscriptionFlag) {
971
+ case SubscriptionFlags.None:
972
+ case SubscriptionFlags.Subscribe:
973
+ case SubscriptionFlags.Unsubscribe:
974
+ break;
975
+ default:
976
+ throw new ProtocolDecodeError(`Invalid SubscriptionFlag: ${subscriptionFlag}`);
977
+ }
978
+ const messages = decodeMessages(input);
979
+ // Only a relay accepts a Request, so only the relay checks this.
980
+ // Deployed relays already store shorter changes, and a client skips a
981
+ // change it cannot read, whereas a check in decodeMessages would make
982
+ // it reject every response holding one.
983
+ for (const { change } of messages)
984
+ if (change.length < minEncryptedDbChangeLength)
985
+ throw new ProtocolDecodeError("EncryptedDbChange is too short");
986
+ const ranges = decodeRanges(input);
756
987
  return ok({
757
- type: "Response",
758
- message: createProtocolMessageBuffer(ownerId, {
759
- messageType: MessageType.Response,
760
- errorCode: ProtocolErrorCode.WriteKeyError,
761
- }).unwrap(),
988
+ type: "Request",
989
+ ownerId,
990
+ writeKey,
991
+ subscriptionFlag,
992
+ messages,
993
+ ranges,
762
994
  });
763
995
  }
764
- try {
765
- const result = await run(storage.writeMessages(ownerIdBytes, messages));
766
- if (!result.ok) {
767
- const message = createProtocolMessageBuffer(ownerId, {
768
- messageType: MessageType.Response,
769
- errorCode: ProtocolErrorCode.QuotaError,
770
- }).unwrap();
771
- return ok({ type: "Response", message });
996
+ case MessageType.Response: {
997
+ const errorCode = input.shift();
998
+ switch (errorCode) {
999
+ case ProtocolErrorCode.NoError: {
1000
+ const messages = decodeMessages(input);
1001
+ const ranges = decodeRanges(input);
1002
+ return ok({ type: "Response", ownerId, messages, ranges });
1003
+ }
1004
+ case ProtocolErrorCode.WriteKeyError:
1005
+ case ProtocolErrorCode.WriteError:
1006
+ case ProtocolErrorCode.QuotaError:
1007
+ case ProtocolErrorCode.SyncError:
1008
+ return ok({ type: "ErrorResponse", ownerId, errorCode });
1009
+ default:
1010
+ throw new ProtocolDecodeError(`Invalid ProtocolErrorCode: ${errorCode}`);
772
1011
  }
773
1012
  }
774
- catch (error) {
775
- if (AbortError.is(error))
776
- throw error;
777
- run.deps.console.error(error);
778
- const message = createProtocolMessageBuffer(ownerId, {
779
- messageType: MessageType.Response,
780
- errorCode: ProtocolErrorCode.WriteError,
781
- }).unwrap();
782
- return ok({ type: "Response", message });
783
- }
784
- /**
785
- * Broadcast messages to all subscribed owners for real-time
786
- * synchronization between clients.
787
- *
788
- * Messages are only broadcasted after successful write to ensure
789
- * devices that can still sync aren't affected by quota errors, and to
790
- * prevent using a half-working relay service (broadcasting without
791
- * persistence).
792
- *
793
- * When a relay's database is deleted or clients migrate to a new relay
794
- * (without data migration), clients will sync their data to the relay,
795
- * and the relay will broadcast those messages to other connected
796
- * clients. Those clients may receive messages they already have, but
797
- * this is safe because Evolu sync is idempotent. As the relay becomes
798
- * more synchronized with clients over time, fewer duplicate messages
799
- * will be broadcasted.
800
- */
801
- if (options.broadcast) {
802
- const broadcastBuffer = createProtocolMessageBuffer(ownerId, {
803
- messageType: MessageType.Broadcast,
804
- totalMaxSize: options.totalMaxSize,
805
- rangesMaxSize: options.rangesMaxSize,
806
- version,
1013
+ case MessageType.Broadcast:
1014
+ return ok({
1015
+ type: "Broadcast",
1016
+ ownerId,
1017
+ messages: decodeMessages(input),
807
1018
  });
808
- for (const message of messages) {
809
- broadcastBuffer.addMessage(message);
810
- }
811
- options.broadcast(ownerId, broadcastBuffer.unwrap());
812
- }
813
- }
814
- const ranges = decodeRanges(input);
815
- const output = createProtocolMessageBuffer(ownerId, {
816
- messageType: MessageType.Response,
817
- errorCode: ProtocolErrorCode.NoError,
818
- totalMaxSize: options.totalMaxSize,
819
- rangesMaxSize: options.rangesMaxSize,
820
- });
821
- // Non-initiators always respond to provide sync completion feedback,
822
- // even when there's nothing to sync.
823
- if (!isNonEmptyArray(ranges)) {
824
- return ok({ type: "Response", message: output.unwrap() });
825
1019
  }
826
- const result = sync(run.deps)(ranges, output, ownerIdBytes);
827
- const message = result.ok
828
- ? output.unwrap()
829
- : createProtocolMessageBuffer(ownerId, {
830
- messageType: MessageType.Response,
831
- errorCode: result.error,
832
- }).unwrap();
833
- // Non-initiators always respond to provide sync completion feedback,
834
- return ok({ type: "Response", message });
835
1020
  }
836
1021
  catch (error) {
837
- if (AbortError.is(error))
838
- throw error;
839
1022
  return err({
840
1023
  type: "ProtocolInvalidDataError",
841
1024
  data: inputMessage,
@@ -846,8 +1029,7 @@ version = protocolVersion) => async (run) => {
846
1029
  const decodeVersionAndOwner = (input) => {
847
1030
  // This structure must never change across protocol versions. The version
848
1031
  // and owner ID must always be the first two fields in every protocol message
849
- // to enable version negotiation and owner identification before any other
850
- // processing occurs.
1032
+ // to route and report a version mismatch before any other processing occurs.
851
1033
  const version = decodeNonNegativeInt(input);
852
1034
  const ownerId = decodeId(input);
853
1035
  return [version, ownerId];
@@ -857,22 +1039,19 @@ const parseProtocolHeaderFromBuffer = (input) => {
857
1039
  if (version !== protocolVersion) {
858
1040
  return { type: "ProtocolHeader", version, ownerId };
859
1041
  }
860
- const messageTypeValue = input.shift();
861
- let messageType;
862
- switch (messageTypeValue) {
1042
+ const messageType = decodeMessageType(input);
1043
+ return { type: "ProtocolHeader", version, ownerId, messageType };
1044
+ };
1045
+ const decodeMessageType = (input) => {
1046
+ const messageType = input.shift();
1047
+ switch (messageType) {
863
1048
  case MessageType.Request:
864
- messageType = MessageType.Request;
865
- break;
866
1049
  case MessageType.Response:
867
- messageType = MessageType.Response;
868
- break;
869
1050
  case MessageType.Broadcast:
870
- messageType = MessageType.Broadcast;
871
- break;
1051
+ return messageType;
872
1052
  default:
873
1053
  throw new ProtocolDecodeError("Invalid MessageType");
874
1054
  }
875
- return { type: "ProtocolHeader", version, ownerId, messageType };
876
1055
  };
877
1056
  /**
878
1057
  * Error thrown for internal protocol validation failures, such as invalid data
@@ -882,7 +1061,6 @@ class ProtocolDecodeError extends Error {
882
1061
  constructor(message) {
883
1062
  super(message);
884
1063
  this.name = this.constructor.name;
885
- Error.captureStackTrace(this, this.constructor);
886
1064
  }
887
1065
  }
888
1066
  const decodeMessages = (buffer) => {
@@ -896,7 +1074,39 @@ const decodeMessages = (buffer) => {
896
1074
  }
897
1075
  return messages;
898
1076
  };
899
- const sync = (deps) => (ranges, output, ownerIdBytes, onMessage) => {
1077
+ // The smallest envelope encodeAndEncryptDbChange can produce: the nonce, a
1078
+ // 1-byte ciphertext length, and the 16-byte Poly1305 tag. Every v1 client sends
1079
+ // at least 77 bytes. The relay quota counts change bytes, so a relay storing
1080
+ // shorter changes, zero-length ones above all, would store rows the quota does
1081
+ // not count.
1082
+ const minEncryptedDbChangeLength = xChaCha20Poly1305NonceLength + 1 + 16;
1083
+ /**
1084
+ * Answers the ranges into `output`, returning whether it has anything to send.
1085
+ *
1086
+ * Every write except a closing one is a trial (see
1087
+ * {@link ProtocolMessageBuffer.tryWrite}), so the frame always has room to close
1088
+ * with one Fingerprint range with {@link InfiniteUpperBound}. When a write does
1089
+ * not fit, that range closes the frame. Its fingerprint covers everything from
1090
+ * the last upper bound the frame states, which every v1 peer assumes, and the
1091
+ * peer reconciles it in the next round.
1092
+ *
1093
+ * A stored change that cannot fit an empty frame with this header, with only
1094
+ * its own timestamp listed after a pending skip, is skipped, and
1095
+ * `onChangeTooLarge` reports it as a {@link ProtocolChangeTooLargeError}. A
1096
+ * later round is guaranteed to reach exactly that frame, so a change that fits
1097
+ * it is sent eventually.
1098
+ *
1099
+ * An answer to a Timestamps range neither sends nor lists a skipped change, but
1100
+ * a request or a split lists its timestamp, so a peer that lacks it may ask for
1101
+ * it once per sync and gets an answer without it. Fingerprints keep disagreeing
1102
+ * about it, so every sync narrows them to it, skips it again, and ends.
1103
+ *
1104
+ * A throw from storage, or from a split's checks of what storage returned, is
1105
+ * logged and returns `SyncError`, except an AbortError from a split's reads,
1106
+ * which is rethrown. Any other throw, such as from writing the frame,
1107
+ * `onChangeTooLarge`, or `onMessage`, is a defect.
1108
+ */
1109
+ const sync = (deps) => (ranges, output, ownerIdBytes, { createEmptyOutput, onChangeTooLarge, onMessage, }) => {
900
1110
  const outputInitialSize = output.getSize();
901
1111
  let storageSize;
902
1112
  try {
@@ -908,11 +1118,18 @@ const sync = (deps) => (ranges, output, ownerIdBytes, onMessage) => {
908
1118
  }
909
1119
  let prevUpperBound = null;
910
1120
  let prevIndex = zeroNonNegativeInt;
1121
+ // The index of the last upper bound the frame states, 0 before any.
1122
+ let statedIndex = zeroNonNegativeInt;
1123
+ // Consecutive skipped ranges become one Skip range, written only before
1124
+ // the next non-skip range.
911
1125
  let skip = false;
912
1126
  let nonSkipRangeAdded = false;
913
1127
  const skipRange = (range) => {
914
1128
  // The last range, if any non skip was added, must have InfiniteUpperBound.
915
1129
  if (nonSkipRangeAdded && range.upperBound === InfiniteUpperBound) {
1130
+ // A closing write. Like the closing Fingerprint range, it encodes no
1131
+ // upper bound, and its type takes 1 byte of the 13 that every kept
1132
+ // trial reserved beyond the ranges count.
916
1133
  output.addRange({
917
1134
  type: RangeType.Skip,
918
1135
  upperBound: InfiniteUpperBound,
@@ -922,36 +1139,50 @@ const sync = (deps) => (ranges, output, ownerIdBytes, onMessage) => {
922
1139
  skip = true;
923
1140
  }
924
1141
  };
925
- const coalesceSkipsBeforeAdd = () => {
926
- // Set to true because we are going to add a non skip range.
927
- nonSkipRangeAdded = true;
928
- if (skip) {
1142
+ /**
1143
+ * Tries to write a non-skip range ending at the index `upper` after the
1144
+ * pending skip, if any.
1145
+ */
1146
+ const tryWriteRange = (upper, write) => {
1147
+ const isKept = output.tryWrite(() => {
1148
+ if (skip) {
1149
+ assertNonNullable(prevUpperBound, "prevUpperBound is null");
1150
+ output.addRange({
1151
+ type: RangeType.Skip,
1152
+ upperBound: prevUpperBound,
1153
+ });
1154
+ }
1155
+ write();
1156
+ });
1157
+ if (isKept) {
929
1158
  skip = false;
930
- assertNonNullable(prevUpperBound, "prevUpperBound is null");
931
- // There is always a space for a skip range before adding.
932
- output.addRange({
933
- type: RangeType.Skip,
934
- upperBound: prevUpperBound,
935
- });
1159
+ nonSkipRangeAdded = true;
1160
+ statedIndex = upper;
936
1161
  }
1162
+ return isKept;
937
1163
  };
938
- // When we don't have a space...
939
- const addFingerprintForRemainingRange = (begin) => {
1164
+ /**
1165
+ * Closes the frame with a Fingerprint range with InfiniteUpperBound over
1166
+ * the items from the last upper bound the frame states. A frame that cannot
1167
+ * fit a range closes without writing the pending Skip range, because only
1168
+ * this range has room reserved, so it covers the skipped items too.
1169
+ */
1170
+ const closeWithFingerprint = () => {
940
1171
  let fingerprint;
941
1172
  try {
942
- fingerprint = deps.storage.fingerprint(ownerIdBytes, begin, storageSize);
1173
+ fingerprint = deps.storage.fingerprint(ownerIdBytes, statedIndex, storageSize);
943
1174
  }
944
1175
  catch (error) {
945
1176
  deps.console.error(error);
946
- return false;
1177
+ return err(ProtocolErrorCode.SyncError);
947
1178
  }
948
- // There is always a space for a ramaining range.
1179
+ // A closing write. Every kept trial reserved room for it.
949
1180
  output.addRange({
950
1181
  type: RangeType.Fingerprint,
951
1182
  upperBound: InfiniteUpperBound,
952
1183
  fingerprint,
953
1184
  });
954
- return true;
1185
+ return ok(true);
955
1186
  };
956
1187
  for (const range of ranges) {
957
1188
  const currentUpperBound = range.upperBound;
@@ -980,24 +1211,24 @@ const sync = (deps) => (ranges, output, ownerIdBytes, onMessage) => {
980
1211
  }
981
1212
  if (eqArrayNumber(range.fingerprint, ourFingerprint)) {
982
1213
  skipRange(range);
1214
+ break;
983
1215
  }
984
- else if (output.canSplitRange()) {
985
- coalesceSkipsBeforeAdd();
986
- try {
987
- splitRange(deps)(ownerIdBytes, lower, upper, currentUpperBound, output);
988
- }
989
- catch (error) {
990
- if (AbortError.is(error))
991
- throw error;
992
- deps.console.error(error);
993
- return err(ProtocolErrorCode.SyncError);
994
- }
1216
+ let splitRanges;
1217
+ try {
1218
+ splitRanges = readSplitRanges(deps)(ownerIdBytes, lower, upper, currentUpperBound);
995
1219
  }
996
- else {
997
- return addFingerprintForRemainingRange(upper)
998
- ? ok(true)
999
- : err(ProtocolErrorCode.SyncError);
1220
+ catch (error) {
1221
+ if (AbortError.is(error))
1222
+ throw error;
1223
+ deps.console.error(error);
1224
+ return err(ProtocolErrorCode.SyncError);
1000
1225
  }
1226
+ const isSplit = tryWriteRange(upper, () => {
1227
+ for (const splitRange of splitRanges)
1228
+ output.addRange(splitRange);
1229
+ });
1230
+ if (!isSplit)
1231
+ return closeWithFingerprint();
1001
1232
  break;
1002
1233
  }
1003
1234
  case RangeType.Timestamps: {
@@ -1006,66 +1237,109 @@ const sync = (deps) => (ranges, output, ownerIdBytes, onMessage) => {
1006
1237
  const ourTimestamps = createTimestampsBuffer();
1007
1238
  let exceeded = false;
1008
1239
  let iterateFailed = false;
1240
+ // A pending skip stays pending until the range is written.
1241
+ const isSkipPending = skip;
1242
+ // Only a throw from the storage itself is a storage failure. A throw
1243
+ // from the callback, such as from onChangeTooLarge or onMessage, is
1244
+ // a defect, rethrown after iterating.
1245
+ let callbackError = null;
1009
1246
  try {
1010
1247
  deps.storage.iterate(ownerIdBytes, lower, upper, (timestamp, index) => {
1011
- const timestampString = timestamp.join();
1012
- const timestampBinary = timestampBytesToTimestamp(timestamp);
1013
- let message = null;
1014
- if (timestampsWeNeed.has(timestampString)) {
1015
- timestampsWeNeed.delete(timestampString);
1016
- }
1017
- else {
1018
- try {
1019
- message = {
1020
- timestamp: timestampBinary,
1021
- change: deps.storage.readDbChange(ownerIdBytes, timestamp),
1022
- };
1248
+ try {
1249
+ const timestampString = timestamp.join();
1250
+ const timestampBinary = timestampBytesToTimestamp(timestamp);
1251
+ let message = null;
1252
+ if (timestampsWeNeed.has(timestampString)) {
1253
+ timestampsWeNeed.delete(timestampString);
1023
1254
  }
1024
- catch (error) {
1025
- deps.console.error(error);
1026
- iterateFailed = true;
1027
- return false;
1255
+ else {
1256
+ try {
1257
+ message = {
1258
+ timestamp: timestampBinary,
1259
+ change: deps.storage.readDbChange(ownerIdBytes, timestamp),
1260
+ };
1261
+ }
1262
+ catch (error) {
1263
+ deps.console.error(error);
1264
+ iterateFailed = true;
1265
+ return false;
1266
+ }
1267
+ }
1268
+ // One trial per timestamp: its entry in ourTimestamps and its
1269
+ // message if the peer lacks it, keeping room to write
1270
+ // ourTimestamps as a range after the pending skip.
1271
+ const restoreOurTimestamps = ourTimestamps.checkpoint();
1272
+ ourTimestamps.add(timestampBinary);
1273
+ if (output.tryWrite(() => {
1274
+ if (message)
1275
+ output.addMessage(message);
1276
+ }, getTimestampsRangeReserve(ourTimestamps, isSkipPending))) {
1277
+ if (message)
1278
+ onMessage?.(message);
1279
+ return true;
1280
+ }
1281
+ restoreOurTimestamps();
1282
+ if (message) {
1283
+ // Whether an empty frame can hold the message in the trial
1284
+ // a Timestamps range makes for it, with only its own
1285
+ // timestamp listed and a skip pending. A later round is
1286
+ // guaranteed to reach exactly that frame, so a message that
1287
+ // fits is sent eventually. A trial without a pending skip
1288
+ // reserves 22 bytes less, so a message up to 22 bytes too
1289
+ // large for this check may still be sent in such a frame.
1290
+ const emptyOutput = createEmptyOutput();
1291
+ const emptyOutputTimestamps = createTimestampsBuffer();
1292
+ emptyOutputTimestamps.add(message.timestamp);
1293
+ const fitsEmptyOutput = emptyOutput.tryWrite(() => {
1294
+ emptyOutput.addMessage(message);
1295
+ }, getTimestampsRangeReserve(emptyOutputTimestamps, true));
1296
+ if (!fitsEmptyOutput) {
1297
+ onChangeTooLarge({
1298
+ type: "ProtocolChangeTooLargeError",
1299
+ timestamp: timestampBinary,
1300
+ size: message.change.length,
1301
+ });
1302
+ return true;
1303
+ }
1028
1304
  }
1029
- }
1030
- if (!output.canAddTimestampsRangeAndMessage(ourTimestamps, message)) {
1031
1305
  exceeded = true;
1032
1306
  endBound = timestamp;
1033
1307
  upper = index;
1034
1308
  return false;
1035
1309
  }
1036
- ourTimestamps.add(timestampBinary);
1037
- if (message) {
1038
- output.addMessage(message);
1039
- onMessage?.(message);
1310
+ catch (error) {
1311
+ callbackError = { error };
1312
+ return false;
1040
1313
  }
1041
- return true;
1042
1314
  });
1043
1315
  }
1044
1316
  catch (error) {
1045
1317
  deps.console.error(error);
1046
- return err(ProtocolErrorCode.SyncError);
1318
+ iterateFailed = true;
1047
1319
  }
1320
+ if (callbackError)
1321
+ throw callbackError.error;
1048
1322
  if (iterateFailed) {
1049
1323
  return err(ProtocolErrorCode.SyncError);
1050
1324
  }
1051
- const addRange = () => {
1052
- coalesceSkipsBeforeAdd();
1325
+ // When any timestamp was kept, its trial reserved room for this
1326
+ // write. Otherwise the range may not fit, and the frame closes
1327
+ // without it.
1328
+ const tryWriteTimestampsRange = () => tryWriteRange(upper, () => {
1053
1329
  output.addRange({
1054
1330
  type: RangeType.Timestamps,
1055
1331
  upperBound: endBound,
1056
1332
  timestamps: ourTimestamps,
1057
1333
  });
1058
- };
1334
+ });
1059
1335
  if (exceeded) {
1060
- addRange();
1061
- if (!addFingerprintForRemainingRange(upper)) {
1062
- return err(ProtocolErrorCode.SyncError);
1063
- }
1064
- return ok(true);
1336
+ tryWriteTimestampsRange();
1337
+ return closeWithFingerprint();
1065
1338
  }
1066
1339
  // If we need something, we have to respond with our timestamps.
1067
1340
  if (timestampsWeNeed.size > 0) {
1068
- addRange();
1341
+ if (!tryWriteTimestampsRange())
1342
+ return closeWithFingerprint();
1069
1343
  }
1070
1344
  else {
1071
1345
  skipRange(range);
@@ -1080,7 +1354,29 @@ const sync = (deps) => (ranges, output, ownerIdBytes, onMessage) => {
1080
1354
  const hasChange = output.getSize() > outputInitialSize;
1081
1355
  return ok(hasChange);
1082
1356
  };
1083
- const splitRange = (deps) => (ownerId, lower, upper, upperBound, buffer) => {
1357
+ // The most a range adds to a frame beyond its payload. Its upper bound adds at
1358
+ // most 20 bytes to the ranges' timestamps: a millis delta varint of up to 7
1359
+ // bytes, as maxMillis has 48 bits; up to 4 for the counter, as a new run is a
1360
+ // varint of up to 3 bytes, Counter being at most 65,535, and a 1-byte run
1361
+ // length, while extending a run adds at most 1 byte; and up to 9 for the
1362
+ // NodeId, as a new run is 8 bytes and a 1-byte run length. Its type takes 1
1363
+ // byte, and the ranges count gains at most 1 byte.
1364
+ const maxRangeOverhead = 22;
1365
+ /**
1366
+ * Returns the bytes a frame must keep to write `timestamps` as a Timestamps
1367
+ * range, after a Skip range when one is pending. A Skip range has no payload,
1368
+ * and the length of `timestamps` is exact.
1369
+ */
1370
+ const getTimestampsRangeReserve = (timestamps, isSkipPending) => (timestamps.getLength() +
1371
+ maxRangeOverhead +
1372
+ (isSkipPending ? maxRangeOverhead : 0));
1373
+ /**
1374
+ * Reads the ranges that split the items from `lower` to `upper`: one Timestamps
1375
+ * range listing them when they are too few for buckets, otherwise Fingerprint
1376
+ * ranges over the buckets. It only reads storage, so a throw is a storage
1377
+ * failure.
1378
+ */
1379
+ const readSplitRanges = (deps) => (ownerId, lower, upper, upperBound) => {
1084
1380
  const itemCount = NonNegativeInt.orThrow(upper - lower);
1085
1381
  const buckets = computeBalancedBuckets(itemCount);
1086
1382
  if (!buckets.ok) {
@@ -1093,30 +1389,43 @@ const splitRange = (deps) => (ownerId, lower, upper, upperBound, buffer) => {
1093
1389
  range.timestamps.add(timestampBytesToTimestamp(timestamp));
1094
1390
  return true;
1095
1391
  });
1096
- buffer.addRange(range);
1097
- return;
1392
+ return [range];
1098
1393
  }
1099
1394
  // Check Storage.ts `fingerprint` and `fingerprintRanges` docs.
1100
1395
  const fingerprintRangesBuckets = lower === 0
1101
1396
  ? buckets.value
1102
- : [
1103
- lower,
1104
- ...buckets.value.map((b) => NonNegativeInt.orThrow(b + lower)),
1105
- ];
1397
+ : [lower, ...buckets.value.map((b) => (b + lower))];
1106
1398
  const fingerprintRanges = deps.storage.fingerprintRanges(ownerId, fingerprintRangesBuckets, upperBound);
1107
- const rangesToUse = lower > 0 ? fingerprintRanges.slice(1) : fingerprintRanges;
1108
- for (const range of rangesToUse) {
1109
- buffer.addRange(range);
1110
- }
1399
+ return lower > 0 ? fingerprintRanges.slice(1) : fingerprintRanges;
1111
1400
  };
1401
+ // Twice the largest rangesMaxSize rather than the receiver's own, because
1402
+ // deployed senders exceed theirs. Up to @evolu/common 8.17, sync answered each
1403
+ // Timestamps range that listed timestamps it lacked, while it held none in that
1404
+ // range, with an empty Timestamps range, without checking the size. Such an
1405
+ // echo reuses the peer's bounds with a 1-byte payload, so the echoes take less
1406
+ // than the peer's ranges that list timestamps, which the peer's size checks
1407
+ // kept within its rangesMaxSize. An empty list asks for nothing, so an echo is
1408
+ // never echoed again.
1409
+ const maxRangesSectionSize = 2 * maxProtocolMessageRangesMaxSize;
1112
1410
  const decodeRanges = (buffer) => {
1411
+ // The ranges section ends the frame, so it is checked before any allocation.
1412
+ if (buffer.getLength() > maxRangesSectionSize)
1413
+ throw new ProtocolDecodeError(`Ranges section exceeds ${maxRangesSectionSize} bytes`);
1113
1414
  if (buffer.getLength() === 0)
1114
1415
  return [];
1115
1416
  const rangesCount = decodeNonNegativeInt(buffer);
1116
1417
  if (rangesCount === 0)
1117
1418
  return [];
1118
- const timestampsCount = NonNegativeInt.orThrow(rangesCount - 1);
1419
+ const timestampsCount = (rangesCount - 1);
1119
1420
  const timestamps = decodeTimestamps(buffer, timestampsCount);
1421
+ // Storage resolves each bound from the owner's first timestamp, so a lower
1422
+ // bound would move back over ranges already answered. Equal bounds are
1423
+ // valid: sync ends a range at the change that did not fit, which can be
1424
+ // where the previous range ended. Timestamps listed in a range are not
1425
+ // checked, because peers before @evolu/common 8.11 list some outside it.
1426
+ for (let i = 1; i < timestamps.length; i++)
1427
+ if (orderTimestamp(timestamps[i - 1], timestamps[i]) > 0)
1428
+ throw new ProtocolDecodeError("Range upper bounds must be non-decreasing");
1120
1429
  const rangeTypes = createMutableArray(rangesCount);
1121
1430
  for (let i = 0; i < rangesCount; i++) {
1122
1431
  const rangeType = decodeNonNegativeInt(buffer);
@@ -1164,6 +1473,9 @@ const decodeRanges = (buffer) => {
1164
1473
  };
1165
1474
  const decodeTimestamps = (buffer, length) => {
1166
1475
  length ??= decodeNonNegativeInt(buffer);
1476
+ // Every timestamp takes at least its 1-byte millis delta.
1477
+ if (length > buffer.getLength())
1478
+ throw new ProtocolDecodeError("Invalid timestamps count");
1167
1479
  let previousMillis = 0;
1168
1480
  const millises = createMutableArray(length);
1169
1481
  for (let i = 0; i < length; i++) {
@@ -1195,13 +1507,27 @@ const decodeId = (buffer) => {
1195
1507
  const bytes = buffer.shiftN(idBytesTypeValueLength);
1196
1508
  return idBytesToId(bytes);
1197
1509
  };
1510
+ /**
1511
+ * The format version that starts every {@link EncryptedDbChange} plaintext.
1512
+ *
1513
+ * It is independent of {@link protocolVersion}, so a new protocol version that
1514
+ * keeps this layout does not make decoders reject changes. 6.0.1-preview.35
1515
+ * wrote 0 with this layout, so decoding accepts every version up to this one.
1516
+ *
1517
+ * Decoders in `@evolu/common` 8.17 and earlier ignore the version. A future
1518
+ * layout must still make them fail, for example with a value they cannot
1519
+ * decode, rather than let them decode wrong values.
1520
+ */
1521
+ const encryptedDbChangeVersion = onePositiveInt;
1198
1522
  /**
1199
1523
  * Encodes and encrypts a {@link DbChange} using the provided owner's encryption
1200
1524
  * key. Returns an encrypted binary representation as {@link EncryptedDbChange}.
1201
1525
  *
1202
- * The format includes the protocol version for backward compatibility and the
1203
- * timestamp for tamper-proof verification that the timestamp matches the change
1204
- * data.
1526
+ * The plaintext starts with the format version of the change, which is
1527
+ * independent of {@link protocolVersion}, and the timestamp, which proves that
1528
+ * the change belongs to the timestamp it is sent with.
1529
+ * {@link decryptAndDecodeDbChange} rejects a newer format version and accepts
1530
+ * older ones.
1205
1531
  */
1206
1532
  export const encodeAndEncryptDbChange = (deps) => (message, key) => {
1207
1533
  const buffer = createBuffer();
@@ -1224,7 +1550,7 @@ export const encodeAndEncryptDbChange = (deps) => (message, key) => {
1224
1550
  * within it fits one protocol message.
1225
1551
  */
1226
1552
  export const encodeDbChange = (buffer, message) => {
1227
- encodeNonNegativeInt(buffer, protocolVersion);
1553
+ encodeNonNegativeInt(buffer, encryptedDbChangeVersion);
1228
1554
  // Encode the timestamp to prevent tampering (e.g., a malicious relay
1229
1555
  // assigning this EncryptedDbChange to a different EncryptedCrdtMessage)
1230
1556
  buffer.extend(timestampToTimestampBytes(message.timestamp));
@@ -1239,8 +1565,9 @@ export const encodeDbChange = (buffer, message) => {
1239
1565
  const entries = objectToEntries(message.change.values);
1240
1566
  encodeLength(buffer, entries);
1241
1567
  for (const [column, value] of entries) {
1242
- assertNotUndefined(value);
1243
1568
  encodeString(buffer, column);
1569
+ // DbChange validated every value as a SqliteValue; only its Partial record
1570
+ // type admits undefined.
1244
1571
  encodeSqliteValue(buffer, value);
1245
1572
  }
1246
1573
  };
@@ -1248,20 +1575,23 @@ export const encodeDbChange = (buffer, message) => {
1248
1575
  * Decrypts and decodes an {@link EncryptedCrdtMessage} using the provided
1249
1576
  * owner's encryption key. Verifies that the embedded timestamp matches the
1250
1577
  * expected timestamp to ensure message integrity.
1578
+ *
1579
+ * A change with a newer format version than {@link encodeAndEncryptDbChange}
1580
+ * writes is a {@link ProtocolInvalidDataError}.
1251
1581
  */
1252
1582
  export const decryptAndDecodeDbChange = (message, key) => {
1253
1583
  try {
1254
1584
  const buffer = createBuffer(message.change);
1255
1585
  const nonce = buffer.shiftN(xChaCha20Poly1305NonceLength);
1256
1586
  const ciphertext = buffer.shiftN(decodeLength(buffer));
1257
- const plaintextBytes = decryptWithXChaCha20Poly1305(XChaCha20Poly1305Ciphertext.orThrow(ciphertext), Entropy24.orThrow(nonce), key);
1587
+ const plaintextBytes = decryptWithXChaCha20Poly1305(ciphertext, nonce, key);
1258
1588
  if (!plaintextBytes.ok)
1259
1589
  return plaintextBytes;
1260
1590
  buffer.reset();
1261
1591
  buffer.extend(plaintextBytes.value);
1262
- // Decode version (for future compatibility, not need yet)
1263
- decodeNonNegativeInt(buffer);
1264
- const timestamp = timestampBytesToTimestamp(TimestampBytes.orThrow(buffer.shiftN(timestampBytesLength)));
1592
+ if (decodeNonNegativeInt(buffer) > encryptedDbChangeVersion)
1593
+ throw new ProtocolDecodeError("Unsupported EncryptedDbChange version");
1594
+ const timestamp = timestampBytesToTimestamp(buffer.shiftN(timestampBytesLength));
1265
1595
  if (!eqTimestamp(timestamp, message.timestamp)) {
1266
1596
  return err({
1267
1597
  type: "ProtocolTimestampMismatchError",
@@ -1269,7 +1599,7 @@ export const decryptAndDecodeDbChange = (message, key) => {
1269
1599
  timestamp,
1270
1600
  });
1271
1601
  }
1272
- const flags = decodeFlags(buffer, PositiveInt.orThrow(3));
1602
+ const flags = decodeFlags(buffer, 3);
1273
1603
  const table = decodeString(buffer);
1274
1604
  const id = decodeId(buffer);
1275
1605
  const length = decodeLength(buffer);
@@ -1296,26 +1626,6 @@ export const decryptAndDecodeDbChange = (message, key) => {
1296
1626
  });
1297
1627
  }
1298
1628
  };
1299
- /**
1300
- * Decodes a ProtocolMessage into a readable JSON object for debugging.
1301
- *
1302
- * Note: This is a stub for future implementation. It should use:
1303
- *
1304
- * - DecodeVersionAndOwner
1305
- * - DecodeError or decodeWriteKeys (depending on context)
1306
- * - DecodeMessages
1307
- * - DecodeRanges
1308
- *
1309
- * If you want to help, please contribute to this function.
1310
- */
1311
- export const decodeProtocolMessageToJson = (_protocolMessage, _isInitiator) => {
1312
- // TODO: Implement using
1313
- // - decodeVersionAndOwner
1314
- // -- decodeError or decodeWriteKeys (should be refactored out),
1315
- // -- decodeMessages, and decodeRanges.
1316
- // This is a stub for PRs and community contributions.
1317
- throw new Error("decodeProtocolMessageToJson is not implemented yet.");
1318
- };
1319
1629
  // Small ints are encoded into ProtocolValueType, saving one byte per int.
1320
1630
  export const ProtocolValueType = {
1321
1631
  // 0-19 small ints
@@ -1363,7 +1673,7 @@ export const encodeSqliteValue = (buffer, value) => {
1363
1673
  }
1364
1674
  else {
1365
1675
  encodeNonNegativeInt(buffer, ProtocolValueType.DateIsoWithNegativeTime);
1366
- encodeNumber(buffer, FiniteNumber.orThrow(time));
1676
+ encodeNumber(buffer, time);
1367
1677
  }
1368
1678
  return;
1369
1679
  }
@@ -1376,7 +1686,7 @@ export const encodeSqliteValue = (buffer, value) => {
1376
1686
  const json = Json.from.parent(value);
1377
1687
  if (json.ok) {
1378
1688
  const jsonValue = jsonToJsonValue(json.value);
1379
- jsonBuffer.reset();
1689
+ const jsonBuffer = createBuffer();
1380
1690
  try {
1381
1691
  // Encoding first rejects nesting deeper than decoding allows, before
1382
1692
  // the recursive JSON.stringify below could overflow the stack. Such
@@ -1398,9 +1708,6 @@ export const encodeSqliteValue = (buffer, value) => {
1398
1708
  if (!(error instanceof BufferError))
1399
1709
  throw error;
1400
1710
  }
1401
- finally {
1402
- jsonBuffer.reset();
1403
- }
1404
1711
  }
1405
1712
  const base64Url = Base64Url.from.parent(value);
1406
1713
  if (base64Url.ok) {
@@ -1410,6 +1717,10 @@ export const encodeSqliteValue = (buffer, value) => {
1410
1717
  buffer.extend(bytes);
1411
1718
  return;
1412
1719
  }
1720
+ // encodeString replaces a lone surrogate with U+FFFD, so the bytes are
1721
+ // always valid UTF-8. Encoding WTF-8 instead would make peers disagree:
1722
+ // deployed decoders read a lone surrogate in WTF-8 as three U+FFFD,
1723
+ // while a decoder that kept it would read the surrogate.
1413
1724
  encodeNonNegativeInt(buffer, ProtocolValueType.String);
1414
1725
  encodeString(buffer, value);
1415
1726
  return;
@@ -1485,5 +1796,4 @@ export const decodeSqliteValue = (buffer) => {
1485
1796
  throw new ProtocolDecodeError("invalid ProtocolValueType");
1486
1797
  }
1487
1798
  };
1488
- const jsonBuffer = createBuffer();
1489
1799
  const isSmallInt = (value) => value >= 0 && value < 20;