@evolu/common 6.0.1-preview.19 → 6.0.1-preview.20

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (140) hide show
  1. package/dist/src/Assert.d.ts.map +1 -1
  2. package/dist/src/Assert.js +1 -1
  3. package/dist/src/Buffer.d.ts +1 -1
  4. package/dist/src/Buffer.d.ts.map +1 -1
  5. package/dist/src/Buffer.js +1 -1
  6. package/dist/src/CallbackRegistry.d.ts +53 -0
  7. package/dist/src/CallbackRegistry.d.ts.map +1 -0
  8. package/dist/src/CallbackRegistry.js +25 -0
  9. package/dist/src/Console.d.ts +31 -6
  10. package/dist/src/Console.d.ts.map +1 -1
  11. package/dist/src/Console.js +72 -9
  12. package/dist/src/Crypto.d.ts +48 -37
  13. package/dist/src/Crypto.d.ts.map +1 -1
  14. package/dist/src/Crypto.js +27 -50
  15. package/dist/src/Evolu/Db.d.ts +138 -66
  16. package/dist/src/Evolu/Db.d.ts.map +1 -1
  17. package/dist/src/Evolu/Db.js +248 -645
  18. package/dist/src/Evolu/Diff.d.ts +3 -3
  19. package/dist/src/Evolu/Diff.d.ts.map +1 -1
  20. package/dist/src/Evolu/Diff.js +7 -5
  21. package/dist/src/Evolu/Evolu.d.ts +79 -116
  22. package/dist/src/Evolu/Evolu.d.ts.map +1 -1
  23. package/dist/src/Evolu/Evolu.js +275 -132
  24. package/dist/src/Evolu/Internal.d.ts +0 -2
  25. package/dist/src/Evolu/Internal.d.ts.map +1 -1
  26. package/dist/src/Evolu/Internal.js +0 -2
  27. package/dist/src/Evolu/LocalAuth.d.ts +144 -0
  28. package/dist/src/Evolu/LocalAuth.d.ts.map +1 -0
  29. package/dist/src/Evolu/LocalAuth.js +171 -0
  30. package/dist/src/Evolu/Owner.d.ts +129 -83
  31. package/dist/src/Evolu/Owner.d.ts.map +1 -1
  32. package/dist/src/Evolu/Owner.js +80 -89
  33. package/dist/src/Evolu/Platform.d.ts +9 -7
  34. package/dist/src/Evolu/Platform.d.ts.map +1 -1
  35. package/dist/src/Evolu/Protocol.d.ts +114 -191
  36. package/dist/src/Evolu/Protocol.d.ts.map +1 -1
  37. package/dist/src/Evolu/Protocol.js +409 -416
  38. package/dist/src/Evolu/Public.d.ts +6 -8
  39. package/dist/src/Evolu/Public.d.ts.map +1 -1
  40. package/dist/src/Evolu/Public.js +2 -3
  41. package/dist/src/Evolu/PublicKysely.js +3 -3
  42. package/dist/src/Evolu/Relay.d.ts +1 -2
  43. package/dist/src/Evolu/Relay.d.ts.map +1 -1
  44. package/dist/src/Evolu/Relay.js +11 -9
  45. package/dist/src/Evolu/Schema.d.ts +88 -27
  46. package/dist/src/Evolu/Schema.d.ts.map +1 -1
  47. package/dist/src/Evolu/Schema.js +141 -24
  48. package/dist/src/Evolu/Storage.d.ts +158 -14
  49. package/dist/src/Evolu/Storage.d.ts.map +1 -1
  50. package/dist/src/Evolu/Storage.js +32 -32
  51. package/dist/src/Evolu/Sync.d.ts +77 -13
  52. package/dist/src/Evolu/Sync.d.ts.map +1 -1
  53. package/dist/src/Evolu/Sync.js +453 -20
  54. package/dist/src/Evolu/Timestamp.d.ts +29 -27
  55. package/dist/src/Evolu/Timestamp.d.ts.map +1 -1
  56. package/dist/src/Evolu/Timestamp.js +20 -18
  57. package/dist/src/ManyToManyMap.d.ts +74 -10
  58. package/dist/src/ManyToManyMap.d.ts.map +1 -1
  59. package/dist/src/ManyToManyMap.js +41 -6
  60. package/dist/src/Random.d.ts +3 -2
  61. package/dist/src/Random.d.ts.map +1 -1
  62. package/dist/src/RefCountedResourceManager.d.ts +119 -0
  63. package/dist/src/RefCountedResourceManager.d.ts.map +1 -0
  64. package/dist/src/RefCountedResourceManager.js +197 -0
  65. package/dist/src/Result.d.ts +144 -22
  66. package/dist/src/Result.d.ts.map +1 -1
  67. package/dist/src/Result.js +5 -2
  68. package/dist/src/Sqlite.d.ts +20 -4
  69. package/dist/src/Sqlite.d.ts.map +1 -1
  70. package/dist/src/Sqlite.js +50 -8
  71. package/dist/src/Task.d.ts +511 -0
  72. package/dist/src/Task.d.ts.map +1 -0
  73. package/dist/src/Task.js +410 -0
  74. package/dist/src/Time.d.ts +59 -0
  75. package/dist/src/Time.d.ts.map +1 -1
  76. package/dist/src/Time.js +87 -4
  77. package/dist/src/Type.d.ts +431 -341
  78. package/dist/src/Type.d.ts.map +1 -1
  79. package/dist/src/Type.js +458 -466
  80. package/dist/src/WebSocket.d.ts +5 -2
  81. package/dist/src/WebSocket.d.ts.map +1 -1
  82. package/dist/src/WebSocket.js +12 -13
  83. package/dist/src/Worker.d.ts +39 -11
  84. package/dist/src/Worker.d.ts.map +1 -1
  85. package/dist/src/Worker.js +22 -4
  86. package/dist/src/index.d.ts +2 -3
  87. package/dist/src/index.d.ts.map +1 -1
  88. package/dist/src/index.js +2 -3
  89. package/package.json +7 -7
  90. package/src/Assert.ts +2 -4
  91. package/src/Buffer.ts +1 -1
  92. package/src/CallbackRegistry.ts +84 -0
  93. package/src/Console.ts +91 -11
  94. package/src/Crypto.ts +78 -91
  95. package/src/Evolu/Db.ts +455 -947
  96. package/src/Evolu/Diff.ts +7 -5
  97. package/src/Evolu/Evolu.ts +545 -307
  98. package/src/Evolu/Internal.ts +0 -2
  99. package/src/Evolu/LocalAuth.ts +422 -0
  100. package/src/Evolu/Owner.ts +191 -131
  101. package/src/Evolu/Platform.ts +9 -9
  102. package/src/Evolu/Protocol.ts +536 -653
  103. package/src/Evolu/Public.ts +7 -9
  104. package/src/Evolu/PublicKysely.ts +3 -3
  105. package/src/Evolu/Relay.ts +17 -12
  106. package/src/Evolu/Schema.ts +271 -66
  107. package/src/Evolu/Storage.ts +263 -55
  108. package/src/Evolu/Sync.ts +758 -37
  109. package/src/Evolu/Timestamp.ts +30 -35
  110. package/src/ManyToManyMap.ts +127 -24
  111. package/src/Random.ts +3 -2
  112. package/src/RefCountedResourceManager.ts +368 -0
  113. package/src/Result.ts +149 -23
  114. package/src/Sqlite.ts +59 -24
  115. package/src/Task.ts +779 -0
  116. package/src/Time.ts +168 -4
  117. package/src/Type.ts +657 -695
  118. package/src/WebSocket.ts +23 -17
  119. package/src/Worker.ts +72 -23
  120. package/src/index.ts +2 -3
  121. package/dist/src/Callbacks.d.ts +0 -20
  122. package/dist/src/Callbacks.d.ts.map +0 -1
  123. package/dist/src/Callbacks.js +0 -18
  124. package/dist/src/Evolu/Config.d.ts +0 -82
  125. package/dist/src/Evolu/Config.d.ts.map +0 -1
  126. package/dist/src/Evolu/Config.js +0 -9
  127. package/dist/src/Evolu/Kysely.d.ts +0 -6
  128. package/dist/src/Evolu/Kysely.d.ts.map +0 -1
  129. package/dist/src/Evolu/Kysely.js +0 -21
  130. package/dist/src/NanoId.d.ts +0 -27
  131. package/dist/src/NanoId.d.ts.map +0 -1
  132. package/dist/src/NanoId.js +0 -6
  133. package/dist/src/Promise.d.ts +0 -180
  134. package/dist/src/Promise.d.ts.map +0 -1
  135. package/dist/src/Promise.js +0 -176
  136. package/src/Callbacks.ts +0 -43
  137. package/src/Evolu/Config.ts +0 -97
  138. package/src/Evolu/Kysely.ts +0 -38
  139. package/src/NanoId.ts +0 -39
  140. package/src/Promise.ts +0 -295
@@ -4,15 +4,22 @@
4
4
  *
5
5
  * Evolu Protocol is a local-first, end-to-end encrypted binary synchronization
6
6
  * protocol optimized for minimal size and maximum speed. It enables data sync
7
- * between a client and a relay, clients in a peer-to-peer (P2P) setup, or
8
- * relays with each other.
7
+ * between a client and a relay. In the future, direct peer-to-peer (P2P) sync
8
+ * between clients will be possible without a relay.
9
+ *
10
+ * Relays don't need to sync with each other—clients using those relays will
11
+ * sync them eventually. If a relay is offline (e.g., for maintenance), it will
12
+ * sync automatically later via client sync logic. For relay backup using
13
+ * SQLite, see https://sqlite.org/rsync.html (uses a similar algorithm to Evolu
14
+ * RBSR).
9
15
  *
10
16
  * Evolu Protocol is designed for SQLite but can be extended to any database. It
11
17
  * implements [Range-Based Set
12
18
  * Reconciliation](https://arxiv.org/abs/2212.13567). To learn how RBSR works,
13
19
  * check [Negentropy](https://logperiodic.com/rbsr.html). Evolu Protocol is
14
20
  * similar to Negentropy but uses different encoding and also provides data
15
- * transfer and ownership.
21
+ * transfer, ownership, real-time broadcasting, request-response semantics, and
22
+ * error handling.
16
23
  *
17
24
  * ### Message Structure
18
25
  *
@@ -21,12 +28,15 @@
21
28
  * | **Header** | |
22
29
  * | - {@link protocolVersion} | |
23
30
  * | - {@link OwnerId} | {@link Owner} |
24
- * | **Initiator** | |
25
- * | - {@link WriteKeyMode} | |
26
- * | - {@link WriteKey} | If WriteKeyMode >= 1 |
27
- * | - {@link WriteKey} | If WriteKeyMode = 2 (new) |
28
- * | **Non-initiator** | |
31
+ * | - messageType | {@link MessageType} |
32
+ * | **Request (messageType=0)** | |
33
+ * | - hasWriteKey | 0 = no, 1 = yes |
34
+ * | - {@link OwnerWriteKey} | If hasWriteKey = 1 |
35
+ * | - subscriptionFlag | {@link SubscriptionFlags} |
36
+ * | **Response (messageType=1)** | |
29
37
  * | - {@link ProtocolErrorCode} | |
38
+ * | **Broadcast (messageType=2)** | |
39
+ * | - (no additional fields) | |
30
40
  * | **Messages** | |
31
41
  * | - {@link NonNegativeInt} | A number of messages. |
32
42
  * | - {@link EncryptedCrdtMessage} | |
@@ -36,19 +46,12 @@
36
46
  *
37
47
  * ### WriteKey Validation
38
48
  *
39
- * The initiator sends WriteKeyMode and optionally one or two WriteKeys. One key
40
- * for write operations and two for key rotation (current and new). Note that
41
- * it's ok to not send any key if initiator is going to be synced with readonly
42
- * owner. The non-initiator validates them immediately after parsing the
43
- * initiator header, before processing any messages or ranges.
44
- *
45
- * ### WriteKey Rotation
46
- *
47
- * When initiator's {@link WriteKeyMode} is `Rotation`, two WriteKeys are
48
- * present:
49
- *
50
- * 1. Current WriteKey (for validation)
51
- * 2. New WriteKey (to be stored)
49
+ * The initiator sends a hasWriteKey flag and optionally a WriteKey. The
50
+ * WriteKey is required when sending messages as a secure token proving the
51
+ * initiator can write changes. It's ok to not send a WriteKey if the initiator
52
+ * is only syncing (read-only) and not sending messages. The non-initiator
53
+ * validates the WriteKey immediately after parsing the initiator header, before
54
+ * processing any messages or ranges.
52
55
  *
53
56
  * ### Synchronization
54
57
  *
@@ -64,18 +67,18 @@
64
67
  *
65
68
  * The **non-initiator always responds** to provide sync completion feedback,
66
69
  * even with empty messages containing only the header and no error. This allows
67
- * the initiator to reliably detect when synchronization is complete.
70
+ * the initiator to detect when synchronization is complete.
68
71
  *
69
72
  * Both **Messages** and **Ranges** are optional, allowing each side to send,
70
73
  * sync, or only subscribe data as needed.
71
74
  *
72
- * When the initiator sends data, the {@link WriteKey} is required in Messages as
73
- * a secure token proving the initiator can write changes. The non-initiator
74
- * responds without a {@link WriteKey}, since the initiator’s request already
75
- * signals it wants data. If the non-initiator detects an issue, it sends an
76
- * error code via the `Error` field in the header back to the initiator. In
77
- * relay-to-relay or P2P sync, both sides may require the {@link WriteKey}
78
- * depending on who is the initiator.
75
+ * When the initiator sends data, the {@link OwnerWriteKey} is required as a
76
+ * secure token proving the initiator can write changes. The non-initiator
77
+ * responds without a {@link OwnerWriteKey}, since the initiator’s request
78
+ * already signals it wants data. If the non-initiator detects an issue, it
79
+ * sends an error code via the `Error` field in the header back to the
80
+ * initiator. In relay-to-relay or P2P sync, both sides may require the
81
+ * {@link OwnerWriteKey} depending on who is the initiator.
79
82
  *
80
83
  * ### Protocol Errors
81
84
  *
@@ -89,15 +92,21 @@
89
92
  * - {@link ProtocolUnsupportedVersionError}: Protocol version mismatch.
90
93
  * - {@link ProtocolInvalidDataError}: The message is malformed or corrupted.
91
94
  *
92
- * All protocol errors except `ProtocolInvalidDataError` include the `ownerId`
95
+ * All protocol errors except `ProtocolInvalidDataError` include the `OwnerId`
93
96
  * to allow clients to associate errors with the correct owner.
94
97
  *
95
98
  * ### Message Size Limit
96
99
  *
97
100
  * The protocol enforces a strict maximum size for all messages, defined by
98
101
  * {@link maxProtocolMessageSize}. This ensures every {@link ProtocolMessage} is
99
- * less than or equal to this limit, eliminating the need for applications to
100
- * fragment and reconstruct messages during transmission.
102
+ * less than or equal to this limit, enabling stateless transports, simplified
103
+ * relay implementation, and predictable memory usage. When all messages don't
104
+ * fit within the limit, the protocol automatically continues synchronization in
105
+ * subsequent rounds using range-based reconciliation.
106
+ *
107
+ * Individual database mutations are limited to `maxMutationSize` (640KB), which
108
+ * is smaller than the protocol message limit to ensure efficient sync with
109
+ * {@link maxProtocolMessageRangesSize}.
101
110
  *
102
111
  * ### Why Binary?
103
112
  *
@@ -105,13 +114,12 @@
105
114
  *
106
115
  * - Encrypted data doesn’t compress well, unlike plain JSON.
107
116
  * - Message size must be controlled during creation.
108
- * - Sequential byte reading is faster than parsing and can avoid conversions.
117
+ * - Sequential byte reading is faster than parsing and avoids conversions.
109
118
  *
110
119
  * It uses structure-aware encoding, significantly outperforming generic binary
111
120
  * serialization formats with the following optimizations:
112
121
  *
113
122
  * - **NonNegativeInt:** Up to 33% smaller than MessagePack.
114
- * - **Base64Url Strings:** Up to 25% size reduction.
115
123
  * - **DateIso:** Up to 75% smaller.
116
124
  * - **Timestamp Encoding:** Delta encoding for milliseconds and run-length
117
125
  * encoding (RLE) for counters and NodeIds.
@@ -143,8 +151,7 @@
143
151
  *
144
152
  * @module
145
153
  */
146
- import { sha256 } from "@noble/hashes/sha2";
147
- import { pack, unpack, unpackMultiple } from "msgpackr";
154
+ import { Packr } from "msgpackr";
148
155
  import { isNonEmptyReadonlyArray } from "../Array.js";
149
156
  import { assert } from "../Assert.js";
150
157
  import { bytesToHex, bytesToUtf8, createBuffer, hexToBytes, utf8ToBytes, } from "../Buffer.js";
@@ -153,16 +160,39 @@ import { eqArrayNumber } from "../Eq.js";
153
160
  import { computeBalancedBuckets } from "../Number.js";
154
161
  import { objectToEntries } from "../Object.js";
155
162
  import { err, ok } from "../Result.js";
156
- import { SqliteValue } from "../Sqlite.js";
157
- import { Base64Url, base64UrlAlphabet, DateIsoString, Id, idTypeValueLength, JsonValueFromString, maxLength, NonNegativeInt, Number, object, record, } from "../Type.js";
158
- import { writeKeyLength } from "./Owner.js";
159
- import { binaryTimestampLength, binaryTimestampToTimestamp, Counter, eqTimestamp, Millis, timestampToBinaryTimestamp, } from "./Timestamp.js";
163
+ import { Base64Url, base64UrlToUint8Array, DateIso, Id, idBytesToId, idBytesTypeValueLength, idToIdBytes, Json, jsonToJsonValue, NonNegativeInt, Number, uint8ArrayToBase64Url, } from "../Type.js";
164
+ import { ownerIdToOwnerIdBytes, ownerWriteKeyLength, } from "./Owner.js";
165
+ import { fingerprintSize, InfiniteUpperBound, RangeType, } from "./Storage.js";
166
+ import { Counter, eqTimestamp, Millis, timestampBytesLength, timestampBytesToTimestamp, timestampToTimestampBytes, } from "./Timestamp.js";
167
+ /**
168
+ * MessagePack serializer for standard compatibility and compact encoding.
169
+ *
170
+ * - `variableMapSize: true` - More compact maps, ~5-10% slower encoding
171
+ * - `useRecords: false` - Standard MessagePack without extensions
172
+ */
173
+ const packr = new Packr({ variableMapSize: true, useRecords: false });
160
174
  /** Maximum size of the entire protocol message in bytes. */
161
175
  export const maxProtocolMessageSize = 1_000_000;
162
176
  /** Maximum size of the ranges in bytes. */
163
177
  export const maxProtocolMessageRangesSize = 30_000;
164
178
  /** Evolu Protocol version. */
165
179
  export const protocolVersion = 0;
180
+ export const MessageType = {
181
+ /** Request message from initiator (client) to non-initiator (relay). */
182
+ Request: 0,
183
+ /** Response message from non-initiator (relay) to initiator (client). */
184
+ Response: 1,
185
+ /** Broadcast message from non-initiator (relay) to subscribed clients. */
186
+ Broadcast: 2,
187
+ };
188
+ export const SubscriptionFlags = {
189
+ /** No subscription changes for this owner. */
190
+ None: 0,
191
+ /** Subscribe to updates for this owner. */
192
+ Subscribe: 1,
193
+ /** Unsubscribe from updates for this owner. */
194
+ Unsubscribe: 2,
195
+ };
166
196
  export const ProtocolErrorCode = {
167
197
  NoError: 0,
168
198
  /** A code for {@link ProtocolWriteKeyError}. */
@@ -172,34 +202,6 @@ export const ProtocolErrorCode = {
172
202
  /** A code for {@link ProtocolSyncError}. */
173
203
  SyncError: 3,
174
204
  };
175
- export const WriteKeyMode = {
176
- None: 0,
177
- Single: 1,
178
- Rotation: 2,
179
- };
180
- /**
181
- * Base64Url string with maximum length of 256 characters. Encoding strings as
182
- * Base64UrlString saves up to 25% in size compared to regular strings.
183
- */
184
- export const Base64Url256 = maxLength(256)(Base64Url);
185
- /**
186
- * A DbChange is a change to a table row. Together with a unique
187
- * {@link Timestamp}, it forms a {@link CrdtMessage}.
188
- */
189
- export const DbChange = object({
190
- table: Base64Url256,
191
- id: Id,
192
- values: record(Base64Url256, SqliteValue),
193
- });
194
- export const RangeType = {
195
- Fingerprint: 1,
196
- Skip: 0,
197
- Timestamps: 2,
198
- };
199
- export const InfiniteUpperBound = Symbol("InfiniteUpperBound");
200
- export const fingerprintSize = 12;
201
- /** A fingerprint of an empty range. */
202
- export const zeroFingerprint = new Uint8Array(fingerprintSize);
203
205
  /**
204
206
  * Creates a {@link ProtocolMessage} from CRDT messages.
205
207
  *
@@ -209,7 +211,7 @@ export const zeroFingerprint = new Uint8Array(fingerprintSize);
209
211
  */
210
212
  export const createProtocolMessageFromCrdtMessages = (deps) => (owner, messages, maxSize) => {
211
213
  const buffer = createProtocolMessageBuffer(owner.id, {
212
- type: "initiator",
214
+ messageType: MessageType.Request,
213
215
  totalMaxSize: maxSize ?? maxProtocolMessageSize,
214
216
  writeKey: owner.writeKey,
215
217
  });
@@ -241,7 +243,7 @@ export const createProtocolMessageFromCrdtMessages = (deps) => (owner, messages,
241
243
  * For now, using a random fingerprint avoids extra complexity and is good
242
244
  * enough for this case.
243
245
  */
244
- const randomFingerprint = deps.createRandomBytes(fingerprintSize);
246
+ const randomFingerprint = deps.randomBytes.create(fingerprintSize);
245
247
  // There is always a space for Fingerprint with InfiniteUpperBound.
246
248
  buffer.addRange({
247
249
  type: RangeType.Fingerprint,
@@ -252,24 +254,23 @@ export const createProtocolMessageFromCrdtMessages = (deps) => (owner, messages,
252
254
  return buffer.unwrap();
253
255
  };
254
256
  /** Creates a {@link ProtocolMessage} for sync. */
255
- export const createProtocolMessageForSync = (deps) => (ownerId) => {
256
- const buffer = createProtocolMessageBuffer(ownerId, { type: "initiator" });
257
- const binaryOwnerId = ownerIdToBinaryOwnerId(ownerId);
258
- const size = deps.storage.getSize(binaryOwnerId);
257
+ export const createProtocolMessageForSync = (deps) => (ownerId, subscriptionFlag) => {
258
+ const buffer = createProtocolMessageBuffer(ownerId, {
259
+ messageType: MessageType.Request,
260
+ subscriptionFlag: subscriptionFlag ?? SubscriptionFlags.None,
261
+ });
262
+ const ownerIdBytes = ownerIdToOwnerIdBytes(ownerId);
263
+ const size = deps.storage.getSize(ownerIdBytes);
259
264
  // Errors are handled by the storage.
260
265
  if (size == null)
261
266
  return null;
262
- splitRange(deps)(binaryOwnerId, 0, size, InfiniteUpperBound, buffer);
263
- return buffer.unwrap();
264
- };
265
- /** Creates a ProtocolMessage for {@link WriteKey} rotation. */
266
- export const createProtocolMessageForWriteKeyRotation = (ownerId, currentWriteKey, newWriteKey) => {
267
- const buffer = createProtocolMessageBuffer(ownerId, {
268
- type: "initiator",
269
- writeKey: [currentWriteKey, newWriteKey],
270
- });
267
+ splitRange(deps)(ownerIdBytes, 0, size, InfiniteUpperBound, buffer);
271
268
  return buffer.unwrap();
272
269
  };
270
+ export const createProtocolMessageForUnsubscribe = (ownerId) => createProtocolMessageBuffer(ownerId, {
271
+ messageType: MessageType.Request,
272
+ subscriptionFlag: SubscriptionFlags.Unsubscribe,
273
+ }).unwrap();
273
274
  export const createProtocolMessageBuffer = (ownerId, options) => {
274
275
  const { totalMaxSize = maxProtocolMessageSize, rangesMaxSize = maxProtocolMessageRangesSize, version = protocolVersion, } = options;
275
276
  const buffers = {
@@ -285,22 +286,20 @@ export const createProtocolMessageBuffer = (ownerId, options) => {
285
286
  },
286
287
  };
287
288
  encodeNonNegativeInt(buffers.header, version);
288
- buffers.header.extend(ownerIdToBinaryOwnerId(ownerId));
289
- if (options.type === "initiator") {
289
+ buffers.header.extend(ownerIdToOwnerIdBytes(ownerId));
290
+ buffers.header.extend([options.messageType]);
291
+ if (options.messageType === MessageType.Request) {
290
292
  if (!options.writeKey) {
291
- buffers.header.extend([WriteKeyMode.None]);
292
- }
293
- else if (!Array.isArray(options.writeKey)) {
294
- buffers.header.extend([WriteKeyMode.Single]);
295
- buffers.header.extend(options.writeKey);
293
+ buffers.header.extend([0]);
296
294
  }
297
295
  else {
298
- buffers.header.extend([WriteKeyMode.Rotation]);
299
- buffers.header.extend(options.writeKey[0]); // current
300
- buffers.header.extend(options.writeKey[1]); // new
296
+ buffers.header.extend([1]);
297
+ buffers.header.extend(options.writeKey);
301
298
  }
299
+ const subscriptionFlag = options.subscriptionFlag ?? SubscriptionFlags.None;
300
+ buffers.header.extend([subscriptionFlag]);
302
301
  }
303
- else {
302
+ else if (options.messageType === MessageType.Response) {
304
303
  buffers.header.extend([options.errorCode]);
305
304
  }
306
305
  let isLastRangeInfinite = false;
@@ -362,6 +361,7 @@ export const createProtocolMessageBuffer = (ownerId, options) => {
362
361
  : true));
363
362
  },
364
363
  addRange: (range) => {
364
+ assert(options.messageType !== MessageType.Broadcast, "Cannot add a range into broadcast message");
365
365
  assert(!isLastRangeInfinite, "Cannot add a range after an InfiniteUpperBound range");
366
366
  isLastRangeInfinite = range.upperBound === InfiniteUpperBound;
367
367
  /**
@@ -370,7 +370,7 @@ export const createProtocolMessageBuffer = (ownerId, options) => {
370
370
  * use SkipRange.
371
371
  */
372
372
  if (range.upperBound !== InfiniteUpperBound)
373
- buffers.ranges.timestamps.add(binaryTimestampToTimestamp(range.upperBound));
373
+ buffers.ranges.timestamps.add(timestampBytesToTimestamp(range.upperBound));
374
374
  else {
375
375
  buffers.ranges.timestamps.addInfinite();
376
376
  }
@@ -471,151 +471,209 @@ const createRunLengthEncoder = (encodeValue) => {
471
471
  unwrap: () => buffer.unwrap(),
472
472
  };
473
473
  };
474
- export const applyProtocolMessageAsClient = (deps) => (inputMessage, { getWriteKey, version = protocolVersion, totalMaxSize, rangesMaxSize, } = {}) => tryDecodeProtocolData(inputMessage, (input) => {
475
- const [requestedVersion, ownerId] = decodeVersionAndOwner(input);
476
- if (requestedVersion !== version) {
477
- return err({
478
- type: "ProtocolUnsupportedVersionError",
479
- unsupportedVersion: requestedVersion,
480
- isInitiator: version < requestedVersion,
481
- ownerId,
474
+ export const applyProtocolMessageAsClient = (deps) => async (inputMessage, options = {}) => {
475
+ // try-catch instead of Result for performance and stacktraces
476
+ try {
477
+ const input = createBuffer(inputMessage);
478
+ const [requestedVersion, ownerId] = decodeVersionAndOwner(input);
479
+ const version = options.version ?? protocolVersion;
480
+ if (requestedVersion !== version) {
481
+ return err({
482
+ type: "ProtocolUnsupportedVersionError",
483
+ unsupportedVersion: requestedVersion,
484
+ isInitiator: version < requestedVersion,
485
+ ownerId,
486
+ });
487
+ }
488
+ const messageType = input.shift();
489
+ assert(messageType === MessageType.Response ||
490
+ messageType === MessageType.Broadcast, "Invalid MessageType");
491
+ if (messageType === MessageType.Response) {
492
+ const errorCode = input.shift();
493
+ if (errorCode !== ProtocolErrorCode.NoError) {
494
+ switch (errorCode) {
495
+ case ProtocolErrorCode.WriteKeyError:
496
+ return err({
497
+ type: "ProtocolWriteKeyError",
498
+ ownerId,
499
+ });
500
+ case ProtocolErrorCode.WriteError:
501
+ return err({
502
+ type: "ProtocolWriteError",
503
+ ownerId,
504
+ });
505
+ case ProtocolErrorCode.SyncError:
506
+ return err({
507
+ type: "ProtocolSyncError",
508
+ ownerId,
509
+ });
510
+ default:
511
+ throw new ProtocolDecodeError(`Invalid ProtocolErrorCode: ${errorCode}`);
512
+ }
513
+ }
514
+ }
515
+ const messages = decodeMessages(input);
516
+ const ownerIdBytes = ownerIdToOwnerIdBytes(ownerId);
517
+ if (isNonEmptyReadonlyArray(messages) &&
518
+ !(await deps.storage.writeMessages(ownerIdBytes, messages))) {
519
+ return ok({ type: "no-response" });
520
+ }
521
+ // Now: No writeKey, no sync.
522
+ // TODO: Allow to sync SharedReadonlyOwner
523
+ // Without local changes, writeKey will not be required.
524
+ // With local changes, writeKey will be required and if not provided,
525
+ // the sync will stop.
526
+ const writeKey = options.getWriteKey?.(ownerId);
527
+ if (writeKey == null) {
528
+ return ok({ type: "no-response" });
529
+ }
530
+ if (messageType === MessageType.Broadcast) {
531
+ return ok({ type: "broadcast" });
532
+ }
533
+ const ranges = decodeRanges(input);
534
+ if (!isNonEmptyReadonlyArray(ranges)) {
535
+ return ok({ type: "no-response" });
536
+ }
537
+ const output = createProtocolMessageBuffer(ownerId, {
538
+ messageType: MessageType.Request,
539
+ writeKey,
540
+ totalMaxSize: options.totalMaxSize,
541
+ rangesMaxSize: options.rangesMaxSize,
482
542
  });
483
- }
484
- const errorCode = input.shift();
485
- if (errorCode !== ProtocolErrorCode.NoError) {
486
- switch (errorCode) {
487
- case ProtocolErrorCode.WriteKeyError:
488
- return err({
489
- type: "ProtocolWriteKeyError",
490
- ownerId,
491
- });
492
- case ProtocolErrorCode.WriteError:
493
- return err({
494
- type: "ProtocolWriteError",
495
- ownerId,
496
- });
497
- case ProtocolErrorCode.SyncError:
498
- return err({
499
- type: "ProtocolSyncError",
500
- ownerId,
501
- });
502
- default:
503
- throw new ProtocolDecodeError(`Invalid ProtocolErrorCode: ${errorCode}`);
543
+ const syncResult = sync(deps)(ranges, output, ownerIdBytes);
544
+ // Client sync error (handled via Storage) or no changes.
545
+ if (!syncResult.ok || !syncResult.value) {
546
+ return ok({ type: "no-response" });
504
547
  }
548
+ return ok({ type: "response", message: output.unwrap() });
505
549
  }
506
- const messages = decodeMessages(input);
507
- const binaryOwnerId = ownerIdToBinaryOwnerId(ownerId);
508
- if (isNonEmptyReadonlyArray(messages) &&
509
- !deps.storage.writeMessages(binaryOwnerId, messages)) {
510
- return ok(null);
550
+ catch (error) {
551
+ return err({
552
+ type: "ProtocolInvalidDataError",
553
+ data: inputMessage,
554
+ error,
555
+ });
511
556
  }
512
- if (!getWriteKey)
513
- return ok(null);
514
- const writeKey = getWriteKey(ownerId);
515
- if (writeKey == null)
516
- return ok(null);
517
- const output = createProtocolMessageBuffer(ownerId, {
518
- type: "initiator",
519
- writeKey,
520
- totalMaxSize,
521
- rangesMaxSize,
522
- });
523
- return sync(deps)("initiator", input, output, binaryOwnerId);
524
- });
525
- export const applyProtocolMessageAsRelay = (deps) => (inputMessage, { subscribe, broadcast, totalMaxSize, rangesMaxSize, } = {},
557
+ };
558
+ export const applyProtocolMessageAsRelay = (deps) => async (inputMessage, options = {},
526
559
  /** For testing purposes only; should not be used in production. */
527
- version = protocolVersion) => tryDecodeProtocolData(inputMessage, (input) => {
528
- const [requestedVersion, ownerId] = decodeVersionAndOwner(input);
529
- const binaryOwnerId = ownerIdToBinaryOwnerId(ownerId);
530
- if (requestedVersion !== version) {
531
- // Non-initiator responds with its version and ownerId.
532
- const output = createBuffer();
533
- encodeNonNegativeInt(output, version);
534
- output.extend(binaryOwnerId);
535
- return ok(output.unwrap());
536
- }
537
- subscribe?.(ownerId);
538
- const writeKeyMode = input.shift();
539
- let writeKey;
540
- let newWriteKey;
541
- if (writeKeyMode !== WriteKeyMode.None) {
542
- writeKey = input.shiftN(writeKeyLength);
543
- switch (writeKeyMode) {
544
- case WriteKeyMode.Single:
560
+ version = protocolVersion) => {
561
+ // try-catch instead of Result for performance and stacktraces
562
+ try {
563
+ const input = createBuffer(inputMessage);
564
+ const [requestedVersion, ownerId] = decodeVersionAndOwner(input);
565
+ const ownerIdBytes = ownerIdToOwnerIdBytes(ownerId);
566
+ if (requestedVersion !== version) {
567
+ // Non-initiator responds with its version and ownerId.
568
+ const output = createBuffer();
569
+ encodeNonNegativeInt(output, version);
570
+ output.extend(ownerIdBytes);
571
+ return ok({
572
+ type: "response",
573
+ message: output.unwrap(),
574
+ });
575
+ }
576
+ const messageType = input.shift();
577
+ assert(messageType === MessageType.Request, "Invalid MessageType");
578
+ const hasWriteKey = input.shift();
579
+ let writeKey;
580
+ if (hasWriteKey === 1) {
581
+ writeKey = input.shiftN(ownerWriteKeyLength);
582
+ }
583
+ const subscriptionFlag = input.shift();
584
+ switch (subscriptionFlag) {
585
+ case SubscriptionFlags.Subscribe:
586
+ options.subscribe?.(ownerId);
545
587
  break;
546
- case WriteKeyMode.Rotation:
547
- newWriteKey = input.shiftN(writeKeyLength);
588
+ case SubscriptionFlags.Unsubscribe:
589
+ options.unsubscribe?.(ownerId);
590
+ break;
591
+ case SubscriptionFlags.None:
548
592
  break;
549
- default:
550
- throw new ProtocolDecodeError(`Invalid WriteKeyMode: ${writeKeyMode}`);
551
- }
552
- }
553
- if (writeKey) {
554
- const isValid = deps.storage.validateWriteKey(binaryOwnerId, writeKey);
555
- if (!isValid) {
556
- return ok(createProtocolMessageBuffer(ownerId, {
557
- type: "non-initiator",
558
- errorCode: ProtocolErrorCode.WriteKeyError,
559
- }).unwrap());
560
593
  }
561
- if (newWriteKey) {
562
- const rotationSuccess = deps.storage.setWriteKey(binaryOwnerId, newWriteKey);
563
- if (!rotationSuccess) {
564
- return ok(createProtocolMessageBuffer(ownerId, {
565
- type: "non-initiator",
566
- errorCode: ProtocolErrorCode.WriteError,
567
- }).unwrap());
594
+ if (writeKey) {
595
+ const isValid = deps.storage.validateWriteKey(ownerIdBytes, writeKey);
596
+ if (!isValid) {
597
+ return ok({
598
+ type: "response",
599
+ message: createProtocolMessageBuffer(ownerId, {
600
+ messageType: MessageType.Response,
601
+ errorCode: ProtocolErrorCode.WriteKeyError,
602
+ }).unwrap(),
603
+ });
568
604
  }
569
605
  }
570
- }
571
- const messages = decodeMessages(input);
572
- if (isNonEmptyReadonlyArray(messages)) {
573
- if (!writeKey)
574
- return ok(createProtocolMessageBuffer(ownerId, {
575
- type: "non-initiator",
576
- errorCode: ProtocolErrorCode.WriteKeyError,
577
- }).unwrap());
578
- // Only broadcast if there's no ranges.
579
- if (broadcast && input.getLength() === 0) {
580
- const broadcastBuffer = createProtocolMessageBuffer(ownerId, {
581
- type: "non-initiator",
582
- errorCode: ProtocolErrorCode.NoError,
583
- totalMaxSize,
584
- rangesMaxSize,
585
- version,
586
- });
587
- for (const message of messages) {
588
- broadcastBuffer.addMessage(message);
606
+ const messages = decodeMessages(input);
607
+ if (isNonEmptyReadonlyArray(messages)) {
608
+ if (!writeKey) {
609
+ return ok({
610
+ type: "response",
611
+ message: createProtocolMessageBuffer(ownerId, {
612
+ messageType: MessageType.Response,
613
+ errorCode: ProtocolErrorCode.WriteKeyError,
614
+ }).unwrap(),
615
+ });
616
+ }
617
+ /**
618
+ * Broadcast messages to all subscribed devices. This ensures real-time
619
+ * synchronization between clients.
620
+ *
621
+ * When a relay's database is deleted or clients migrate to a new relay
622
+ * (without data migration), clients will sync their data to the relay,
623
+ * and the relay will broadcast those messages to other connected
624
+ * clients. Those clients may receive messages they already have, but
625
+ * this is safe because `applyMessages` is idempotent. As the relay
626
+ * becomes more synchronized with clients over time, fewer duplicate
627
+ * messages will be broadcasted.
628
+ */
629
+ if (options.broadcast) {
630
+ const broadcastBuffer = createProtocolMessageBuffer(ownerId, {
631
+ messageType: MessageType.Broadcast,
632
+ totalMaxSize: options.totalMaxSize,
633
+ rangesMaxSize: options.rangesMaxSize,
634
+ version,
635
+ });
636
+ for (const message of messages) {
637
+ broadcastBuffer.addMessage(message);
638
+ }
639
+ options.broadcast(ownerId, broadcastBuffer.unwrap());
640
+ }
641
+ if (!(await deps.storage.writeMessages(ownerIdBytes, messages))) {
642
+ return ok({
643
+ type: "response",
644
+ message: createProtocolMessageBuffer(ownerId, {
645
+ messageType: MessageType.Response,
646
+ errorCode: ProtocolErrorCode.WriteError,
647
+ }).unwrap(),
648
+ });
589
649
  }
590
- broadcast(ownerId, broadcastBuffer.unwrap());
591
650
  }
592
- if (!deps.storage.writeMessages(binaryOwnerId, messages))
593
- return ok(createProtocolMessageBuffer(ownerId, {
594
- type: "non-initiator",
595
- errorCode: ProtocolErrorCode.WriteError,
596
- }).unwrap());
597
- }
598
- const output = createProtocolMessageBuffer(ownerId, {
599
- type: "non-initiator",
600
- errorCode: ProtocolErrorCode.NoError,
601
- totalMaxSize,
602
- rangesMaxSize,
603
- });
604
- return sync(deps)("non-initiator", input, output, binaryOwnerId);
605
- });
606
- /**
607
- * Wraps Evolu Protocol decoding functions, which use exceptions instead of
608
- * {@link Result} to provide stack traces for debugging and reduce allocation
609
- * overhead in success cases.
610
- */
611
- const tryDecodeProtocolData = (data, callback) => {
612
- try {
613
- return callback(createBuffer(data));
651
+ const ranges = decodeRanges(input);
652
+ const output = createProtocolMessageBuffer(ownerId, {
653
+ messageType: MessageType.Response,
654
+ errorCode: ProtocolErrorCode.NoError,
655
+ totalMaxSize: options.totalMaxSize,
656
+ rangesMaxSize: options.rangesMaxSize,
657
+ });
658
+ // Non-initiators always respond to provide sync completion feedback,
659
+ // even when there's nothing to sync.
660
+ if (!isNonEmptyReadonlyArray(ranges)) {
661
+ return ok({ type: "response", message: output.unwrap() });
662
+ }
663
+ const syncResult = sync(deps)(ranges, output, ownerIdBytes);
664
+ const message = syncResult.ok
665
+ ? output.unwrap()
666
+ : createProtocolMessageBuffer(ownerId, {
667
+ messageType: MessageType.Response,
668
+ errorCode: syncResult.error,
669
+ }).unwrap();
670
+ // Non-initiators always respond to provide sync completion feedback,
671
+ return ok({ type: "response", message });
614
672
  }
615
673
  catch (error) {
616
674
  return err({
617
675
  type: "ProtocolInvalidDataError",
618
- data,
676
+ data: inputMessage,
619
677
  error,
620
678
  });
621
679
  }
@@ -626,7 +684,7 @@ const decodeVersionAndOwner = (input) => {
626
684
  // to enable version negotiation and owner identification before any other
627
685
  // processing occurs.
628
686
  const version = decodeNonNegativeInt(input);
629
- const ownerId = decodeOwnerId(input);
687
+ const ownerId = decodeId(input);
630
688
  return [version, ownerId];
631
689
  };
632
690
  /**
@@ -650,33 +708,11 @@ const decodeMessages = (buffer) => {
650
708
  }
651
709
  return messages;
652
710
  };
653
- const sync = (deps) => (role, input, output, ownerId) => {
654
- const ranges = decodeRanges(input);
655
- if (!isNonEmptyReadonlyArray(ranges)) {
656
- // Non-initiators always respond to provide sync completion feedback,
657
- // even when there's nothing to sync.
658
- if (role === "non-initiator") {
659
- return ok(output.unwrap());
660
- }
661
- // Nothing to sync.
662
- return ok(null);
663
- }
664
- const binaryOwnerId = binaryOwnerIdToOwnerId(ownerId);
711
+ const sync = (deps) => (ranges, output, ownerIdBytes) => {
665
712
  const outputInitialSize = output.getSize();
666
- const syncFail = () => {
667
- // Only the relay (non-initiator) reports sync errors, not the client (initiator).
668
- if (role === "initiator") {
669
- return ok(null);
670
- }
671
- const message = createProtocolMessageBuffer(binaryOwnerId, {
672
- type: "non-initiator",
673
- errorCode: ProtocolErrorCode.SyncError,
674
- });
675
- return ok(message.unwrap());
676
- };
677
- const storageSize = deps.storage.getSize(ownerId);
713
+ const storageSize = deps.storage.getSize(ownerIdBytes);
678
714
  if (storageSize == null)
679
- return syncFail();
715
+ return err(ProtocolErrorCode.SyncError);
680
716
  let prevUpperBound = null;
681
717
  let prevIndex = 0;
682
718
  let skip = false;
@@ -708,7 +744,7 @@ const sync = (deps) => (role, input, output, ownerId) => {
708
744
  };
709
745
  // When we don't have a space...
710
746
  const addFingerprintForRemainingRange = (begin) => {
711
- const fingerprint = deps.storage.fingerprint(ownerId, begin, storageSize);
747
+ const fingerprint = deps.storage.fingerprint(ownerIdBytes, begin, storageSize);
712
748
  if (!fingerprint)
713
749
  return false;
714
750
  // There is always a space for a ramaining range.
@@ -722,30 +758,30 @@ const sync = (deps) => (role, input, output, ownerId) => {
722
758
  for (const range of ranges) {
723
759
  const currentUpperBound = range.upperBound;
724
760
  const lower = prevIndex;
725
- let upper = deps.storage.findLowerBound(ownerId, prevIndex, storageSize, currentUpperBound);
761
+ let upper = deps.storage.findLowerBound(ownerIdBytes, prevIndex, storageSize, currentUpperBound);
726
762
  if (upper == null)
727
- return syncFail();
763
+ return err(ProtocolErrorCode.SyncError);
728
764
  switch (range.type) {
729
765
  case RangeType.Skip: {
730
766
  skipRange(range);
731
767
  break;
732
768
  }
733
769
  case RangeType.Fingerprint: {
734
- const ourFingerprint = deps.storage.fingerprint(ownerId, lower, upper);
770
+ const ourFingerprint = deps.storage.fingerprint(ownerIdBytes, lower, upper);
735
771
  if (ourFingerprint == null)
736
- return syncFail();
772
+ return err(ProtocolErrorCode.SyncError);
737
773
  if (eqArrayNumber(range.fingerprint, ourFingerprint)) {
738
774
  skipRange(range);
739
775
  }
740
776
  else {
741
777
  if (output.canSplitRange()) {
742
778
  coalesceSkipsBeforeAdd();
743
- splitRange(deps)(ownerId, lower, upper, currentUpperBound, output);
779
+ splitRange(deps)(ownerIdBytes, lower, upper, currentUpperBound, output);
744
780
  }
745
781
  else {
746
- if (!addFingerprintForRemainingRange(upper))
747
- return syncFail();
748
- return ok(output.unwrap());
782
+ return addFingerprintForRemainingRange(upper)
783
+ ? ok(true)
784
+ : err(ProtocolErrorCode.SyncError);
749
785
  }
750
786
  }
751
787
  break;
@@ -754,19 +790,19 @@ const sync = (deps) => (role, input, output, ownerId) => {
754
790
  let endBound = currentUpperBound;
755
791
  const timestampsWeNeed = new Map(range.timestamps.map((t) => [t.join(), true]));
756
792
  const ourTimestamps = createTimestampsBuffer();
757
- let storageError = false;
793
+ let cantReadDbChange = false;
758
794
  let exceeded = false;
759
- deps.storage.iterate(ownerId, lower, upper, (timestamp, index) => {
795
+ deps.storage.iterate(ownerIdBytes, lower, upper, (timestamp, index) => {
760
796
  const timestampString = timestamp.join();
761
- const timestampBinary = binaryTimestampToTimestamp(timestamp);
797
+ const timestampBinary = timestampBytesToTimestamp(timestamp);
762
798
  let message = null;
763
799
  if (timestampsWeNeed.has(timestampString)) {
764
800
  timestampsWeNeed.delete(timestampString);
765
801
  }
766
802
  else {
767
- const dbChange = deps.storage.readDbChange(ownerId, timestamp);
803
+ const dbChange = deps.storage.readDbChange(ownerIdBytes, timestamp);
768
804
  if (dbChange == null) {
769
- storageError = true;
805
+ cantReadDbChange = true;
770
806
  return false;
771
807
  }
772
808
  message = {
@@ -785,8 +821,8 @@ const sync = (deps) => (role, input, output, ownerId) => {
785
821
  output.addMessage(message);
786
822
  return true;
787
823
  });
788
- if (storageError) {
789
- return syncFail();
824
+ if (cantReadDbChange) {
825
+ return err(ProtocolErrorCode.SyncError);
790
826
  }
791
827
  const addRange = () => {
792
828
  coalesceSkipsBeforeAdd();
@@ -799,9 +835,9 @@ const sync = (deps) => (role, input, output, ownerId) => {
799
835
  if (exceeded) {
800
836
  addRange();
801
837
  if (!addFingerprintForRemainingRange(upper)) {
802
- return syncFail();
838
+ return err(ProtocolErrorCode.SyncError);
803
839
  }
804
- return ok(output.unwrap());
840
+ return ok(true);
805
841
  }
806
842
  // If we need something, we have to respond with our timestamps.
807
843
  if (timestampsWeNeed.size > 0) {
@@ -818,12 +854,7 @@ const sync = (deps) => (role, input, output, ownerId) => {
818
854
  }
819
855
  // If all ranges were skipped, there are no changes and sync is complete.
820
856
  const hasChange = output.getSize() > outputInitialSize;
821
- // Non-initiators always respond to provide sync completion feedback,
822
- // even with empty messages. This allows clients to detect sync completion.
823
- if (role === "non-initiator" && !hasChange) {
824
- return ok(output.unwrap());
825
- }
826
- return ok(hasChange ? output.unwrap() : null);
857
+ return ok(hasChange);
827
858
  };
828
859
  const splitRange = (deps) => (ownerId, lower, upper, upperBound, buffer) => {
829
860
  const itemCount = (upper - lower);
@@ -835,7 +866,7 @@ const splitRange = (deps) => (ownerId, lower, upper, upperBound, buffer) => {
835
866
  timestamps: createTimestampsBuffer(),
836
867
  };
837
868
  deps.storage.iterate(ownerId, 0, itemCount, (timestamp) => {
838
- range.timestamps.add(binaryTimestampToTimestamp(timestamp));
869
+ range.timestamps.add(timestampBytesToTimestamp(timestamp));
839
870
  return true;
840
871
  });
841
872
  buffer.addRange(range);
@@ -878,7 +909,7 @@ const decodeRanges = (buffer) => {
878
909
  const ranges = [];
879
910
  for (let i = 0; i < rangesCount; i++) {
880
911
  const upperBound = i < timestampsCount
881
- ? timestampToBinaryTimestamp(timestamps[i])
912
+ ? timestampToTimestampBytes(timestamps[i])
882
913
  : InfiniteUpperBound;
883
914
  const rangeType = rangeTypes[i];
884
915
  switch (rangeType) {
@@ -895,7 +926,7 @@ const decodeRanges = (buffer) => {
895
926
  break;
896
927
  }
897
928
  case RangeType.Timestamps: {
898
- const timestamps = decodeTimestamps(buffer).map(timestampToBinaryTimestamp);
929
+ const timestamps = decodeTimestamps(buffer).map(timestampToTimestampBytes);
899
930
  ranges.push({
900
931
  type: RangeType.Timestamps,
901
932
  upperBound,
@@ -952,74 +983,21 @@ const decodeTimestamps = (buffer, length) => {
952
983
  }
953
984
  return timestamps;
954
985
  };
955
- export const binaryIdLength = 16;
956
- export const idToBinaryId = (id) => base64Url256ToBytes(id);
957
- export const binaryIdToId = (binaryId) => decodeId(createBuffer(binaryId));
958
- export const ownerIdToBinaryOwnerId = (ownerId) => base64Url256ToBytes(ownerId);
959
- export const binaryOwnerIdToOwnerId = (binaryOwnerId) => decodeOwnerId(createBuffer(binaryOwnerId));
960
- /**
961
- * Converts a Base64Url string to a Uint8Array for binary storage. This encoding
962
- * is more space-efficient than UTF-8 for Base64Url strings.
963
- */
964
- export const base64Url256ToBytes = (string) => {
965
- const totalBits = string.length * 6; // 6 bits per character
966
- const byteLength = Math.ceil(totalBits / 8);
967
- const value = new globalThis.Uint8Array(byteLength);
968
- let bitBuffer = 0;
969
- let bitsInBuffer = 0;
970
- let byteIndex = 0;
971
- for (const char of string) {
972
- const charValue = base64UrlAlphabet.indexOf(char);
973
- bitBuffer = (bitBuffer << 6) | charValue;
974
- bitsInBuffer += 6;
975
- while (bitsInBuffer >= 8) {
976
- bitsInBuffer -= 8;
977
- value[byteIndex++] = (bitBuffer >> bitsInBuffer) & 0xff;
978
- }
979
- }
980
- if (bitsInBuffer > 0 && byteIndex < byteLength) {
981
- value[byteIndex] = (bitBuffer << (8 - bitsInBuffer)) & 0xff;
982
- }
983
- return value;
984
- };
985
- export const decodeBase64Url256 = (buffer, stringLength) => {
986
- const bytes = buffer.shiftN(Math.ceil((stringLength * 6) / 8));
987
- let bitBuffer = 0;
988
- let bitsInBuffer = 0;
989
- let string = "";
990
- for (const byte of bytes) {
991
- bitBuffer = (bitBuffer << 8) | byte;
992
- bitsInBuffer += 8;
993
- while (bitsInBuffer >= 6) {
994
- bitsInBuffer -= 6;
995
- if (string.length < stringLength) {
996
- const charValue = (bitBuffer >> bitsInBuffer) & 0x3f;
997
- if (charValue < 0 || charValue >= base64UrlAlphabet.length) {
998
- throw new ProtocolDecodeError("invalid charValue");
999
- }
1000
- string += base64UrlAlphabet[charValue];
1001
- }
1002
- }
1003
- }
1004
- const result = Base64Url256.from(string);
1005
- if (!result.ok)
1006
- throw new ProtocolDecodeError(result.error.type);
1007
- return result.value;
986
+ const decodeId = (buffer) => {
987
+ const bytes = buffer.shiftN(idBytesTypeValueLength);
988
+ return idBytesToId(bytes);
1008
989
  };
1009
- const decodeId = (buffer) => decodeBase64Url256(buffer, idTypeValueLength);
1010
- /** Not all 16 bytes are valid {@link OwnerId}. */
1011
- const decodeOwnerId = (buffer) => decodeId(buffer);
1012
990
  /**
1013
991
  * Evolu uses MessagePack to handle all number variants except for
1014
992
  * NonNegativeInt. For NonNegativeInt, Evolu provides more efficient encoding.
1015
993
  */
1016
994
  export const encodeNumber = (buffer, number) => {
1017
- buffer.extend(pack(number));
995
+ buffer.extend(packr.pack(number));
1018
996
  };
1019
997
  export const decodeNumber = (buffer) => {
1020
998
  let number;
1021
999
  let end;
1022
- unpackMultiple(buffer.unwrap(), (n, _, e) => {
1000
+ packr.unpackMultiple(buffer.unwrap(), (n, _, e) => {
1023
1001
  number = n;
1024
1002
  end = e;
1025
1003
  return false;
@@ -1033,10 +1011,6 @@ export const decodeNumber = (buffer) => {
1033
1011
  buffer.shiftN(endResult.value);
1034
1012
  return numberResult.value;
1035
1013
  };
1036
- export const binaryTimestampToFingerprint = (timestamp) => {
1037
- const hash = sha256(timestamp).slice(0, fingerprintSize);
1038
- return hash;
1039
- };
1040
1014
  /**
1041
1015
  * Encodes and encrypts a {@link DbChange} using the provided owner's encryption
1042
1016
  * key. Returns an encrypted binary representation as {@link EncryptedDbChange}.
@@ -1051,16 +1025,16 @@ export const encodeAndEncryptDbChange = (deps) => (message, key) => {
1051
1025
  // Encode protocol version first for backward compatibility
1052
1026
  encodeNonNegativeInt(buffer, protocolVersion);
1053
1027
  // Encode the timestamp (after version) for tamper verification
1054
- const binaryTimestamp = timestampToBinaryTimestamp(message.timestamp);
1055
- buffer.extend(binaryTimestamp);
1056
- encodeBase64Url256(buffer, change.table);
1057
- buffer.extend(idToBinaryId(change.id));
1028
+ const timestampBytes = timestampToTimestampBytes(message.timestamp);
1029
+ buffer.extend(timestampBytes);
1030
+ encodeString(buffer, change.table);
1031
+ buffer.extend(idToIdBytes(change.id));
1058
1032
  const entries = objectToEntries(change.values).map(([column, value]) => {
1059
1033
  return [column, value];
1060
1034
  });
1061
1035
  encodeLength(buffer, entries);
1062
1036
  for (const [column, value] of entries) {
1063
- encodeBase64Url256(buffer, column);
1037
+ encodeString(buffer, column);
1064
1038
  encodeSqliteValue(buffer, value);
1065
1039
  }
1066
1040
  const paddingLength = padmePaddingLength(buffer.getLength());
@@ -1078,40 +1052,51 @@ export const encodeAndEncryptDbChange = (deps) => (message, key) => {
1078
1052
  * owner's encryption key. Verifies that the embedded timestamp matches the
1079
1053
  * expected timestamp to ensure message integrity.
1080
1054
  */
1081
- export const decryptAndDecodeDbChange = (deps) => (message, key) => tryDecodeProtocolData(message.change, (buffer) => {
1082
- const nonce = buffer.shiftN(deps.symmetricCrypto.nonceLength);
1083
- const ciphertextLength = decodeLength(buffer);
1084
- const ciphertext = buffer.shiftN(ciphertextLength);
1085
- const plaintextBytes = deps.symmetricCrypto.decrypt(ciphertext, key, nonce);
1086
- if (!plaintextBytes.ok)
1087
- return plaintextBytes;
1088
- buffer.reset();
1089
- buffer.extend(plaintextBytes.value);
1090
- // Decode version (for future compatibility, no validation needed for now)
1091
- decodeNonNegativeInt(buffer);
1092
- // Decode and verify the embedded timestamp
1093
- const embeddedBinaryTimestamp = buffer.shiftN(binaryTimestampLength);
1094
- const embeddedTimestamp = binaryTimestampToTimestamp(embeddedBinaryTimestamp);
1095
- // Verify timestamp integrity
1096
- if (!eqTimestamp(embeddedTimestamp, message.timestamp)) {
1055
+ export const decryptAndDecodeDbChange = (deps) => (message, key) => {
1056
+ // try-catch instead of Result for performance and stacktraces
1057
+ try {
1058
+ const buffer = createBuffer(message.change);
1059
+ const nonce = buffer.shiftN(deps.symmetricCrypto.nonceLength);
1060
+ const ciphertextLength = decodeLength(buffer);
1061
+ const ciphertext = buffer.shiftN(ciphertextLength);
1062
+ const plaintextBytes = deps.symmetricCrypto.decrypt(ciphertext, key, nonce);
1063
+ if (!plaintextBytes.ok)
1064
+ return plaintextBytes;
1065
+ buffer.reset();
1066
+ buffer.extend(plaintextBytes.value);
1067
+ // Decode version (for future compatibility, no validation needed for now)
1068
+ decodeNonNegativeInt(buffer);
1069
+ // Decode and verify the embedded timestamp
1070
+ const embeddedTimestampBytes = buffer.shiftN(timestampBytesLength);
1071
+ const embeddedTimestamp = timestampBytesToTimestamp(embeddedTimestampBytes);
1072
+ // Verify timestamp integrity
1073
+ if (!eqTimestamp(embeddedTimestamp, message.timestamp)) {
1074
+ return err({
1075
+ type: "ProtocolTimestampMismatchError",
1076
+ expected: message.timestamp,
1077
+ embedded: embeddedTimestamp,
1078
+ });
1079
+ }
1080
+ const table = decodeString(buffer);
1081
+ const id = decodeId(buffer);
1082
+ const length = decodeLength(buffer);
1083
+ const values = Object.create(null);
1084
+ for (let i = 0; i < length; i++) {
1085
+ const column = decodeString(buffer);
1086
+ const value = decodeSqliteValue(buffer);
1087
+ values[column] = value;
1088
+ }
1089
+ const dbChange = { table, id, values };
1090
+ return ok(dbChange);
1091
+ }
1092
+ catch (error) {
1097
1093
  return err({
1098
- type: "ProtocolTimestampMismatchError",
1099
- expected: message.timestamp,
1100
- embedded: embeddedTimestamp,
1094
+ type: "ProtocolInvalidDataError",
1095
+ data: message.change,
1096
+ error,
1101
1097
  });
1102
1098
  }
1103
- const table = decodeBase64Url256WithLength(buffer);
1104
- const id = decodeId(buffer);
1105
- const length = decodeLength(buffer);
1106
- const values = Object.create(null);
1107
- for (let i = 0; i < length; i++) {
1108
- const column = decodeBase64Url256WithLength(buffer);
1109
- const value = decodeSqliteValue(buffer);
1110
- values[column] = value;
1111
- }
1112
- const dbChange = { table, id, values };
1113
- return ok(dbChange);
1114
- });
1099
+ };
1115
1100
  /**
1116
1101
  * Encodes a non-negative integer into a variable-length integer format. It's
1117
1102
  * more efficient than encoding via {@link encodeNumber}.
@@ -1178,14 +1163,6 @@ export const decodeNodeId = (buffer) => {
1178
1163
  const bytes = buffer.shiftN(8);
1179
1164
  return bytesToHex(bytes);
1180
1165
  };
1181
- export const encodeBase64Url256 = (buffer, string) => {
1182
- encodeLength(buffer, string);
1183
- buffer.extend(base64Url256ToBytes(string));
1184
- };
1185
- export const decodeBase64Url256WithLength = (buffer) => {
1186
- const length = decodeLength(buffer);
1187
- return decodeBase64Url256(buffer, length);
1188
- };
1189
1166
  // Small ints are encoded into ProtocolValueType, saving one byte per int.
1190
1167
  const isSmallInt = (value) => value >= 0 && value < 20;
1191
1168
  export const ProtocolValueType = {
@@ -1194,18 +1171,20 @@ export const ProtocolValueType = {
1194
1171
  String: 20,
1195
1172
  Number: 21,
1196
1173
  Null: 22,
1197
- Binary: 23,
1174
+ Bytes: 23,
1198
1175
  // We can add more types for other DBs or anything else later.
1199
1176
  // Optimized types
1200
- Id: 30,
1201
- Base64Url256: 31,
1202
- NonNegativeInt: 32,
1203
- Json: 33,
1177
+ NonNegativeInt: 30,
1178
+ // String optimizations
1179
+ EmptyString: 31, // 1 byte vs 2 bytes (50% reduction)
1180
+ Base64Url: 32,
1181
+ Id: 33,
1182
+ Json: 34,
1204
1183
  // new Date().toISOString() - 24 bytes
1205
1184
  // encoded with fixed length - 8 bytes
1206
1185
  // encode as NonNegativeInt - 6 bytes (additional 25% reduction)
1207
- DateIsoWithNonNegativeTime: 34,
1208
- DateIsoWithNegativeTime: 35, // 9 bytes
1186
+ DateIsoWithNonNegativeTime: 35,
1187
+ DateIsoWithNegativeTime: 36, // 9 bytes
1209
1188
  // TODO: Operations (from 40)
1210
1189
  // Increment, Decrement, Patch, whatever.
1211
1190
  };
@@ -1216,9 +1195,13 @@ export const encodeSqliteValue = (buffer, value) => {
1216
1195
  }
1217
1196
  switch (typeof value) {
1218
1197
  case "string": {
1219
- const dateIsoString = DateIsoString.from(value);
1220
- if (dateIsoString.ok) {
1221
- const time = new Date(dateIsoString.value).getTime();
1198
+ if (value === "") {
1199
+ encodeNonNegativeInt(buffer, ProtocolValueType.EmptyString);
1200
+ return;
1201
+ }
1202
+ const dateIso = DateIso.fromParent(value);
1203
+ if (dateIso.ok) {
1204
+ const time = new Date(dateIso.value).getTime();
1222
1205
  if (NonNegativeInt.is(time)) {
1223
1206
  encodeNonNegativeInt(buffer, ProtocolValueType.DateIsoWithNonNegativeTime);
1224
1207
  encodeNonNegativeInt(buffer, time);
@@ -1229,25 +1212,31 @@ export const encodeSqliteValue = (buffer, value) => {
1229
1212
  }
1230
1213
  return;
1231
1214
  }
1232
- const base64Url256 = Base64Url256.from(value);
1233
- if (base64Url256.ok) {
1234
- if (base64Url256.value.length === idTypeValueLength) {
1235
- encodeNonNegativeInt(buffer, ProtocolValueType.Id);
1236
- buffer.extend(base64Url256ToBytes(base64Url256.value));
1237
- return;
1238
- }
1239
- encodeNonNegativeInt(buffer, ProtocolValueType.Base64Url256);
1240
- encodeBase64Url256(buffer, base64Url256.value);
1215
+ const id = Id.fromParent(value);
1216
+ if (id.ok) {
1217
+ encodeNonNegativeInt(buffer, ProtocolValueType.Id);
1218
+ buffer.extend(idToIdBytes(id.value));
1241
1219
  return;
1242
1220
  }
1243
- const jsonValue = JsonValueFromString.fromParent(value);
1244
- if (jsonValue.ok) {
1245
- const jsonBytes = pack(jsonValue.value);
1221
+ const json = Json.fromParent(value);
1222
+ // Only encode as Json if it survives JSON.parse/JSON.stringify round-trip.
1223
+ // Some valid JSON strings like "-0E0" get normalized to "0" during parsing,
1224
+ // which would cause data corruption if we don't verify round-trip safety.
1225
+ if (json.ok && JSON.stringify(jsonToJsonValue(json.value)) === value) {
1226
+ const jsonBytes = packr.pack(jsonToJsonValue(json.value));
1246
1227
  encodeNonNegativeInt(buffer, ProtocolValueType.Json);
1247
1228
  encodeLength(buffer, jsonBytes);
1248
1229
  buffer.extend(jsonBytes);
1249
1230
  return;
1250
1231
  }
1232
+ const base64Url = Base64Url.fromParent(value);
1233
+ if (base64Url.ok) {
1234
+ encodeNonNegativeInt(buffer, ProtocolValueType.Base64Url);
1235
+ const bytes = base64UrlToUint8Array(base64Url.value);
1236
+ encodeLength(buffer, bytes);
1237
+ buffer.extend(bytes);
1238
+ return;
1239
+ }
1251
1240
  encodeNonNegativeInt(buffer, ProtocolValueType.String);
1252
1241
  encodeString(buffer, value);
1253
1242
  return;
@@ -1267,7 +1256,7 @@ export const encodeSqliteValue = (buffer, value) => {
1267
1256
  return;
1268
1257
  }
1269
1258
  }
1270
- encodeNonNegativeInt(buffer, ProtocolValueType.Binary);
1259
+ encodeNonNegativeInt(buffer, ProtocolValueType.Bytes);
1271
1260
  encodeLength(buffer, value);
1272
1261
  buffer.extend(value);
1273
1262
  };
@@ -1283,31 +1272,35 @@ export const decodeSqliteValue = (buffer) => {
1283
1272
  return decodeNumber(buffer);
1284
1273
  case ProtocolValueType.Null:
1285
1274
  return null;
1286
- case ProtocolValueType.Binary: {
1275
+ case ProtocolValueType.Bytes: {
1287
1276
  const length = decodeLength(buffer);
1288
1277
  return buffer.shiftN(length);
1289
1278
  }
1290
- case ProtocolValueType.Id: {
1279
+ case ProtocolValueType.Id:
1291
1280
  return decodeId(buffer);
1292
- }
1293
- case ProtocolValueType.Base64Url256:
1294
- return decodeBase64Url256WithLength(buffer);
1295
1281
  case ProtocolValueType.NonNegativeInt:
1296
1282
  return decodeNonNegativeInt(buffer);
1297
1283
  case ProtocolValueType.Json: {
1298
1284
  const length = decodeLength(buffer);
1299
1285
  const bytes = buffer.shiftN(length);
1300
- return JSON.stringify(unpack(bytes));
1286
+ return JSON.stringify(packr.unpack(bytes));
1301
1287
  }
1302
1288
  case ProtocolValueType.DateIsoWithNonNegativeTime:
1303
1289
  case ProtocolValueType.DateIsoWithNegativeTime: {
1304
1290
  const time = type === ProtocolValueType.DateIsoWithNonNegativeTime
1305
1291
  ? decodeNonNegativeInt(buffer)
1306
1292
  : decodeNumber(buffer);
1307
- const dateIsoString = DateIsoString.fromParent(new Date(time).toISOString());
1308
- if (!dateIsoString.ok)
1309
- throw new ProtocolDecodeError(dateIsoString.error.type);
1310
- return dateIsoString.value;
1293
+ const dateIso = DateIso.fromParent(new Date(time).toISOString());
1294
+ if (!dateIso.ok)
1295
+ throw new ProtocolDecodeError(dateIso.error.type);
1296
+ return dateIso.value;
1297
+ }
1298
+ case ProtocolValueType.EmptyString:
1299
+ return "";
1300
+ case ProtocolValueType.Base64Url: {
1301
+ const length = decodeLength(buffer);
1302
+ const bytes = buffer.shiftN(length);
1303
+ return uint8ArrayToBase64Url(bytes);
1311
1304
  }
1312
1305
  default:
1313
1306
  throw new ProtocolDecodeError("invalid ProtocolValueType");