@evolu/common 6.0.1-preview.8 → 7.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (184) hide show
  1. package/README.md +2 -2
  2. package/dist/src/Array.d.ts +256 -12
  3. package/dist/src/Array.d.ts.map +1 -1
  4. package/dist/src/Array.js +215 -9
  5. package/dist/src/Assert.d.ts +0 -13
  6. package/dist/src/Assert.d.ts.map +1 -1
  7. package/dist/src/Assert.js +0 -15
  8. package/dist/src/Brand.d.ts +75 -0
  9. package/dist/src/Brand.d.ts.map +1 -0
  10. package/dist/src/Brand.js +1 -0
  11. package/dist/src/Buffer.d.ts +1 -1
  12. package/dist/src/Buffer.d.ts.map +1 -1
  13. package/dist/src/Buffer.js +8 -7
  14. package/dist/src/Cache.d.ts +44 -0
  15. package/dist/src/Cache.d.ts.map +1 -0
  16. package/dist/src/Cache.js +52 -0
  17. package/dist/src/Callbacks.d.ts +45 -12
  18. package/dist/src/Callbacks.d.ts.map +1 -1
  19. package/dist/src/Callbacks.js +14 -7
  20. package/dist/src/Console.d.ts +31 -6
  21. package/dist/src/Console.d.ts.map +1 -1
  22. package/dist/src/Console.js +72 -9
  23. package/dist/src/Crypto.d.ts +56 -42
  24. package/dist/src/Crypto.d.ts.map +1 -1
  25. package/dist/src/Crypto.js +40 -53
  26. package/dist/src/Evolu/Db.d.ts +162 -74
  27. package/dist/src/Evolu/Db.d.ts.map +1 -1
  28. package/dist/src/Evolu/Db.js +284 -702
  29. package/dist/src/Evolu/Diff.d.ts +3 -3
  30. package/dist/src/Evolu/Diff.d.ts.map +1 -1
  31. package/dist/src/Evolu/Diff.js +7 -5
  32. package/dist/src/Evolu/Evolu.d.ts +214 -134
  33. package/dist/src/Evolu/Evolu.d.ts.map +1 -1
  34. package/dist/src/Evolu/Evolu.js +189 -180
  35. package/dist/src/Evolu/Internal.d.ts +0 -2
  36. package/dist/src/Evolu/Internal.d.ts.map +1 -1
  37. package/dist/src/Evolu/Internal.js +0 -2
  38. package/dist/src/Evolu/LocalAuth.d.ts +150 -0
  39. package/dist/src/Evolu/LocalAuth.d.ts.map +1 -0
  40. package/dist/src/Evolu/LocalAuth.js +174 -0
  41. package/dist/src/Evolu/Owner.d.ts +273 -120
  42. package/dist/src/Evolu/Owner.d.ts.map +1 -1
  43. package/dist/src/Evolu/Owner.js +130 -104
  44. package/dist/src/Evolu/Platform.d.ts +9 -7
  45. package/dist/src/Evolu/Platform.d.ts.map +1 -1
  46. package/dist/src/Evolu/Protocol.d.ts +268 -240
  47. package/dist/src/Evolu/Protocol.d.ts.map +1 -1
  48. package/dist/src/Evolu/Protocol.js +600 -454
  49. package/dist/src/Evolu/Public.d.ts +6 -8
  50. package/dist/src/Evolu/Public.d.ts.map +1 -1
  51. package/dist/src/Evolu/Public.js +2 -3
  52. package/dist/src/Evolu/PublicKysely.d.ts.map +1 -1
  53. package/dist/src/Evolu/PublicKysely.js +3 -4
  54. package/dist/src/Evolu/Query.d.ts +2 -1
  55. package/dist/src/Evolu/Query.d.ts.map +1 -1
  56. package/dist/src/Evolu/Query.js +1 -1
  57. package/dist/src/Evolu/Relay.d.ts +91 -8
  58. package/dist/src/Evolu/Relay.d.ts.map +1 -1
  59. package/dist/src/Evolu/Relay.js +214 -88
  60. package/dist/src/Evolu/Schema.d.ts +125 -47
  61. package/dist/src/Evolu/Schema.d.ts.map +1 -1
  62. package/dist/src/Evolu/Schema.js +175 -31
  63. package/dist/src/Evolu/Storage.d.ts +249 -27
  64. package/dist/src/Evolu/Storage.d.ts.map +1 -1
  65. package/dist/src/Evolu/Storage.js +198 -92
  66. package/dist/src/Evolu/Sync.d.ts +68 -13
  67. package/dist/src/Evolu/Sync.d.ts.map +1 -1
  68. package/dist/src/Evolu/Sync.js +469 -20
  69. package/dist/src/Evolu/Timestamp.d.ts +83 -30
  70. package/dist/src/Evolu/Timestamp.d.ts.map +1 -1
  71. package/dist/src/Evolu/Timestamp.js +79 -34
  72. package/dist/src/Identicon.d.ts +35 -0
  73. package/dist/src/Identicon.d.ts.map +1 -0
  74. package/dist/src/Identicon.js +143 -0
  75. package/dist/src/Instances.d.ts +34 -0
  76. package/dist/src/Instances.d.ts.map +1 -0
  77. package/dist/src/Instances.js +44 -0
  78. package/dist/src/Number.d.ts +4 -3
  79. package/dist/src/Number.d.ts.map +1 -1
  80. package/dist/src/Number.js +5 -4
  81. package/dist/src/Object.d.ts +10 -4
  82. package/dist/src/Object.d.ts.map +1 -1
  83. package/dist/src/Object.js +9 -3
  84. package/dist/src/Platform.d.ts +20 -0
  85. package/dist/src/Platform.d.ts.map +1 -0
  86. package/dist/src/Platform.js +22 -0
  87. package/dist/src/Random.d.ts +3 -2
  88. package/dist/src/Random.d.ts.map +1 -1
  89. package/dist/src/Relation.d.ts +79 -0
  90. package/dist/src/Relation.d.ts.map +1 -0
  91. package/dist/src/Relation.js +127 -0
  92. package/dist/src/Resources.d.ts +118 -0
  93. package/dist/src/Resources.d.ts.map +1 -0
  94. package/dist/src/Resources.js +197 -0
  95. package/dist/src/Result.d.ts +184 -52
  96. package/dist/src/Result.d.ts.map +1 -1
  97. package/dist/src/Result.js +30 -241
  98. package/dist/src/Skiplist.js +2 -1
  99. package/dist/src/Sqlite.d.ts +89 -5
  100. package/dist/src/Sqlite.d.ts.map +1 -1
  101. package/dist/src/Sqlite.js +136 -9
  102. package/dist/src/Task.d.ts +586 -0
  103. package/dist/src/Task.d.ts.map +1 -0
  104. package/dist/src/Task.js +469 -0
  105. package/dist/src/Time.d.ts +66 -1
  106. package/dist/src/Time.d.ts.map +1 -1
  107. package/dist/src/Time.js +99 -5
  108. package/dist/src/Type.d.ts +676 -343
  109. package/dist/src/Type.d.ts.map +1 -1
  110. package/dist/src/Type.js +718 -467
  111. package/dist/src/Types.d.ts +1 -75
  112. package/dist/src/Types.d.ts.map +1 -1
  113. package/dist/src/WebSocket.d.ts +5 -2
  114. package/dist/src/WebSocket.d.ts.map +1 -1
  115. package/dist/src/WebSocket.js +12 -18
  116. package/dist/src/Worker.d.ts +39 -11
  117. package/dist/src/Worker.d.ts.map +1 -1
  118. package/dist/src/Worker.js +22 -4
  119. package/dist/src/index.d.ts +8 -3
  120. package/dist/src/index.d.ts.map +1 -1
  121. package/dist/src/index.js +8 -3
  122. package/package.json +15 -14
  123. package/src/Array.ts +301 -19
  124. package/src/Assert.ts +0 -21
  125. package/src/Brand.ts +75 -0
  126. package/src/Buffer.ts +7 -7
  127. package/src/Cache.ts +85 -0
  128. package/src/Callbacks.ts +62 -22
  129. package/src/Console.ts +91 -11
  130. package/src/Crypto.ts +94 -90
  131. package/src/Evolu/Db.ts +519 -1026
  132. package/src/Evolu/Diff.ts +7 -5
  133. package/src/Evolu/Evolu.ts +477 -361
  134. package/src/Evolu/Internal.ts +0 -2
  135. package/src/Evolu/LocalAuth.ts +463 -0
  136. package/src/Evolu/Owner.ts +355 -228
  137. package/src/Evolu/Platform.ts +9 -9
  138. package/src/Evolu/Protocol.ts +842 -742
  139. package/src/Evolu/Public.ts +7 -14
  140. package/src/Evolu/PublicKysely.ts +4 -5
  141. package/src/Evolu/Query.ts +3 -2
  142. package/src/Evolu/Relay.ts +406 -103
  143. package/src/Evolu/Schema.ts +323 -91
  144. package/src/Evolu/Storage.ts +559 -137
  145. package/src/Evolu/Sync.ts +819 -36
  146. package/src/Evolu/Timestamp.ts +90 -58
  147. package/src/Identicon.ts +197 -0
  148. package/src/Instances.ts +90 -0
  149. package/src/Number.ts +6 -10
  150. package/src/Object.ts +13 -5
  151. package/src/Platform.ts +26 -0
  152. package/src/Random.ts +3 -2
  153. package/src/Relation.ts +234 -0
  154. package/src/Resources.ts +367 -0
  155. package/src/Result.ts +191 -54
  156. package/src/Skiplist.ts +1 -1
  157. package/src/Sqlite.ts +152 -17
  158. package/src/Task.ts +901 -0
  159. package/src/Time.ts +180 -5
  160. package/src/Type.ts +1135 -730
  161. package/src/Types.ts +1 -77
  162. package/src/WebSocket.ts +27 -25
  163. package/src/Worker.ts +72 -23
  164. package/src/index.ts +8 -3
  165. package/dist/src/Evolu/Config.d.ts +0 -69
  166. package/dist/src/Evolu/Config.d.ts.map +0 -1
  167. package/dist/src/Evolu/Config.js +0 -9
  168. package/dist/src/Evolu/Kysely.d.ts +0 -6
  169. package/dist/src/Evolu/Kysely.d.ts.map +0 -1
  170. package/dist/src/Evolu/Kysely.js +0 -21
  171. package/dist/src/ManyToManyMap.d.ts +0 -26
  172. package/dist/src/ManyToManyMap.d.ts.map +0 -1
  173. package/dist/src/ManyToManyMap.js +0 -92
  174. package/dist/src/NanoId.d.ts +0 -27
  175. package/dist/src/NanoId.d.ts.map +0 -1
  176. package/dist/src/NanoId.js +0 -6
  177. package/dist/src/Promise.d.ts +0 -180
  178. package/dist/src/Promise.d.ts.map +0 -1
  179. package/dist/src/Promise.js +0 -176
  180. package/src/Evolu/Config.ts +0 -83
  181. package/src/Evolu/Kysely.ts +0 -38
  182. package/src/ManyToManyMap.ts +0 -140
  183. package/src/NanoId.ts +0 -39
  184. package/src/Promise.ts +0 -295
@@ -3,29 +3,39 @@
3
3
  *
4
4
  * Evolu Protocol is a local-first, end-to-end encrypted binary synchronization
5
5
  * protocol optimized for minimal size and maximum speed. It enables data sync
6
- * between a client and a relay, clients in a peer-to-peer (P2P) setup, or
7
- * relays with each other.
6
+ * between a client and a relay. In the future, direct peer-to-peer (P2P) sync
7
+ * between clients will be possible without a relay.
8
+ *
9
+ * Relays don't need to sync with each other—clients using those relays will
10
+ * sync them eventually. If a relay is offline (e.g., for maintenance), it will
11
+ * sync automatically later via client sync logic. For relay backup using
12
+ * SQLite, see https://sqlite.org/rsync.html (uses a similar algorithm to Evolu
13
+ * RBSR).
8
14
  *
9
15
  * Evolu Protocol is designed for SQLite but can be extended to any database. It
10
16
  * implements [Range-Based Set
11
17
  * Reconciliation](https://arxiv.org/abs/2212.13567). To learn how RBSR works,
12
18
  * check [Negentropy](https://logperiodic.com/rbsr.html). Evolu Protocol is
13
19
  * similar to Negentropy but uses different encoding and also provides data
14
- * transfer and ownership.
20
+ * transfer, ownership, real-time broadcasting, request-response semantics, and
21
+ * error handling.
15
22
  *
16
- * ### Message Structure
23
+ * ### Message structure
17
24
  *
18
25
  * | Field | Notes |
19
26
  * | :----------------------------- | :------------------------ |
20
27
  * | **Header** | |
21
28
  * | - {@link protocolVersion} | |
22
29
  * | - {@link OwnerId} | {@link Owner} |
23
- * | **Initiator** | |
24
- * | - {@link WriteKeyMode} | |
25
- * | - {@link WriteKey} | If WriteKeyMode >= 1 |
26
- * | - {@link WriteKey} | If WriteKeyMode = 2 (new) |
27
- * | **Non-initiator** | |
30
+ * | - messageType | {@link MessageType} |
31
+ * | **Request (messageType=0)** | |
32
+ * | - hasWriteKey | 0 = no, 1 = yes |
33
+ * | - {@link OwnerWriteKey} | If hasWriteKey = 1 |
34
+ * | - subscriptionFlag | {@link SubscriptionFlags} |
35
+ * | **Response (messageType=1)** | |
28
36
  * | - {@link ProtocolErrorCode} | |
37
+ * | **Broadcast (messageType=2)** | |
38
+ * | - (no additional fields) | |
29
39
  * | **Messages** | |
30
40
  * | - {@link NonNegativeInt} | A number of messages. |
31
41
  * | - {@link EncryptedCrdtMessage} | |
@@ -33,21 +43,14 @@
33
43
  * | - {@link NonNegativeInt} | Number of ranges. |
34
44
  * | - {@link Range} | |
35
45
  *
36
- * ### WriteKey Validation
37
- *
38
- * The initiator sends WriteKeyMode and optionally one or two WriteKeys. One key
39
- * for write operations and two for key rotation (current and new). Note that
40
- * it's ok to not send any key if initiator is going to be synced with readonly
41
- * owner. The non-initiator validates them immediately after parsing the
42
- * initiator header, before processing any messages or ranges.
43
- *
44
- * ### WriteKey Rotation
46
+ * ### WriteKey validation
45
47
  *
46
- * When initiator's {@link WriteKeyMode} is `Rotation`, two WriteKeys are
47
- * present:
48
- *
49
- * 1. Current WriteKey (for validation)
50
- * 2. New WriteKey (to be stored)
48
+ * The initiator sends a hasWriteKey flag and optionally a WriteKey. The
49
+ * WriteKey is required when sending messages as a secure token proving the
50
+ * initiator can write changes. It's ok to not send a WriteKey if the initiator
51
+ * is only syncing (read-only) and not sending messages. The non-initiator
52
+ * validates the WriteKey immediately after parsing the initiator header, before
53
+ * processing any messages or ranges.
51
54
  *
52
55
  * ### Synchronization
53
56
  *
@@ -61,38 +64,48 @@
61
64
  * if further sync is needed or possible, continuing until both sides are
62
65
  * synchronized.
63
66
  *
67
+ * The **non-initiator always responds** to provide sync completion feedback,
68
+ * even with empty messages containing only the header and no error. This allows
69
+ * the initiator to detect when synchronization is complete.
70
+ *
64
71
  * Both **Messages** and **Ranges** are optional, allowing each side to send,
65
72
  * sync, or only subscribe data as needed.
66
73
  *
67
- * When the initiator sends data, the {@link WriteKey} is required in Messages as
68
- * a secure token proving the initiator can write changes. The non-initiator
69
- * responds without a {@link WriteKey}, since the initiator’s request already
70
- * signals it wants data. If the non-initiator detects an issue, it sends an
71
- * error code via the `Error` field in the header back to the initiator. In
72
- * relay-to-relay or P2P sync, both sides may require the {@link WriteKey}
73
- * depending on who is the initiator.
74
+ * When the initiator sends data, the {@link OwnerWriteKey} is required as a
75
+ * secure token proving the initiator can write changes. The non-initiator
76
+ * responds without a {@link OwnerWriteKey}, since the initiator’s request
77
+ * already signals it wants data. If the non-initiator detects an issue, it
78
+ * sends an error code via the `Error` field in the header back to the
79
+ * initiator. In relay-to-relay or P2P sync, both sides may require the
80
+ * {@link OwnerWriteKey} depending on who is the initiator.
74
81
  *
75
- * ### Protocol Errors
82
+ * ### Protocol errors
76
83
  *
77
84
  * The protocol uses error codes in the header to signal issues:
78
85
  *
79
86
  * - {@link ProtocolWriteKeyError}: The provided WriteKey is invalid or missing.
80
- * - {@link ProtocolWriteError}: A write operation failed (e.g., due to storage
81
- * limits or billing).
82
- * - {@link ProtocolSyncError}: A generic or unexpected synchronization failure
87
+ * - {@link ProtocolWriteError}: A serious relay-side write failure occurred.
88
+ * - {@link ProtocolQuotaError}: Storage or billing quota exceeded.
89
+ * - {@link ProtocolSyncError}: A serious relay-side synchronization failure
83
90
  * occurred.
84
- * - {@link ProtocolUnsupportedVersionError}: Protocol version mismatch.
91
+ * - {@link ProtocolVersionError}: Protocol version mismatch.
85
92
  * - {@link ProtocolInvalidDataError}: The message is malformed or corrupted.
86
93
  *
87
- * All protocol errors except `ProtocolInvalidDataError` include the `ownerId`
94
+ * All protocol errors except `ProtocolInvalidDataError` include the `OwnerId`
88
95
  * to allow clients to associate errors with the correct owner.
89
96
  *
90
- * ### Message Size Limit
97
+ * ### Message size limit
91
98
  *
92
99
  * The protocol enforces a strict maximum size for all messages, defined by
93
- * {@link maxProtocolMessageSize}. This ensures every {@link ProtocolMessage} is
94
- * less than or equal to this limit, eliminating the need for applications to
95
- * fragment and reconstruct messages during transmission.
100
+ * {@link ProtocolMessageMaxSize}. This ensures every {@link ProtocolMessage} is
101
+ * less than or equal to this limit, enabling stateless transports, simplified
102
+ * relay implementation, and predictable memory usage. When all messages don't
103
+ * fit within the limit, the protocol automatically continues synchronization in
104
+ * subsequent rounds using range-based reconciliation.
105
+ *
106
+ * Database mutations are limited to 640KB, which is smaller than the protocol
107
+ * message limit to ensure efficient sync with
108
+ * {@link defaultProtocolMessageRangesMaxSize}.
96
109
  *
97
110
  * ### Why Binary?
98
111
  *
@@ -100,13 +113,12 @@
100
113
  *
101
114
  * - Encrypted data doesn’t compress well, unlike plain JSON.
102
115
  * - Message size must be controlled during creation.
103
- * - Sequential byte reading is faster than parsing and can avoid conversions.
116
+ * - Sequential byte reading is faster than parsing and avoids conversions.
104
117
  *
105
118
  * It uses structure-aware encoding, significantly outperforming generic binary
106
119
  * serialization formats with the following optimizations:
107
120
  *
108
121
  * - **NonNegativeInt:** Up to 33% smaller than MessagePack.
109
- * - **Base64Url Strings:** Up to 25% size reduction.
110
122
  * - **DateIso:** Up to 75% smaller.
111
123
  * - **Timestamp Encoding:** Delta encoding for milliseconds and run-length
112
124
  * encoding (RLE) for counters and NodeIds.
@@ -136,16 +148,41 @@
136
148
  * Version negotiation is per-owner, allowing Evolu Protocol to evolve safely
137
149
  * over time and provide clear feedback about version mismatches.
138
150
  *
151
+ * ### Credible exit
152
+ *
153
+ * The protocol specification is intentionally non-configurable to ensure
154
+ * universal compatibility. This design allows applications (users) to switch
155
+ * between any compliant relay without negotiation or compatibility checks
156
+ * beyond version matching. Relays are generic infrastructure that any
157
+ * application can use interchangeably making exit from any single provider
158
+ * technically feasible and economically viable.
159
+ *
139
160
  * @module
140
161
  */
141
162
 
142
- import { sha256 } from "@noble/hashes/sha2";
143
- import { pack, unpack, unpackMultiple } from "msgpackr";
163
+ /**
164
+ * TODO:
165
+ *
166
+ * - The client-relay naming convention in functions like
167
+ * `applyProtocolMessageAsClient` and `applyProtocolMessageAsRelay` is not
168
+ * ideal. In the future, clients will be able to sync directly with each other
169
+ * (P2P), making the current naming misleading. Consider using
170
+ * initiator/non-initiator terminology instead, and consolidate into a single
171
+ * `applyProtocolMessage` function with conditional arguments to reduce code
172
+ * duplication.
173
+ * - Replace try-catch with Result + new Error (to preserve stacktraces). Measure
174
+ * Result overhead, it should be super small.
175
+ * - Allow clients to broadcast messages that are not persisted by relays. This
176
+ * would enable real-time ephemeral data (like cursor positions, typing
177
+ * indicators) to be forwarded by relays without storage overhead.
178
+ */
179
+
180
+ import { Packr } from "msgpackr";
144
181
  import { isNonEmptyReadonlyArray, NonEmptyReadonlyArray } from "../Array.js";
145
182
  import { assert } from "../Assert.js";
183
+ import { Brand } from "../Brand.js";
146
184
  import {
147
185
  Buffer,
148
- BufferError,
149
186
  bytesToHex,
150
187
  bytesToUtf8,
151
188
  createBuffer,
@@ -153,9 +190,9 @@ import {
153
190
  utf8ToBytes,
154
191
  } from "../Buffer.js";
155
192
  import {
156
- CreateRandomBytesDep,
193
+ createPadmePadding,
157
194
  EncryptionKey,
158
- padmePaddingLength,
195
+ RandomBytesDep,
159
196
  SymmetricCryptoDecryptError,
160
197
  SymmetricCryptoDep,
161
198
  } from "../Crypto.js";
@@ -166,253 +203,188 @@ import { err, ok, Result } from "../Result.js";
166
203
  import { SqliteValue } from "../Sqlite.js";
167
204
  import {
168
205
  Base64Url,
169
- DateIsoString,
206
+ base64UrlToUint8Array,
207
+ between,
208
+ DateIso,
170
209
  Id,
171
- idTypeValueLength,
172
- JsonValueFromString,
173
- maxLength,
174
- NanoId,
210
+ IdBytes,
211
+ idBytesToId,
212
+ idBytesTypeValueLength,
213
+ idToIdBytes,
214
+ Int,
215
+ Json,
216
+ jsonToJsonValue,
175
217
  NonNegativeInt,
176
218
  Number,
177
- object,
178
219
  PositiveInt,
179
- record,
220
+ uint8ArrayToBase64Url,
180
221
  } from "../Type.js";
181
- import { Brand, Predicate } from "../Types.js";
222
+ import { Predicate } from "../Types.js";
182
223
  import {
224
+ BaseOwnerError,
183
225
  Owner,
184
226
  OwnerId,
185
- OwnerWithWriteAccess,
186
- WriteKey,
187
- writeKeyLength,
227
+ OwnerIdBytes,
228
+ ownerIdToOwnerIdBytes,
229
+ OwnerWriteKey,
230
+ ownerWriteKeyLength,
188
231
  } from "./Owner.js";
189
232
  import {
190
- BinaryTimestamp,
191
- binaryTimestampToTimestamp,
233
+ BaseRange,
234
+ CrdtMessage,
235
+ DbChange,
236
+ EncryptedCrdtMessage,
237
+ EncryptedDbChange,
238
+ Fingerprint,
239
+ FingerprintRange,
240
+ fingerprintSize,
241
+ InfiniteUpperBound,
242
+ Range,
243
+ RangeType,
244
+ RangeUpperBound,
245
+ SkipRange,
246
+ StorageDep,
247
+ TimestampsRange,
248
+ } from "./Storage.js";
249
+ import {
192
250
  Counter,
251
+ eqTimestamp,
193
252
  Millis,
194
253
  NodeId,
195
254
  Timestamp,
196
- timestampToBinaryTimestamp,
255
+ TimestampBytes,
256
+ timestampBytesLength,
257
+ timestampBytesToTimestamp,
258
+ timestampToTimestampBytes,
197
259
  } from "./Timestamp.js";
198
260
 
199
- /** Maximum size of the entire protocol message in bytes. */
200
- export const maxProtocolMessageSize = 1_000_000 as PositiveInt;
201
-
202
- /** Maximum size of the ranges in bytes. */
203
- export const maxProtocolMessageRangesSize = 30_000 as PositiveInt;
204
-
205
- /** Evolu Protocol Message. */
206
- export type ProtocolMessage = Uint8Array & Brand<"ProtocolMessage">;
207
-
208
- /** Evolu Protocol version. */
209
- export const protocolVersion = 0 as NonNegativeInt;
210
-
211
- export const ProtocolErrorCode = {
212
- NoError: 0,
213
- /** A code for {@link ProtocolWriteKeyError}. */
214
- WriteKeyError: 1,
215
- /** A code for {@link ProtocolWriteError}. */
216
- WriteError: 2,
217
- /** A code for {@link ProtocolSyncError}. */
218
- SyncError: 3,
219
- } as const;
220
-
221
- type ProtocolErrorCode =
222
- (typeof ProtocolErrorCode)[keyof typeof ProtocolErrorCode];
223
-
224
- export const WriteKeyMode = {
225
- None: 0,
226
- Single: 1,
227
- Rotation: 2,
228
- } as const;
229
-
230
- type WriteKeyMode = (typeof WriteKeyMode)[keyof typeof WriteKeyMode];
231
-
232
261
  /**
233
- * Evolu Protocol Storage
262
+ * Evolu uses MessagePack for numbers and JSONs.
234
263
  *
235
- * The protocol is agnostic to storage implementation details—any storage can be
236
- * plugged in, as long as it implements this interface. Implementations must
237
- * handle their own errors; return values only indicates overall success or
238
- * failure.
264
+ * - `variableMapSize: true` - More compact maps, ~5-10% slower encoding
265
+ * - `useRecords: false` - Standard MessagePack without extensions
239
266
  */
240
- export interface Storage {
241
- readonly getSize: (ownerId: BinaryOwnerId) => NonNegativeInt | null;
267
+ const packr = new Packr({ variableMapSize: true, useRecords: false });
242
268
 
243
- readonly fingerprint: (
244
- ownerId: BinaryOwnerId,
245
- begin: NonNegativeInt,
246
- end: NonNegativeInt,
247
- ) => Fingerprint | null;
248
-
249
- /**
250
- * Computes fingerprints with their upper bounds in one call.
251
- *
252
- * This function can be replaced with many fingerprint/findLowerBound calls,
253
- * but implementations can leverage it for batching and more efficient
254
- * fingerprint computation.
255
- */
256
- readonly fingerprintRanges: (
257
- ownerId: BinaryOwnerId,
258
- buckets: ReadonlyArray<NonNegativeInt>,
259
- upperBound?: RangeUpperBound,
260
- ) => ReadonlyArray<FingerprintRange> | null;
261
-
262
- readonly findLowerBound: (
263
- ownerId: BinaryOwnerId,
264
- begin: NonNegativeInt,
265
- end: NonNegativeInt,
266
- upperBound: RangeUpperBound,
267
- ) => NonNegativeInt | null;
268
-
269
- readonly iterate: (
270
- ownerId: BinaryOwnerId,
271
- begin: NonNegativeInt,
272
- end: NonNegativeInt,
273
- callback: (timestamp: BinaryTimestamp, index: NonNegativeInt) => boolean,
274
- ) => void;
275
-
276
- /** Validates the {@link WriteKey} for the given {@link Owner}. */
277
- readonly validateWriteKey: (
278
- ownerId: BinaryOwnerId,
279
- writeKey: WriteKey,
280
- ) => boolean;
281
-
282
- /** Sets the {@link WriteKey} for the given {@link Owner}. */
283
- readonly setWriteKey: (ownerId: BinaryOwnerId, writeKey: WriteKey) => boolean;
284
-
285
- /** Write encrypted {@link CrdtMessage}s to storage. */
286
- readonly writeMessages: (
287
- ownerId: BinaryOwnerId,
288
- messages: NonEmptyReadonlyArray<EncryptedCrdtMessage>,
289
- ) => boolean;
290
-
291
- /** Read encrypted {@link DbChange}s from storage. */
292
- readonly readDbChange: (
293
- ownerId: BinaryOwnerId,
294
- timestamp: BinaryTimestamp,
295
- ) => EncryptedDbChange | null;
296
-
297
- /** Delete all data for the given {@link Owner}. */
298
- readonly deleteOwner: (ownerId: BinaryOwnerId) => boolean;
299
- }
300
-
301
- export interface StorageDep {
302
- readonly storage: Storage;
303
- }
304
-
305
- /** An encrypted {@link CrdtMessage}. */
306
- export interface EncryptedCrdtMessage {
307
- readonly timestamp: Timestamp;
308
- readonly change: EncryptedDbChange;
309
- }
310
-
311
- /** Encrypted DbChange */
312
- export type EncryptedDbChange = Uint8Array & Brand<"EncryptedDbChange">;
269
+ const minProtocolMessageMaxSize = 1_000_000;
270
+ const maxProtocolMessageMaxSize = 100_000_000;
313
271
 
314
272
  /**
315
- * A CRDT message that combines a unique {@link Timestamp} with a
316
- * {@link DbChange}.
273
+ * Protocol message maximum size.
274
+ *
275
+ * Defines the upper limit for how large a single protocol message can be.
276
+ * Implementations must enforce a maximum size between 1MB and 100MB to ensure
277
+ * compatibility across all Evolu implementations (the maximum size of mutation
278
+ * change is hardcoded and enforced hence the maximum size can't be smaller).
279
+ *
280
+ * Larger maximum sizes can be configured by relays to reduce roundtrips. For
281
+ * example, a dedicated relay with ample resources could configure a 100MB
282
+ * maximum to minimize roundtrips for large syncs.
283
+ *
284
+ * Only relays can safely configure larger sizes, as clients will handle them.
285
+ * Increasing this value on the client side would break compatibility with
286
+ * relays that enforce smaller limits.
317
287
  */
318
- export interface CrdtMessage {
319
- readonly timestamp: Timestamp;
320
- readonly change: DbChange;
321
- }
288
+ export const ProtocolMessageMaxSize = between(
289
+ minProtocolMessageMaxSize,
290
+ maxProtocolMessageMaxSize,
291
+ )(Int);
322
292
 
323
- /**
324
- * Base64Url string with maximum length of 256 characters. Encoding strings as
325
- * Base64UrlString saves up to 25% in size compared to regular strings.
326
- */
327
- export const Base64Url256 = maxLength(256)(Base64Url);
328
- export type Base64Url256 = typeof Base64Url256.Type;
293
+ export type ProtocolMessageMaxSize = typeof ProtocolMessageMaxSize.Type;
329
294
 
330
295
  /**
331
- * A DbChange is a change to a table row. Together with a unique
332
- * {@link Timestamp}, it forms a {@link CrdtMessage}.
296
+ * Default {@link ProtocolMessageMaxSize} (1MB).
297
+ *
298
+ * The standard size used across Evolu implementations. Relays with more
299
+ * resources can configure larger sizes to reduce roundtrips.
333
300
  */
334
- export const DbChange = object({
335
- table: Base64Url256,
336
- id: Id,
337
- values: record(Base64Url256, SqliteValue),
338
- });
339
- export type DbChange = typeof DbChange.Type;
340
-
341
- export const RangeType = {
342
- Fingerprint: 1,
343
- Skip: 0,
344
- Timestamps: 2,
345
- } as const;
346
-
347
- export type RangeType = (typeof RangeType)[keyof typeof RangeType];
348
-
349
- export const InfiniteUpperBound = Symbol("InfiniteUpperBound");
350
- export type InfiniteUpperBound = typeof InfiniteUpperBound;
301
+ export const defaultProtocolMessageMaxSize =
302
+ minProtocolMessageMaxSize as ProtocolMessageMaxSize;
351
303
 
352
304
  /**
353
- * Union type for Range's upperBound: either a {@link BinaryTimestamp} or
354
- * {@link InfiniteUpperBound}.
305
+ * Protocol message ranges maximum size.
306
+ *
307
+ * Defines the upper limit for how large the ranges section of a protocol
308
+ * message can be. Implementations must enforce a maximum size between 3KB and
309
+ * 100KB to ensure compatibility.
310
+ *
311
+ * The upper bound is set to ensure ranges fit within the default 1MB
312
+ * {@link defaultProtocolMessageMaxSize}, maintaining compatibility between all
313
+ * clients and relays.
355
314
  */
356
- export type RangeUpperBound = BinaryTimestamp | InfiniteUpperBound;
357
-
358
- interface BaseRange {
359
- readonly upperBound: RangeUpperBound;
360
- }
361
-
362
- export interface SkipRange extends BaseRange {
363
- readonly type: typeof RangeType.Skip;
364
- }
365
-
366
- export interface FingerprintRange extends BaseRange {
367
- readonly type: typeof RangeType.Fingerprint;
368
- readonly fingerprint: Fingerprint;
369
- }
315
+ export const ProtocolMessageRangesMaxSize = between(3_000, 100_000)(Int);
316
+ export type ProtocolMessageRangesMaxSize =
317
+ typeof ProtocolMessageRangesMaxSize.Type;
370
318
 
371
319
  /**
372
- * A cryptographic hash used for efficiently comparing collections of
373
- * {@link BinaryTimestamp}s.
320
+ * Default {@link ProtocolMessageRangesMaxSize} (30KB).
374
321
  *
375
- * It consists of the first {@link fingerprintSize} bytes of the SHA-256 hash of
376
- * one or more timestamps.
322
+ * The standard size used across Evolu implementations. Relays with more
323
+ * resources can configure larger sizes to reduce roundtrips.
377
324
  */
378
- export type Fingerprint = Uint8Array & Brand<"Fingerprint">;
325
+ export const defaultProtocolMessageRangesMaxSize =
326
+ 30_000 as ProtocolMessageRangesMaxSize;
379
327
 
380
- export const fingerprintSize = 12 as NonNegativeInt;
328
+ /** Evolu Protocol Message. */
329
+ export type ProtocolMessage = Uint8Array & Brand<"ProtocolMessage">;
381
330
 
382
- /** A fingerprint of an empty range. */
383
- export const zeroFingerprint = new Uint8Array(fingerprintSize) as Fingerprint;
331
+ /** Evolu Protocol version. */
332
+ export const protocolVersion = NonNegativeInt.orThrow(1);
333
+
334
+ export const MessageType = {
335
+ /** Request message from initiator (client) to non-initiator (relay). */
336
+ Request: 0,
337
+ /** Response message from non-initiator (relay) to initiator (client). */
338
+ Response: 1,
339
+ /** Broadcast message from non-initiator (relay) to subscribed clients. */
340
+ Broadcast: 2,
341
+ } as const;
384
342
 
385
- export interface TimestampsRange extends BaseRange {
386
- readonly type: typeof RangeType.Timestamps;
387
- readonly timestamps: ReadonlyArray<BinaryTimestamp>;
388
- }
343
+ export type MessageType = (typeof MessageType)[keyof typeof MessageType];
389
344
 
390
- export interface TimestampsRangeWithTimestampsBuffer extends BaseRange {
391
- readonly type: typeof RangeType.Timestamps;
392
- readonly timestamps: TimestampsBuffer;
393
- }
345
+ export const SubscriptionFlags = {
346
+ /** No subscription changes for this owner. */
347
+ None: 0,
348
+ /** Subscribe to updates for this owner. */
349
+ Subscribe: 1,
350
+ /** Unsubscribe from updates for this owner. */
351
+ Unsubscribe: 2,
352
+ } as const;
353
+
354
+ export type SubscriptionFlag =
355
+ (typeof SubscriptionFlags)[keyof typeof SubscriptionFlags];
356
+
357
+ export const ProtocolErrorCode = {
358
+ NoError: 0,
359
+ /** A code for {@link ProtocolWriteKeyError}. */
360
+ WriteKeyError: 1,
361
+ /** A code for {@link ProtocolWriteError}. */
362
+ WriteError: 2,
363
+ /** A code for {@link ProtocolQuotaError}. */
364
+ QuotaError: 3,
365
+ /** A code for {@link ProtocolSyncError}. */
366
+ SyncError: 4,
367
+ } as const;
394
368
 
395
- export type Range = SkipRange | FingerprintRange | TimestampsRange;
369
+ type ProtocolErrorCode =
370
+ (typeof ProtocolErrorCode)[keyof typeof ProtocolErrorCode];
396
371
 
397
372
  export type ProtocolError =
398
- | ProtocolUnsupportedVersionError
373
+ | ProtocolVersionError
399
374
  | ProtocolInvalidDataError
400
375
  | ProtocolWriteKeyError
401
376
  | ProtocolWriteError
402
- | ProtocolSyncError;
403
-
404
- /** Base interface for all protocol errors. */
405
- export interface ProtocolErrorBase {
406
- readonly ownerId: OwnerId;
407
- }
377
+ | ProtocolSyncError
378
+ | ProtocolQuotaError
379
+ | ProtocolTimestampMismatchError;
408
380
 
409
381
  /**
410
382
  * Represents a version mismatch in the Evolu Protocol. Occurs when the
411
383
  * initiator and non-initiator are using incompatible protocol versions.
412
384
  */
413
- export interface ProtocolUnsupportedVersionError extends ProtocolErrorBase {
414
- readonly type: "ProtocolUnsupportedVersionError";
415
- readonly unsupportedVersion: NonNegativeInt;
385
+ export interface ProtocolVersionError extends BaseOwnerError {
386
+ readonly type: "ProtocolVersionError";
387
+ readonly version: NonNegativeInt;
416
388
  /** Indicates which side is obsolete and should update. */
417
389
  readonly isInitiator: boolean;
418
390
  }
@@ -424,44 +396,72 @@ export interface ProtocolInvalidDataError {
424
396
  readonly error: unknown;
425
397
  }
426
398
 
427
- /** Error when a {@link WriteKey} is invalid, missing, or fails validation. */
428
- export interface ProtocolWriteKeyError extends ProtocolErrorBase {
399
+ /** Error when a {@link OwnerWriteKey} is invalid, missing, or fails validation. */
400
+ export interface ProtocolWriteKeyError extends BaseOwnerError {
429
401
  readonly type: "ProtocolWriteKeyError";
430
402
  }
431
403
 
432
404
  /**
433
- * Error when a write fails due to storage limits or billing requirements.
434
- * Indicates the need to expand capacity or resolve payment issues.
405
+ * Error indicating a serious relay-side write failure. Clients should log this
406
+ * error and show a generic sync error to the user.
435
407
  */
436
- export interface ProtocolWriteError extends ProtocolErrorBase {
408
+ export interface ProtocolWriteError extends BaseOwnerError {
437
409
  readonly type: "ProtocolWriteError";
438
410
  }
439
411
 
440
412
  /**
441
- * Error indicating a synchronization failure during the protocol exchange. Used
442
- * for unexpected or generic sync errors not covered by other error types.
413
+ * Error when storage or billing quota is exceeded.
414
+ *
415
+ * When relay rejects writes due to quota, the affected device stops syncing
416
+ * because RBSR requires both sides to converge—if the relay won't accept the
417
+ * client's data, they can never reach the same state. Only the device with
418
+ * excess local data is affected. Other devices that haven't exceeded quota can
419
+ * still sync normally.
420
+ *
421
+ * Clients should prompt the user to contact the relay provider or upgrade their
422
+ * plan. Quota monitoring and management is the relay provider's
423
+ * responsibility.
443
424
  */
444
- export interface ProtocolSyncError extends ProtocolErrorBase {
425
+ export interface ProtocolQuotaError extends BaseOwnerError {
426
+ readonly type: "ProtocolQuotaError";
427
+ }
428
+
429
+ /**
430
+ * Error indicating a serious relay-side synchronization failure. Clients should
431
+ * log this error and show a generic sync error to the user.
432
+ */
433
+ export interface ProtocolSyncError extends BaseOwnerError {
445
434
  readonly type: "ProtocolSyncError";
446
435
  }
447
436
 
437
+ /**
438
+ * Error when embedded timestamp doesn't match expected timestamp in
439
+ * EncryptedDbChange. Indicates potential tampering or corruption of CRDT
440
+ * messages.
441
+ */
442
+ export interface ProtocolTimestampMismatchError {
443
+ readonly type: "ProtocolTimestampMismatchError";
444
+ readonly expected: Timestamp;
445
+ readonly timestamp: Timestamp;
446
+ }
447
+
448
448
  /**
449
449
  * Creates a {@link ProtocolMessage} from CRDT messages.
450
450
  *
451
- * If the message size would exceed {@link maxProtocolMessageSize}, the protocol
452
- * ensures all messages will be sent in the next round(s) even over
451
+ * If the message size would exceed {@link defaultProtocolMessageMaxSize}, the
452
+ * protocol ensures all messages will be sent in the next round(s) even over
453
453
  * unidirectional and stateless transports.
454
454
  */
455
455
  export const createProtocolMessageFromCrdtMessages =
456
- (deps: SymmetricCryptoDep & CreateRandomBytesDep) =>
456
+ (deps: RandomBytesDep & SymmetricCryptoDep) =>
457
457
  (
458
- owner: OwnerWithWriteAccess,
458
+ owner: Owner,
459
459
  messages: NonEmptyReadonlyArray<CrdtMessage>,
460
- maxSize?: PositiveInt,
460
+ maxSize?: ProtocolMessageMaxSize,
461
461
  ): ProtocolMessage => {
462
462
  const buffer = createProtocolMessageBuffer(owner.id, {
463
- type: "initiator",
464
- totalMaxSize: maxSize ?? maxProtocolMessageSize,
463
+ messageType: MessageType.Request,
464
+ totalMaxSize: maxSize ?? defaultProtocolMessageMaxSize,
465
465
  writeKey: owner.writeKey,
466
466
  });
467
467
 
@@ -469,7 +469,7 @@ export const createProtocolMessageFromCrdtMessages =
469
469
 
470
470
  for (const message of messages) {
471
471
  const change = encodeAndEncryptDbChange(deps)(
472
- message.change,
472
+ message,
473
473
  owner.encryptionKey,
474
474
  );
475
475
  const encryptedCrdtMessage = { timestamp: message.timestamp, change };
@@ -490,14 +490,14 @@ export const createProtocolMessageFromCrdtMessages =
490
490
  *
491
491
  * The ideal approach would be to send three ranges (skip, fingerprint,
492
492
  * skip) where the fingerprint of unsent messages would act as narrow sync
493
- * probe. I think we can send {@link zeroFingerprint} which can be
494
- * interpreted as an indication that the other side should reply with
493
+ * probe. I think we can send `zeroFingerprint` which can be interpreted
494
+ * as an indication that the other side should reply with
495
495
  * {@link TimestampsRange}, so no need to restart syncing.
496
496
  *
497
497
  * For now, using a random fingerprint avoids extra complexity and is good
498
498
  * enough for this case.
499
499
  */
500
- const randomFingerprint = deps.createRandomBytes(
500
+ const randomFingerprint = deps.randomBytes.create(
501
501
  fingerprintSize,
502
502
  ) as unknown as Fingerprint;
503
503
 
@@ -515,17 +515,23 @@ export const createProtocolMessageFromCrdtMessages =
515
515
  /** Creates a {@link ProtocolMessage} for sync. */
516
516
  export const createProtocolMessageForSync =
517
517
  (deps: StorageDep) =>
518
- (ownerId: OwnerId): ProtocolMessage | null => {
519
- const buffer = createProtocolMessageBuffer(ownerId, { type: "initiator" });
520
- const binaryOwnerId = ownerIdToBinaryOwnerId(ownerId);
518
+ (
519
+ ownerId: OwnerId,
520
+ subscriptionFlag?: SubscriptionFlag,
521
+ ): ProtocolMessage | null => {
522
+ const buffer = createProtocolMessageBuffer(ownerId, {
523
+ messageType: MessageType.Request,
524
+ subscriptionFlag: subscriptionFlag ?? SubscriptionFlags.None,
525
+ });
526
+ const ownerIdBytes = ownerIdToOwnerIdBytes(ownerId);
521
527
 
522
- const size = deps.storage.getSize(binaryOwnerId);
528
+ const size = deps.storage.getSize(ownerIdBytes);
523
529
  // Errors are handled by the storage.
524
530
  if (size == null) return null;
525
531
 
526
532
  splitRange(deps)(
527
- binaryOwnerId,
528
- 0 as NonNegativeInt,
533
+ ownerIdBytes,
534
+ NonNegativeInt.orThrow(0),
529
535
  size,
530
536
  InfiniteUpperBound,
531
537
  buffer,
@@ -534,18 +540,13 @@ export const createProtocolMessageForSync =
534
540
  return buffer.unwrap();
535
541
  };
536
542
 
537
- /** Creates a ProtocolMessage for {@link WriteKey} rotation. */
538
- export const createProtocolMessageForWriteKeyRotation = (
543
+ export const createProtocolMessageForUnsubscribe = (
539
544
  ownerId: OwnerId,
540
- currentWriteKey: WriteKey,
541
- newWriteKey: WriteKey,
542
- ): ProtocolMessage => {
543
- const buffer = createProtocolMessageBuffer(ownerId, {
544
- type: "initiator",
545
- writeKey: [currentWriteKey, newWriteKey],
546
- });
547
- return buffer.unwrap();
548
- };
545
+ ): ProtocolMessage =>
546
+ createProtocolMessageBuffer(ownerId, {
547
+ messageType: MessageType.Request,
548
+ subscriptionFlag: SubscriptionFlags.Unsubscribe,
549
+ }).unwrap();
549
550
 
550
551
  /**
551
552
  * Mutable builder for constructing {@link ProtocolMessage} respecting size
@@ -574,24 +575,27 @@ export interface ProtocolMessageBuffer {
574
575
  export const createProtocolMessageBuffer = (
575
576
  ownerId: OwnerId,
576
577
  options: {
577
- readonly totalMaxSize?: PositiveInt | undefined;
578
- readonly rangesMaxSize?: PositiveInt | undefined;
578
+ readonly totalMaxSize?: ProtocolMessageMaxSize | undefined;
579
+ readonly rangesMaxSize?: ProtocolMessageRangesMaxSize | undefined;
579
580
  readonly version?: NonNegativeInt;
580
581
  } & (
581
582
  | {
582
- readonly type: "initiator";
583
- /** Single key or [current, new] for rotation. */
584
- readonly writeKey?: WriteKey | readonly [WriteKey, WriteKey];
583
+ readonly messageType: typeof MessageType.Request;
584
+ readonly writeKey?: OwnerWriteKey;
585
+ readonly subscriptionFlag?: SubscriptionFlag;
585
586
  }
586
587
  | {
587
- readonly type: "non-initiator";
588
+ readonly messageType: typeof MessageType.Response;
588
589
  readonly errorCode: ProtocolErrorCode;
589
590
  }
591
+ | {
592
+ readonly messageType: typeof MessageType.Broadcast;
593
+ }
590
594
  ),
591
595
  ): ProtocolMessageBuffer => {
592
596
  const {
593
- totalMaxSize = maxProtocolMessageSize,
594
- rangesMaxSize = maxProtocolMessageRangesSize,
597
+ totalMaxSize = defaultProtocolMessageMaxSize,
598
+ rangesMaxSize = defaultProtocolMessageRangesMaxSize,
595
599
  version = protocolVersion,
596
600
  } = options;
597
601
 
@@ -609,20 +613,19 @@ export const createProtocolMessageBuffer = (
609
613
  };
610
614
 
611
615
  encodeNonNegativeInt(buffers.header, version);
612
- buffers.header.extend(ownerIdToBinaryOwnerId(ownerId));
616
+ buffers.header.extend(ownerIdToOwnerIdBytes(ownerId));
617
+ buffers.header.extend([options.messageType]);
613
618
 
614
- if (options.type === "initiator") {
619
+ if (options.messageType === MessageType.Request) {
615
620
  if (!options.writeKey) {
616
- buffers.header.extend([WriteKeyMode.None]);
617
- } else if (!Array.isArray(options.writeKey)) {
618
- buffers.header.extend([WriteKeyMode.Single]);
619
- buffers.header.extend(options.writeKey as WriteKey);
621
+ buffers.header.extend([0]);
620
622
  } else {
621
- buffers.header.extend([WriteKeyMode.Rotation]);
622
- buffers.header.extend(options.writeKey[0] as WriteKey); // current
623
- buffers.header.extend(options.writeKey[1] as WriteKey); // new
623
+ buffers.header.extend([1]);
624
+ buffers.header.extend(options.writeKey);
624
625
  }
625
- } else {
626
+ const subscriptionFlag = options.subscriptionFlag ?? SubscriptionFlags.None;
627
+ buffers.header.extend([subscriptionFlag]);
628
+ } else if (options.messageType === MessageType.Response) {
626
629
  buffers.header.extend([options.errorCode]);
627
630
  }
628
631
 
@@ -631,7 +634,7 @@ export const createProtocolMessageBuffer = (
631
634
  const isWithinSizeLimits = () => getSize() <= totalMaxSize;
632
635
 
633
636
  const getSize = () =>
634
- (getHeaderAndMessagesSize() + getRangesSize()) as PositiveInt;
637
+ PositiveInt.orThrow(getHeaderAndMessagesSize() + getRangesSize());
635
638
 
636
639
  const getHeaderAndMessagesSize = () =>
637
640
  buffers.header.getLength() +
@@ -705,10 +708,15 @@ export const createProtocolMessageBuffer = (
705
708
  },
706
709
 
707
710
  addRange: (range) => {
711
+ assert(
712
+ options.messageType !== MessageType.Broadcast,
713
+ "Cannot add a range into broadcast message",
714
+ );
708
715
  assert(
709
716
  !isLastRangeInfinite,
710
717
  "Cannot add a range after an InfiniteUpperBound range",
711
718
  );
719
+
712
720
  isLastRangeInfinite = range.upperBound === InfiniteUpperBound;
713
721
 
714
722
  /**
@@ -718,13 +726,16 @@ export const createProtocolMessageBuffer = (
718
726
  */
719
727
  if (range.upperBound !== InfiniteUpperBound)
720
728
  buffers.ranges.timestamps.add(
721
- binaryTimestampToTimestamp(range.upperBound),
729
+ timestampBytesToTimestamp(range.upperBound),
722
730
  );
723
731
  else {
724
732
  buffers.ranges.timestamps.addInfinite();
725
733
  }
726
734
 
727
- encodeNonNegativeInt(buffers.ranges.types, range.type as NonNegativeInt);
735
+ encodeNonNegativeInt(
736
+ buffers.ranges.types,
737
+ NonNegativeInt.orThrow(range.type),
738
+ );
728
739
 
729
740
  switch (range.type) {
730
741
  case RangeType.Skip:
@@ -765,6 +776,11 @@ export const createProtocolMessageBuffer = (
765
776
  };
766
777
  };
767
778
 
779
+ export interface TimestampsRangeWithTimestampsBuffer extends BaseRange {
780
+ readonly type: typeof RangeType.Timestamps;
781
+ readonly timestamps: TimestampsBuffer;
782
+ }
783
+
768
784
  export interface TimestampsBuffer {
769
785
  readonly add: (timestamp: Timestamp) => void;
770
786
  readonly addInfinite: () => void;
@@ -774,7 +790,7 @@ export interface TimestampsBuffer {
774
790
  }
775
791
 
776
792
  export const createTimestampsBuffer = (): TimestampsBuffer => {
777
- let count = 0 as NonNegativeInt;
793
+ let count = NonNegativeInt.orThrow(0);
778
794
  const countBuffer = createBuffer();
779
795
 
780
796
  const syncCount = () => {
@@ -841,9 +857,9 @@ const createRunLengthEncoder = <T>(
841
857
  encodeValue: (buffer: Buffer, value: T) => void,
842
858
  ): RunLengthEncoder<T> => {
843
859
  const buffer = createBuffer();
844
- let previousLength = 0 as NonNegativeInt;
860
+ let previousLength = NonNegativeInt.orThrow(0);
845
861
  let previousValue = null as T | null;
846
- let runLength = 0 as NonNegativeInt;
862
+ let runLength = NonNegativeInt.orThrow(0);
847
863
 
848
864
  return {
849
865
  add: (value) => {
@@ -852,7 +868,7 @@ const createRunLengthEncoder = <T>(
852
868
  buffer.truncate(previousLength);
853
869
  } else {
854
870
  previousValue = value;
855
- runLength = 1 as NonNegativeInt;
871
+ runLength = NonNegativeInt.orThrow(1);
856
872
  }
857
873
  previousLength = buffer.getLength();
858
874
  encodeValue(buffer, value);
@@ -866,40 +882,61 @@ const createRunLengthEncoder = <T>(
866
882
  };
867
883
 
868
884
  export interface ApplyProtocolMessageAsClientOptions {
869
- getWriteKey?: (ownerId: OwnerId) => WriteKey | null;
885
+ getWriteKey?: (ownerId: OwnerId) => OwnerWriteKey | null;
870
886
 
871
- /** For testing purposes only; should not be used in production. */
872
- version?: NonNegativeInt;
887
+ rangesMaxSize?: ProtocolMessageRangesMaxSize;
873
888
 
874
- totalMaxSize?: PositiveInt;
875
- rangesMaxSize?: PositiveInt;
889
+ /** For tests only. */
890
+ version?: NonNegativeInt;
876
891
  }
877
892
 
893
+ /**
894
+ * Result type for {@link applyProtocolMessageAsClient} that distinguishes
895
+ * between responses to client requests and broadcast messages.
896
+ */
897
+ export type ApplyProtocolMessageAsClientResult =
898
+ | { readonly type: "response"; readonly message: ProtocolMessage }
899
+ | { readonly type: "no-response" }
900
+ | { readonly type: "broadcast" };
901
+
878
902
  export const applyProtocolMessageAsClient =
879
903
  (deps: StorageDep) =>
880
- (
904
+ async (
881
905
  inputMessage: Uint8Array,
882
- {
883
- getWriteKey,
884
- version = protocolVersion,
885
- totalMaxSize,
886
- rangesMaxSize,
887
- }: ApplyProtocolMessageAsClientOptions = {},
888
- ): Result<ProtocolMessage | null, ProtocolError> =>
889
- tryDecodeProtocolData<ProtocolMessage | null, ProtocolError>(
890
- inputMessage,
891
- (input) => {
892
- const [requestedVersion, ownerId] = decodeVersionAndOwner(input);
893
-
894
- if (requestedVersion !== version) {
895
- return err<ProtocolUnsupportedVersionError>({
896
- type: "ProtocolUnsupportedVersionError",
897
- unsupportedVersion: requestedVersion,
898
- isInitiator: version < requestedVersion,
899
- ownerId,
900
- });
901
- }
906
+ options: ApplyProtocolMessageAsClientOptions = {},
907
+ ): Promise<
908
+ Result<
909
+ ApplyProtocolMessageAsClientResult,
910
+ | ProtocolInvalidDataError
911
+ | ProtocolSyncError
912
+ | ProtocolVersionError
913
+ | ProtocolWriteError
914
+ | ProtocolWriteKeyError
915
+ | ProtocolQuotaError
916
+ >
917
+ > => {
918
+ try {
919
+ const input = createBuffer(inputMessage);
920
+ const [requestedVersion, ownerId] = decodeVersionAndOwner(input);
921
+ const version = options.version ?? protocolVersion;
922
+
923
+ if (requestedVersion !== version) {
924
+ return err<ProtocolVersionError>({
925
+ type: "ProtocolVersionError",
926
+ version: requestedVersion,
927
+ isInitiator: version < requestedVersion,
928
+ ownerId,
929
+ });
930
+ }
931
+
932
+ const messageType = input.shift() as MessageType;
933
+ assert(
934
+ messageType === MessageType.Response ||
935
+ messageType === MessageType.Broadcast,
936
+ "Invalid MessageType",
937
+ );
902
938
 
939
+ if (messageType === MessageType.Response) {
903
940
  const errorCode = input.shift() as ProtocolErrorCode;
904
941
  if (errorCode !== ProtocolErrorCode.NoError) {
905
942
  switch (errorCode) {
@@ -913,6 +950,11 @@ export const applyProtocolMessageAsClient =
913
950
  type: "ProtocolWriteError",
914
951
  ownerId,
915
952
  });
953
+ case ProtocolErrorCode.QuotaError:
954
+ return err<ProtocolQuotaError>({
955
+ type: "ProtocolQuotaError",
956
+ ownerId,
957
+ });
916
958
  case ProtocolErrorCode.SyncError:
917
959
  return err<ProtocolSyncError>({
918
960
  type: "ProtocolSyncError",
@@ -924,183 +966,248 @@ export const applyProtocolMessageAsClient =
924
966
  );
925
967
  }
926
968
  }
969
+ }
927
970
 
928
- const messages = decodeMessages(input);
929
- const binaryOwnerId = ownerIdToBinaryOwnerId(ownerId);
971
+ const messages = decodeMessages(input);
972
+ const ownerIdBytes = ownerIdToOwnerIdBytes(ownerId);
930
973
 
931
- if (
932
- isNonEmptyReadonlyArray(messages) &&
933
- !deps.storage.writeMessages(binaryOwnerId, messages)
934
- ) {
935
- return ok(null);
936
- }
974
+ if (isNonEmptyReadonlyArray(messages)) {
975
+ const writeResult = await deps.storage.writeMessages(
976
+ ownerIdBytes,
977
+ messages,
978
+ );
979
+ // Errors are handled by the Storage. Here we just stop syncing.
980
+ if (!writeResult.ok) return ok({ type: "no-response" });
981
+ }
937
982
 
938
- if (!getWriteKey) return ok(null);
939
- const writeKey = getWriteKey(ownerId);
940
- if (writeKey == null) return ok(null);
983
+ // Now: No writeKey, no sync.
984
+ // TODO: Allow to sync SharedReadonlyOwner
985
+ // Without local changes, writeKey will not be required.
986
+ // With local changes, writeKey will be required and if not provided,
987
+ // the sync will stop.
988
+ const writeKey = options.getWriteKey?.(ownerId);
989
+ if (writeKey == null) {
990
+ return ok({ type: "no-response" });
991
+ }
941
992
 
942
- const output = createProtocolMessageBuffer(ownerId, {
943
- type: "initiator",
944
- writeKey,
945
- totalMaxSize,
946
- rangesMaxSize,
947
- });
993
+ if (messageType === MessageType.Broadcast) {
994
+ return ok({ type: "broadcast" });
995
+ }
948
996
 
949
- return sync(deps)("initiator", input, output, binaryOwnerId);
950
- },
951
- );
997
+ const ranges = decodeRanges(input);
998
+
999
+ if (!isNonEmptyReadonlyArray(ranges)) {
1000
+ return ok({ type: "no-response" });
1001
+ }
1002
+
1003
+ const output = createProtocolMessageBuffer(ownerId, {
1004
+ messageType: MessageType.Request,
1005
+ writeKey,
1006
+ rangesMaxSize: options.rangesMaxSize,
1007
+ });
1008
+
1009
+ const syncResult = sync(deps)(ranges, output, ownerIdBytes);
1010
+
1011
+ // Client sync error (handled via Storage) or no changes.
1012
+ if (!syncResult.ok || !syncResult.value) {
1013
+ return ok({ type: "no-response" });
1014
+ }
1015
+
1016
+ return ok({ type: "response", message: output.unwrap() });
1017
+ } catch (error) {
1018
+ return err<ProtocolInvalidDataError>({
1019
+ type: "ProtocolInvalidDataError",
1020
+ data: inputMessage,
1021
+ error,
1022
+ });
1023
+ }
1024
+ };
952
1025
 
953
1026
  export interface ApplyProtocolMessageAsRelayOptions {
954
1027
  /** To subscribe an owner for broadcasting. */
955
1028
  subscribe?: (ownerId: OwnerId) => void;
956
1029
 
1030
+ /** To unsubscribe an owner from broadcasting. */
1031
+ unsubscribe?: (ownerId: OwnerId) => void;
1032
+
957
1033
  /** To broadcast a protocol message to all subscribers. */
958
1034
  broadcast?: (ownerId: OwnerId, message: ProtocolMessage) => void;
959
1035
 
960
- totalMaxSize?: PositiveInt;
961
- rangesMaxSize?: PositiveInt;
1036
+ totalMaxSize?: ProtocolMessageMaxSize;
1037
+ rangesMaxSize?: ProtocolMessageRangesMaxSize;
1038
+ }
1039
+
1040
+ /**
1041
+ * Result type for {@link applyProtocolMessageAsRelay}.
1042
+ *
1043
+ * Unlike {@link ApplyProtocolMessageAsClientResult}, relays always respond with
1044
+ * a message to provide sync completion feedback. This ensures the initiator can
1045
+ * reliably detect when synchronization is complete, even when there's nothing
1046
+ * to sync. Clients may choose not to respond in certain cases (like when they
1047
+ * receive broadcast messages or when they lack a write key for syncing).
1048
+ */
1049
+ export interface ApplyProtocolMessageAsRelayResult {
1050
+ readonly type: "response";
1051
+ readonly message: ProtocolMessage;
962
1052
  }
963
1053
 
964
1054
  export const applyProtocolMessageAsRelay =
965
1055
  (deps: StorageDep) =>
966
- (
1056
+ async (
967
1057
  inputMessage: Uint8Array,
968
- {
969
- subscribe,
970
- broadcast,
971
- totalMaxSize,
972
- rangesMaxSize,
973
- }: ApplyProtocolMessageAsRelayOptions = {},
974
- /** For testing purposes only; should not be used in production. */
1058
+ options: ApplyProtocolMessageAsRelayOptions = {},
1059
+ /** For tests only. */
975
1060
  version = protocolVersion,
976
- ): Result<ProtocolMessage | null, ProtocolInvalidDataError> =>
977
- tryDecodeProtocolData(inputMessage, (input) => {
1061
+ ): Promise<
1062
+ Result<ApplyProtocolMessageAsRelayResult, ProtocolInvalidDataError>
1063
+ > => {
1064
+ try {
1065
+ const input = createBuffer(inputMessage);
978
1066
  const [requestedVersion, ownerId] = decodeVersionAndOwner(input);
979
- const binaryOwnerId = ownerIdToBinaryOwnerId(ownerId);
1067
+ const ownerIdBytes = ownerIdToOwnerIdBytes(ownerId);
980
1068
 
981
1069
  if (requestedVersion !== version) {
982
1070
  // Non-initiator responds with its version and ownerId.
983
1071
  const output = createBuffer();
984
1072
  encodeNonNegativeInt(output, version);
985
- output.extend(binaryOwnerId);
986
- return ok(output.unwrap() as ProtocolMessage);
1073
+ output.extend(ownerIdBytes);
1074
+ return ok({
1075
+ type: "response",
1076
+ message: output.unwrap() as ProtocolMessage,
1077
+ });
987
1078
  }
988
1079
 
989
- subscribe?.(ownerId);
990
-
991
- const writeKeyMode = input.shift() as WriteKeyMode;
992
- let writeKey: WriteKey | undefined;
993
- let newWriteKey: WriteKey | undefined;
994
-
995
- if (writeKeyMode !== WriteKeyMode.None) {
996
- writeKey = input.shiftN(writeKeyLength) as WriteKey;
997
- switch (writeKeyMode) {
998
- case WriteKeyMode.Single:
999
- break;
1000
- case WriteKeyMode.Rotation:
1001
- newWriteKey = input.shiftN(writeKeyLength) as WriteKey;
1002
- break;
1003
- default:
1004
- throw new ProtocolDecodeError(
1005
- `Invalid WriteKeyMode: ${writeKeyMode}`,
1006
- );
1007
- }
1080
+ const messageType = input.shift() as MessageType;
1081
+ assert(messageType === MessageType.Request, "Invalid MessageType");
1082
+
1083
+ const hasWriteKey = input.shift();
1084
+ let writeKey: OwnerWriteKey | undefined;
1085
+
1086
+ if (hasWriteKey === 1) {
1087
+ writeKey = input.shiftN(ownerWriteKeyLength) as OwnerWriteKey;
1088
+ }
1089
+
1090
+ const subscriptionFlag = input.shift() as SubscriptionFlag;
1091
+
1092
+ switch (subscriptionFlag) {
1093
+ case SubscriptionFlags.Subscribe:
1094
+ options.subscribe?.(ownerId);
1095
+ break;
1096
+ case SubscriptionFlags.Unsubscribe:
1097
+ options.unsubscribe?.(ownerId);
1098
+ break;
1099
+ case SubscriptionFlags.None:
1100
+ break;
1008
1101
  }
1009
1102
 
1010
1103
  if (writeKey) {
1011
- const isValid = deps.storage.validateWriteKey(binaryOwnerId, writeKey);
1104
+ const isValid = deps.storage.validateWriteKey(ownerIdBytes, writeKey);
1012
1105
  if (!isValid) {
1013
- return ok(
1014
- createProtocolMessageBuffer(ownerId, {
1015
- type: "non-initiator",
1106
+ return ok({
1107
+ type: "response",
1108
+ message: createProtocolMessageBuffer(ownerId, {
1109
+ messageType: MessageType.Response,
1016
1110
  errorCode: ProtocolErrorCode.WriteKeyError,
1017
1111
  }).unwrap(),
1018
- );
1019
- }
1020
-
1021
- if (newWriteKey) {
1022
- const rotationSuccess = deps.storage.setWriteKey(
1023
- binaryOwnerId,
1024
- newWriteKey,
1025
- );
1026
- if (!rotationSuccess) {
1027
- return ok(
1028
- createProtocolMessageBuffer(ownerId, {
1029
- type: "non-initiator",
1030
- errorCode: ProtocolErrorCode.WriteError,
1031
- }).unwrap(),
1032
- );
1033
- }
1112
+ });
1034
1113
  }
1035
1114
  }
1036
1115
 
1037
1116
  const messages = decodeMessages(input);
1038
1117
 
1039
1118
  if (isNonEmptyReadonlyArray(messages)) {
1040
- if (!writeKey)
1041
- return ok(
1042
- createProtocolMessageBuffer(ownerId, {
1043
- type: "non-initiator",
1119
+ if (!writeKey) {
1120
+ return ok({
1121
+ type: "response",
1122
+ message: createProtocolMessageBuffer(ownerId, {
1123
+ messageType: MessageType.Response,
1044
1124
  errorCode: ProtocolErrorCode.WriteKeyError,
1045
1125
  }).unwrap(),
1046
- );
1126
+ });
1127
+ }
1128
+
1129
+ const writeResult = await deps.storage.writeMessages(
1130
+ ownerIdBytes,
1131
+ messages,
1132
+ );
1047
1133
 
1048
- // Only broadcast if there's no ranges.
1049
- if (broadcast && input.getLength() === 0) {
1134
+ if (!writeResult.ok) {
1135
+ const errorCode =
1136
+ writeResult.error.type === "StorageWriteError"
1137
+ ? ProtocolErrorCode.WriteError
1138
+ : ProtocolErrorCode.QuotaError;
1139
+ const message = createProtocolMessageBuffer(ownerId, {
1140
+ messageType: MessageType.Response,
1141
+ errorCode,
1142
+ }).unwrap();
1143
+ return ok({ type: "response", message });
1144
+ }
1145
+
1146
+ /**
1147
+ * Broadcast messages to all subscribed owners for real-time
1148
+ * synchronization between clients.
1149
+ *
1150
+ * Messages are only broadcasted after successful write to ensure
1151
+ * devices that can still sync aren't affected by quota errors, and to
1152
+ * prevent using a half-working relay service (broadcasting without
1153
+ * persistence).
1154
+ *
1155
+ * When a relay's database is deleted or clients migrate to a new relay
1156
+ * (without data migration), clients will sync their data to the relay,
1157
+ * and the relay will broadcast those messages to other connected
1158
+ * clients. Those clients may receive messages they already have, but
1159
+ * this is safe because Evolu sync is idempotent. As the relay becomes
1160
+ * more synchronized with clients over time, fewer duplicate messages
1161
+ * will be broadcasted.
1162
+ */
1163
+ if (options.broadcast) {
1050
1164
  const broadcastBuffer = createProtocolMessageBuffer(ownerId, {
1051
- type: "non-initiator",
1052
- errorCode: ProtocolErrorCode.NoError,
1053
- totalMaxSize,
1054
- rangesMaxSize,
1165
+ messageType: MessageType.Broadcast,
1166
+ totalMaxSize: options.totalMaxSize,
1167
+ rangesMaxSize: options.rangesMaxSize,
1055
1168
  version,
1056
1169
  });
1057
1170
  for (const message of messages) {
1058
1171
  broadcastBuffer.addMessage(message);
1059
1172
  }
1060
- broadcast(ownerId, broadcastBuffer.unwrap());
1173
+ options.broadcast(ownerId, broadcastBuffer.unwrap());
1061
1174
  }
1062
-
1063
- if (!deps.storage.writeMessages(binaryOwnerId, messages))
1064
- return ok(
1065
- createProtocolMessageBuffer(ownerId, {
1066
- type: "non-initiator",
1067
- errorCode: ProtocolErrorCode.WriteError,
1068
- }).unwrap(),
1069
- );
1070
1175
  }
1071
1176
 
1177
+ const ranges = decodeRanges(input);
1178
+
1072
1179
  const output = createProtocolMessageBuffer(ownerId, {
1073
- type: "non-initiator",
1180
+ messageType: MessageType.Response,
1074
1181
  errorCode: ProtocolErrorCode.NoError,
1075
- totalMaxSize,
1076
- rangesMaxSize,
1182
+ totalMaxSize: options.totalMaxSize,
1183
+ rangesMaxSize: options.rangesMaxSize,
1077
1184
  });
1078
1185
 
1079
- return sync(deps)("non-initiator", input, output, binaryOwnerId);
1080
- });
1186
+ // Non-initiators always respond to provide sync completion feedback,
1187
+ // even when there's nothing to sync.
1188
+ if (!isNonEmptyReadonlyArray(ranges)) {
1189
+ return ok({ type: "response", message: output.unwrap() });
1190
+ }
1081
1191
 
1082
- /**
1083
- * Wraps Evolu Protocol decoding functions, which use exceptions instead of
1084
- * {@link Result} to provide stack traces for debugging and reduce allocation
1085
- * overhead in success cases.
1086
- */
1087
- const tryDecodeProtocolData = <T, E>(
1088
- data: Uint8Array,
1089
- callback: (buffer: Buffer) => Result<T, E | ProtocolInvalidDataError>,
1090
- ) => {
1091
- try {
1092
- return callback(createBuffer(data));
1093
- } catch (error: unknown) {
1094
- if (error instanceof ProtocolDecodeError || error instanceof BufferError)
1192
+ const syncResult = sync(deps)(ranges, output, ownerIdBytes);
1193
+
1194
+ const message = syncResult.ok
1195
+ ? output.unwrap()
1196
+ : createProtocolMessageBuffer(ownerId, {
1197
+ messageType: MessageType.Response,
1198
+ errorCode: syncResult.error,
1199
+ }).unwrap();
1200
+
1201
+ // Non-initiators always respond to provide sync completion feedback,
1202
+ return ok({ type: "response", message });
1203
+ } catch (error) {
1095
1204
  return err<ProtocolInvalidDataError>({
1096
1205
  type: "ProtocolInvalidDataError",
1097
- data,
1206
+ data: inputMessage,
1098
1207
  error,
1099
1208
  });
1100
-
1101
- throw error;
1102
- }
1103
- };
1209
+ }
1210
+ };
1104
1211
 
1105
1212
  const decodeVersionAndOwner = (input: Buffer): [NonNegativeInt, OwnerId] => {
1106
1213
  // This structure must never change across protocol versions. The version
@@ -1108,7 +1215,7 @@ const decodeVersionAndOwner = (input: Buffer): [NonNegativeInt, OwnerId] => {
1108
1215
  // to enable version negotiation and owner identification before any other
1109
1216
  // processing occurs.
1110
1217
  const version = decodeNonNegativeInt(input);
1111
- const ownerId = decodeOwnerId(input);
1218
+ const ownerId = decodeId(input) as OwnerId;
1112
1219
  return [version, ownerId];
1113
1220
  };
1114
1221
 
@@ -1143,38 +1250,17 @@ const decodeMessages = (
1143
1250
  const sync =
1144
1251
  (deps: StorageDep) =>
1145
1252
  (
1146
- role: "initiator" | "non-initiator",
1147
- input: Buffer,
1253
+ ranges: NonEmptyReadonlyArray<Range>,
1148
1254
  output: ProtocolMessageBuffer,
1149
- ownerId: BinaryOwnerId,
1150
- ): Result<ProtocolMessage | null, never> => {
1151
- const ranges = decodeRanges(input);
1152
-
1153
- if (!isNonEmptyReadonlyArray(ranges)) {
1154
- // Nothing to sync.
1155
- return ok(null);
1156
- }
1157
-
1158
- const binaryOwnerId = binaryOwnerIdToOwnerId(ownerId);
1255
+ ownerIdBytes: OwnerIdBytes,
1256
+ ): Result<boolean, typeof ProtocolErrorCode.SyncError> => {
1159
1257
  const outputInitialSize = output.getSize();
1160
1258
 
1161
- const syncFail = () => {
1162
- // Only the relay (non-initiator) reports sync errors, not the client (initiator).
1163
- if (role === "initiator") {
1164
- return ok(null);
1165
- }
1166
- const message = createProtocolMessageBuffer(binaryOwnerId, {
1167
- type: "non-initiator",
1168
- errorCode: ProtocolErrorCode.SyncError,
1169
- });
1170
- return ok(message.unwrap());
1171
- };
1172
-
1173
- const storageSize = deps.storage.getSize(ownerId);
1174
- if (storageSize == null) return syncFail();
1259
+ const storageSize = deps.storage.getSize(ownerIdBytes);
1260
+ if (storageSize == null) return err(ProtocolErrorCode.SyncError);
1175
1261
 
1176
1262
  let prevUpperBound: RangeUpperBound | null = null;
1177
- let prevIndex = 0 as NonNegativeInt;
1263
+ let prevIndex = NonNegativeInt.orThrow(0);
1178
1264
 
1179
1265
  let skip = false;
1180
1266
  let nonSkipRangeAdded = false;
@@ -1211,7 +1297,11 @@ const sync =
1211
1297
  const addFingerprintForRemainingRange = (
1212
1298
  begin: NonNegativeInt,
1213
1299
  ): boolean => {
1214
- const fingerprint = deps.storage.fingerprint(ownerId, begin, storageSize);
1300
+ const fingerprint = deps.storage.fingerprint(
1301
+ ownerIdBytes,
1302
+ begin,
1303
+ storageSize,
1304
+ );
1215
1305
  if (!fingerprint) return false;
1216
1306
  // There is always a space for a ramaining range.
1217
1307
  output.addRange({
@@ -1227,12 +1317,12 @@ const sync =
1227
1317
 
1228
1318
  const lower = prevIndex;
1229
1319
  let upper = deps.storage.findLowerBound(
1230
- ownerId,
1320
+ ownerIdBytes,
1231
1321
  prevIndex,
1232
1322
  storageSize,
1233
1323
  currentUpperBound,
1234
1324
  );
1235
- if (upper == null) return syncFail();
1325
+ if (upper == null) return err(ProtocolErrorCode.SyncError);
1236
1326
 
1237
1327
  switch (range.type) {
1238
1328
  case RangeType.Skip: {
@@ -1242,11 +1332,11 @@ const sync =
1242
1332
 
1243
1333
  case RangeType.Fingerprint: {
1244
1334
  const ourFingerprint = deps.storage.fingerprint(
1245
- ownerId,
1335
+ ownerIdBytes,
1246
1336
  lower,
1247
1337
  upper,
1248
1338
  );
1249
- if (ourFingerprint == null) return syncFail();
1339
+ if (ourFingerprint == null) return err(ProtocolErrorCode.SyncError);
1250
1340
 
1251
1341
  if (eqArrayNumber(range.fingerprint, ourFingerprint)) {
1252
1342
  skipRange(range);
@@ -1254,15 +1344,16 @@ const sync =
1254
1344
  if (output.canSplitRange()) {
1255
1345
  coalesceSkipsBeforeAdd();
1256
1346
  splitRange(deps)(
1257
- ownerId,
1347
+ ownerIdBytes,
1258
1348
  lower,
1259
1349
  upper,
1260
1350
  currentUpperBound,
1261
1351
  output,
1262
1352
  );
1263
1353
  } else {
1264
- if (!addFingerprintForRemainingRange(upper)) return syncFail();
1265
- return ok(output.unwrap());
1354
+ return addFingerprintForRemainingRange(upper)
1355
+ ? ok(true)
1356
+ : err(ProtocolErrorCode.SyncError);
1266
1357
  }
1267
1358
  }
1268
1359
  break;
@@ -1276,45 +1367,53 @@ const sync =
1276
1367
  );
1277
1368
  const ourTimestamps = createTimestampsBuffer();
1278
1369
 
1279
- let storageError = false as boolean;
1370
+ let cantReadDbChange = false as boolean;
1280
1371
  let exceeded = false as boolean;
1281
1372
 
1282
- deps.storage.iterate(ownerId, lower, upper, (timestamp, index) => {
1283
- const timestampString = timestamp.join();
1284
- const timestampBinary = binaryTimestampToTimestamp(timestamp);
1285
-
1286
- let message: EncryptedCrdtMessage | null = null;
1373
+ deps.storage.iterate(
1374
+ ownerIdBytes,
1375
+ lower,
1376
+ upper,
1377
+ (timestamp, index) => {
1378
+ const timestampString = timestamp.join();
1379
+ const timestampBinary = timestampBytesToTimestamp(timestamp);
1380
+
1381
+ let message: EncryptedCrdtMessage | null = null;
1382
+
1383
+ if (timestampsWeNeed.has(timestampString)) {
1384
+ timestampsWeNeed.delete(timestampString);
1385
+ } else {
1386
+ const dbChange = deps.storage.readDbChange(
1387
+ ownerIdBytes,
1388
+ timestamp,
1389
+ );
1390
+ if (dbChange == null) {
1391
+ cantReadDbChange = true;
1392
+ return false;
1393
+ }
1394
+ message = {
1395
+ timestamp: timestampBinary,
1396
+ change: dbChange,
1397
+ };
1398
+ }
1287
1399
 
1288
- if (timestampsWeNeed.has(timestampString)) {
1289
- timestampsWeNeed.delete(timestampString);
1290
- } else {
1291
- const dbChange = deps.storage.readDbChange(ownerId, timestamp);
1292
- if (dbChange == null) {
1293
- storageError = true;
1400
+ if (
1401
+ !output.canAddTimestampsRangeAndMessage(ourTimestamps, message)
1402
+ ) {
1403
+ exceeded = true;
1404
+ endBound = timestamp;
1405
+ upper = index;
1294
1406
  return false;
1295
1407
  }
1296
- message = {
1297
- timestamp: timestampBinary,
1298
- change: dbChange,
1299
- };
1300
- }
1301
-
1302
- if (
1303
- !output.canAddTimestampsRangeAndMessage(ourTimestamps, message)
1304
- ) {
1305
- exceeded = true;
1306
- endBound = timestamp;
1307
- upper = index;
1308
- return false;
1309
- }
1310
1408
 
1311
- ourTimestamps.add(timestampBinary);
1312
- if (message) output.addMessage(message);
1313
- return true;
1314
- });
1409
+ ourTimestamps.add(timestampBinary);
1410
+ if (message) output.addMessage(message);
1411
+ return true;
1412
+ },
1413
+ );
1315
1414
 
1316
- if (storageError) {
1317
- return syncFail();
1415
+ if (cantReadDbChange) {
1416
+ return err(ProtocolErrorCode.SyncError);
1318
1417
  }
1319
1418
 
1320
1419
  const addRange = () => {
@@ -1329,9 +1428,9 @@ const sync =
1329
1428
  if (exceeded) {
1330
1429
  addRange();
1331
1430
  if (!addFingerprintForRemainingRange(upper)) {
1332
- return syncFail();
1431
+ return err(ProtocolErrorCode.SyncError);
1333
1432
  }
1334
- return ok(output.unwrap());
1433
+ return ok(true);
1335
1434
  }
1336
1435
 
1337
1436
  // If we need something, we have to respond with our timestamps.
@@ -1351,19 +1450,20 @@ const sync =
1351
1450
 
1352
1451
  // If all ranges were skipped, there are no changes and sync is complete.
1353
1452
  const hasChange = output.getSize() > outputInitialSize;
1354
- return ok(hasChange ? output.unwrap() : null);
1453
+
1454
+ return ok(hasChange);
1355
1455
  };
1356
1456
 
1357
1457
  const splitRange =
1358
1458
  (deps: StorageDep) =>
1359
1459
  (
1360
- ownerId: BinaryOwnerId,
1460
+ ownerId: OwnerIdBytes,
1361
1461
  lower: NonNegativeInt,
1362
1462
  upper: NonNegativeInt,
1363
1463
  upperBound: RangeUpperBound,
1364
1464
  buffer: ProtocolMessageBuffer,
1365
1465
  ): void => {
1366
- const itemCount = (upper - lower) as NonNegativeInt;
1466
+ const itemCount = NonNegativeInt.orThrow(upper - lower);
1367
1467
  const buckets = computeBalancedBuckets(itemCount);
1368
1468
 
1369
1469
  if (!buckets.ok) {
@@ -1375,10 +1475,10 @@ const splitRange =
1375
1475
 
1376
1476
  deps.storage.iterate(
1377
1477
  ownerId,
1378
- 0 as NonNegativeInt,
1478
+ NonNegativeInt.orThrow(0),
1379
1479
  itemCount,
1380
1480
  (timestamp) => {
1381
- range.timestamps.add(binaryTimestampToTimestamp(timestamp));
1481
+ range.timestamps.add(timestampBytesToTimestamp(timestamp));
1382
1482
  return true;
1383
1483
  },
1384
1484
  );
@@ -1391,7 +1491,10 @@ const splitRange =
1391
1491
  const fingerprintRangesBuckets =
1392
1492
  lower === 0
1393
1493
  ? buckets.value
1394
- : [lower, ...buckets.value.map((b) => (b + lower) as NonNegativeInt)];
1494
+ : [
1495
+ lower,
1496
+ ...buckets.value.map((b) => NonNegativeInt.orThrow(b + lower)),
1497
+ ];
1395
1498
 
1396
1499
  const fingerprintRanges = deps.storage.fingerprintRanges(
1397
1500
  ownerId,
@@ -1415,7 +1518,7 @@ const decodeRanges = (buffer: Buffer): ReadonlyArray<Range> => {
1415
1518
  const rangesCount = decodeNonNegativeInt(buffer);
1416
1519
  if (rangesCount === 0) return [];
1417
1520
 
1418
- const timestampsCount = (rangesCount - 1) as NonNegativeInt;
1521
+ const timestampsCount = NonNegativeInt.orThrow(rangesCount - 1);
1419
1522
  const timestamps = decodeTimestamps(buffer, timestampsCount);
1420
1523
  const rangeTypes: Array<RangeType> = [];
1421
1524
 
@@ -1437,7 +1540,7 @@ const decodeRanges = (buffer: Buffer): ReadonlyArray<Range> => {
1437
1540
  for (let i = 0; i < rangesCount; i++) {
1438
1541
  const upperBound =
1439
1542
  i < timestampsCount
1440
- ? timestampToBinaryTimestamp(timestamps[i])
1543
+ ? timestampToTimestampBytes(timestamps[i])
1441
1544
  : InfiniteUpperBound;
1442
1545
 
1443
1546
  const rangeType = rangeTypes[i];
@@ -1459,7 +1562,7 @@ const decodeRanges = (buffer: Buffer): ReadonlyArray<Range> => {
1459
1562
 
1460
1563
  case RangeType.Timestamps: {
1461
1564
  const timestamps = decodeTimestamps(buffer).map(
1462
- timestampToBinaryTimestamp,
1565
+ timestampToTimestampBytes,
1463
1566
  );
1464
1567
  ranges.push({
1465
1568
  type: RangeType.Timestamps,
@@ -1487,7 +1590,7 @@ const decodeTimestamps = (
1487
1590
  for (let i = 0; i < length; i++) {
1488
1591
  const deltaMillis = decodeNonNegativeInt(buffer);
1489
1592
  const millis = Millis.from(previousMillis + deltaMillis);
1490
- if (!millis.ok) throw new Error(millis.error.type);
1593
+ if (!millis.ok) throw new ProtocolDecodeError(millis.error.type);
1491
1594
  millises.push(millis.value);
1492
1595
  previousMillis = millis.value;
1493
1596
  }
@@ -1496,7 +1599,7 @@ const decodeTimestamps = (
1496
1599
  let counterIndex = 0;
1497
1600
  while (counterIndex < length) {
1498
1601
  const counter = Counter.from(decodeNonNegativeInt(buffer));
1499
- if (!counter.ok) throw new Error(counter.error.type);
1602
+ if (!counter.ok) throw new ProtocolDecodeError(counter.error.type);
1500
1603
  const runLength = decodeNonNegativeInt(buffer);
1501
1604
  for (let i = 0; i < runLength; i++) {
1502
1605
  counters.push(counter.value);
@@ -1527,127 +1630,31 @@ const decodeTimestamps = (
1527
1630
  return timestamps;
1528
1631
  };
1529
1632
 
1530
- /** Binary representation of {@link Id}. */
1531
- export type BinaryId = Uint8Array & Brand<"BinaryId">;
1532
-
1533
- export const binaryIdLength = 16 as NonNegativeInt;
1534
-
1535
- export const idToBinaryId = (id: Id): BinaryId =>
1536
- base64Url256ToBytes(id) as BinaryId;
1537
-
1538
- export const binaryIdToId = (binaryId: BinaryId): Id =>
1539
- decodeId(createBuffer(binaryId));
1540
-
1541
- /** Binary representation of {@link OwnerId}. */
1542
- export type BinaryOwnerId = Uint8Array & Brand<"BinaryOwnerId">;
1543
-
1544
- export const ownerIdToBinaryOwnerId = (ownerId: OwnerId): BinaryOwnerId =>
1545
- base64Url256ToBytes(ownerId) as BinaryOwnerId;
1546
-
1547
- export const binaryOwnerIdToOwnerId = (binaryOwnerId: BinaryOwnerId): OwnerId =>
1548
- decodeOwnerId(createBuffer(binaryOwnerId));
1549
-
1550
- /**
1551
- * Union type for all variants of Base64Url strings with limited length. All
1552
- * these types use Base64Url alphabet and are < 256 characters.
1553
- */
1554
- export type Base64Url256Variant = Base64Url256 | Id | NanoId | OwnerId;
1555
-
1556
- /**
1557
- * Alphabet used for Base64Url encoding. This is copied from the `nanoid`
1558
- * library to avoid dependency on a specific version of `nanoid`.
1559
- */
1560
- const urlAlphabet =
1561
- "useandom-26T198340PX75pxJACKVERYMINDBUSHWOLF_GQZbfghjklqvwyzrict";
1562
-
1563
- /**
1564
- * Converts a Base64Url string to a Uint8Array for binary storage. This encoding
1565
- * is more space-efficient than UTF-8 for Base64Url strings.
1566
- */
1567
- export const base64Url256ToBytes = (
1568
- string: Base64Url256Variant,
1569
- ): globalThis.Uint8Array => {
1570
- const totalBits = string.length * 6; // 6 bits per character
1571
- const byteLength = Math.ceil(totalBits / 8);
1572
- const value = new globalThis.Uint8Array(byteLength);
1573
-
1574
- let bitBuffer = 0;
1575
- let bitsInBuffer = 0;
1576
- let byteIndex = 0;
1577
-
1578
- for (const char of string) {
1579
- const charValue = urlAlphabet.indexOf(char);
1580
- bitBuffer = (bitBuffer << 6) | charValue;
1581
- bitsInBuffer += 6;
1582
- while (bitsInBuffer >= 8) {
1583
- bitsInBuffer -= 8;
1584
- value[byteIndex++] = (bitBuffer >> bitsInBuffer) & 0xff;
1585
- }
1586
- }
1587
-
1588
- if (bitsInBuffer > 0 && byteIndex < byteLength) {
1589
- value[byteIndex] = (bitBuffer << (8 - bitsInBuffer)) & 0xff;
1590
- }
1591
-
1592
- return value;
1633
+ const decodeId = (buffer: Buffer): Id => {
1634
+ const bytes = buffer.shiftN(idBytesTypeValueLength);
1635
+ return idBytesToId(bytes as IdBytes);
1593
1636
  };
1594
1637
 
1595
- export const decodeBase64Url256 = (
1596
- buffer: Buffer,
1597
- stringLength: number,
1598
- ): Base64Url256Variant => {
1599
- const bytes = buffer.shiftN(
1600
- Math.ceil((stringLength * 6) / 8) as NonNegativeInt,
1601
- );
1602
-
1603
- let bitBuffer = 0;
1604
- let bitsInBuffer = 0;
1605
- let string = "";
1606
-
1607
- for (const byte of bytes) {
1608
- bitBuffer = (bitBuffer << 8) | byte;
1609
- bitsInBuffer += 8;
1610
- while (bitsInBuffer >= 6) {
1611
- bitsInBuffer -= 6;
1612
- if (string.length < stringLength) {
1613
- const charValue = (bitBuffer >> bitsInBuffer) & 0x3f;
1614
- if (charValue < 0 || charValue >= urlAlphabet.length) {
1615
- throw new ProtocolDecodeError("invalid charValue");
1616
- }
1617
- string += urlAlphabet[charValue];
1618
- }
1619
- }
1620
- }
1621
-
1622
- const result = Base64Url256.from(string);
1623
- if (!result.ok) throw new ProtocolDecodeError(result.error.type);
1624
-
1625
- return result.value;
1626
- };
1627
-
1628
- const decodeId = (buffer: Buffer): Id =>
1629
- decodeBase64Url256(buffer, idTypeValueLength) as Id;
1630
-
1631
- /** Not all 16 bytes are valid {@link OwnerId}. */
1632
- const decodeOwnerId = (buffer: Buffer): OwnerId => decodeId(buffer) as OwnerId;
1633
-
1634
1638
  /**
1635
1639
  * Evolu uses MessagePack to handle all number variants except for
1636
1640
  * NonNegativeInt. For NonNegativeInt, Evolu provides more efficient encoding.
1637
1641
  */
1638
1642
  export const encodeNumber = (buffer: Buffer, number: number): void => {
1639
- buffer.extend(pack(number));
1643
+ buffer.extend(packr.pack(number));
1640
1644
  };
1641
1645
 
1642
1646
  export const decodeNumber = (buffer: Buffer): number => {
1643
1647
  let number: unknown;
1644
1648
  let end: unknown;
1645
1649
 
1646
- unpackMultiple(buffer.unwrap(), (n, _, e) => {
1647
- number = n;
1648
- end = e;
1649
- return false;
1650
- });
1650
+ packr.unpackMultiple(
1651
+ buffer.unwrap(),
1652
+ (n: unknown, _: unknown, e: unknown) => {
1653
+ number = n;
1654
+ end = e;
1655
+ return false;
1656
+ },
1657
+ );
1651
1658
 
1652
1659
  const endResult = NonNegativeInt.fromUnknown(end);
1653
1660
  if (!endResult.ok) throw new ProtocolDecodeError(endResult.error.type);
@@ -1659,42 +1666,90 @@ export const decodeNumber = (buffer: Buffer): number => {
1659
1666
  return numberResult.value;
1660
1667
  };
1661
1668
 
1662
- export const binaryTimestampToFingerprint = (
1663
- timestamp: BinaryTimestamp,
1664
- ): Fingerprint => {
1665
- const hash = sha256(timestamp).slice(0, fingerprintSize);
1666
- return hash as Fingerprint;
1669
+ /**
1670
+ * Encodes an array of boolean flags into a single byte.
1671
+ *
1672
+ * Each element in the array corresponds to a bit (0-7). Array can have 0-8
1673
+ * elements.
1674
+ *
1675
+ * ### Example
1676
+ *
1677
+ * ```ts
1678
+ * encodeFlags(buffer, [true, false, true]); // Encodes bits 0, 1, 2
1679
+ * ```
1680
+ */
1681
+ export const encodeFlags = (
1682
+ buffer: Buffer,
1683
+ flags: ReadonlyArray<boolean>,
1684
+ ): void => {
1685
+ let byte = 0;
1686
+ for (let i = 0; i < flags.length && i < 8; i++) {
1687
+ if (flags[i]) {
1688
+ byte |= 1 << i;
1689
+ }
1690
+ }
1691
+ buffer.extend([byte]);
1692
+ };
1693
+
1694
+ /**
1695
+ * Decodes a byte into an array of boolean flags.
1696
+ *
1697
+ * ### Example
1698
+ *
1699
+ * ```ts
1700
+ * const flags = decodeFlags(buffer, 3); // Decode 3 flags
1701
+ * ```
1702
+ */
1703
+ export const decodeFlags = (
1704
+ buffer: Buffer,
1705
+ count: PositiveInt,
1706
+ ): ReadonlyArray<boolean> => {
1707
+ const byte = buffer.shift();
1708
+ const flags: Array<boolean> = [];
1709
+ for (let i = 0; i < count && i < 8; i++) {
1710
+ flags.push((byte & (1 << i)) !== 0);
1711
+ }
1712
+ return flags;
1667
1713
  };
1668
1714
 
1669
1715
  /**
1670
1716
  * Encodes and encrypts a {@link DbChange} using the provided owner's encryption
1671
1717
  * key. Returns an encrypted binary representation as {@link EncryptedDbChange}.
1718
+ *
1719
+ * The format includes the protocol version for backward compatibility and the
1720
+ * timestamp for tamper-proof verification that the timestamp matches the change
1721
+ * data.
1672
1722
  */
1673
1723
  export const encodeAndEncryptDbChange =
1674
1724
  (deps: SymmetricCryptoDep) =>
1675
- (change: DbChange, key: EncryptionKey): EncryptedDbChange => {
1725
+ (message: CrdtMessage, key: EncryptionKey): EncryptedDbChange => {
1676
1726
  const buffer = createBuffer();
1677
1727
 
1678
- encodeBase64Url256(buffer, change.table);
1728
+ encodeNonNegativeInt(buffer, protocolVersion);
1679
1729
 
1680
- buffer.extend(idToBinaryId(change.id));
1730
+ // Encode the timestamp to prevent tampering (e.g., a malicious relay
1731
+ // assigning this EncryptedDbChange to a different EncryptedCrdtMessage)
1732
+ buffer.extend(timestampToTimestampBytes(message.timestamp));
1681
1733
 
1682
- const entries = objectToEntries(change.values).map(
1683
- ([column, value]): [Base64Url256, SqliteValue] => {
1684
- return [column, value];
1685
- },
1686
- );
1734
+ encodeFlags(buffer, [
1735
+ message.change.isInsert,
1736
+ message.change.isDelete != null,
1737
+ message.change.isDelete ?? false,
1738
+ ]);
1687
1739
 
1688
- encodeLength(buffer, entries);
1740
+ encodeString(buffer, message.change.table);
1741
+ buffer.extend(idToIdBytes(message.change.id));
1742
+
1743
+ const entries = objectToEntries(message.change.values);
1689
1744
 
1745
+ encodeLength(buffer, entries);
1690
1746
  for (const [column, value] of entries) {
1691
- encodeBase64Url256(buffer, column);
1747
+ encodeString(buffer, column);
1692
1748
  encodeSqliteValue(buffer, value);
1693
1749
  }
1694
1750
 
1695
- const paddingLength = padmePaddingLength(buffer.getLength());
1696
- // Add zero bytes as PADMÉ padding - these will be ignored during decoding.
1697
- buffer.extend(new Uint8Array(paddingLength));
1751
+ // Add PADMÉ padding (ignored during decoding)
1752
+ buffer.extend(createPadmePadding(buffer.getLength()));
1698
1753
 
1699
1754
  const { nonce, ciphertext } = deps.symmetricCrypto.encrypt(
1700
1755
  buffer.unwrap(),
@@ -1710,50 +1765,82 @@ export const encodeAndEncryptDbChange =
1710
1765
  };
1711
1766
 
1712
1767
  /**
1713
- * Decrypts and decodes an {@link EncryptedDbChange} using the provided owner's
1714
- * encryption key.
1768
+ * Decrypts and decodes an {@link EncryptedCrdtMessage} using the provided
1769
+ * owner's encryption key. Verifies that the embedded timestamp matches the
1770
+ * expected timestamp to ensure message integrity.
1715
1771
  */
1716
1772
  export const decryptAndDecodeDbChange =
1717
1773
  (deps: SymmetricCryptoDep) =>
1718
1774
  (
1719
- change: EncryptedDbChange,
1775
+ message: EncryptedCrdtMessage,
1720
1776
  key: EncryptionKey,
1721
- ): Result<DbChange, SymmetricCryptoDecryptError | ProtocolInvalidDataError> =>
1722
- tryDecodeProtocolData<DbChange, SymmetricCryptoDecryptError>(
1723
- change,
1724
- (buffer) => {
1725
- const nonce = buffer.shiftN(deps.symmetricCrypto.nonceLength);
1726
-
1727
- const ciphertextLength = decodeLength(buffer);
1728
- const ciphertext = buffer.shiftN(ciphertextLength);
1729
-
1730
- const plaintextBytes = deps.symmetricCrypto.decrypt(
1731
- ciphertext,
1732
- key,
1733
- nonce,
1734
- );
1735
- if (!plaintextBytes.ok) return plaintextBytes;
1777
+ ): Result<
1778
+ DbChange,
1779
+ | SymmetricCryptoDecryptError
1780
+ | ProtocolInvalidDataError
1781
+ | ProtocolTimestampMismatchError
1782
+ > => {
1783
+ try {
1784
+ const buffer = createBuffer(message.change);
1785
+
1786
+ const nonce = buffer.shiftN(deps.symmetricCrypto.nonceLength);
1787
+ const ciphertext = buffer.shiftN(decodeLength(buffer));
1788
+
1789
+ const plaintextBytes = deps.symmetricCrypto.decrypt(
1790
+ ciphertext,
1791
+ key,
1792
+ nonce,
1793
+ );
1794
+ if (!plaintextBytes.ok) return plaintextBytes;
1736
1795
 
1737
- buffer.reset();
1738
- buffer.extend(plaintextBytes.value);
1796
+ buffer.reset();
1797
+ buffer.extend(plaintextBytes.value);
1739
1798
 
1740
- const table = decodeBase64Url256WithLength(buffer);
1741
- const id = decodeId(buffer);
1799
+ // Decode version (for future compatibility, not need yet)
1800
+ decodeNonNegativeInt(buffer);
1742
1801
 
1743
- const length = decodeLength(buffer);
1744
- const values = Object.create(null) as Record<string, SqliteValue>;
1802
+ const timestamp = timestampBytesToTimestamp(
1803
+ buffer.shiftN(timestampBytesLength) as TimestampBytes,
1804
+ );
1745
1805
 
1746
- for (let i = 0; i < length; i++) {
1747
- const column = decodeBase64Url256WithLength(buffer);
1748
- const value = decodeSqliteValue(buffer);
1749
- values[column] = value;
1750
- }
1806
+ if (!eqTimestamp(timestamp, message.timestamp)) {
1807
+ return err<ProtocolTimestampMismatchError>({
1808
+ type: "ProtocolTimestampMismatchError",
1809
+ expected: message.timestamp,
1810
+ timestamp,
1811
+ });
1812
+ }
1751
1813
 
1752
- const dbChange = { table, id, values };
1814
+ const flags = decodeFlags(buffer, PositiveInt.orThrow(3));
1815
+ const table = decodeString(buffer);
1816
+ const id = decodeId(buffer);
1753
1817
 
1754
- return ok(dbChange);
1755
- },
1756
- );
1818
+ const length = decodeLength(buffer);
1819
+ const values = Object.create(null) as Record<string, SqliteValue>;
1820
+
1821
+ for (let i = 0; i < length; i++) {
1822
+ const column = decodeString(buffer);
1823
+ const value = decodeSqliteValue(buffer);
1824
+ values[column] = value;
1825
+ }
1826
+
1827
+ const dbChange = DbChange.orThrow({
1828
+ table,
1829
+ id,
1830
+ values,
1831
+ isInsert: flags[0],
1832
+ isDelete: flags[1] ? flags[2] : null,
1833
+ });
1834
+
1835
+ return ok(dbChange);
1836
+ } catch (error) {
1837
+ return err<ProtocolInvalidDataError>({
1838
+ type: "ProtocolInvalidDataError",
1839
+ data: message.change,
1840
+ error,
1841
+ });
1842
+ }
1843
+ };
1757
1844
 
1758
1845
  /**
1759
1846
  * Encodes a non-negative integer into a variable-length integer format. It's
@@ -1811,7 +1898,7 @@ export const decodeNonNegativeInt = (buffer: Buffer): NonNegativeInt => {
1811
1898
  };
1812
1899
 
1813
1900
  export const encodeLength = (buffer: Buffer, value: ArrayLike<any>): void => {
1814
- encodeNonNegativeInt(buffer, value.length as NonNegativeInt);
1901
+ encodeNonNegativeInt(buffer, NonNegativeInt.orThrow(value.length));
1815
1902
  };
1816
1903
 
1817
1904
  export const decodeLength = decodeNonNegativeInt;
@@ -1833,23 +1920,10 @@ export const encodeNodeId = (buffer: Buffer, nodeId: NodeId): void => {
1833
1920
  };
1834
1921
 
1835
1922
  export const decodeNodeId = (buffer: Buffer): NodeId => {
1836
- const bytes = buffer.shiftN(8 as NonNegativeInt);
1923
+ const bytes = buffer.shiftN(NonNegativeInt.orThrow(8));
1837
1924
  return bytesToHex(bytes) as NodeId;
1838
1925
  };
1839
1926
 
1840
- export const encodeBase64Url256 = (
1841
- buffer: Buffer,
1842
- string: Base64Url256Variant,
1843
- ): void => {
1844
- encodeLength(buffer, string);
1845
- buffer.extend(base64Url256ToBytes(string));
1846
- };
1847
-
1848
- export const decodeBase64Url256WithLength = (buffer: Buffer): Base64Url256 => {
1849
- const length = decodeLength(buffer);
1850
- return decodeBase64Url256(buffer, length) as Base64Url256;
1851
- };
1852
-
1853
1927
  // Small ints are encoded into ProtocolValueType, saving one byte per int.
1854
1928
  const isSmallInt: Predicate<number> = (value: number) =>
1855
1929
  value >= 0 && value < 20;
@@ -1858,23 +1932,26 @@ export const ProtocolValueType = {
1858
1932
  // 0-19 small ints
1859
1933
 
1860
1934
  // SQLite types
1861
- String: 20 as NonNegativeInt,
1862
- Number: 21 as NonNegativeInt,
1863
- Null: 22 as NonNegativeInt,
1864
- Binary: 23 as NonNegativeInt,
1935
+ String: NonNegativeInt.orThrow(20),
1936
+ Number: NonNegativeInt.orThrow(21),
1937
+ Null: NonNegativeInt.orThrow(22),
1938
+ Bytes: NonNegativeInt.orThrow(23),
1865
1939
  // We can add more types for other DBs or anything else later.
1866
1940
 
1867
1941
  // Optimized types
1868
- Id: 30 as NonNegativeInt,
1869
- Base64Url256: 31 as NonNegativeInt,
1870
- NonNegativeInt: 32 as NonNegativeInt,
1871
- Json: 33 as NonNegativeInt,
1942
+ NonNegativeInt: NonNegativeInt.orThrow(30),
1943
+
1944
+ // String optimizations
1945
+ EmptyString: NonNegativeInt.orThrow(31), // 1 byte vs 2 bytes (50% reduction)
1946
+ Base64Url: NonNegativeInt.orThrow(32),
1947
+ Id: NonNegativeInt.orThrow(33),
1948
+ Json: NonNegativeInt.orThrow(34),
1872
1949
 
1873
1950
  // new Date().toISOString() - 24 bytes
1874
1951
  // encoded with fixed length - 8 bytes
1875
1952
  // encode as NonNegativeInt - 6 bytes (additional 25% reduction)
1876
- DateIsoWithNonNegativeTime: 34 as NonNegativeInt,
1877
- DateIsoWithNegativeTime: 35 as NonNegativeInt, // 9 bytes
1953
+ DateIsoWithNonNegativeTime: NonNegativeInt.orThrow(35),
1954
+ DateIsoWithNegativeTime: NonNegativeInt.orThrow(36), // 9 bytes
1878
1955
 
1879
1956
  // TODO: Operations (from 40)
1880
1957
  // Increment, Decrement, Patch, whatever.
@@ -1888,9 +1965,14 @@ export const encodeSqliteValue = (buffer: Buffer, value: SqliteValue): void => {
1888
1965
 
1889
1966
  switch (typeof value) {
1890
1967
  case "string": {
1891
- const dateIsoString = DateIsoString.from(value);
1892
- if (dateIsoString.ok) {
1893
- const time = new Date(dateIsoString.value).getTime();
1968
+ if (value === "") {
1969
+ encodeNonNegativeInt(buffer, ProtocolValueType.EmptyString);
1970
+ return;
1971
+ }
1972
+
1973
+ const dateIso = DateIso.fromParent(value);
1974
+ if (dateIso.ok) {
1975
+ const time = new Date(dateIso.value).getTime();
1894
1976
  if (NonNegativeInt.is(time)) {
1895
1977
  encodeNonNegativeInt(
1896
1978
  buffer,
@@ -1907,27 +1989,34 @@ export const encodeSqliteValue = (buffer: Buffer, value: SqliteValue): void => {
1907
1989
  return;
1908
1990
  }
1909
1991
 
1910
- const base64Url256 = Base64Url256.from(value);
1911
- if (base64Url256.ok) {
1912
- if (base64Url256.value.length === idTypeValueLength) {
1913
- encodeNonNegativeInt(buffer, ProtocolValueType.Id);
1914
- buffer.extend(base64Url256ToBytes(base64Url256.value));
1915
- return;
1916
- }
1917
- encodeNonNegativeInt(buffer, ProtocolValueType.Base64Url256);
1918
- encodeBase64Url256(buffer, base64Url256.value);
1992
+ const id = Id.fromParent(value);
1993
+ if (id.ok) {
1994
+ encodeNonNegativeInt(buffer, ProtocolValueType.Id);
1995
+ buffer.extend(idToIdBytes(id.value));
1919
1996
  return;
1920
1997
  }
1921
1998
 
1922
- const jsonValue = JsonValueFromString.fromParent(value);
1923
- if (jsonValue.ok) {
1924
- const jsonBytes = pack(jsonValue.value);
1999
+ const json = Json.fromParent(value);
2000
+ // Only encode as Json if it survives JSON.parse/JSON.stringify round-trip.
2001
+ // Some valid JSON strings like "-0E0" get normalized to "0" during parsing,
2002
+ // which would cause data corruption if we don't verify round-trip safety.
2003
+ if (json.ok && JSON.stringify(jsonToJsonValue(json.value)) === value) {
2004
+ const jsonBytes = packr.pack(jsonToJsonValue(json.value));
1925
2005
  encodeNonNegativeInt(buffer, ProtocolValueType.Json);
1926
2006
  encodeLength(buffer, jsonBytes);
1927
2007
  buffer.extend(jsonBytes);
1928
2008
  return;
1929
2009
  }
1930
2010
 
2011
+ const base64Url = Base64Url.fromParent(value);
2012
+ if (base64Url.ok) {
2013
+ encodeNonNegativeInt(buffer, ProtocolValueType.Base64Url);
2014
+ const bytes = base64UrlToUint8Array(base64Url.value);
2015
+ encodeLength(buffer, bytes);
2016
+ buffer.extend(bytes);
2017
+ return;
2018
+ }
2019
+
1931
2020
  encodeNonNegativeInt(buffer, ProtocolValueType.String);
1932
2021
  encodeString(buffer, value);
1933
2022
  return;
@@ -1949,7 +2038,7 @@ export const encodeSqliteValue = (buffer: Buffer, value: SqliteValue): void => {
1949
2038
  }
1950
2039
  }
1951
2040
 
1952
- encodeNonNegativeInt(buffer, ProtocolValueType.Binary);
2041
+ encodeNonNegativeInt(buffer, ProtocolValueType.Bytes);
1953
2042
  encodeLength(buffer, value);
1954
2043
  buffer.extend(value);
1955
2044
  };
@@ -1964,39 +2053,50 @@ export const decodeSqliteValue = (buffer: Buffer): SqliteValue => {
1964
2053
  switch (type) {
1965
2054
  case ProtocolValueType.String:
1966
2055
  return decodeString(buffer);
2056
+
1967
2057
  case ProtocolValueType.Number:
1968
2058
  return decodeNumber(buffer);
2059
+
1969
2060
  case ProtocolValueType.Null:
1970
2061
  return null;
1971
- case ProtocolValueType.Binary: {
2062
+
2063
+ case ProtocolValueType.Bytes: {
1972
2064
  const length = decodeLength(buffer);
1973
2065
  return buffer.shiftN(length);
1974
2066
  }
1975
- case ProtocolValueType.Id: {
2067
+
2068
+ case ProtocolValueType.Id:
1976
2069
  return decodeId(buffer);
1977
- }
1978
- case ProtocolValueType.Base64Url256:
1979
- return decodeBase64Url256WithLength(buffer);
2070
+
1980
2071
  case ProtocolValueType.NonNegativeInt:
1981
2072
  return decodeNonNegativeInt(buffer);
2073
+
1982
2074
  case ProtocolValueType.Json: {
1983
2075
  const length = decodeLength(buffer);
1984
2076
  const bytes = buffer.shiftN(length);
1985
- return JSON.stringify(unpack(bytes));
2077
+ return JSON.stringify(packr.unpack(bytes));
1986
2078
  }
2079
+
1987
2080
  case ProtocolValueType.DateIsoWithNonNegativeTime:
1988
2081
  case ProtocolValueType.DateIsoWithNegativeTime: {
1989
2082
  const time =
1990
2083
  type === ProtocolValueType.DateIsoWithNonNegativeTime
1991
2084
  ? decodeNonNegativeInt(buffer)
1992
2085
  : decodeNumber(buffer);
1993
- const dateIsoString = DateIsoString.fromParent(
1994
- new Date(time).toISOString(),
1995
- );
1996
- if (!dateIsoString.ok)
1997
- throw new ProtocolDecodeError(dateIsoString.error.type);
1998
- return dateIsoString.value;
2086
+ const dateIso = DateIso.fromParent(new Date(time).toISOString());
2087
+ if (!dateIso.ok) throw new ProtocolDecodeError(dateIso.error.type);
2088
+ return dateIso.value;
2089
+ }
2090
+
2091
+ case ProtocolValueType.EmptyString:
2092
+ return "";
2093
+
2094
+ case ProtocolValueType.Base64Url: {
2095
+ const length = decodeLength(buffer);
2096
+ const bytes = buffer.shiftN(length);
2097
+ return uint8ArrayToBase64Url(bytes);
1999
2098
  }
2099
+
2000
2100
  default:
2001
2101
  throw new ProtocolDecodeError("invalid ProtocolValueType");
2002
2102
  }