@evolu/common 6.0.1-preview.3 → 6.0.1-preview.31

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (173) hide show
  1. package/dist/src/Array.d.ts +69 -5
  2. package/dist/src/Array.d.ts.map +1 -1
  3. package/dist/src/Array.js +64 -5
  4. package/dist/src/Assert.d.ts +6 -16
  5. package/dist/src/Assert.d.ts.map +1 -1
  6. package/dist/src/Assert.js +6 -18
  7. package/dist/src/Brand.d.ts +75 -0
  8. package/dist/src/Brand.d.ts.map +1 -0
  9. package/dist/src/Brand.js +1 -0
  10. package/dist/src/Buffer.d.ts +1 -1
  11. package/dist/src/Buffer.d.ts.map +1 -1
  12. package/dist/src/Buffer.js +8 -7
  13. package/dist/src/Cache.d.ts +44 -0
  14. package/dist/src/Cache.d.ts.map +1 -0
  15. package/dist/src/Cache.js +52 -0
  16. package/dist/src/Callbacks.d.ts +45 -12
  17. package/dist/src/Callbacks.d.ts.map +1 -1
  18. package/dist/src/Callbacks.js +14 -7
  19. package/dist/src/Console.d.ts +31 -6
  20. package/dist/src/Console.d.ts.map +1 -1
  21. package/dist/src/Console.js +72 -9
  22. package/dist/src/Crypto.d.ts +61 -34
  23. package/dist/src/Crypto.d.ts.map +1 -1
  24. package/dist/src/Crypto.js +32 -45
  25. package/dist/src/Evolu/Db.d.ts +161 -65
  26. package/dist/src/Evolu/Db.d.ts.map +1 -1
  27. package/dist/src/Evolu/Db.js +286 -694
  28. package/dist/src/Evolu/Diff.d.ts +3 -3
  29. package/dist/src/Evolu/Diff.d.ts.map +1 -1
  30. package/dist/src/Evolu/Diff.js +7 -5
  31. package/dist/src/Evolu/Evolu.d.ts +208 -133
  32. package/dist/src/Evolu/Evolu.d.ts.map +1 -1
  33. package/dist/src/Evolu/Evolu.js +188 -183
  34. package/dist/src/Evolu/Internal.d.ts +0 -2
  35. package/dist/src/Evolu/Internal.d.ts.map +1 -1
  36. package/dist/src/Evolu/Internal.js +0 -2
  37. package/dist/src/Evolu/LocalAuth.d.ts +150 -0
  38. package/dist/src/Evolu/LocalAuth.d.ts.map +1 -0
  39. package/dist/src/Evolu/LocalAuth.js +174 -0
  40. package/dist/src/Evolu/Owner.d.ts +273 -120
  41. package/dist/src/Evolu/Owner.d.ts.map +1 -1
  42. package/dist/src/Evolu/Owner.js +130 -104
  43. package/dist/src/Evolu/Platform.d.ts +9 -7
  44. package/dist/src/Evolu/Platform.d.ts.map +1 -1
  45. package/dist/src/Evolu/Protocol.d.ts +277 -232
  46. package/dist/src/Evolu/Protocol.d.ts.map +1 -1
  47. package/dist/src/Evolu/Protocol.js +603 -378
  48. package/dist/src/Evolu/Public.d.ts +6 -8
  49. package/dist/src/Evolu/Public.d.ts.map +1 -1
  50. package/dist/src/Evolu/Public.js +2 -3
  51. package/dist/src/Evolu/PublicKysely.js +3 -3
  52. package/dist/src/Evolu/Query.d.ts +2 -1
  53. package/dist/src/Evolu/Query.d.ts.map +1 -1
  54. package/dist/src/Evolu/Relay.d.ts +92 -7
  55. package/dist/src/Evolu/Relay.d.ts.map +1 -1
  56. package/dist/src/Evolu/Relay.js +238 -76
  57. package/dist/src/Evolu/Schema.d.ts +129 -73
  58. package/dist/src/Evolu/Schema.d.ts.map +1 -1
  59. package/dist/src/Evolu/Schema.js +169 -89
  60. package/dist/src/Evolu/Storage.d.ts +240 -26
  61. package/dist/src/Evolu/Storage.d.ts.map +1 -1
  62. package/dist/src/Evolu/Storage.js +189 -91
  63. package/dist/src/Evolu/Sync.d.ts +67 -13
  64. package/dist/src/Evolu/Sync.d.ts.map +1 -1
  65. package/dist/src/Evolu/Sync.js +441 -20
  66. package/dist/src/Evolu/Timestamp.d.ts +85 -27
  67. package/dist/src/Evolu/Timestamp.d.ts.map +1 -1
  68. package/dist/src/Evolu/Timestamp.js +77 -18
  69. package/dist/src/Identicon.d.ts +35 -0
  70. package/dist/src/Identicon.d.ts.map +1 -0
  71. package/dist/src/Identicon.js +143 -0
  72. package/dist/src/Instances.d.ts +34 -0
  73. package/dist/src/Instances.d.ts.map +1 -0
  74. package/dist/src/Instances.js +44 -0
  75. package/dist/src/ManyToManyMap.d.ts +71 -10
  76. package/dist/src/ManyToManyMap.d.ts.map +1 -1
  77. package/dist/src/ManyToManyMap.js +41 -6
  78. package/dist/src/Number.d.ts +4 -3
  79. package/dist/src/Number.d.ts.map +1 -1
  80. package/dist/src/Number.js +5 -4
  81. package/dist/src/Platform.d.ts +20 -0
  82. package/dist/src/Platform.d.ts.map +1 -0
  83. package/dist/src/Platform.js +22 -0
  84. package/dist/src/Random.d.ts +3 -2
  85. package/dist/src/Random.d.ts.map +1 -1
  86. package/dist/src/Resources.d.ts +118 -0
  87. package/dist/src/Resources.d.ts.map +1 -0
  88. package/dist/src/Resources.js +197 -0
  89. package/dist/src/Result.d.ts +184 -52
  90. package/dist/src/Result.d.ts.map +1 -1
  91. package/dist/src/Result.js +30 -241
  92. package/dist/src/Skiplist.js +2 -1
  93. package/dist/src/Sqlite.d.ts +63 -5
  94. package/dist/src/Sqlite.d.ts.map +1 -1
  95. package/dist/src/Sqlite.js +110 -9
  96. package/dist/src/Task.d.ts +586 -0
  97. package/dist/src/Task.d.ts.map +1 -0
  98. package/dist/src/Task.js +469 -0
  99. package/dist/src/Time.d.ts +66 -1
  100. package/dist/src/Time.d.ts.map +1 -1
  101. package/dist/src/Time.js +99 -5
  102. package/dist/src/Type.d.ts +622 -340
  103. package/dist/src/Type.d.ts.map +1 -1
  104. package/dist/src/Type.js +666 -464
  105. package/dist/src/Types.d.ts +1 -75
  106. package/dist/src/Types.d.ts.map +1 -1
  107. package/dist/src/WebSocket.d.ts +5 -2
  108. package/dist/src/WebSocket.d.ts.map +1 -1
  109. package/dist/src/WebSocket.js +12 -18
  110. package/dist/src/Worker.d.ts +39 -11
  111. package/dist/src/Worker.d.ts.map +1 -1
  112. package/dist/src/Worker.js +22 -4
  113. package/dist/src/index.d.ts +7 -2
  114. package/dist/src/index.d.ts.map +1 -1
  115. package/dist/src/index.js +7 -2
  116. package/package.json +14 -13
  117. package/src/Array.ts +90 -11
  118. package/src/Assert.ts +6 -24
  119. package/src/Brand.ts +75 -0
  120. package/src/Buffer.ts +7 -7
  121. package/src/Cache.ts +85 -0
  122. package/src/Callbacks.ts +62 -22
  123. package/src/Console.ts +91 -11
  124. package/src/Crypto.ts +97 -82
  125. package/src/Evolu/Db.ts +517 -1020
  126. package/src/Evolu/Diff.ts +7 -5
  127. package/src/Evolu/Evolu.ts +464 -355
  128. package/src/Evolu/Internal.ts +0 -2
  129. package/src/Evolu/LocalAuth.ts +463 -0
  130. package/src/Evolu/Owner.ts +355 -228
  131. package/src/Evolu/Platform.ts +9 -9
  132. package/src/Evolu/Protocol.ts +859 -676
  133. package/src/Evolu/Public.ts +7 -14
  134. package/src/Evolu/PublicKysely.ts +3 -3
  135. package/src/Evolu/Query.ts +2 -1
  136. package/src/Evolu/Relay.ts +437 -93
  137. package/src/Evolu/Schema.ts +391 -191
  138. package/src/Evolu/Storage.ts +532 -135
  139. package/src/Evolu/Sync.ts +766 -37
  140. package/src/Evolu/Timestamp.ts +88 -35
  141. package/src/Identicon.ts +197 -0
  142. package/src/Instances.ts +90 -0
  143. package/src/ManyToManyMap.ts +124 -24
  144. package/src/Number.ts +6 -10
  145. package/src/Platform.ts +26 -0
  146. package/src/Random.ts +3 -2
  147. package/src/Resources.ts +367 -0
  148. package/src/Result.ts +191 -54
  149. package/src/Skiplist.ts +1 -1
  150. package/src/Sqlite.ts +122 -17
  151. package/src/Task.ts +901 -0
  152. package/src/Time.ts +180 -5
  153. package/src/Type.ts +1084 -727
  154. package/src/Types.ts +1 -77
  155. package/src/WebSocket.ts +27 -25
  156. package/src/Worker.ts +72 -23
  157. package/src/index.ts +7 -2
  158. package/dist/src/Evolu/Config.d.ts +0 -69
  159. package/dist/src/Evolu/Config.d.ts.map +0 -1
  160. package/dist/src/Evolu/Config.js +0 -9
  161. package/dist/src/Evolu/Kysely.d.ts +0 -6
  162. package/dist/src/Evolu/Kysely.d.ts.map +0 -1
  163. package/dist/src/Evolu/Kysely.js +0 -21
  164. package/dist/src/NanoId.d.ts +0 -27
  165. package/dist/src/NanoId.d.ts.map +0 -1
  166. package/dist/src/NanoId.js +0 -6
  167. package/dist/src/Promise.d.ts +0 -180
  168. package/dist/src/Promise.d.ts.map +0 -1
  169. package/dist/src/Promise.js +0 -176
  170. package/src/Evolu/Config.ts +0 -83
  171. package/src/Evolu/Kysely.ts +0 -38
  172. package/src/NanoId.ts +0 -39
  173. package/src/Promise.ts +0 -295
@@ -3,33 +3,54 @@
3
3
  *
4
4
  * Evolu Protocol is a local-first, end-to-end encrypted binary synchronization
5
5
  * protocol optimized for minimal size and maximum speed. It enables data sync
6
- * between a client and a relay, clients in a peer-to-peer (P2P) setup, or
7
- * relays with each other.
6
+ * between a client and a relay. In the future, direct peer-to-peer (P2P) sync
7
+ * between clients will be possible without a relay.
8
+ *
9
+ * Relays don't need to sync with each other—clients using those relays will
10
+ * sync them eventually. If a relay is offline (e.g., for maintenance), it will
11
+ * sync automatically later via client sync logic. For relay backup using
12
+ * SQLite, see https://sqlite.org/rsync.html (uses a similar algorithm to Evolu
13
+ * RBSR).
8
14
  *
9
15
  * Evolu Protocol is designed for SQLite but can be extended to any database. It
10
16
  * implements [Range-Based Set
11
17
  * Reconciliation](https://arxiv.org/abs/2212.13567). To learn how RBSR works,
12
18
  * check [Negentropy](https://logperiodic.com/rbsr.html). Evolu Protocol is
13
19
  * similar to Negentropy but uses different encoding and also provides data
14
- * transfer and ownership.
20
+ * transfer, ownership, real-time broadcasting, request-response semantics, and
21
+ * error handling.
22
+ *
23
+ * ### Message structure
15
24
  *
16
- * ### Message Structure
25
+ * | Field | Notes |
26
+ * | :----------------------------- | :------------------------ |
27
+ * | **Header** | |
28
+ * | - {@link protocolVersion} | |
29
+ * | - {@link OwnerId} | {@link Owner} |
30
+ * | - messageType | {@link MessageType} |
31
+ * | **Request (messageType=0)** | |
32
+ * | - hasWriteKey | 0 = no, 1 = yes |
33
+ * | - {@link OwnerWriteKey} | If hasWriteKey = 1 |
34
+ * | - subscriptionFlag | {@link SubscriptionFlags} |
35
+ * | **Response (messageType=1)** | |
36
+ * | - {@link ProtocolErrorCode} | |
37
+ * | **Broadcast (messageType=2)** | |
38
+ * | - (no additional fields) | |
39
+ * | **Messages** | |
40
+ * | - {@link NonNegativeInt} | A number of messages. |
41
+ * | - {@link EncryptedCrdtMessage} | |
42
+ * | **Ranges** | |
43
+ * | - {@link NonNegativeInt} | Number of ranges. |
44
+ * | - {@link Range} | |
17
45
  *
18
- * | Field | Notes |
19
- * | :----------------------------- | :------------------------- |
20
- * | **Header** | |
21
- * | - {@link protocolVersion} | |
22
- * | - {@link OwnerId} | |
23
- * | - {@link ProtocolErrorCode} | In non-initiator response. |
24
- * | **Messages** | |
25
- * | - {@link NonNegativeInt} | A number of messages. |
26
- * | - {@link EncryptedCrdtMessage} | |
27
- * | - {@link WriteKey} | In initiator request. |
28
- * | **Ranges** | |
29
- * | - {@link NonNegativeInt} | Number of ranges. |
30
- * | - {@link Range} | |
46
+ * ### WriteKey validation
31
47
  *
32
- * Every protocol message belongs to an {@link Owner}.
48
+ * The initiator sends a hasWriteKey flag and optionally a WriteKey. The
49
+ * WriteKey is required when sending messages as a secure token proving the
50
+ * initiator can write changes. It's ok to not send a WriteKey if the initiator
51
+ * is only syncing (read-only) and not sending messages. The non-initiator
52
+ * validates the WriteKey immediately after parsing the initiator header, before
53
+ * processing any messages or ranges.
33
54
  *
34
55
  * ### Synchronization
35
56
  *
@@ -43,38 +64,48 @@
43
64
  * if further sync is needed or possible, continuing until both sides are
44
65
  * synchronized.
45
66
  *
67
+ * The **non-initiator always responds** to provide sync completion feedback,
68
+ * even with empty messages containing only the header and no error. This allows
69
+ * the initiator to detect when synchronization is complete.
70
+ *
46
71
  * Both **Messages** and **Ranges** are optional, allowing each side to send,
47
72
  * sync, or only subscribe data as needed.
48
73
  *
49
- * When the initiator sends data, the {@link WriteKey} is required in Messages as
50
- * a secure token proving the initiator can write changes. The non-initiator
51
- * responds without a {@link WriteKey}, since the initiator’s request already
52
- * signals it wants data. If the non-initiator detects an issue, it sends an
53
- * error code via the `Error` field in the header back to the initiator. In
54
- * relay-to-relay or P2P sync, both sides may require the {@link WriteKey}
55
- * depending on who is the initiator.
74
+ * When the initiator sends data, the {@link OwnerWriteKey} is required as a
75
+ * secure token proving the initiator can write changes. The non-initiator
76
+ * responds without a {@link OwnerWriteKey}, since the initiator’s request
77
+ * already signals it wants data. If the non-initiator detects an issue, it
78
+ * sends an error code via the `Error` field in the header back to the
79
+ * initiator. In relay-to-relay or P2P sync, both sides may require the
80
+ * {@link OwnerWriteKey} depending on who is the initiator.
56
81
  *
57
- * ### Protocol Errors
82
+ * ### Protocol errors
58
83
  *
59
84
  * The protocol uses error codes in the header to signal issues:
60
85
  *
61
86
  * - {@link ProtocolWriteKeyError}: The provided WriteKey is invalid or missing.
62
- * - {@link ProtocolWriteError}: A write operation failed (e.g., due to storage
63
- * limits or billing).
64
- * - {@link ProtocolSyncError}: A generic or unexpected synchronization failure
87
+ * - {@link ProtocolWriteError}: A serious relay-side write failure occurred.
88
+ * - {@link ProtocolQuotaError}: Storage or billing quota exceeded.
89
+ * - {@link ProtocolSyncError}: A serious relay-side synchronization failure
65
90
  * occurred.
66
- * - {@link ProtocolUnsupportedVersionError}: Protocol version mismatch.
91
+ * - {@link ProtocolVersionError}: Protocol version mismatch.
67
92
  * - {@link ProtocolInvalidDataError}: The message is malformed or corrupted.
68
93
  *
69
- * All protocol errors except `ProtocolInvalidDataError` include the `ownerId`
94
+ * All protocol errors except `ProtocolInvalidDataError` include the `OwnerId`
70
95
  * to allow clients to associate errors with the correct owner.
71
96
  *
72
- * ### Message Size Limit
97
+ * ### Message size limit
73
98
  *
74
99
  * The protocol enforces a strict maximum size for all messages, defined by
75
- * {@link maxProtocolMessageSize}. This ensures every {@link ProtocolMessage} is
76
- * less than or equal to this limit, eliminating the need for applications to
77
- * fragment and reconstruct messages during transmission.
100
+ * {@link ProtocolMessageMaxSize}. This ensures every {@link ProtocolMessage} is
101
+ * less than or equal to this limit, enabling stateless transports, simplified
102
+ * relay implementation, and predictable memory usage. When all messages don't
103
+ * fit within the limit, the protocol automatically continues synchronization in
104
+ * subsequent rounds using range-based reconciliation.
105
+ *
106
+ * Database mutations are limited to 640KB, which is smaller than the protocol
107
+ * message limit to ensure efficient sync with
108
+ * {@link defaultProtocolMessageRangesMaxSize}.
78
109
  *
79
110
  * ### Why Binary?
80
111
  *
@@ -82,13 +113,12 @@
82
113
  *
83
114
  * - Encrypted data doesn’t compress well, unlike plain JSON.
84
115
  * - Message size must be controlled during creation.
85
- * - Sequential byte reading is faster than parsing and can avoid conversions.
116
+ * - Sequential byte reading is faster than parsing and avoids conversions.
86
117
  *
87
118
  * It uses structure-aware encoding, significantly outperforming generic binary
88
119
  * serialization formats with the following optimizations:
89
120
  *
90
121
  * - **NonNegativeInt:** Up to 33% smaller than MessagePack.
91
- * - **Base64Url Strings:** Up to 25% size reduction.
92
122
  * - **DateIso:** Up to 75% smaller.
93
123
  * - **Timestamp Encoding:** Delta encoding for milliseconds and run-length
94
124
  * encoding (RLE) for counters and NodeIds.
@@ -118,266 +148,239 @@
118
148
  * Version negotiation is per-owner, allowing Evolu Protocol to evolve safely
119
149
  * over time and provide clear feedback about version mismatches.
120
150
  *
151
+ * ### Credible exit
152
+ *
153
+ * The protocol specification is intentionally non-configurable to ensure
154
+ * universal compatibility. This design allows applications (users) to switch
155
+ * between any compliant relay without negotiation or compatibility checks
156
+ * beyond version matching. Relays are generic infrastructure that any
157
+ * application can use interchangeably making exit from any single provider
158
+ * technically feasible and economically viable.
159
+ *
121
160
  * @module
122
161
  */
123
162
 
124
- import { sha256 } from "@noble/hashes/sha2";
125
- import { pack, unpack, unpackMultiple } from "msgpackr";
163
+ /**
164
+ * TODO:
165
+ *
166
+ * - The client-relay naming convention in functions like
167
+ * `applyProtocolMessageAsClient` and `applyProtocolMessageAsRelay` is not
168
+ * ideal. In the future, clients will be able to sync directly with each other
169
+ * (P2P), making the current naming misleading. Consider using
170
+ * initiator/non-initiator terminology instead, and consolidate into a single
171
+ * `applyProtocolMessage` function with conditional arguments to reduce code
172
+ * duplication.
173
+ * - ProtocolQuotaError should return storedBytes and actual quota.
174
+ */
175
+
176
+ import { Packr } from "msgpackr";
126
177
  import { isNonEmptyReadonlyArray, NonEmptyReadonlyArray } from "../Array.js";
127
178
  import { assert } from "../Assert.js";
179
+ import { Brand } from "../Brand.js";
128
180
  import {
129
181
  Buffer,
130
- BufferError,
131
182
  bytesToHex,
132
183
  bytesToUtf8,
133
- concatBytes,
134
184
  createBuffer,
135
185
  hexToBytes,
136
186
  utf8ToBytes,
137
187
  } from "../Buffer.js";
138
188
  import {
139
- CreateRandomBytesDep,
140
189
  EncryptionKey,
141
190
  padmePaddingLength,
191
+ RandomBytesDep,
142
192
  SymmetricCryptoDecryptError,
143
193
  SymmetricCryptoDep,
144
194
  } from "../Crypto.js";
145
195
  import { eqArrayNumber } from "../Eq.js";
146
196
  import { computeBalancedBuckets } from "../Number.js";
147
- import { objectToEntries, ReadonlyRecord } from "../Object.js";
197
+ import { objectToEntries } from "../Object.js";
148
198
  import { err, ok, Result } from "../Result.js";
149
199
  import { SqliteValue } from "../Sqlite.js";
150
200
  import {
151
201
  Base64Url,
152
- DateIsoString,
202
+ base64UrlToUint8Array,
203
+ between,
204
+ DateIso,
153
205
  Id,
154
- idTypeValueLength,
155
- JsonValueFromString,
156
- maxLength,
157
- NanoId,
206
+ IdBytes,
207
+ idBytesToId,
208
+ idBytesTypeValueLength,
209
+ idToIdBytes,
210
+ Int,
211
+ Json,
212
+ jsonToJsonValue,
158
213
  NonNegativeInt,
159
214
  Number,
160
215
  PositiveInt,
216
+ uint8ArrayToBase64Url,
161
217
  } from "../Type.js";
162
- import { Brand, Predicate } from "../Types.js";
218
+ import { Predicate } from "../Types.js";
163
219
  import {
164
- // eslint-disable-next-line @typescript-eslint/no-unused-vars
220
+ BaseOwnerError,
165
221
  Owner,
166
222
  OwnerId,
167
- OwnerWithWriteAccess,
168
- WriteKey,
169
- writeKeyLength,
223
+ OwnerIdBytes,
224
+ ownerIdToOwnerIdBytes,
225
+ OwnerWriteKey,
226
+ ownerWriteKeyLength,
170
227
  } from "./Owner.js";
171
228
  import {
172
- BinaryTimestamp,
173
- binaryTimestampToTimestamp,
229
+ BaseRange,
230
+ CrdtMessage,
231
+ DbChange,
232
+ EncryptedCrdtMessage,
233
+ EncryptedDbChange,
234
+ Fingerprint,
235
+ FingerprintRange,
236
+ fingerprintSize,
237
+ InfiniteUpperBound,
238
+ Range,
239
+ RangeType,
240
+ RangeUpperBound,
241
+ SkipRange,
242
+ StorageDep,
243
+ TimestampsRange,
244
+ } from "./Storage.js";
245
+ import {
174
246
  Counter,
247
+ eqTimestamp,
175
248
  Millis,
176
249
  NodeId,
177
250
  Timestamp,
178
- timestampToBinaryTimestamp,
251
+ TimestampBytes,
252
+ timestampBytesLength,
253
+ timestampBytesToTimestamp,
254
+ timestampToTimestampBytes,
179
255
  } from "./Timestamp.js";
180
256
 
181
- /** Maximum size of the entire protocol message in bytes. */
182
- export const maxProtocolMessageSize = 1_000_000 as PositiveInt;
183
-
184
- /** Maximum size of the ranges in bytes. */
185
- export const maxProtocolMessageRangesSize = 30_000 as PositiveInt;
186
-
187
- /** Evolu Protocol Message. */
188
- export type ProtocolMessage = Uint8Array & Brand<"ProtocolMessage">;
189
-
190
- /** Evolu Protocol version. */
191
- export const protocolVersion = 0 as NonNegativeInt;
192
-
193
- export const ProtocolErrorCode = {
194
- NoError: 0,
195
- /** A code for {@link ProtocolWriteKeyError}. */
196
- WriteKeyError: 1,
197
- /** A code for {@link ProtocolWriteError}. */
198
- WriteError: 2,
199
- /** A code for {@link ProtocolSyncError}. */
200
- SyncError: 3,
201
- } as const;
202
-
203
- type ProtocolErrorCode =
204
- (typeof ProtocolErrorCode)[keyof typeof ProtocolErrorCode];
205
-
206
257
  /**
207
- * Evolu Protocol Storage
258
+ * Evolu uses MessagePack for numbers and JSONs.
208
259
  *
209
- * The protocol is agnostic to storage implementation details—any storage can be
210
- * plugged in, as long as it implements this interface. Implementations must
211
- * handle their own errors; return values only indicates overall success or
212
- * failure.
260
+ * - `variableMapSize: true` - More compact maps, ~5-10% slower encoding
261
+ * - `useRecords: false` - Standard MessagePack without extensions
213
262
  */
214
- export interface Storage {
215
- readonly getSize: (ownerId: BinaryOwnerId) => NonNegativeInt | null;
216
-
217
- readonly fingerprint: (
218
- ownerId: BinaryOwnerId,
219
- begin: NonNegativeInt,
220
- end: NonNegativeInt,
221
- ) => Fingerprint | null;
222
-
223
- /**
224
- * Computes fingerprints with their upper bounds in one call.
225
- *
226
- * This function can be replaced with many fingerprint/findLowerBound calls,
227
- * but implementations can leverage it for batching and more efficient
228
- * fingerprint computation.
229
- */
230
- readonly fingerprintRanges: (
231
- ownerId: BinaryOwnerId,
232
- buckets: ReadonlyArray<NonNegativeInt>,
233
- upperBound?: RangeUpperBound,
234
- ) => ReadonlyArray<FingerprintRange> | null;
235
-
236
- readonly findLowerBound: (
237
- ownerId: BinaryOwnerId,
238
- begin: NonNegativeInt,
239
- end: NonNegativeInt,
240
- upperBound: RangeUpperBound,
241
- ) => NonNegativeInt | null;
242
-
243
- readonly iterate: (
244
- ownerId: BinaryOwnerId,
245
- begin: NonNegativeInt,
246
- end: NonNegativeInt,
247
- callback: (timestamp: BinaryTimestamp, index: NonNegativeInt) => boolean,
248
- ) => void;
249
-
250
- /**
251
- * Authorizes the initiator's {@link WriteKey} for the given
252
- * {@link BinaryOwnerId}.
253
- *
254
- * For a client that does not expect foreign writes, return `false`.
255
- */
256
- readonly validateWriteKey: (
257
- ownerId: BinaryOwnerId,
258
- writeKey: WriteKey,
259
- ) => boolean;
260
-
261
- /** Write encrypted {@link CrdtMessage}s to storage. */
262
- readonly writeMessages: (
263
- ownerId: BinaryOwnerId,
264
- messages: NonEmptyReadonlyArray<EncryptedCrdtMessage>,
265
- ) => boolean;
263
+ const packr = new Packr({ variableMapSize: true, useRecords: false });
266
264
 
267
- /** Read encrypted {@link DbChange}s from storage. */
268
- readonly readDbChange: (
269
- ownerId: BinaryOwnerId,
270
- timestamp: BinaryTimestamp,
271
- ) => EncryptedDbChange | null;
272
- }
273
-
274
- export interface StorageDep {
275
- readonly storage: Storage;
276
- }
277
-
278
- /** An encrypted {@link CrdtMessage}. */
279
- export interface EncryptedCrdtMessage {
280
- readonly timestamp: Timestamp;
281
- readonly change: EncryptedDbChange;
282
- }
283
-
284
- /** Encrypted DbChange */
285
- export type EncryptedDbChange = Uint8Array & Brand<"EncryptedDbChange">;
265
+ const minProtocolMessageMaxSize = 1_000_000;
266
+ const maxProtocolMessageMaxSize = 100_000_000;
286
267
 
287
268
  /**
288
- * A CRDT message that combines a unique {@link Timestamp} with a
289
- * {@link DbChange}.
269
+ * Protocol message maximum size.
270
+ *
271
+ * Defines the upper limit for how large a single protocol message can be.
272
+ * Implementations must enforce a maximum size between 1MB and 100MB to ensure
273
+ * compatibility across all Evolu implementations (the maximum size of mutation
274
+ * change is hardcoded and enforced hence the maximum size can't be smaller).
275
+ *
276
+ * Larger maximum sizes can be configured by relays to reduce roundtrips. For
277
+ * example, a dedicated relay with ample resources could configure a 100MB
278
+ * maximum to minimize roundtrips for large syncs.
279
+ *
280
+ * Only relays can safely configure larger sizes, as clients will handle them.
281
+ * Increasing this value on the client side would break compatibility with
282
+ * relays that enforce smaller limits.
290
283
  */
291
- export interface CrdtMessage {
292
- readonly timestamp: Timestamp;
293
- readonly change: DbChange;
294
- }
284
+ export const ProtocolMessageMaxSize = between(
285
+ minProtocolMessageMaxSize,
286
+ maxProtocolMessageMaxSize,
287
+ )(Int);
288
+
289
+ export type ProtocolMessageMaxSize = typeof ProtocolMessageMaxSize.Type;
295
290
 
296
291
  /**
297
- * A DbChange is a change to a table row. Together with a unique
298
- * {@link Timestamp}, it forms a {@link CrdtMessage}.
292
+ * Default {@link ProtocolMessageMaxSize} (1MB).
293
+ *
294
+ * The standard size used across Evolu implementations. Relays with more
295
+ * resources can configure larger sizes to reduce roundtrips.
299
296
  */
300
- export interface DbChange {
301
- readonly table: Base64Url256;
302
- readonly id: Id;
303
- readonly values: ReadonlyRecord<Base64Url256, SqliteValue>;
304
- }
305
-
306
- export const RangeType = {
307
- Fingerprint: 1,
308
- Skip: 0,
309
- Timestamps: 2,
310
- } as const;
311
-
312
- export type RangeType = (typeof RangeType)[keyof typeof RangeType];
313
-
314
- export const InfiniteUpperBound = Symbol("InfiniteUpperBound");
315
- export type InfiniteUpperBound = typeof InfiniteUpperBound;
297
+ export const defaultProtocolMessageMaxSize =
298
+ minProtocolMessageMaxSize as ProtocolMessageMaxSize;
316
299
 
317
300
  /**
318
- * Union type for Range's upperBound: either a {@link BinaryTimestamp} or
319
- * {@link InfiniteUpperBound}.
301
+ * Protocol message ranges maximum size.
302
+ *
303
+ * Defines the upper limit for how large the ranges section of a protocol
304
+ * message can be. Implementations must enforce a maximum size between 3KB and
305
+ * 100KB to ensure compatibility.
306
+ *
307
+ * The upper bound is set to ensure ranges fit within the default 1MB
308
+ * {@link defaultProtocolMessageMaxSize}, maintaining compatibility between all
309
+ * clients and relays.
320
310
  */
321
- export type RangeUpperBound = BinaryTimestamp | InfiniteUpperBound;
322
-
323
- interface BaseRange {
324
- readonly upperBound: RangeUpperBound;
325
- }
326
-
327
- export interface SkipRange extends BaseRange {
328
- readonly type: typeof RangeType.Skip;
329
- }
330
-
331
- export interface FingerprintRange extends BaseRange {
332
- readonly type: typeof RangeType.Fingerprint;
333
- readonly fingerprint: Fingerprint;
334
- }
311
+ export const ProtocolMessageRangesMaxSize = between(3_000, 100_000)(Int);
312
+ export type ProtocolMessageRangesMaxSize =
313
+ typeof ProtocolMessageRangesMaxSize.Type;
335
314
 
336
315
  /**
337
- * A cryptographic hash used for efficiently comparing collections of
338
- * {@link BinaryTimestamp}s.
316
+ * Default {@link ProtocolMessageRangesMaxSize} (30KB).
339
317
  *
340
- * It consists of the first {@link fingerprintSize} bytes of the SHA-256 hash of
341
- * one or more timestamps.
318
+ * The standard size used across Evolu implementations. Relays with more
319
+ * resources can configure larger sizes to reduce roundtrips.
342
320
  */
343
- export type Fingerprint = Uint8Array & Brand<"Fingerprint">;
321
+ export const defaultProtocolMessageRangesMaxSize =
322
+ 30_000 as ProtocolMessageRangesMaxSize;
344
323
 
345
- export const fingerprintSize = 12 as NonNegativeInt;
324
+ /** Evolu Protocol Message. */
325
+ export type ProtocolMessage = Uint8Array & Brand<"ProtocolMessage">;
346
326
 
347
- /** A fingerprint of an empty range. */
348
- export const zeroFingerprint = new Uint8Array(fingerprintSize) as Fingerprint;
327
+ /** Evolu Protocol version. */
328
+ export const protocolVersion = NonNegativeInt.orThrow(0);
329
+
330
+ export const MessageType = {
331
+ /** Request message from initiator (client) to non-initiator (relay). */
332
+ Request: 0,
333
+ /** Response message from non-initiator (relay) to initiator (client). */
334
+ Response: 1,
335
+ /** Broadcast message from non-initiator (relay) to subscribed clients. */
336
+ Broadcast: 2,
337
+ } as const;
349
338
 
350
- export interface TimestampsRange extends BaseRange {
351
- readonly type: typeof RangeType.Timestamps;
352
- readonly timestamps: ReadonlyArray<BinaryTimestamp>;
353
- }
339
+ export type MessageType = (typeof MessageType)[keyof typeof MessageType];
354
340
 
355
- export interface TimestampsRangeWithTimestampsBuffer extends BaseRange {
356
- readonly type: typeof RangeType.Timestamps;
357
- readonly timestamps: TimestampsBuffer;
358
- }
341
+ export const SubscriptionFlags = {
342
+ /** No subscription changes for this owner. */
343
+ None: 0,
344
+ /** Subscribe to updates for this owner. */
345
+ Subscribe: 1,
346
+ /** Unsubscribe from updates for this owner. */
347
+ Unsubscribe: 2,
348
+ } as const;
349
+
350
+ export type SubscriptionFlag =
351
+ (typeof SubscriptionFlags)[keyof typeof SubscriptionFlags];
352
+
353
+ export const ProtocolErrorCode = {
354
+ NoError: 0,
355
+ /** A code for {@link ProtocolWriteKeyError}. */
356
+ WriteKeyError: 1,
357
+ /** A code for {@link ProtocolWriteError}. */
358
+ WriteError: 2,
359
+ /** A code for {@link ProtocolQuotaError}. */
360
+ QuotaError: 3,
361
+ /** A code for {@link ProtocolSyncError}. */
362
+ SyncError: 4,
363
+ } as const;
359
364
 
360
- export type Range = SkipRange | FingerprintRange | TimestampsRange;
365
+ type ProtocolErrorCode =
366
+ (typeof ProtocolErrorCode)[keyof typeof ProtocolErrorCode];
361
367
 
362
368
  export type ProtocolError =
363
- | ProtocolUnsupportedVersionError
369
+ | ProtocolVersionError
364
370
  | ProtocolInvalidDataError
365
371
  | ProtocolWriteKeyError
366
372
  | ProtocolWriteError
367
- | ProtocolSyncError;
368
-
369
- /** Base interface for all protocol errors. */
370
- export interface ProtocolErrorBase {
371
- readonly ownerId: OwnerId;
372
- }
373
+ | ProtocolSyncError
374
+ | ProtocolQuotaError
375
+ | ProtocolTimestampMismatchError;
373
376
 
374
377
  /**
375
378
  * Represents a version mismatch in the Evolu Protocol. Occurs when the
376
379
  * initiator and non-initiator are using incompatible protocol versions.
377
380
  */
378
- export interface ProtocolUnsupportedVersionError extends ProtocolErrorBase {
379
- readonly type: "ProtocolUnsupportedVersionError";
380
- readonly unsupportedVersion: NonNegativeInt;
381
+ export interface ProtocolVersionError extends BaseOwnerError {
382
+ readonly type: "ProtocolVersionError";
383
+ readonly version: NonNegativeInt;
381
384
  /** Indicates which side is obsolete and should update. */
382
385
  readonly isInitiator: boolean;
383
386
  }
@@ -389,43 +392,70 @@ export interface ProtocolInvalidDataError {
389
392
  readonly error: unknown;
390
393
  }
391
394
 
392
- /** Error when a {@link WriteKey} is invalid, missing, or fails validation. */
393
- export interface ProtocolWriteKeyError extends ProtocolErrorBase {
395
+ /** Error when a {@link OwnerWriteKey} is invalid, missing, or fails validation. */
396
+ export interface ProtocolWriteKeyError extends BaseOwnerError {
394
397
  readonly type: "ProtocolWriteKeyError";
395
398
  }
396
399
 
397
400
  /**
398
- * Error when a write fails due to storage limits or billing requirements.
399
- * Indicates the need to expand capacity or resolve payment issues.
401
+ * Error indicating a serious relay-side write failure. Clients should log this
402
+ * error and show a generic sync error to the user.
400
403
  */
401
- export interface ProtocolWriteError extends ProtocolErrorBase {
404
+ export interface ProtocolWriteError extends BaseOwnerError {
402
405
  readonly type: "ProtocolWriteError";
403
406
  }
404
407
 
405
408
  /**
406
- * Error indicating a synchronization failure during the protocol exchange. Used
407
- * for unexpected or generic sync errors not covered by other error types.
409
+ * Error when storage or billing quota is exceeded.
410
+ *
411
+ * When relay rejects writes due to quota, the affected device stops syncing
412
+ * because RBSR requires both sides to converge—if the relay won't accept the
413
+ * client's data, they can never reach the same state. Only the device with
414
+ * excess local data is affected. Other devices that haven't exceeded quota can
415
+ * still sync normally.
416
+ *
417
+ * Clients should prompt the user to upgrade their plan.
418
+ */
419
+ export interface ProtocolQuotaError extends BaseOwnerError {
420
+ readonly type: "ProtocolQuotaError";
421
+ }
422
+
423
+ /**
424
+ * Error indicating a serious relay-side synchronization failure. Clients should
425
+ * log this error and show a generic sync error to the user.
408
426
  */
409
- export interface ProtocolSyncError extends ProtocolErrorBase {
427
+ export interface ProtocolSyncError extends BaseOwnerError {
410
428
  readonly type: "ProtocolSyncError";
411
429
  }
412
430
 
431
+ /**
432
+ * Error when embedded timestamp doesn't match expected timestamp in
433
+ * EncryptedDbChange. Indicates potential tampering or corruption of CRDT
434
+ * messages.
435
+ */
436
+ export interface ProtocolTimestampMismatchError {
437
+ readonly type: "ProtocolTimestampMismatchError";
438
+ readonly expected: Timestamp;
439
+ readonly embedded: Timestamp;
440
+ }
441
+
413
442
  /**
414
443
  * Creates a {@link ProtocolMessage} from CRDT messages.
415
444
  *
416
- * If the message size would exceed {@link maxProtocolMessageSize}, the protocol
417
- * ensures all messages will be sent in the next round(s) even over
445
+ * If the message size would exceed {@link defaultProtocolMessageMaxSize}, the
446
+ * protocol ensures all messages will be sent in the next round(s) even over
418
447
  * unidirectional and stateless transports.
419
448
  */
420
449
  export const createProtocolMessageFromCrdtMessages =
421
- (deps: SymmetricCryptoDep & CreateRandomBytesDep) =>
450
+ (deps: RandomBytesDep & SymmetricCryptoDep) =>
422
451
  (
423
- owner: OwnerWithWriteAccess,
452
+ owner: Owner,
424
453
  messages: NonEmptyReadonlyArray<CrdtMessage>,
425
- maxSize?: PositiveInt,
454
+ maxSize?: ProtocolMessageMaxSize,
426
455
  ): ProtocolMessage => {
427
456
  const buffer = createProtocolMessageBuffer(owner.id, {
428
- totalMaxSize: maxSize ?? maxProtocolMessageSize,
457
+ messageType: MessageType.Request,
458
+ totalMaxSize: maxSize ?? defaultProtocolMessageMaxSize,
429
459
  writeKey: owner.writeKey,
430
460
  });
431
461
 
@@ -433,7 +463,7 @@ export const createProtocolMessageFromCrdtMessages =
433
463
 
434
464
  for (const message of messages) {
435
465
  const change = encodeAndEncryptDbChange(deps)(
436
- message.change,
466
+ message,
437
467
  owner.encryptionKey,
438
468
  );
439
469
  const encryptedCrdtMessage = { timestamp: message.timestamp, change };
@@ -454,14 +484,14 @@ export const createProtocolMessageFromCrdtMessages =
454
484
  *
455
485
  * The ideal approach would be to send three ranges (skip, fingerprint,
456
486
  * skip) where the fingerprint of unsent messages would act as narrow sync
457
- * probe. I think we can send {@link zeroFingerprint} which can be
458
- * interpreted as an indication that the other side should reply with
487
+ * probe. I think we can send `zeroFingerprint` which can be interpreted
488
+ * as an indication that the other side should reply with
459
489
  * {@link TimestampsRange}, so no need to restart syncing.
460
490
  *
461
491
  * For now, using a random fingerprint avoids extra complexity and is good
462
492
  * enough for this case.
463
493
  */
464
- const randomFingerprint = deps.createRandomBytes(
494
+ const randomFingerprint = deps.randomBytes.create(
465
495
  fingerprintSize,
466
496
  ) as unknown as Fingerprint;
467
497
 
@@ -479,17 +509,23 @@ export const createProtocolMessageFromCrdtMessages =
479
509
  /** Creates a {@link ProtocolMessage} for sync. */
480
510
  export const createProtocolMessageForSync =
481
511
  (deps: StorageDep) =>
482
- (ownerId: OwnerId): ProtocolMessage | null => {
483
- const buffer = createProtocolMessageBuffer(ownerId);
484
- const binaryOwnerId = ownerIdToBinaryOwnerId(ownerId);
512
+ (
513
+ ownerId: OwnerId,
514
+ subscriptionFlag?: SubscriptionFlag,
515
+ ): ProtocolMessage | null => {
516
+ const buffer = createProtocolMessageBuffer(ownerId, {
517
+ messageType: MessageType.Request,
518
+ subscriptionFlag: subscriptionFlag ?? SubscriptionFlags.None,
519
+ });
520
+ const ownerIdBytes = ownerIdToOwnerIdBytes(ownerId);
485
521
 
486
- const size = deps.storage.getSize(binaryOwnerId);
522
+ const size = deps.storage.getSize(ownerIdBytes);
487
523
  // Errors are handled by the storage.
488
524
  if (size == null) return null;
489
525
 
490
526
  splitRange(deps)(
491
- binaryOwnerId,
492
- 0 as NonNegativeInt,
527
+ ownerIdBytes,
528
+ NonNegativeInt.orThrow(0),
493
529
  size,
494
530
  InfiniteUpperBound,
495
531
  buffer,
@@ -498,6 +534,14 @@ export const createProtocolMessageForSync =
498
534
  return buffer.unwrap();
499
535
  };
500
536
 
537
+ export const createProtocolMessageForUnsubscribe = (
538
+ ownerId: OwnerId,
539
+ ): ProtocolMessage =>
540
+ createProtocolMessageBuffer(ownerId, {
541
+ messageType: MessageType.Request,
542
+ subscriptionFlag: SubscriptionFlags.Unsubscribe,
543
+ }).unwrap();
544
+
501
545
  /**
502
546
  * Mutable builder for constructing {@link ProtocolMessage} respecting size
503
547
  * limits.
@@ -525,18 +569,27 @@ export interface ProtocolMessageBuffer {
525
569
  export const createProtocolMessageBuffer = (
526
570
  ownerId: OwnerId,
527
571
  options: {
528
- readonly errorCode?: ProtocolErrorCode;
529
- readonly writeKey?: WriteKey;
530
- readonly totalMaxSize?: PositiveInt | undefined;
531
- readonly rangesMaxSize?: PositiveInt | undefined;
572
+ readonly totalMaxSize?: ProtocolMessageMaxSize | undefined;
573
+ readonly rangesMaxSize?: ProtocolMessageRangesMaxSize | undefined;
532
574
  readonly version?: NonNegativeInt;
533
- } = {},
575
+ } & (
576
+ | {
577
+ readonly messageType: typeof MessageType.Request;
578
+ readonly writeKey?: OwnerWriteKey;
579
+ readonly subscriptionFlag?: SubscriptionFlag;
580
+ }
581
+ | {
582
+ readonly messageType: typeof MessageType.Response;
583
+ readonly errorCode: ProtocolErrorCode;
584
+ }
585
+ | {
586
+ readonly messageType: typeof MessageType.Broadcast;
587
+ }
588
+ ),
534
589
  ): ProtocolMessageBuffer => {
535
590
  const {
536
- errorCode,
537
- writeKey,
538
- totalMaxSize = maxProtocolMessageSize,
539
- rangesMaxSize = maxProtocolMessageRangesSize,
591
+ totalMaxSize = defaultProtocolMessageMaxSize,
592
+ rangesMaxSize = defaultProtocolMessageRangesMaxSize,
540
593
  version = protocolVersion,
541
594
  } = options;
542
595
 
@@ -554,23 +607,33 @@ export const createProtocolMessageBuffer = (
554
607
  };
555
608
 
556
609
  encodeNonNegativeInt(buffers.header, version);
557
- buffers.header.extend(ownerIdToBinaryOwnerId(ownerId));
558
- if (errorCode != null) buffers.header.extend([errorCode]);
610
+ buffers.header.extend(ownerIdToOwnerIdBytes(ownerId));
611
+ buffers.header.extend([options.messageType]);
612
+
613
+ if (options.messageType === MessageType.Request) {
614
+ if (!options.writeKey) {
615
+ buffers.header.extend([0]);
616
+ } else {
617
+ buffers.header.extend([1]);
618
+ buffers.header.extend(options.writeKey);
619
+ }
620
+ const subscriptionFlag = options.subscriptionFlag ?? SubscriptionFlags.None;
621
+ buffers.header.extend([subscriptionFlag]);
622
+ } else if (options.messageType === MessageType.Response) {
623
+ buffers.header.extend([options.errorCode]);
624
+ }
559
625
 
560
626
  let isLastRangeInfinite = false;
561
627
 
562
628
  const isWithinSizeLimits = () => getSize() <= totalMaxSize;
563
629
 
564
630
  const getSize = () =>
565
- (getHeaderAndMessagesSize() + getRangesSize()) as PositiveInt;
631
+ PositiveInt.orThrow(getHeaderAndMessagesSize() + getRangesSize());
566
632
 
567
633
  const getHeaderAndMessagesSize = () =>
568
634
  buffers.header.getLength() +
569
635
  buffers.messages.timestamps.getLength() +
570
- buffers.messages.dbChanges.getLength() +
571
- (buffers.messages.timestamps.getCount() > 0 && writeKey
572
- ? writeKeyLength
573
- : 0);
636
+ buffers.messages.dbChanges.getLength();
574
637
 
575
638
  const getRangesSize = () =>
576
639
  buffers.ranges.timestamps.getCount() > 0
@@ -639,10 +702,15 @@ export const createProtocolMessageBuffer = (
639
702
  },
640
703
 
641
704
  addRange: (range) => {
705
+ assert(
706
+ options.messageType !== MessageType.Broadcast,
707
+ "Cannot add a range into broadcast message",
708
+ );
642
709
  assert(
643
710
  !isLastRangeInfinite,
644
711
  "Cannot add a range after an InfiniteUpperBound range",
645
712
  );
713
+
646
714
  isLastRangeInfinite = range.upperBound === InfiniteUpperBound;
647
715
 
648
716
  /**
@@ -652,13 +720,16 @@ export const createProtocolMessageBuffer = (
652
720
  */
653
721
  if (range.upperBound !== InfiniteUpperBound)
654
722
  buffers.ranges.timestamps.add(
655
- binaryTimestampToTimestamp(range.upperBound),
723
+ timestampBytesToTimestamp(range.upperBound),
656
724
  );
657
725
  else {
658
726
  buffers.ranges.timestamps.addInfinite();
659
727
  }
660
728
 
661
- encodeNonNegativeInt(buffers.ranges.types, range.type as NonNegativeInt);
729
+ encodeNonNegativeInt(
730
+ buffers.ranges.types,
731
+ NonNegativeInt.orThrow(range.type),
732
+ );
662
733
 
663
734
  switch (range.type) {
664
735
  case RangeType.Skip:
@@ -685,8 +756,6 @@ export const createProtocolMessageBuffer = (
685
756
 
686
757
  buffers.messages.timestamps.append(buffers.header);
687
758
  buffers.header.extend(buffers.messages.dbChanges.unwrap());
688
- if (buffers.messages.timestamps.getCount() > 0 && writeKey)
689
- buffers.header.extend(writeKey);
690
759
 
691
760
  if (buffers.ranges.timestamps.getCount() > 0) {
692
761
  buffers.ranges.timestamps.append(buffers.header);
@@ -701,6 +770,11 @@ export const createProtocolMessageBuffer = (
701
770
  };
702
771
  };
703
772
 
773
+ export interface TimestampsRangeWithTimestampsBuffer extends BaseRange {
774
+ readonly type: typeof RangeType.Timestamps;
775
+ readonly timestamps: TimestampsBuffer;
776
+ }
777
+
704
778
  export interface TimestampsBuffer {
705
779
  readonly add: (timestamp: Timestamp) => void;
706
780
  readonly addInfinite: () => void;
@@ -710,7 +784,7 @@ export interface TimestampsBuffer {
710
784
  }
711
785
 
712
786
  export const createTimestampsBuffer = (): TimestampsBuffer => {
713
- let count = 0 as NonNegativeInt;
787
+ let count = NonNegativeInt.orThrow(0);
714
788
  const countBuffer = createBuffer();
715
789
 
716
790
  const syncCount = () => {
@@ -777,9 +851,9 @@ const createRunLengthEncoder = <T>(
777
851
  encodeValue: (buffer: Buffer, value: T) => void,
778
852
  ): RunLengthEncoder<T> => {
779
853
  const buffer = createBuffer();
780
- let previousLength = 0 as NonNegativeInt;
854
+ let previousLength = NonNegativeInt.orThrow(0);
781
855
  let previousValue = null as T | null;
782
- let runLength = 0 as NonNegativeInt;
856
+ let runLength = NonNegativeInt.orThrow(0);
783
857
 
784
858
  return {
785
859
  add: (value) => {
@@ -788,7 +862,7 @@ const createRunLengthEncoder = <T>(
788
862
  buffer.truncate(previousLength);
789
863
  } else {
790
864
  previousValue = value;
791
- runLength = 1 as NonNegativeInt;
865
+ runLength = NonNegativeInt.orThrow(1);
792
866
  }
793
867
  previousLength = buffer.getLength();
794
868
  encodeValue(buffer, value);
@@ -802,42 +876,63 @@ const createRunLengthEncoder = <T>(
802
876
  };
803
877
 
804
878
  export interface ApplyProtocolMessageAsClientOptions {
805
- getWriteKey?: (ownerId: OwnerId) => WriteKey | null;
879
+ getWriteKey?: (ownerId: OwnerId) => OwnerWriteKey | null;
806
880
 
807
- /** For testing purposes only; should not be used in production. */
808
- version?: NonNegativeInt;
881
+ rangesMaxSize?: ProtocolMessageRangesMaxSize;
809
882
 
810
- totalMaxSize?: PositiveInt;
811
- rangesMaxSize?: PositiveInt;
883
+ /** For tests only. */
884
+ version?: NonNegativeInt;
812
885
  }
813
886
 
887
+ /**
888
+ * Result type for {@link applyProtocolMessageAsClient} that distinguishes
889
+ * between responses to client requests and broadcast messages.
890
+ */
891
+ export type ApplyProtocolMessageAsClientResult =
892
+ | { readonly type: "response"; readonly message: ProtocolMessage }
893
+ | { readonly type: "no-response" }
894
+ | { readonly type: "broadcast" };
895
+
814
896
  export const applyProtocolMessageAsClient =
815
897
  (deps: StorageDep) =>
816
- (
898
+ async (
817
899
  inputMessage: Uint8Array,
818
- {
819
- getWriteKey,
820
- version = protocolVersion,
821
- totalMaxSize,
822
- rangesMaxSize,
823
- }: ApplyProtocolMessageAsClientOptions = {},
824
- ): Result<ProtocolMessage | null, ProtocolError> =>
825
- tryDecodeProtocolData<ProtocolMessage | null, ProtocolError>(
826
- inputMessage,
827
- (input) => {
828
- const [requestedVersion, ownerId] = decodeVersionAndOwner(input);
829
-
830
- if (requestedVersion !== version) {
831
- return err<ProtocolUnsupportedVersionError>({
832
- type: "ProtocolUnsupportedVersionError",
833
- unsupportedVersion: requestedVersion,
834
- isInitiator: version < requestedVersion,
835
- ownerId,
836
- });
837
- }
900
+ options: ApplyProtocolMessageAsClientOptions = {},
901
+ ): Promise<
902
+ Result<
903
+ ApplyProtocolMessageAsClientResult,
904
+ | ProtocolInvalidDataError
905
+ | ProtocolSyncError
906
+ | ProtocolVersionError
907
+ | ProtocolWriteError
908
+ | ProtocolWriteKeyError
909
+ | ProtocolQuotaError
910
+ >
911
+ > => {
912
+ // try-catch instead of Result for performance and stacktraces
913
+ // DEV: Measure it again, I think we should use Result with new Error.
914
+ try {
915
+ const input = createBuffer(inputMessage);
916
+ const [requestedVersion, ownerId] = decodeVersionAndOwner(input);
917
+ const version = options.version ?? protocolVersion;
838
918
 
839
- const binaryOwnerId = ownerIdToBinaryOwnerId(ownerId);
919
+ if (requestedVersion !== version) {
920
+ return err<ProtocolVersionError>({
921
+ type: "ProtocolVersionError",
922
+ version: requestedVersion,
923
+ isInitiator: version < requestedVersion,
924
+ ownerId,
925
+ });
926
+ }
927
+
928
+ const messageType = input.shift() as MessageType;
929
+ assert(
930
+ messageType === MessageType.Response ||
931
+ messageType === MessageType.Broadcast,
932
+ "Invalid MessageType",
933
+ );
840
934
 
935
+ if (messageType === MessageType.Response) {
841
936
  const errorCode = input.shift() as ProtocolErrorCode;
842
937
  if (errorCode !== ProtocolErrorCode.NoError) {
843
938
  switch (errorCode) {
@@ -851,6 +946,11 @@ export const applyProtocolMessageAsClient =
851
946
  type: "ProtocolWriteError",
852
947
  ownerId,
853
948
  });
949
+ case ProtocolErrorCode.QuotaError:
950
+ return err<ProtocolQuotaError>({
951
+ type: "ProtocolQuotaError",
952
+ ownerId,
953
+ });
854
954
  case ProtocolErrorCode.SyncError:
855
955
  return err<ProtocolSyncError>({
856
956
  type: "ProtocolSyncError",
@@ -862,141 +962,258 @@ export const applyProtocolMessageAsClient =
862
962
  );
863
963
  }
864
964
  }
965
+ }
865
966
 
866
- const messages = decodeMessages(input);
967
+ const messages = decodeMessages(input);
968
+ const ownerIdBytes = ownerIdToOwnerIdBytes(ownerId);
867
969
 
868
- if (
869
- isNonEmptyReadonlyArray(messages) &&
870
- !deps.storage.writeMessages(binaryOwnerId, messages)
871
- ) {
872
- return ok(null);
873
- }
970
+ if (isNonEmptyReadonlyArray(messages)) {
971
+ const writeResult = await deps.storage.writeMessages(
972
+ ownerIdBytes,
973
+ messages,
974
+ );
975
+ // Errors are handled by the Storage. Here we just stop syncing.
976
+ if (!writeResult.ok) return ok({ type: "no-response" });
977
+ }
874
978
 
875
- if (!getWriteKey) return ok(null);
876
- const writeKey = getWriteKey(ownerId);
877
- if (writeKey == null) return ok(null);
979
+ // Now: No writeKey, no sync.
980
+ // TODO: Allow to sync SharedReadonlyOwner
981
+ // Without local changes, writeKey will not be required.
982
+ // With local changes, writeKey will be required and if not provided,
983
+ // the sync will stop.
984
+ const writeKey = options.getWriteKey?.(ownerId);
985
+ if (writeKey == null) {
986
+ return ok({ type: "no-response" });
987
+ }
878
988
 
879
- const output = createProtocolMessageBuffer(ownerId, {
880
- writeKey,
881
- totalMaxSize,
882
- rangesMaxSize,
883
- });
989
+ if (messageType === MessageType.Broadcast) {
990
+ return ok({ type: "broadcast" });
991
+ }
884
992
 
885
- return sync(deps)("initiator", input, output, binaryOwnerId);
886
- },
887
- );
993
+ const ranges = decodeRanges(input);
994
+
995
+ if (!isNonEmptyReadonlyArray(ranges)) {
996
+ return ok({ type: "no-response" });
997
+ }
998
+
999
+ const output = createProtocolMessageBuffer(ownerId, {
1000
+ messageType: MessageType.Request,
1001
+ writeKey,
1002
+ rangesMaxSize: options.rangesMaxSize,
1003
+ });
1004
+
1005
+ const syncResult = sync(deps)(ranges, output, ownerIdBytes);
1006
+
1007
+ // Client sync error (handled via Storage) or no changes.
1008
+ if (!syncResult.ok || !syncResult.value) {
1009
+ return ok({ type: "no-response" });
1010
+ }
1011
+
1012
+ return ok({ type: "response", message: output.unwrap() });
1013
+ } catch (error) {
1014
+ return err<ProtocolInvalidDataError>({
1015
+ type: "ProtocolInvalidDataError",
1016
+ data: inputMessage,
1017
+ error,
1018
+ });
1019
+ }
1020
+ };
888
1021
 
889
1022
  export interface ApplyProtocolMessageAsRelayOptions {
890
1023
  /** To subscribe an owner for broadcasting. */
891
1024
  subscribe?: (ownerId: OwnerId) => void;
892
1025
 
1026
+ /** To unsubscribe an owner from broadcasting. */
1027
+ unsubscribe?: (ownerId: OwnerId) => void;
1028
+
893
1029
  /** To broadcast a protocol message to all subscribers. */
894
1030
  broadcast?: (ownerId: OwnerId, message: ProtocolMessage) => void;
895
1031
 
896
- totalMaxSize?: PositiveInt;
897
- rangesMaxSize?: PositiveInt;
1032
+ totalMaxSize?: ProtocolMessageMaxSize;
1033
+ rangesMaxSize?: ProtocolMessageRangesMaxSize;
1034
+ }
1035
+
1036
+ /**
1037
+ * Result type for {@link applyProtocolMessageAsRelay}.
1038
+ *
1039
+ * Unlike {@link ApplyProtocolMessageAsClientResult}, relays always respond with
1040
+ * a message to provide sync completion feedback. This ensures the initiator can
1041
+ * reliably detect when synchronization is complete, even when there's nothing
1042
+ * to sync. Clients may choose not to respond in certain cases (like when they
1043
+ * receive broadcast messages or when they lack a write key for syncing).
1044
+ */
1045
+ export interface ApplyProtocolMessageAsRelayResult {
1046
+ readonly type: "response";
1047
+ readonly message: ProtocolMessage;
898
1048
  }
899
1049
 
900
1050
  export const applyProtocolMessageAsRelay =
901
1051
  (deps: StorageDep) =>
902
- (
1052
+ async (
903
1053
  inputMessage: Uint8Array,
904
- {
905
- subscribe,
906
- broadcast,
907
- totalMaxSize,
908
- rangesMaxSize,
909
- }: ApplyProtocolMessageAsRelayOptions = {},
910
- /** For testing purposes only; should not be used in production. */
1054
+ options: ApplyProtocolMessageAsRelayOptions = {},
1055
+ /** For tests only. */
911
1056
  version = protocolVersion,
912
- ): Result<ProtocolMessage | null, ProtocolInvalidDataError> =>
913
- tryDecodeProtocolData(inputMessage, (input) => {
914
- const requestedVersion = decodeNonNegativeInt(input);
915
- const ownerId = decodeOwnerId(input);
916
- const binaryOwnerId = ownerIdToBinaryOwnerId(ownerId);
1057
+ ): Promise<
1058
+ Result<ApplyProtocolMessageAsRelayResult, ProtocolInvalidDataError>
1059
+ > => {
1060
+ // try-catch instead of Result for performance and stacktraces
1061
+ // DEV: Measure it again, I think we should use Result with new Error.
1062
+ try {
1063
+ const input = createBuffer(inputMessage);
1064
+ const [requestedVersion, ownerId] = decodeVersionAndOwner(input);
1065
+ const ownerIdBytes = ownerIdToOwnerIdBytes(ownerId);
917
1066
 
918
1067
  if (requestedVersion !== version) {
919
1068
  // Non-initiator responds with its version and ownerId.
920
1069
  const output = createBuffer();
921
1070
  encodeNonNegativeInt(output, version);
922
- output.extend(binaryOwnerId);
923
- return ok(output.unwrap() as ProtocolMessage);
1071
+ output.extend(ownerIdBytes);
1072
+ return ok({
1073
+ type: "response",
1074
+ message: output.unwrap() as ProtocolMessage,
1075
+ });
924
1076
  }
925
1077
 
926
- subscribe?.(ownerId);
1078
+ const messageType = input.shift() as MessageType;
1079
+ assert(messageType === MessageType.Request, "Invalid MessageType");
927
1080
 
928
- const messages = decodeMessages(input);
1081
+ const hasWriteKey = input.shift();
1082
+ let writeKey: OwnerWriteKey | undefined;
929
1083
 
930
- if (isNonEmptyReadonlyArray(messages)) {
931
- const messagesEnd = inputMessage.length - input.getLength();
932
- const writeKey = input.shiftN(writeKeyLength) as WriteKey;
1084
+ if (hasWriteKey === 1) {
1085
+ writeKey = input.shiftN(ownerWriteKeyLength) as OwnerWriteKey;
1086
+ }
933
1087
 
934
- const writeKeyIsValid = deps.storage.validateWriteKey(
935
- binaryOwnerId,
936
- writeKey,
937
- );
1088
+ const subscriptionFlag = input.shift() as SubscriptionFlag;
938
1089
 
939
- if (!writeKeyIsValid)
940
- return ok(
941
- createProtocolMessageBuffer(ownerId, {
1090
+ switch (subscriptionFlag) {
1091
+ case SubscriptionFlags.Subscribe:
1092
+ options.subscribe?.(ownerId);
1093
+ break;
1094
+ case SubscriptionFlags.Unsubscribe:
1095
+ options.unsubscribe?.(ownerId);
1096
+ break;
1097
+ case SubscriptionFlags.None:
1098
+ break;
1099
+ }
1100
+
1101
+ if (writeKey) {
1102
+ const isValid = deps.storage.validateWriteKey(ownerIdBytes, writeKey);
1103
+ if (!isValid) {
1104
+ return ok({
1105
+ type: "response",
1106
+ message: createProtocolMessageBuffer(ownerId, {
1107
+ messageType: MessageType.Response,
942
1108
  errorCode: ProtocolErrorCode.WriteKeyError,
943
1109
  }).unwrap(),
944
- );
1110
+ });
1111
+ }
1112
+ }
1113
+
1114
+ const messages = decodeMessages(input);
1115
+
1116
+ if (isNonEmptyReadonlyArray(messages)) {
1117
+ if (!writeKey) {
1118
+ return ok({
1119
+ type: "response",
1120
+ message: createProtocolMessageBuffer(ownerId, {
1121
+ messageType: MessageType.Response,
1122
+ errorCode: ProtocolErrorCode.WriteKeyError,
1123
+ }).unwrap(),
1124
+ });
1125
+ }
945
1126
 
946
- if (broadcast) {
947
- // Instead of encoding a new protocol message, we reuse the inputMessage.
948
- const broadcastMessage = concatBytes(
949
- inputMessage.slice(0, 17),
950
- new Uint8Array([ProtocolErrorCode.NoError]),
951
- inputMessage.slice(17, messagesEnd),
952
- ) as ProtocolMessage;
1127
+ const writeResult = await deps.storage.writeMessages(
1128
+ ownerIdBytes,
1129
+ messages,
1130
+ );
953
1131
 
954
- broadcast(ownerId, broadcastMessage);
1132
+ if (!writeResult.ok) {
1133
+ const errorCode =
1134
+ writeResult.error.type === "StorageWriteError"
1135
+ ? ProtocolErrorCode.WriteError
1136
+ : ProtocolErrorCode.QuotaError;
1137
+ const message = createProtocolMessageBuffer(ownerId, {
1138
+ messageType: MessageType.Response,
1139
+ errorCode,
1140
+ }).unwrap();
1141
+ return ok({ type: "response", message });
955
1142
  }
956
1143
 
957
- if (!deps.storage.writeMessages(binaryOwnerId, messages))
958
- return ok(
959
- createProtocolMessageBuffer(ownerId, {
960
- errorCode: ProtocolErrorCode.WriteError,
961
- }).unwrap(),
962
- );
1144
+ /**
1145
+ * Broadcast messages to all subscribed owners for real-time
1146
+ * synchronization between clients.
1147
+ *
1148
+ * Messages are only broadcasted after successful write to ensure
1149
+ * devices that can still sync aren't affected by quota errors, and to
1150
+ * prevent using a half-working relay service (broadcasting without
1151
+ * persistence).
1152
+ *
1153
+ * When a relay's database is deleted or clients migrate to a new relay
1154
+ * (without data migration), clients will sync their data to the relay,
1155
+ * and the relay will broadcast those messages to other connected
1156
+ * clients. Those clients may receive messages they already have, but
1157
+ * this is safe because Evolu sync is idempotent. As the relay becomes
1158
+ * more synchronized with clients over time, fewer duplicate messages
1159
+ * will be broadcasted.
1160
+ */
1161
+ if (options.broadcast) {
1162
+ const broadcastBuffer = createProtocolMessageBuffer(ownerId, {
1163
+ messageType: MessageType.Broadcast,
1164
+ totalMaxSize: options.totalMaxSize,
1165
+ rangesMaxSize: options.rangesMaxSize,
1166
+ version,
1167
+ });
1168
+ for (const message of messages) {
1169
+ broadcastBuffer.addMessage(message);
1170
+ }
1171
+ options.broadcast(ownerId, broadcastBuffer.unwrap());
1172
+ }
963
1173
  }
964
1174
 
1175
+ const ranges = decodeRanges(input);
1176
+
965
1177
  const output = createProtocolMessageBuffer(ownerId, {
1178
+ messageType: MessageType.Response,
966
1179
  errorCode: ProtocolErrorCode.NoError,
967
- totalMaxSize,
968
- rangesMaxSize,
1180
+ totalMaxSize: options.totalMaxSize,
1181
+ rangesMaxSize: options.rangesMaxSize,
969
1182
  });
970
1183
 
971
- return sync(deps)("non-initiator", input, output, binaryOwnerId);
972
- });
1184
+ // Non-initiators always respond to provide sync completion feedback,
1185
+ // even when there's nothing to sync.
1186
+ if (!isNonEmptyReadonlyArray(ranges)) {
1187
+ return ok({ type: "response", message: output.unwrap() });
1188
+ }
973
1189
 
974
- /**
975
- * Wraps Evolu Protocol decoding functions, which use exceptions instead of
976
- * {@link Result} to provide stack traces for debugging and reduce allocation
977
- * overhead in success cases.
978
- */
979
- const tryDecodeProtocolData = <T, E>(
980
- data: Uint8Array,
981
- callback: (buffer: Buffer) => Result<T, E | ProtocolInvalidDataError>,
982
- ) => {
983
- try {
984
- return callback(createBuffer(data));
985
- } catch (error: unknown) {
986
- if (error instanceof ProtocolDecodeError || error instanceof BufferError)
1190
+ const syncResult = sync(deps)(ranges, output, ownerIdBytes);
1191
+
1192
+ const message = syncResult.ok
1193
+ ? output.unwrap()
1194
+ : createProtocolMessageBuffer(ownerId, {
1195
+ messageType: MessageType.Response,
1196
+ errorCode: syncResult.error,
1197
+ }).unwrap();
1198
+
1199
+ // Non-initiators always respond to provide sync completion feedback,
1200
+ return ok({ type: "response", message });
1201
+ } catch (error) {
987
1202
  return err<ProtocolInvalidDataError>({
988
1203
  type: "ProtocolInvalidDataError",
989
- data,
1204
+ data: inputMessage,
990
1205
  error,
991
1206
  });
992
-
993
- throw error;
994
- }
995
- };
1207
+ }
1208
+ };
996
1209
 
997
1210
  const decodeVersionAndOwner = (input: Buffer): [NonNegativeInt, OwnerId] => {
1211
+ // This structure must never change across protocol versions. The version
1212
+ // and owner ID must always be the first two fields in every protocol message
1213
+ // to enable version negotiation and owner identification before any other
1214
+ // processing occurs.
998
1215
  const version = decodeNonNegativeInt(input);
999
- const ownerId = decodeOwnerId(input);
1216
+ const ownerId = decodeId(input) as OwnerId;
1000
1217
  return [version, ownerId];
1001
1218
  };
1002
1219
 
@@ -1031,37 +1248,17 @@ const decodeMessages = (
1031
1248
  const sync =
1032
1249
  (deps: StorageDep) =>
1033
1250
  (
1034
- role: "initiator" | "non-initiator",
1035
- input: Buffer,
1251
+ ranges: NonEmptyReadonlyArray<Range>,
1036
1252
  output: ProtocolMessageBuffer,
1037
- ownerId: BinaryOwnerId,
1038
- ): Result<ProtocolMessage | null, never> => {
1039
- const ranges = decodeRanges(input);
1040
-
1041
- if (!isNonEmptyReadonlyArray(ranges)) {
1042
- // Nothing to sync.
1043
- return ok(null);
1044
- }
1045
-
1046
- const binaryOwnerId = binaryOwnerIdToOwnerId(ownerId);
1253
+ ownerIdBytes: OwnerIdBytes,
1254
+ ): Result<boolean, typeof ProtocolErrorCode.SyncError> => {
1047
1255
  const outputInitialSize = output.getSize();
1048
1256
 
1049
- const syncFail = () => {
1050
- // Only the relay (non-initiator) reports sync errors, not the client (initiator).
1051
- if (role === "initiator") {
1052
- return ok(null);
1053
- }
1054
- const message = createProtocolMessageBuffer(binaryOwnerId, {
1055
- errorCode: ProtocolErrorCode.SyncError,
1056
- });
1057
- return ok(message.unwrap());
1058
- };
1059
-
1060
- const storageSize = deps.storage.getSize(ownerId);
1061
- if (storageSize == null) return syncFail();
1257
+ const storageSize = deps.storage.getSize(ownerIdBytes);
1258
+ if (storageSize == null) return err(ProtocolErrorCode.SyncError);
1062
1259
 
1063
1260
  let prevUpperBound: RangeUpperBound | null = null;
1064
- let prevIndex = 0 as NonNegativeInt;
1261
+ let prevIndex = NonNegativeInt.orThrow(0);
1065
1262
 
1066
1263
  let skip = false;
1067
1264
  let nonSkipRangeAdded = false;
@@ -1098,7 +1295,11 @@ const sync =
1098
1295
  const addFingerprintForRemainingRange = (
1099
1296
  begin: NonNegativeInt,
1100
1297
  ): boolean => {
1101
- const fingerprint = deps.storage.fingerprint(ownerId, begin, storageSize);
1298
+ const fingerprint = deps.storage.fingerprint(
1299
+ ownerIdBytes,
1300
+ begin,
1301
+ storageSize,
1302
+ );
1102
1303
  if (!fingerprint) return false;
1103
1304
  // There is always a space for a ramaining range.
1104
1305
  output.addRange({
@@ -1114,12 +1315,12 @@ const sync =
1114
1315
 
1115
1316
  const lower = prevIndex;
1116
1317
  let upper = deps.storage.findLowerBound(
1117
- ownerId,
1318
+ ownerIdBytes,
1118
1319
  prevIndex,
1119
1320
  storageSize,
1120
1321
  currentUpperBound,
1121
1322
  );
1122
- if (upper == null) return syncFail();
1323
+ if (upper == null) return err(ProtocolErrorCode.SyncError);
1123
1324
 
1124
1325
  switch (range.type) {
1125
1326
  case RangeType.Skip: {
@@ -1129,11 +1330,11 @@ const sync =
1129
1330
 
1130
1331
  case RangeType.Fingerprint: {
1131
1332
  const ourFingerprint = deps.storage.fingerprint(
1132
- ownerId,
1333
+ ownerIdBytes,
1133
1334
  lower,
1134
1335
  upper,
1135
1336
  );
1136
- if (ourFingerprint == null) return syncFail();
1337
+ if (ourFingerprint == null) return err(ProtocolErrorCode.SyncError);
1137
1338
 
1138
1339
  if (eqArrayNumber(range.fingerprint, ourFingerprint)) {
1139
1340
  skipRange(range);
@@ -1141,15 +1342,16 @@ const sync =
1141
1342
  if (output.canSplitRange()) {
1142
1343
  coalesceSkipsBeforeAdd();
1143
1344
  splitRange(deps)(
1144
- ownerId,
1345
+ ownerIdBytes,
1145
1346
  lower,
1146
1347
  upper,
1147
1348
  currentUpperBound,
1148
1349
  output,
1149
1350
  );
1150
1351
  } else {
1151
- if (!addFingerprintForRemainingRange(upper)) return syncFail();
1152
- return ok(output.unwrap());
1352
+ return addFingerprintForRemainingRange(upper)
1353
+ ? ok(true)
1354
+ : err(ProtocolErrorCode.SyncError);
1153
1355
  }
1154
1356
  }
1155
1357
  break;
@@ -1163,45 +1365,53 @@ const sync =
1163
1365
  );
1164
1366
  const ourTimestamps = createTimestampsBuffer();
1165
1367
 
1166
- let storageError = false as boolean;
1368
+ let cantReadDbChange = false as boolean;
1167
1369
  let exceeded = false as boolean;
1168
1370
 
1169
- deps.storage.iterate(ownerId, lower, upper, (timestamp, index) => {
1170
- const timestampString = timestamp.join();
1171
- const timestampBinary = binaryTimestampToTimestamp(timestamp);
1172
-
1173
- let message: EncryptedCrdtMessage | null = null;
1371
+ deps.storage.iterate(
1372
+ ownerIdBytes,
1373
+ lower,
1374
+ upper,
1375
+ (timestamp, index) => {
1376
+ const timestampString = timestamp.join();
1377
+ const timestampBinary = timestampBytesToTimestamp(timestamp);
1378
+
1379
+ let message: EncryptedCrdtMessage | null = null;
1380
+
1381
+ if (timestampsWeNeed.has(timestampString)) {
1382
+ timestampsWeNeed.delete(timestampString);
1383
+ } else {
1384
+ const dbChange = deps.storage.readDbChange(
1385
+ ownerIdBytes,
1386
+ timestamp,
1387
+ );
1388
+ if (dbChange == null) {
1389
+ cantReadDbChange = true;
1390
+ return false;
1391
+ }
1392
+ message = {
1393
+ timestamp: timestampBinary,
1394
+ change: dbChange,
1395
+ };
1396
+ }
1174
1397
 
1175
- if (timestampsWeNeed.has(timestampString)) {
1176
- timestampsWeNeed.delete(timestampString);
1177
- } else {
1178
- const dbChange = deps.storage.readDbChange(ownerId, timestamp);
1179
- if (dbChange == null) {
1180
- storageError = true;
1398
+ if (
1399
+ !output.canAddTimestampsRangeAndMessage(ourTimestamps, message)
1400
+ ) {
1401
+ exceeded = true;
1402
+ endBound = timestamp;
1403
+ upper = index;
1181
1404
  return false;
1182
1405
  }
1183
- message = {
1184
- timestamp: timestampBinary,
1185
- change: dbChange,
1186
- };
1187
- }
1188
-
1189
- if (
1190
- !output.canAddTimestampsRangeAndMessage(ourTimestamps, message)
1191
- ) {
1192
- exceeded = true;
1193
- endBound = timestamp;
1194
- upper = index;
1195
- return false;
1196
- }
1197
1406
 
1198
- ourTimestamps.add(timestampBinary);
1199
- if (message) output.addMessage(message);
1200
- return true;
1201
- });
1407
+ ourTimestamps.add(timestampBinary);
1408
+ if (message) output.addMessage(message);
1409
+ return true;
1410
+ },
1411
+ );
1202
1412
 
1203
- if (storageError) {
1204
- return syncFail();
1413
+ if (cantReadDbChange) {
1414
+ return err(ProtocolErrorCode.SyncError);
1205
1415
  }
1206
1416
 
1207
1417
  const addRange = () => {
@@ -1216,9 +1426,9 @@ const sync =
1216
1426
  if (exceeded) {
1217
1427
  addRange();
1218
1428
  if (!addFingerprintForRemainingRange(upper)) {
1219
- return syncFail();
1429
+ return err(ProtocolErrorCode.SyncError);
1220
1430
  }
1221
- return ok(output.unwrap());
1431
+ return ok(true);
1222
1432
  }
1223
1433
 
1224
1434
  // If we need something, we have to respond with our timestamps.
@@ -1238,19 +1448,20 @@ const sync =
1238
1448
 
1239
1449
  // If all ranges were skipped, there are no changes and sync is complete.
1240
1450
  const hasChange = output.getSize() > outputInitialSize;
1241
- return ok(hasChange ? output.unwrap() : null);
1451
+
1452
+ return ok(hasChange);
1242
1453
  };
1243
1454
 
1244
1455
  const splitRange =
1245
1456
  (deps: StorageDep) =>
1246
1457
  (
1247
- ownerId: BinaryOwnerId,
1458
+ ownerId: OwnerIdBytes,
1248
1459
  lower: NonNegativeInt,
1249
1460
  upper: NonNegativeInt,
1250
1461
  upperBound: RangeUpperBound,
1251
1462
  buffer: ProtocolMessageBuffer,
1252
1463
  ): void => {
1253
- const itemCount = (upper - lower) as NonNegativeInt;
1464
+ const itemCount = NonNegativeInt.orThrow(upper - lower);
1254
1465
  const buckets = computeBalancedBuckets(itemCount);
1255
1466
 
1256
1467
  if (!buckets.ok) {
@@ -1262,10 +1473,10 @@ const splitRange =
1262
1473
 
1263
1474
  deps.storage.iterate(
1264
1475
  ownerId,
1265
- 0 as NonNegativeInt,
1476
+ NonNegativeInt.orThrow(0),
1266
1477
  itemCount,
1267
1478
  (timestamp) => {
1268
- range.timestamps.add(binaryTimestampToTimestamp(timestamp));
1479
+ range.timestamps.add(timestampBytesToTimestamp(timestamp));
1269
1480
  return true;
1270
1481
  },
1271
1482
  );
@@ -1278,7 +1489,10 @@ const splitRange =
1278
1489
  const fingerprintRangesBuckets =
1279
1490
  lower === 0
1280
1491
  ? buckets.value
1281
- : [lower, ...buckets.value.map((b) => (b + lower) as NonNegativeInt)];
1492
+ : [
1493
+ lower,
1494
+ ...buckets.value.map((b) => NonNegativeInt.orThrow(b + lower)),
1495
+ ];
1282
1496
 
1283
1497
  const fingerprintRanges = deps.storage.fingerprintRanges(
1284
1498
  ownerId,
@@ -1302,7 +1516,7 @@ const decodeRanges = (buffer: Buffer): ReadonlyArray<Range> => {
1302
1516
  const rangesCount = decodeNonNegativeInt(buffer);
1303
1517
  if (rangesCount === 0) return [];
1304
1518
 
1305
- const timestampsCount = (rangesCount - 1) as NonNegativeInt;
1519
+ const timestampsCount = NonNegativeInt.orThrow(rangesCount - 1);
1306
1520
  const timestamps = decodeTimestamps(buffer, timestampsCount);
1307
1521
  const rangeTypes: Array<RangeType> = [];
1308
1522
 
@@ -1324,7 +1538,7 @@ const decodeRanges = (buffer: Buffer): ReadonlyArray<Range> => {
1324
1538
  for (let i = 0; i < rangesCount; i++) {
1325
1539
  const upperBound =
1326
1540
  i < timestampsCount
1327
- ? timestampToBinaryTimestamp(timestamps[i])
1541
+ ? timestampToTimestampBytes(timestamps[i])
1328
1542
  : InfiniteUpperBound;
1329
1543
 
1330
1544
  const rangeType = rangeTypes[i];
@@ -1346,7 +1560,7 @@ const decodeRanges = (buffer: Buffer): ReadonlyArray<Range> => {
1346
1560
 
1347
1561
  case RangeType.Timestamps: {
1348
1562
  const timestamps = decodeTimestamps(buffer).map(
1349
- timestampToBinaryTimestamp,
1563
+ timestampToTimestampBytes,
1350
1564
  );
1351
1565
  ranges.push({
1352
1566
  type: RangeType.Timestamps,
@@ -1374,7 +1588,7 @@ const decodeTimestamps = (
1374
1588
  for (let i = 0; i < length; i++) {
1375
1589
  const deltaMillis = decodeNonNegativeInt(buffer);
1376
1590
  const millis = Millis.from(previousMillis + deltaMillis);
1377
- if (!millis.ok) throw new Error(millis.error.type);
1591
+ if (!millis.ok) throw new ProtocolDecodeError(millis.error.type);
1378
1592
  millises.push(millis.value);
1379
1593
  previousMillis = millis.value;
1380
1594
  }
@@ -1383,7 +1597,7 @@ const decodeTimestamps = (
1383
1597
  let counterIndex = 0;
1384
1598
  while (counterIndex < length) {
1385
1599
  const counter = Counter.from(decodeNonNegativeInt(buffer));
1386
- if (!counter.ok) throw new Error(counter.error.type);
1600
+ if (!counter.ok) throw new ProtocolDecodeError(counter.error.type);
1387
1601
  const runLength = decodeNonNegativeInt(buffer);
1388
1602
  for (let i = 0; i < runLength; i++) {
1389
1603
  counters.push(counter.value);
@@ -1414,134 +1628,31 @@ const decodeTimestamps = (
1414
1628
  return timestamps;
1415
1629
  };
1416
1630
 
1417
- /** Binary representation of {@link Id}. */
1418
- export type BinaryId = Uint8Array & Brand<"BinaryId">;
1419
-
1420
- export const binaryIdLength = 16 as NonNegativeInt;
1421
-
1422
- export const idToBinaryId = (id: Id): BinaryId =>
1423
- base64Url256ToBytes(id) as BinaryId;
1424
-
1425
- export const binaryIdToId = (binaryId: BinaryId): Id =>
1426
- decodeId(createBuffer(binaryId));
1427
-
1428
- /** Binary representation of {@link OwnerId}. */
1429
- export type BinaryOwnerId = Uint8Array & Brand<"BinaryOwnerId">;
1430
-
1431
- export const ownerIdToBinaryOwnerId = (ownerId: OwnerId): BinaryOwnerId =>
1432
- base64Url256ToBytes(ownerId) as BinaryOwnerId;
1433
-
1434
- export const binaryOwnerIdToOwnerId = (binaryOwnerId: BinaryOwnerId): OwnerId =>
1435
- decodeOwnerId(createBuffer(binaryOwnerId));
1436
-
1437
- /**
1438
- * Base64Url string with maximum length of 256 characters. Encoding strings as
1439
- * Base64UrlString saves up to 25% in size compared to regular strings.
1440
- */
1441
- export const Base64Url256 = maxLength(256)(Base64Url);
1442
- export type Base64Url256 = typeof Base64Url256.Type;
1443
-
1444
- /**
1445
- * Union type for all variants of Base64Url strings with limited length. All
1446
- * these types use Base64Url alphabet and are < 256 characters.
1447
- */
1448
- export type Base64Url256Variant = Base64Url256 | Id | NanoId | OwnerId;
1449
-
1450
- /**
1451
- * Alphabet used for Base64Url encoding. This is copied from the `nanoid`
1452
- * library to avoid dependency on a specific version of `nanoid`.
1453
- */
1454
- const urlAlphabet =
1455
- "useandom-26T198340PX75pxJACKVERYMINDBUSHWOLF_GQZbfghjklqvwyzrict";
1456
-
1457
- /**
1458
- * Converts a Base64Url string to a Uint8Array for binary storage. This encoding
1459
- * is more space-efficient than UTF-8 for Base64Url strings.
1460
- */
1461
- export const base64Url256ToBytes = (
1462
- string: Base64Url256Variant,
1463
- ): globalThis.Uint8Array => {
1464
- const totalBits = string.length * 6; // 6 bits per character
1465
- const byteLength = Math.ceil(totalBits / 8);
1466
- const value = new globalThis.Uint8Array(byteLength);
1467
-
1468
- let bitBuffer = 0;
1469
- let bitsInBuffer = 0;
1470
- let byteIndex = 0;
1471
-
1472
- for (const char of string) {
1473
- const charValue = urlAlphabet.indexOf(char);
1474
- bitBuffer = (bitBuffer << 6) | charValue;
1475
- bitsInBuffer += 6;
1476
- while (bitsInBuffer >= 8) {
1477
- bitsInBuffer -= 8;
1478
- value[byteIndex++] = (bitBuffer >> bitsInBuffer) & 0xff;
1479
- }
1480
- }
1481
-
1482
- if (bitsInBuffer > 0 && byteIndex < byteLength) {
1483
- value[byteIndex] = (bitBuffer << (8 - bitsInBuffer)) & 0xff;
1484
- }
1485
-
1486
- return value;
1487
- };
1488
-
1489
- export const decodeBase64Url256 = (
1490
- buffer: Buffer,
1491
- stringLength: number,
1492
- ): Base64Url256Variant => {
1493
- const bytes = buffer.shiftN(
1494
- Math.ceil((stringLength * 6) / 8) as NonNegativeInt,
1495
- );
1496
-
1497
- let bitBuffer = 0;
1498
- let bitsInBuffer = 0;
1499
- let string = "";
1500
-
1501
- for (const byte of bytes) {
1502
- bitBuffer = (bitBuffer << 8) | byte;
1503
- bitsInBuffer += 8;
1504
- while (bitsInBuffer >= 6) {
1505
- bitsInBuffer -= 6;
1506
- if (string.length < stringLength) {
1507
- const charValue = (bitBuffer >> bitsInBuffer) & 0x3f;
1508
- if (charValue < 0 || charValue >= urlAlphabet.length) {
1509
- throw new ProtocolDecodeError("invalid charValue");
1510
- }
1511
- string += urlAlphabet[charValue];
1512
- }
1513
- }
1514
- }
1515
-
1516
- const result = Base64Url256.from(string);
1517
- if (!result.ok) throw new ProtocolDecodeError(result.error.type);
1518
-
1519
- return result.value;
1631
+ const decodeId = (buffer: Buffer): Id => {
1632
+ const bytes = buffer.shiftN(idBytesTypeValueLength);
1633
+ return idBytesToId(bytes as IdBytes);
1520
1634
  };
1521
1635
 
1522
- const decodeId = (buffer: Buffer): Id =>
1523
- decodeBase64Url256(buffer, idTypeValueLength) as Id;
1524
-
1525
- /** Not all 16 bytes are valid {@link OwnerId}. */
1526
- const decodeOwnerId = (buffer: Buffer): OwnerId => decodeId(buffer) as OwnerId;
1527
-
1528
1636
  /**
1529
1637
  * Evolu uses MessagePack to handle all number variants except for
1530
1638
  * NonNegativeInt. For NonNegativeInt, Evolu provides more efficient encoding.
1531
1639
  */
1532
1640
  export const encodeNumber = (buffer: Buffer, number: number): void => {
1533
- buffer.extend(pack(number));
1641
+ buffer.extend(packr.pack(number));
1534
1642
  };
1535
1643
 
1536
1644
  export const decodeNumber = (buffer: Buffer): number => {
1537
1645
  let number: unknown;
1538
1646
  let end: unknown;
1539
1647
 
1540
- unpackMultiple(buffer.unwrap(), (n, _, e) => {
1541
- number = n;
1542
- end = e;
1543
- return false;
1544
- });
1648
+ packr.unpackMultiple(
1649
+ buffer.unwrap(),
1650
+ (n: unknown, _: unknown, e: unknown) => {
1651
+ number = n;
1652
+ end = e;
1653
+ return false;
1654
+ },
1655
+ );
1545
1656
 
1546
1657
  const endResult = NonNegativeInt.fromUnknown(end);
1547
1658
  if (!endResult.ok) throw new ProtocolDecodeError(endResult.error.type);
@@ -1553,28 +1664,33 @@ export const decodeNumber = (buffer: Buffer): number => {
1553
1664
  return numberResult.value;
1554
1665
  };
1555
1666
 
1556
- export const binaryTimestampToFingerprint = (
1557
- timestamp: BinaryTimestamp,
1558
- ): Fingerprint => {
1559
- const hash = sha256(timestamp).slice(0, fingerprintSize);
1560
- return hash as Fingerprint;
1561
- };
1562
-
1563
1667
  /**
1564
1668
  * Encodes and encrypts a {@link DbChange} using the provided owner's encryption
1565
1669
  * key. Returns an encrypted binary representation as {@link EncryptedDbChange}.
1670
+ *
1671
+ * The format includes the protocol version for backward compatibility and the
1672
+ * timestamp for tamper-proof verification that the timestamp matches the change
1673
+ * data.
1566
1674
  */
1567
1675
  export const encodeAndEncryptDbChange =
1568
1676
  (deps: SymmetricCryptoDep) =>
1569
- (change: DbChange, key: EncryptionKey): EncryptedDbChange => {
1677
+ (message: CrdtMessage, key: EncryptionKey): EncryptedDbChange => {
1678
+ const change = message.change;
1570
1679
  const buffer = createBuffer();
1571
1680
 
1572
- encodeBase64Url256(buffer, change.table);
1681
+ // Encode protocol version first for backward compatibility
1682
+ encodeNonNegativeInt(buffer, protocolVersion);
1683
+
1684
+ // Encode the timestamp (after version) for tamper verification
1685
+ const timestampBytes = timestampToTimestampBytes(message.timestamp);
1686
+ buffer.extend(timestampBytes);
1687
+
1688
+ encodeString(buffer, change.table);
1573
1689
 
1574
- buffer.extend(idToBinaryId(change.id));
1690
+ buffer.extend(idToIdBytes(change.id));
1575
1691
 
1576
1692
  const entries = objectToEntries(change.values).map(
1577
- ([column, value]): [Base64Url256, SqliteValue] => {
1693
+ ([column, value]): [string, SqliteValue] => {
1578
1694
  return [column, value];
1579
1695
  },
1580
1696
  );
@@ -1582,7 +1698,7 @@ export const encodeAndEncryptDbChange =
1582
1698
  encodeLength(buffer, entries);
1583
1699
 
1584
1700
  for (const [column, value] of entries) {
1585
- encodeBase64Url256(buffer, column);
1701
+ encodeString(buffer, column);
1586
1702
  encodeSqliteValue(buffer, value);
1587
1703
  }
1588
1704
 
@@ -1604,50 +1720,80 @@ export const encodeAndEncryptDbChange =
1604
1720
  };
1605
1721
 
1606
1722
  /**
1607
- * Decrypts and decodes an {@link EncryptedDbChange} using the provided owner's
1608
- * encryption key.
1723
+ * Decrypts and decodes an {@link EncryptedCrdtMessage} using the provided
1724
+ * owner's encryption key. Verifies that the embedded timestamp matches the
1725
+ * expected timestamp to ensure message integrity.
1609
1726
  */
1610
1727
  export const decryptAndDecodeDbChange =
1611
1728
  (deps: SymmetricCryptoDep) =>
1612
1729
  (
1613
- change: EncryptedDbChange,
1730
+ message: EncryptedCrdtMessage,
1614
1731
  key: EncryptionKey,
1615
- ): Result<DbChange, SymmetricCryptoDecryptError | ProtocolInvalidDataError> =>
1616
- tryDecodeProtocolData<DbChange, SymmetricCryptoDecryptError>(
1617
- change,
1618
- (buffer) => {
1619
- const nonce = buffer.shiftN(deps.symmetricCrypto.nonceLength);
1620
-
1621
- const ciphertextLength = decodeLength(buffer);
1622
- const ciphertext = buffer.shiftN(ciphertextLength);
1623
-
1624
- const plaintextBytes = deps.symmetricCrypto.decrypt(
1625
- ciphertext,
1626
- key,
1627
- nonce,
1628
- );
1629
- if (!plaintextBytes.ok) return plaintextBytes;
1732
+ ): Result<
1733
+ DbChange,
1734
+ | SymmetricCryptoDecryptError
1735
+ | ProtocolInvalidDataError
1736
+ | ProtocolTimestampMismatchError
1737
+ > => {
1738
+ // try-catch instead of Result for performance and stacktraces
1739
+ try {
1740
+ const buffer = createBuffer(message.change);
1741
+ const nonce = buffer.shiftN(deps.symmetricCrypto.nonceLength);
1742
+
1743
+ const ciphertextLength = decodeLength(buffer);
1744
+ const ciphertext = buffer.shiftN(ciphertextLength);
1745
+
1746
+ const plaintextBytes = deps.symmetricCrypto.decrypt(
1747
+ ciphertext,
1748
+ key,
1749
+ nonce,
1750
+ );
1751
+ if (!plaintextBytes.ok) return plaintextBytes;
1630
1752
 
1631
- buffer.reset();
1632
- buffer.extend(plaintextBytes.value);
1753
+ buffer.reset();
1754
+ buffer.extend(plaintextBytes.value);
1633
1755
 
1634
- const table = decodeBase64Url256WithLength(buffer);
1635
- const id = decodeId(buffer);
1756
+ // Decode version (for future compatibility, no validation needed for now)
1757
+ decodeNonNegativeInt(buffer);
1636
1758
 
1637
- const length = decodeLength(buffer);
1638
- const values = Object.create(null) as Record<string, SqliteValue>;
1759
+ // Decode and verify the embedded timestamp
1760
+ const embeddedTimestampBytes = buffer.shiftN(timestampBytesLength);
1761
+ const embeddedTimestamp = timestampBytesToTimestamp(
1762
+ embeddedTimestampBytes as TimestampBytes,
1763
+ );
1639
1764
 
1640
- for (let i = 0; i < length; i++) {
1641
- const column = decodeBase64Url256WithLength(buffer);
1642
- const value = decodeSqliteValue(buffer);
1643
- values[column] = value;
1644
- }
1765
+ // Verify timestamp integrity
1766
+ if (!eqTimestamp(embeddedTimestamp, message.timestamp)) {
1767
+ return err<ProtocolTimestampMismatchError>({
1768
+ type: "ProtocolTimestampMismatchError",
1769
+ expected: message.timestamp,
1770
+ embedded: embeddedTimestamp,
1771
+ });
1772
+ }
1645
1773
 
1646
- const dbChange = { table, id, values };
1774
+ const table = decodeString(buffer);
1775
+ const id = decodeId(buffer);
1647
1776
 
1648
- return ok(dbChange);
1649
- },
1650
- );
1777
+ const length = decodeLength(buffer);
1778
+ const values = Object.create(null) as Record<string, SqliteValue>;
1779
+
1780
+ for (let i = 0; i < length; i++) {
1781
+ const column = decodeString(buffer);
1782
+ const value = decodeSqliteValue(buffer);
1783
+ values[column] = value;
1784
+ }
1785
+
1786
+ const dbChange = { table, id, values };
1787
+
1788
+ return ok(dbChange);
1789
+ } catch (error) {
1790
+ return err<ProtocolInvalidDataError>({
1791
+ type: "ProtocolInvalidDataError",
1792
+ data: message.change,
1793
+ error,
1794
+ });
1795
+ }
1796
+ };
1651
1797
 
1652
1798
  /**
1653
1799
  * Encodes a non-negative integer into a variable-length integer format. It's
@@ -1705,7 +1851,7 @@ export const decodeNonNegativeInt = (buffer: Buffer): NonNegativeInt => {
1705
1851
  };
1706
1852
 
1707
1853
  export const encodeLength = (buffer: Buffer, value: ArrayLike<any>): void => {
1708
- encodeNonNegativeInt(buffer, value.length as NonNegativeInt);
1854
+ encodeNonNegativeInt(buffer, NonNegativeInt.orThrow(value.length));
1709
1855
  };
1710
1856
 
1711
1857
  export const decodeLength = decodeNonNegativeInt;
@@ -1727,23 +1873,10 @@ export const encodeNodeId = (buffer: Buffer, nodeId: NodeId): void => {
1727
1873
  };
1728
1874
 
1729
1875
  export const decodeNodeId = (buffer: Buffer): NodeId => {
1730
- const bytes = buffer.shiftN(8 as NonNegativeInt);
1876
+ const bytes = buffer.shiftN(NonNegativeInt.orThrow(8));
1731
1877
  return bytesToHex(bytes) as NodeId;
1732
1878
  };
1733
1879
 
1734
- export const encodeBase64Url256 = (
1735
- buffer: Buffer,
1736
- string: Base64Url256Variant,
1737
- ): void => {
1738
- encodeLength(buffer, string);
1739
- buffer.extend(base64Url256ToBytes(string));
1740
- };
1741
-
1742
- export const decodeBase64Url256WithLength = (buffer: Buffer): Base64Url256 => {
1743
- const length = decodeLength(buffer);
1744
- return decodeBase64Url256(buffer, length) as Base64Url256;
1745
- };
1746
-
1747
1880
  // Small ints are encoded into ProtocolValueType, saving one byte per int.
1748
1881
  const isSmallInt: Predicate<number> = (value: number) =>
1749
1882
  value >= 0 && value < 20;
@@ -1752,23 +1885,26 @@ export const ProtocolValueType = {
1752
1885
  // 0-19 small ints
1753
1886
 
1754
1887
  // SQLite types
1755
- String: 20 as NonNegativeInt,
1756
- Number: 21 as NonNegativeInt,
1757
- Null: 22 as NonNegativeInt,
1758
- Binary: 23 as NonNegativeInt,
1888
+ String: NonNegativeInt.orThrow(20),
1889
+ Number: NonNegativeInt.orThrow(21),
1890
+ Null: NonNegativeInt.orThrow(22),
1891
+ Bytes: NonNegativeInt.orThrow(23),
1759
1892
  // We can add more types for other DBs or anything else later.
1760
1893
 
1761
1894
  // Optimized types
1762
- Id: 30 as NonNegativeInt,
1763
- Base64Url256: 31 as NonNegativeInt,
1764
- NonNegativeInt: 32 as NonNegativeInt,
1765
- Json: 33 as NonNegativeInt,
1895
+ NonNegativeInt: NonNegativeInt.orThrow(30),
1896
+
1897
+ // String optimizations
1898
+ EmptyString: NonNegativeInt.orThrow(31), // 1 byte vs 2 bytes (50% reduction)
1899
+ Base64Url: NonNegativeInt.orThrow(32),
1900
+ Id: NonNegativeInt.orThrow(33),
1901
+ Json: NonNegativeInt.orThrow(34),
1766
1902
 
1767
1903
  // new Date().toISOString() - 24 bytes
1768
1904
  // encoded with fixed length - 8 bytes
1769
1905
  // encode as NonNegativeInt - 6 bytes (additional 25% reduction)
1770
- DateIsoWithNonNegativeTime: 34 as NonNegativeInt,
1771
- DateIsoWithNegativeTime: 35 as NonNegativeInt, // 9 bytes
1906
+ DateIsoWithNonNegativeTime: NonNegativeInt.orThrow(35),
1907
+ DateIsoWithNegativeTime: NonNegativeInt.orThrow(36), // 9 bytes
1772
1908
 
1773
1909
  // TODO: Operations (from 40)
1774
1910
  // Increment, Decrement, Patch, whatever.
@@ -1782,9 +1918,14 @@ export const encodeSqliteValue = (buffer: Buffer, value: SqliteValue): void => {
1782
1918
 
1783
1919
  switch (typeof value) {
1784
1920
  case "string": {
1785
- const dateIsoString = DateIsoString.from(value);
1786
- if (dateIsoString.ok) {
1787
- const time = new Date(dateIsoString.value).getTime();
1921
+ if (value === "") {
1922
+ encodeNonNegativeInt(buffer, ProtocolValueType.EmptyString);
1923
+ return;
1924
+ }
1925
+
1926
+ const dateIso = DateIso.fromParent(value);
1927
+ if (dateIso.ok) {
1928
+ const time = new Date(dateIso.value).getTime();
1788
1929
  if (NonNegativeInt.is(time)) {
1789
1930
  encodeNonNegativeInt(
1790
1931
  buffer,
@@ -1801,27 +1942,34 @@ export const encodeSqliteValue = (buffer: Buffer, value: SqliteValue): void => {
1801
1942
  return;
1802
1943
  }
1803
1944
 
1804
- const base64Url256 = Base64Url256.from(value);
1805
- if (base64Url256.ok) {
1806
- if (base64Url256.value.length === idTypeValueLength) {
1807
- encodeNonNegativeInt(buffer, ProtocolValueType.Id);
1808
- buffer.extend(base64Url256ToBytes(base64Url256.value));
1809
- return;
1810
- }
1811
- encodeNonNegativeInt(buffer, ProtocolValueType.Base64Url256);
1812
- encodeBase64Url256(buffer, base64Url256.value);
1945
+ const id = Id.fromParent(value);
1946
+ if (id.ok) {
1947
+ encodeNonNegativeInt(buffer, ProtocolValueType.Id);
1948
+ buffer.extend(idToIdBytes(id.value));
1813
1949
  return;
1814
1950
  }
1815
1951
 
1816
- const jsonValue = JsonValueFromString.fromParent(value);
1817
- if (jsonValue.ok) {
1818
- const jsonBytes = pack(jsonValue.value);
1952
+ const json = Json.fromParent(value);
1953
+ // Only encode as Json if it survives JSON.parse/JSON.stringify round-trip.
1954
+ // Some valid JSON strings like "-0E0" get normalized to "0" during parsing,
1955
+ // which would cause data corruption if we don't verify round-trip safety.
1956
+ if (json.ok && JSON.stringify(jsonToJsonValue(json.value)) === value) {
1957
+ const jsonBytes = packr.pack(jsonToJsonValue(json.value));
1819
1958
  encodeNonNegativeInt(buffer, ProtocolValueType.Json);
1820
1959
  encodeLength(buffer, jsonBytes);
1821
1960
  buffer.extend(jsonBytes);
1822
1961
  return;
1823
1962
  }
1824
1963
 
1964
+ const base64Url = Base64Url.fromParent(value);
1965
+ if (base64Url.ok) {
1966
+ encodeNonNegativeInt(buffer, ProtocolValueType.Base64Url);
1967
+ const bytes = base64UrlToUint8Array(base64Url.value);
1968
+ encodeLength(buffer, bytes);
1969
+ buffer.extend(bytes);
1970
+ return;
1971
+ }
1972
+
1825
1973
  encodeNonNegativeInt(buffer, ProtocolValueType.String);
1826
1974
  encodeString(buffer, value);
1827
1975
  return;
@@ -1843,7 +1991,7 @@ export const encodeSqliteValue = (buffer: Buffer, value: SqliteValue): void => {
1843
1991
  }
1844
1992
  }
1845
1993
 
1846
- encodeNonNegativeInt(buffer, ProtocolValueType.Binary);
1994
+ encodeNonNegativeInt(buffer, ProtocolValueType.Bytes);
1847
1995
  encodeLength(buffer, value);
1848
1996
  buffer.extend(value);
1849
1997
  };
@@ -1858,40 +2006,75 @@ export const decodeSqliteValue = (buffer: Buffer): SqliteValue => {
1858
2006
  switch (type) {
1859
2007
  case ProtocolValueType.String:
1860
2008
  return decodeString(buffer);
2009
+
1861
2010
  case ProtocolValueType.Number:
1862
2011
  return decodeNumber(buffer);
2012
+
1863
2013
  case ProtocolValueType.Null:
1864
2014
  return null;
1865
- case ProtocolValueType.Binary: {
2015
+
2016
+ case ProtocolValueType.Bytes: {
1866
2017
  const length = decodeLength(buffer);
1867
2018
  return buffer.shiftN(length);
1868
2019
  }
1869
- case ProtocolValueType.Id: {
2020
+
2021
+ case ProtocolValueType.Id:
1870
2022
  return decodeId(buffer);
1871
- }
1872
- case ProtocolValueType.Base64Url256:
1873
- return decodeBase64Url256WithLength(buffer);
2023
+
1874
2024
  case ProtocolValueType.NonNegativeInt:
1875
2025
  return decodeNonNegativeInt(buffer);
2026
+
1876
2027
  case ProtocolValueType.Json: {
1877
2028
  const length = decodeLength(buffer);
1878
2029
  const bytes = buffer.shiftN(length);
1879
- return JSON.stringify(unpack(bytes));
2030
+ return JSON.stringify(packr.unpack(bytes));
1880
2031
  }
2032
+
1881
2033
  case ProtocolValueType.DateIsoWithNonNegativeTime:
1882
2034
  case ProtocolValueType.DateIsoWithNegativeTime: {
1883
2035
  const time =
1884
2036
  type === ProtocolValueType.DateIsoWithNonNegativeTime
1885
2037
  ? decodeNonNegativeInt(buffer)
1886
2038
  : decodeNumber(buffer);
1887
- const dateIsoString = DateIsoString.fromParent(
1888
- new Date(time).toISOString(),
1889
- );
1890
- if (!dateIsoString.ok)
1891
- throw new ProtocolDecodeError(dateIsoString.error.type);
1892
- return dateIsoString.value;
2039
+ const dateIso = DateIso.fromParent(new Date(time).toISOString());
2040
+ if (!dateIso.ok) throw new ProtocolDecodeError(dateIso.error.type);
2041
+ return dateIso.value;
1893
2042
  }
2043
+
2044
+ case ProtocolValueType.EmptyString:
2045
+ return "";
2046
+
2047
+ case ProtocolValueType.Base64Url: {
2048
+ const length = decodeLength(buffer);
2049
+ const bytes = buffer.shiftN(length);
2050
+ return uint8ArrayToBase64Url(bytes);
2051
+ }
2052
+
1894
2053
  default:
1895
2054
  throw new ProtocolDecodeError("invalid ProtocolValueType");
1896
2055
  }
1897
2056
  };
2057
+
2058
+ /**
2059
+ * Decodes a ProtocolMessage into a readable JSON object for debugging.
2060
+ *
2061
+ * Note: This is a stub for future implementation. It should use:
2062
+ *
2063
+ * - DecodeVersionAndOwner
2064
+ * - DecodeError or decodeWriteKeys (depending on context)
2065
+ * - DecodeMessages
2066
+ * - DecodeRanges
2067
+ *
2068
+ * If you want to help, please contribute to this function.
2069
+ */
2070
+ export const decodeProtocolMessageToJson = (
2071
+ _protocolMessage: ProtocolMessage,
2072
+ _isInitiator: boolean,
2073
+ ): unknown => {
2074
+ // TODO: Implement using
2075
+ // - decodeVersionAndOwner
2076
+ // -- decodeError or decodeWriteKeys (should be refactored out),
2077
+ // -- decodeMessages, and decodeRanges.
2078
+ // This is a stub for PRs and community contributions.
2079
+ throw new Error("decodeProtocolMessageToJson is not implemented yet.");
2080
+ };