@evolu/common 6.0.1-preview.2 → 6.0.1-preview.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/Assert.d.ts +6 -3
- package/dist/src/Assert.d.ts.map +1 -1
- package/dist/src/Assert.js +7 -4
- package/dist/src/Brand.d.ts +75 -0
- package/dist/src/Brand.d.ts.map +1 -0
- package/dist/src/Brand.js +1 -0
- package/dist/src/Buffer.d.ts +1 -1
- package/dist/src/Buffer.d.ts.map +1 -1
- package/dist/src/Buffer.js +1 -1
- package/dist/src/CallbackRegistry.d.ts +53 -0
- package/dist/src/CallbackRegistry.d.ts.map +1 -0
- package/dist/src/CallbackRegistry.js +25 -0
- package/dist/src/Console.d.ts +31 -6
- package/dist/src/Console.d.ts.map +1 -1
- package/dist/src/Console.js +72 -9
- package/dist/src/Crypto.d.ts +61 -34
- package/dist/src/Crypto.d.ts.map +1 -1
- package/dist/src/Crypto.js +29 -42
- package/dist/src/Evolu/Db.d.ts +159 -66
- package/dist/src/Evolu/Db.d.ts.map +1 -1
- package/dist/src/Evolu/Db.js +300 -705
- package/dist/src/Evolu/Diff.d.ts +3 -3
- package/dist/src/Evolu/Diff.d.ts.map +1 -1
- package/dist/src/Evolu/Diff.js +7 -5
- package/dist/src/Evolu/Evolu.d.ts +200 -115
- package/dist/src/Evolu/Evolu.d.ts.map +1 -1
- package/dist/src/Evolu/Evolu.js +287 -157
- package/dist/src/Evolu/Internal.d.ts +0 -2
- package/dist/src/Evolu/Internal.d.ts.map +1 -1
- package/dist/src/Evolu/Internal.js +0 -2
- package/dist/src/Evolu/LocalAuth.d.ts +144 -0
- package/dist/src/Evolu/LocalAuth.d.ts.map +1 -0
- package/dist/src/Evolu/LocalAuth.js +171 -0
- package/dist/src/Evolu/Owner.d.ts +209 -124
- package/dist/src/Evolu/Owner.d.ts.map +1 -1
- package/dist/src/Evolu/Owner.js +118 -104
- package/dist/src/Evolu/Platform.d.ts +9 -7
- package/dist/src/Evolu/Platform.d.ts.map +1 -1
- package/dist/src/Evolu/Protocol.d.ts +256 -211
- package/dist/src/Evolu/Protocol.d.ts.map +1 -1
- package/dist/src/Evolu/Protocol.js +558 -359
- package/dist/src/Evolu/Public.d.ts +6 -8
- package/dist/src/Evolu/Public.d.ts.map +1 -1
- package/dist/src/Evolu/Public.js +2 -3
- package/dist/src/Evolu/PublicKysely.js +3 -3
- package/dist/src/Evolu/Query.d.ts +2 -1
- package/dist/src/Evolu/Query.d.ts.map +1 -1
- package/dist/src/Evolu/Relay.d.ts +72 -4
- package/dist/src/Evolu/Relay.d.ts.map +1 -1
- package/dist/src/Evolu/Relay.js +124 -13
- package/dist/src/Evolu/Schema.d.ts +129 -73
- package/dist/src/Evolu/Schema.d.ts.map +1 -1
- package/dist/src/Evolu/Schema.js +169 -89
- package/dist/src/Evolu/Storage.d.ts +159 -14
- package/dist/src/Evolu/Storage.d.ts.map +1 -1
- package/dist/src/Evolu/Storage.js +42 -32
- package/dist/src/Evolu/Sync.d.ts +68 -13
- package/dist/src/Evolu/Sync.d.ts.map +1 -1
- package/dist/src/Evolu/Sync.js +414 -20
- package/dist/src/Evolu/Timestamp.d.ts +35 -27
- package/dist/src/Evolu/Timestamp.d.ts.map +1 -1
- package/dist/src/Evolu/Timestamp.js +27 -18
- package/dist/src/ManyToManyMap.d.ts +74 -10
- package/dist/src/ManyToManyMap.d.ts.map +1 -1
- package/dist/src/ManyToManyMap.js +41 -6
- package/dist/src/Number.d.ts +2 -1
- package/dist/src/Number.d.ts.map +1 -1
- package/dist/src/Random.d.ts +3 -2
- package/dist/src/Random.d.ts.map +1 -1
- package/dist/src/RefCountedResourceManager.d.ts +119 -0
- package/dist/src/RefCountedResourceManager.d.ts.map +1 -0
- package/dist/src/RefCountedResourceManager.js +197 -0
- package/dist/src/Result.d.ts +176 -51
- package/dist/src/Result.d.ts.map +1 -1
- package/dist/src/Result.js +30 -241
- package/dist/src/Sqlite.d.ts +21 -4
- package/dist/src/Sqlite.d.ts.map +1 -1
- package/dist/src/Sqlite.js +50 -8
- package/dist/src/Task.d.ts +511 -0
- package/dist/src/Task.d.ts.map +1 -0
- package/dist/src/Task.js +410 -0
- package/dist/src/Time.d.ts +59 -0
- package/dist/src/Time.d.ts.map +1 -1
- package/dist/src/Time.js +87 -4
- package/dist/src/Type.d.ts +474 -327
- package/dist/src/Type.d.ts.map +1 -1
- package/dist/src/Type.js +502 -436
- package/dist/src/Types.d.ts +0 -74
- package/dist/src/Types.d.ts.map +1 -1
- package/dist/src/WebSocket.d.ts +5 -2
- package/dist/src/WebSocket.d.ts.map +1 -1
- package/dist/src/WebSocket.js +12 -13
- package/dist/src/Worker.d.ts +39 -11
- package/dist/src/Worker.d.ts.map +1 -1
- package/dist/src/Worker.js +22 -4
- package/dist/src/index.d.ts +3 -3
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +3 -3
- package/package.json +14 -13
- package/src/Assert.ts +8 -7
- package/src/Brand.ts +75 -0
- package/src/Buffer.ts +1 -1
- package/src/CallbackRegistry.ts +84 -0
- package/src/Console.ts +91 -11
- package/src/Crypto.ts +94 -79
- package/src/Evolu/Db.ts +528 -1027
- package/src/Evolu/Diff.ts +7 -5
- package/src/Evolu/Evolu.ts +682 -329
- package/src/Evolu/Internal.ts +0 -2
- package/src/Evolu/LocalAuth.ts +422 -0
- package/src/Evolu/Owner.ts +282 -227
- package/src/Evolu/Platform.ts +9 -9
- package/src/Evolu/Protocol.ts +788 -640
- package/src/Evolu/Public.ts +7 -14
- package/src/Evolu/PublicKysely.ts +3 -3
- package/src/Evolu/Query.ts +2 -1
- package/src/Evolu/Relay.ts +238 -18
- package/src/Evolu/Schema.ts +391 -191
- package/src/Evolu/Storage.ts +275 -55
- package/src/Evolu/Sync.ts +708 -37
- package/src/Evolu/Timestamp.ts +38 -35
- package/src/ManyToManyMap.ts +127 -24
- package/src/Number.ts +2 -6
- package/src/Random.ts +3 -2
- package/src/RefCountedResourceManager.ts +368 -0
- package/src/Result.ts +183 -53
- package/src/Sqlite.ts +59 -23
- package/src/Task.ts +779 -0
- package/src/Time.ts +168 -4
- package/src/Type.ts +708 -660
- package/src/Types.ts +0 -76
- package/src/WebSocket.ts +23 -17
- package/src/Worker.ts +72 -23
- package/src/index.ts +3 -3
- package/dist/src/Callbacks.d.ts +0 -20
- package/dist/src/Callbacks.d.ts.map +0 -1
- package/dist/src/Callbacks.js +0 -18
- package/dist/src/Evolu/Config.d.ts +0 -69
- package/dist/src/Evolu/Config.d.ts.map +0 -1
- package/dist/src/Evolu/Config.js +0 -9
- package/dist/src/Evolu/Kysely.d.ts +0 -6
- package/dist/src/Evolu/Kysely.d.ts.map +0 -1
- package/dist/src/Evolu/Kysely.js +0 -21
- package/dist/src/NanoId.d.ts +0 -27
- package/dist/src/NanoId.d.ts.map +0 -1
- package/dist/src/NanoId.js +0 -6
- package/dist/src/Promise.d.ts +0 -180
- package/dist/src/Promise.d.ts.map +0 -1
- package/dist/src/Promise.js +0 -176
- package/src/Callbacks.ts +0 -43
- package/src/Evolu/Config.ts +0 -83
- package/src/Evolu/Kysely.ts +0 -38
- package/src/NanoId.ts +0 -39
- package/src/Promise.ts +0 -295
package/src/Evolu/Protocol.ts
CHANGED
|
@@ -3,33 +3,54 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Evolu Protocol is a local-first, end-to-end encrypted binary synchronization
|
|
5
5
|
* protocol optimized for minimal size and maximum speed. It enables data sync
|
|
6
|
-
* between a client and a relay
|
|
7
|
-
*
|
|
6
|
+
* between a client and a relay. In the future, direct peer-to-peer (P2P) sync
|
|
7
|
+
* between clients will be possible without a relay.
|
|
8
|
+
*
|
|
9
|
+
* Relays don't need to sync with each other—clients using those relays will
|
|
10
|
+
* sync them eventually. If a relay is offline (e.g., for maintenance), it will
|
|
11
|
+
* sync automatically later via client sync logic. For relay backup using
|
|
12
|
+
* SQLite, see https://sqlite.org/rsync.html (uses a similar algorithm to Evolu
|
|
13
|
+
* RBSR).
|
|
8
14
|
*
|
|
9
15
|
* Evolu Protocol is designed for SQLite but can be extended to any database. It
|
|
10
16
|
* implements [Range-Based Set
|
|
11
17
|
* Reconciliation](https://arxiv.org/abs/2212.13567). To learn how RBSR works,
|
|
12
18
|
* check [Negentropy](https://logperiodic.com/rbsr.html). Evolu Protocol is
|
|
13
19
|
* similar to Negentropy but uses different encoding and also provides data
|
|
14
|
-
* transfer and
|
|
20
|
+
* transfer, ownership, real-time broadcasting, request-response semantics, and
|
|
21
|
+
* error handling.
|
|
22
|
+
*
|
|
23
|
+
* ### Message structure
|
|
15
24
|
*
|
|
16
|
-
*
|
|
25
|
+
* | Field | Notes |
|
|
26
|
+
* | :----------------------------- | :------------------------ |
|
|
27
|
+
* | **Header** | |
|
|
28
|
+
* | - {@link protocolVersion} | |
|
|
29
|
+
* | - {@link OwnerId} | {@link Owner} |
|
|
30
|
+
* | - messageType | {@link MessageType} |
|
|
31
|
+
* | **Request (messageType=0)** | |
|
|
32
|
+
* | - hasWriteKey | 0 = no, 1 = yes |
|
|
33
|
+
* | - {@link OwnerWriteKey} | If hasWriteKey = 1 |
|
|
34
|
+
* | - subscriptionFlag | {@link SubscriptionFlags} |
|
|
35
|
+
* | **Response (messageType=1)** | |
|
|
36
|
+
* | - {@link ProtocolErrorCode} | |
|
|
37
|
+
* | **Broadcast (messageType=2)** | |
|
|
38
|
+
* | - (no additional fields) | |
|
|
39
|
+
* | **Messages** | |
|
|
40
|
+
* | - {@link NonNegativeInt} | A number of messages. |
|
|
41
|
+
* | - {@link EncryptedCrdtMessage} | |
|
|
42
|
+
* | **Ranges** | |
|
|
43
|
+
* | - {@link NonNegativeInt} | Number of ranges. |
|
|
44
|
+
* | - {@link Range} | |
|
|
17
45
|
*
|
|
18
|
-
*
|
|
19
|
-
* | :----------------------------- | :------------------------- |
|
|
20
|
-
* | **Header** | |
|
|
21
|
-
* | - {@link protocolVersion} | |
|
|
22
|
-
* | - {@link OwnerId} | |
|
|
23
|
-
* | - {@link ProtocolErrorCode} | In non-initiator response. |
|
|
24
|
-
* | **Messages** | |
|
|
25
|
-
* | - {@link NonNegativeInt} | A number of messages. |
|
|
26
|
-
* | - {@link EncryptedCrdtMessage} | |
|
|
27
|
-
* | - {@link WriteKey} | In initiator request. |
|
|
28
|
-
* | **Ranges** | |
|
|
29
|
-
* | - {@link NonNegativeInt} | Number of ranges. |
|
|
30
|
-
* | - {@link Range} | |
|
|
46
|
+
* ### WriteKey validation
|
|
31
47
|
*
|
|
32
|
-
*
|
|
48
|
+
* The initiator sends a hasWriteKey flag and optionally a WriteKey. The
|
|
49
|
+
* WriteKey is required when sending messages as a secure token proving the
|
|
50
|
+
* initiator can write changes. It's ok to not send a WriteKey if the initiator
|
|
51
|
+
* is only syncing (read-only) and not sending messages. The non-initiator
|
|
52
|
+
* validates the WriteKey immediately after parsing the initiator header, before
|
|
53
|
+
* processing any messages or ranges.
|
|
33
54
|
*
|
|
34
55
|
* ### Synchronization
|
|
35
56
|
*
|
|
@@ -43,38 +64,48 @@
|
|
|
43
64
|
* if further sync is needed or possible, continuing until both sides are
|
|
44
65
|
* synchronized.
|
|
45
66
|
*
|
|
67
|
+
* The **non-initiator always responds** to provide sync completion feedback,
|
|
68
|
+
* even with empty messages containing only the header and no error. This allows
|
|
69
|
+
* the initiator to detect when synchronization is complete.
|
|
70
|
+
*
|
|
46
71
|
* Both **Messages** and **Ranges** are optional, allowing each side to send,
|
|
47
72
|
* sync, or only subscribe data as needed.
|
|
48
73
|
*
|
|
49
|
-
* When the initiator sends data, the {@link
|
|
50
|
-
*
|
|
51
|
-
* responds without a {@link
|
|
52
|
-
* signals it wants data. If the non-initiator detects an issue, it
|
|
53
|
-
* error code via the `Error` field in the header back to the
|
|
54
|
-
* relay-to-relay or P2P sync, both sides may require the
|
|
55
|
-
* depending on who is the initiator.
|
|
74
|
+
* When the initiator sends data, the {@link OwnerWriteKey} is required as a
|
|
75
|
+
* secure token proving the initiator can write changes. The non-initiator
|
|
76
|
+
* responds without a {@link OwnerWriteKey}, since the initiator’s request
|
|
77
|
+
* already signals it wants data. If the non-initiator detects an issue, it
|
|
78
|
+
* sends an error code via the `Error` field in the header back to the
|
|
79
|
+
* initiator. In relay-to-relay or P2P sync, both sides may require the
|
|
80
|
+
* {@link OwnerWriteKey} depending on who is the initiator.
|
|
56
81
|
*
|
|
57
|
-
* ### Protocol
|
|
82
|
+
* ### Protocol errors
|
|
58
83
|
*
|
|
59
84
|
* The protocol uses error codes in the header to signal issues:
|
|
60
85
|
*
|
|
61
86
|
* - {@link ProtocolWriteKeyError}: The provided WriteKey is invalid or missing.
|
|
62
|
-
* - {@link ProtocolWriteError}: A write
|
|
63
|
-
*
|
|
64
|
-
* - {@link ProtocolSyncError}: A generic or unexpected synchronization failure
|
|
87
|
+
* - {@link ProtocolWriteError}: A serious relay-side write failure occurred.
|
|
88
|
+
* - {@link ProtocolSyncError}: A serious relay-side synchronization failure
|
|
65
89
|
* occurred.
|
|
90
|
+
* - {@link ProtocolQuotaExceededError}: Storage or billing quota exceeded.
|
|
66
91
|
* - {@link ProtocolUnsupportedVersionError}: Protocol version mismatch.
|
|
67
92
|
* - {@link ProtocolInvalidDataError}: The message is malformed or corrupted.
|
|
68
93
|
*
|
|
69
|
-
* All protocol errors except `ProtocolInvalidDataError` include the `
|
|
94
|
+
* All protocol errors except `ProtocolInvalidDataError` include the `OwnerId`
|
|
70
95
|
* to allow clients to associate errors with the correct owner.
|
|
71
96
|
*
|
|
72
|
-
* ### Message
|
|
97
|
+
* ### Message size limit
|
|
73
98
|
*
|
|
74
99
|
* The protocol enforces a strict maximum size for all messages, defined by
|
|
75
|
-
* {@link
|
|
76
|
-
* less than or equal to this limit,
|
|
77
|
-
*
|
|
100
|
+
* {@link ProtocolMessageMaxSize}. This ensures every {@link ProtocolMessage} is
|
|
101
|
+
* less than or equal to this limit, enabling stateless transports, simplified
|
|
102
|
+
* relay implementation, and predictable memory usage. When all messages don't
|
|
103
|
+
* fit within the limit, the protocol automatically continues synchronization in
|
|
104
|
+
* subsequent rounds using range-based reconciliation.
|
|
105
|
+
*
|
|
106
|
+
* Database mutations are limited to 640KB, which is smaller than the protocol
|
|
107
|
+
* message limit to ensure efficient sync with
|
|
108
|
+
* {@link defaultProtocolMessageRangesMaxSize}.
|
|
78
109
|
*
|
|
79
110
|
* ### Why Binary?
|
|
80
111
|
*
|
|
@@ -82,13 +113,12 @@
|
|
|
82
113
|
*
|
|
83
114
|
* - Encrypted data doesn’t compress well, unlike plain JSON.
|
|
84
115
|
* - Message size must be controlled during creation.
|
|
85
|
-
* - Sequential byte reading is faster than parsing and
|
|
116
|
+
* - Sequential byte reading is faster than parsing and avoids conversions.
|
|
86
117
|
*
|
|
87
118
|
* It uses structure-aware encoding, significantly outperforming generic binary
|
|
88
119
|
* serialization formats with the following optimizations:
|
|
89
120
|
*
|
|
90
121
|
* - **NonNegativeInt:** Up to 33% smaller than MessagePack.
|
|
91
|
-
* - **Base64Url Strings:** Up to 25% size reduction.
|
|
92
122
|
* - **DateIso:** Up to 75% smaller.
|
|
93
123
|
* - **Timestamp Encoding:** Delta encoding for milliseconds and run-length
|
|
94
124
|
* encoding (RLE) for counters and NodeIds.
|
|
@@ -118,253 +148,217 @@
|
|
|
118
148
|
* Version negotiation is per-owner, allowing Evolu Protocol to evolve safely
|
|
119
149
|
* over time and provide clear feedback about version mismatches.
|
|
120
150
|
*
|
|
151
|
+
* ### Credible exit
|
|
152
|
+
*
|
|
153
|
+
* The protocol specification is intentionally non-configurable to ensure
|
|
154
|
+
* universal compatibility. This design allows applications (users) to switch
|
|
155
|
+
* between any compliant relay without negotiation or compatibility checks
|
|
156
|
+
* beyond version matching. Relays are generic infrastructure that any
|
|
157
|
+
* application can use interchangeably making exit from any single provider
|
|
158
|
+
* technically feasible and economically viable.
|
|
159
|
+
*
|
|
121
160
|
* @module
|
|
122
161
|
*/
|
|
123
162
|
|
|
124
|
-
import {
|
|
125
|
-
import { pack, unpack, unpackMultiple } from "msgpackr";
|
|
163
|
+
import { Packr } from "msgpackr";
|
|
126
164
|
import { isNonEmptyReadonlyArray, NonEmptyReadonlyArray } from "../Array.js";
|
|
127
165
|
import { assert } from "../Assert.js";
|
|
166
|
+
import { Brand } from "../Brand.js";
|
|
128
167
|
import {
|
|
129
168
|
Buffer,
|
|
130
|
-
BufferError,
|
|
131
169
|
bytesToHex,
|
|
132
170
|
bytesToUtf8,
|
|
133
|
-
concatBytes,
|
|
134
171
|
createBuffer,
|
|
135
172
|
hexToBytes,
|
|
136
173
|
utf8ToBytes,
|
|
137
174
|
} from "../Buffer.js";
|
|
138
175
|
import {
|
|
139
|
-
CreateRandomBytesDep,
|
|
140
176
|
EncryptionKey,
|
|
141
177
|
padmePaddingLength,
|
|
178
|
+
RandomBytesDep,
|
|
142
179
|
SymmetricCryptoDecryptError,
|
|
143
180
|
SymmetricCryptoDep,
|
|
144
181
|
} from "../Crypto.js";
|
|
145
182
|
import { eqArrayNumber } from "../Eq.js";
|
|
146
183
|
import { computeBalancedBuckets } from "../Number.js";
|
|
147
|
-
import { objectToEntries
|
|
184
|
+
import { objectToEntries } from "../Object.js";
|
|
148
185
|
import { err, ok, Result } from "../Result.js";
|
|
149
186
|
import { SqliteValue } from "../Sqlite.js";
|
|
150
187
|
import {
|
|
151
188
|
Base64Url,
|
|
152
|
-
|
|
189
|
+
base64UrlToUint8Array,
|
|
190
|
+
between,
|
|
191
|
+
DateIso,
|
|
153
192
|
Id,
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
193
|
+
IdBytes,
|
|
194
|
+
idBytesToId,
|
|
195
|
+
idBytesTypeValueLength,
|
|
196
|
+
idToIdBytes,
|
|
197
|
+
Int,
|
|
198
|
+
Json,
|
|
199
|
+
jsonToJsonValue,
|
|
158
200
|
NonNegativeInt,
|
|
159
201
|
Number,
|
|
160
202
|
PositiveInt,
|
|
203
|
+
uint8ArrayToBase64Url,
|
|
161
204
|
} from "../Type.js";
|
|
162
|
-
import {
|
|
205
|
+
import { Predicate } from "../Types.js";
|
|
163
206
|
import {
|
|
164
|
-
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
165
207
|
Owner,
|
|
166
208
|
OwnerId,
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
209
|
+
OwnerIdBytes,
|
|
210
|
+
ownerIdToOwnerIdBytes,
|
|
211
|
+
OwnerWriteKey,
|
|
212
|
+
ownerWriteKeyLength,
|
|
170
213
|
} from "./Owner.js";
|
|
171
214
|
import {
|
|
172
|
-
|
|
173
|
-
|
|
215
|
+
BaseRange,
|
|
216
|
+
CrdtMessage,
|
|
217
|
+
DbChange,
|
|
218
|
+
EncryptedCrdtMessage,
|
|
219
|
+
EncryptedDbChange,
|
|
220
|
+
Fingerprint,
|
|
221
|
+
FingerprintRange,
|
|
222
|
+
fingerprintSize,
|
|
223
|
+
InfiniteUpperBound,
|
|
224
|
+
Range,
|
|
225
|
+
RangeType,
|
|
226
|
+
RangeUpperBound,
|
|
227
|
+
SkipRange,
|
|
228
|
+
StorageDep,
|
|
229
|
+
TimestampsRange,
|
|
230
|
+
} from "./Storage.js";
|
|
231
|
+
import {
|
|
174
232
|
Counter,
|
|
233
|
+
eqTimestamp,
|
|
175
234
|
Millis,
|
|
176
235
|
NodeId,
|
|
177
236
|
Timestamp,
|
|
178
|
-
|
|
237
|
+
TimestampBytes,
|
|
238
|
+
timestampBytesLength,
|
|
239
|
+
timestampBytesToTimestamp,
|
|
240
|
+
timestampToTimestampBytes,
|
|
179
241
|
} from "./Timestamp.js";
|
|
180
242
|
|
|
181
|
-
/** Maximum size of the entire protocol message in bytes. */
|
|
182
|
-
export const maxProtocolMessageSize = 1_000_000 as PositiveInt;
|
|
183
|
-
|
|
184
|
-
/** Maximum size of the ranges in bytes. */
|
|
185
|
-
export const maxProtocolMessageRangesSize = 30_000 as PositiveInt;
|
|
186
|
-
|
|
187
|
-
/** Evolu Protocol Message. */
|
|
188
|
-
export type ProtocolMessage = Uint8Array & Brand<"ProtocolMessage">;
|
|
189
|
-
|
|
190
|
-
/** Evolu Protocol version. */
|
|
191
|
-
export const protocolVersion = 0 as NonNegativeInt;
|
|
192
|
-
|
|
193
|
-
export const ProtocolErrorCode = {
|
|
194
|
-
NoError: 0,
|
|
195
|
-
/** A code for {@link ProtocolWriteKeyError}. */
|
|
196
|
-
WriteKeyError: 1,
|
|
197
|
-
/** A code for {@link ProtocolWriteError}. */
|
|
198
|
-
WriteError: 2,
|
|
199
|
-
/** A code for {@link ProtocolSyncError}. */
|
|
200
|
-
SyncError: 3,
|
|
201
|
-
} as const;
|
|
202
|
-
|
|
203
|
-
type ProtocolErrorCode =
|
|
204
|
-
(typeof ProtocolErrorCode)[keyof typeof ProtocolErrorCode];
|
|
205
|
-
|
|
206
243
|
/**
|
|
207
|
-
* Evolu
|
|
244
|
+
* Evolu uses MessagePack for numbers and JSONs.
|
|
208
245
|
*
|
|
209
|
-
*
|
|
210
|
-
*
|
|
211
|
-
* handle their own errors; return values only indicates overall success or
|
|
212
|
-
* failure.
|
|
246
|
+
* - `variableMapSize: true` - More compact maps, ~5-10% slower encoding
|
|
247
|
+
* - `useRecords: false` - Standard MessagePack without extensions
|
|
213
248
|
*/
|
|
214
|
-
|
|
215
|
-
readonly getSize: (ownerId: BinaryOwnerId) => NonNegativeInt | null;
|
|
216
|
-
|
|
217
|
-
readonly fingerprint: (
|
|
218
|
-
ownerId: BinaryOwnerId,
|
|
219
|
-
begin: NonNegativeInt,
|
|
220
|
-
end: NonNegativeInt,
|
|
221
|
-
) => Fingerprint | null;
|
|
222
|
-
|
|
223
|
-
/**
|
|
224
|
-
* Computes fingerprints with their upper bounds in one call.
|
|
225
|
-
*
|
|
226
|
-
* This function can be replaced with many fingerprint/findLowerBound calls,
|
|
227
|
-
* but implementations can leverage it for batching and more efficient
|
|
228
|
-
* fingerprint computation.
|
|
229
|
-
*/
|
|
230
|
-
readonly fingerprintRanges: (
|
|
231
|
-
ownerId: BinaryOwnerId,
|
|
232
|
-
buckets: ReadonlyArray<NonNegativeInt>,
|
|
233
|
-
upperBound?: RangeUpperBound,
|
|
234
|
-
) => ReadonlyArray<FingerprintRange> | null;
|
|
235
|
-
|
|
236
|
-
readonly findLowerBound: (
|
|
237
|
-
ownerId: BinaryOwnerId,
|
|
238
|
-
begin: NonNegativeInt,
|
|
239
|
-
end: NonNegativeInt,
|
|
240
|
-
upperBound: RangeUpperBound,
|
|
241
|
-
) => NonNegativeInt | null;
|
|
242
|
-
|
|
243
|
-
readonly iterate: (
|
|
244
|
-
ownerId: BinaryOwnerId,
|
|
245
|
-
begin: NonNegativeInt,
|
|
246
|
-
end: NonNegativeInt,
|
|
247
|
-
callback: (timestamp: BinaryTimestamp, index: NonNegativeInt) => boolean,
|
|
248
|
-
) => void;
|
|
249
|
+
const packr = new Packr({ variableMapSize: true, useRecords: false });
|
|
249
250
|
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
* {@link BinaryOwnerId}.
|
|
253
|
-
*
|
|
254
|
-
* For a client that does not expect foreign writes, return `false`.
|
|
255
|
-
*/
|
|
256
|
-
readonly validateWriteKey: (
|
|
257
|
-
ownerId: BinaryOwnerId,
|
|
258
|
-
writeKey: WriteKey,
|
|
259
|
-
) => boolean;
|
|
251
|
+
const minProtocolMessageMaxSize = 1_000_000;
|
|
252
|
+
const maxProtocolMessageMaxSize = 100_000_000;
|
|
260
253
|
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
readonly change: EncryptedDbChange;
|
|
282
|
-
}
|
|
254
|
+
/**
|
|
255
|
+
* Protocol message maximum size.
|
|
256
|
+
*
|
|
257
|
+
* Defines the upper limit for how large a single protocol message can be.
|
|
258
|
+
* Implementations must enforce a maximum size between 1MB and 100MB to ensure
|
|
259
|
+
* compatibility across all Evolu implementations (the maximum size of mutation
|
|
260
|
+
* change is hardcoded and enforced hence the maximum size can't be smaller).
|
|
261
|
+
*
|
|
262
|
+
* Larger maximum sizes can be configured by relays to reduce roundtrips. For
|
|
263
|
+
* example, a dedicated relay with ample resources could configure a 100MB
|
|
264
|
+
* maximum to minimize roundtrips for large syncs.
|
|
265
|
+
*
|
|
266
|
+
* Only relays can safely configure larger sizes, as clients will handle them.
|
|
267
|
+
* Increasing this value on the client side would break compatibility with
|
|
268
|
+
* relays that enforce smaller limits.
|
|
269
|
+
*/
|
|
270
|
+
export const ProtocolMessageMaxSize = between(
|
|
271
|
+
minProtocolMessageMaxSize,
|
|
272
|
+
maxProtocolMessageMaxSize,
|
|
273
|
+
)(Int);
|
|
283
274
|
|
|
284
|
-
|
|
285
|
-
export type EncryptedDbChange = Uint8Array & Brand<"EncryptedDbChange">;
|
|
275
|
+
export type ProtocolMessageMaxSize = typeof ProtocolMessageMaxSize.Type;
|
|
286
276
|
|
|
287
277
|
/**
|
|
288
|
-
*
|
|
289
|
-
*
|
|
278
|
+
* Default {@link ProtocolMessageMaxSize} (1MB).
|
|
279
|
+
*
|
|
280
|
+
* The standard size used across Evolu implementations. Relays with more
|
|
281
|
+
* resources can configure larger sizes to reduce roundtrips.
|
|
290
282
|
*/
|
|
291
|
-
export
|
|
292
|
-
|
|
293
|
-
readonly change: DbChange;
|
|
294
|
-
}
|
|
283
|
+
export const defaultProtocolMessageMaxSize =
|
|
284
|
+
minProtocolMessageMaxSize as ProtocolMessageMaxSize;
|
|
295
285
|
|
|
296
286
|
/**
|
|
297
|
-
*
|
|
298
|
-
*
|
|
287
|
+
* Protocol message ranges maximum size.
|
|
288
|
+
*
|
|
289
|
+
* Defines the upper limit for how large the ranges section of a protocol
|
|
290
|
+
* message can be. Implementations must enforce a maximum size between 3KB and
|
|
291
|
+
* 100KB to ensure compatibility.
|
|
292
|
+
*
|
|
293
|
+
* The upper bound is set to ensure ranges fit within the default 1MB
|
|
294
|
+
* {@link defaultProtocolMessageMaxSize}, maintaining compatibility between all
|
|
295
|
+
* clients and relays.
|
|
299
296
|
*/
|
|
300
|
-
export
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
readonly values: ReadonlyRecord<Base64Url256, SqliteValue>;
|
|
304
|
-
}
|
|
305
|
-
|
|
306
|
-
export const RangeType = {
|
|
307
|
-
Fingerprint: 1,
|
|
308
|
-
Skip: 0,
|
|
309
|
-
Timestamps: 2,
|
|
310
|
-
} as const;
|
|
311
|
-
|
|
312
|
-
export type RangeType = (typeof RangeType)[keyof typeof RangeType];
|
|
313
|
-
|
|
314
|
-
export const InfiniteUpperBound = Symbol("InfiniteUpperBound");
|
|
315
|
-
export type InfiniteUpperBound = typeof InfiniteUpperBound;
|
|
297
|
+
export const ProtocolMessageRangesMaxSize = between(3_000, 100_000)(Int);
|
|
298
|
+
export type ProtocolMessageRangesMaxSize =
|
|
299
|
+
typeof ProtocolMessageRangesMaxSize.Type;
|
|
316
300
|
|
|
317
301
|
/**
|
|
318
|
-
*
|
|
319
|
-
*
|
|
302
|
+
* Default {@link ProtocolMessageRangesMaxSize} (30KB).
|
|
303
|
+
*
|
|
304
|
+
* The standard size used across Evolu implementations. Relays with more
|
|
305
|
+
* resources can configure larger sizes to reduce roundtrips.
|
|
320
306
|
*/
|
|
321
|
-
export
|
|
322
|
-
|
|
323
|
-
interface BaseRange {
|
|
324
|
-
readonly upperBound: RangeUpperBound;
|
|
325
|
-
}
|
|
307
|
+
export const defaultProtocolMessageRangesMaxSize =
|
|
308
|
+
30_000 as ProtocolMessageRangesMaxSize;
|
|
326
309
|
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
}
|
|
310
|
+
/** Evolu Protocol Message. */
|
|
311
|
+
export type ProtocolMessage = Uint8Array & Brand<"ProtocolMessage">;
|
|
330
312
|
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
readonly fingerprint: Fingerprint;
|
|
334
|
-
}
|
|
313
|
+
/** Evolu Protocol version. */
|
|
314
|
+
export const protocolVersion = 0 as NonNegativeInt;
|
|
335
315
|
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
316
|
+
export const MessageType = {
|
|
317
|
+
/** Request message from initiator (client) to non-initiator (relay). */
|
|
318
|
+
Request: 0,
|
|
319
|
+
/** Response message from non-initiator (relay) to initiator (client). */
|
|
320
|
+
Response: 1,
|
|
321
|
+
/** Broadcast message from non-initiator (relay) to subscribed clients. */
|
|
322
|
+
Broadcast: 2,
|
|
323
|
+
} as const;
|
|
344
324
|
|
|
345
|
-
export
|
|
325
|
+
export type MessageType = (typeof MessageType)[keyof typeof MessageType];
|
|
346
326
|
|
|
347
|
-
|
|
348
|
-
|
|
327
|
+
export const SubscriptionFlags = {
|
|
328
|
+
/** No subscription changes for this owner. */
|
|
329
|
+
None: 0,
|
|
330
|
+
/** Subscribe to updates for this owner. */
|
|
331
|
+
Subscribe: 1,
|
|
332
|
+
/** Unsubscribe from updates for this owner. */
|
|
333
|
+
Unsubscribe: 2,
|
|
334
|
+
} as const;
|
|
349
335
|
|
|
350
|
-
export
|
|
351
|
-
|
|
352
|
-
readonly timestamps: ReadonlyArray<BinaryTimestamp>;
|
|
353
|
-
}
|
|
336
|
+
export type SubscriptionFlag =
|
|
337
|
+
(typeof SubscriptionFlags)[keyof typeof SubscriptionFlags];
|
|
354
338
|
|
|
355
|
-
export
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
339
|
+
export const ProtocolErrorCode = {
|
|
340
|
+
NoError: 0,
|
|
341
|
+
/** A code for {@link ProtocolWriteKeyError}. */
|
|
342
|
+
WriteKeyError: 1,
|
|
343
|
+
/** A code for {@link ProtocolWriteError}. */
|
|
344
|
+
WriteError: 2,
|
|
345
|
+
/** A code for {@link ProtocolSyncError}. */
|
|
346
|
+
SyncError: 3,
|
|
347
|
+
/** A code for {@link ProtocolQuotaExceededError}. */
|
|
348
|
+
QuotaExceededError: 4,
|
|
349
|
+
} as const;
|
|
359
350
|
|
|
360
|
-
|
|
351
|
+
type ProtocolErrorCode =
|
|
352
|
+
(typeof ProtocolErrorCode)[keyof typeof ProtocolErrorCode];
|
|
361
353
|
|
|
362
354
|
export type ProtocolError =
|
|
363
355
|
| ProtocolUnsupportedVersionError
|
|
364
356
|
| ProtocolInvalidDataError
|
|
365
357
|
| ProtocolWriteKeyError
|
|
366
358
|
| ProtocolWriteError
|
|
367
|
-
| ProtocolSyncError
|
|
359
|
+
| ProtocolSyncError
|
|
360
|
+
| ProtocolQuotaExceededError
|
|
361
|
+
| ProtocolTimestampMismatchError;
|
|
368
362
|
|
|
369
363
|
/** Base interface for all protocol errors. */
|
|
370
364
|
export interface ProtocolErrorBase {
|
|
@@ -389,43 +383,66 @@ export interface ProtocolInvalidDataError {
|
|
|
389
383
|
readonly error: unknown;
|
|
390
384
|
}
|
|
391
385
|
|
|
392
|
-
/** Error when a {@link
|
|
386
|
+
/** Error when a {@link OwnerWriteKey} is invalid, missing, or fails validation. */
|
|
393
387
|
export interface ProtocolWriteKeyError extends ProtocolErrorBase {
|
|
394
388
|
readonly type: "ProtocolWriteKeyError";
|
|
395
389
|
}
|
|
396
390
|
|
|
397
391
|
/**
|
|
398
|
-
* Error
|
|
399
|
-
*
|
|
392
|
+
* Error indicating a serious relay-side write failure. Clients should log this
|
|
393
|
+
* error and show a generic sync error to the user.
|
|
400
394
|
*/
|
|
401
395
|
export interface ProtocolWriteError extends ProtocolErrorBase {
|
|
402
396
|
readonly type: "ProtocolWriteError";
|
|
403
397
|
}
|
|
404
398
|
|
|
405
399
|
/**
|
|
406
|
-
* Error indicating a synchronization failure
|
|
407
|
-
*
|
|
400
|
+
* Error indicating a serious relay-side synchronization failure. Clients should
|
|
401
|
+
* log this error and show a generic sync error to the user.
|
|
408
402
|
*/
|
|
409
403
|
export interface ProtocolSyncError extends ProtocolErrorBase {
|
|
410
404
|
readonly type: "ProtocolSyncError";
|
|
411
405
|
}
|
|
412
406
|
|
|
407
|
+
/**
|
|
408
|
+
* Error when storage or billing quota is exceeded. Clients should prompt the
|
|
409
|
+
* user to upgrade their plan or expand capacity.
|
|
410
|
+
*
|
|
411
|
+
* TODO: Add callback to relay config to check quota and return this error when
|
|
412
|
+
* limits are reached.
|
|
413
|
+
*/
|
|
414
|
+
export interface ProtocolQuotaExceededError extends ProtocolErrorBase {
|
|
415
|
+
readonly type: "ProtocolQuotaExceededError";
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
/**
|
|
419
|
+
* Error when embedded timestamp doesn't match expected timestamp in
|
|
420
|
+
* EncryptedDbChange. Indicates potential tampering or corruption of CRDT
|
|
421
|
+
* messages.
|
|
422
|
+
*/
|
|
423
|
+
export interface ProtocolTimestampMismatchError {
|
|
424
|
+
readonly type: "ProtocolTimestampMismatchError";
|
|
425
|
+
readonly expected: Timestamp;
|
|
426
|
+
readonly embedded: Timestamp;
|
|
427
|
+
}
|
|
428
|
+
|
|
413
429
|
/**
|
|
414
430
|
* Creates a {@link ProtocolMessage} from CRDT messages.
|
|
415
431
|
*
|
|
416
|
-
* If the message size would exceed {@link
|
|
417
|
-
* ensures all messages will be sent in the next round(s) even over
|
|
432
|
+
* If the message size would exceed {@link defaultProtocolMessageMaxSize}, the
|
|
433
|
+
* protocol ensures all messages will be sent in the next round(s) even over
|
|
418
434
|
* unidirectional and stateless transports.
|
|
419
435
|
*/
|
|
420
436
|
export const createProtocolMessageFromCrdtMessages =
|
|
421
|
-
(deps:
|
|
437
|
+
(deps: RandomBytesDep & SymmetricCryptoDep) =>
|
|
422
438
|
(
|
|
423
|
-
owner:
|
|
439
|
+
owner: Owner,
|
|
424
440
|
messages: NonEmptyReadonlyArray<CrdtMessage>,
|
|
425
|
-
maxSize?:
|
|
441
|
+
maxSize?: ProtocolMessageMaxSize,
|
|
426
442
|
): ProtocolMessage => {
|
|
427
443
|
const buffer = createProtocolMessageBuffer(owner.id, {
|
|
428
|
-
|
|
444
|
+
messageType: MessageType.Request,
|
|
445
|
+
totalMaxSize: maxSize ?? defaultProtocolMessageMaxSize,
|
|
429
446
|
writeKey: owner.writeKey,
|
|
430
447
|
});
|
|
431
448
|
|
|
@@ -433,7 +450,7 @@ export const createProtocolMessageFromCrdtMessages =
|
|
|
433
450
|
|
|
434
451
|
for (const message of messages) {
|
|
435
452
|
const change = encodeAndEncryptDbChange(deps)(
|
|
436
|
-
message
|
|
453
|
+
message,
|
|
437
454
|
owner.encryptionKey,
|
|
438
455
|
);
|
|
439
456
|
const encryptedCrdtMessage = { timestamp: message.timestamp, change };
|
|
@@ -454,14 +471,14 @@ export const createProtocolMessageFromCrdtMessages =
|
|
|
454
471
|
*
|
|
455
472
|
* The ideal approach would be to send three ranges (skip, fingerprint,
|
|
456
473
|
* skip) where the fingerprint of unsent messages would act as narrow sync
|
|
457
|
-
* probe. I think we can send
|
|
458
|
-
*
|
|
474
|
+
* probe. I think we can send `zeroFingerprint` which can be interpreted
|
|
475
|
+
* as an indication that the other side should reply with
|
|
459
476
|
* {@link TimestampsRange}, so no need to restart syncing.
|
|
460
477
|
*
|
|
461
478
|
* For now, using a random fingerprint avoids extra complexity and is good
|
|
462
479
|
* enough for this case.
|
|
463
480
|
*/
|
|
464
|
-
const randomFingerprint = deps.
|
|
481
|
+
const randomFingerprint = deps.randomBytes.create(
|
|
465
482
|
fingerprintSize,
|
|
466
483
|
) as unknown as Fingerprint;
|
|
467
484
|
|
|
@@ -479,16 +496,22 @@ export const createProtocolMessageFromCrdtMessages =
|
|
|
479
496
|
/** Creates a {@link ProtocolMessage} for sync. */
|
|
480
497
|
export const createProtocolMessageForSync =
|
|
481
498
|
(deps: StorageDep) =>
|
|
482
|
-
(
|
|
483
|
-
|
|
484
|
-
|
|
499
|
+
(
|
|
500
|
+
ownerId: OwnerId,
|
|
501
|
+
subscriptionFlag?: SubscriptionFlag,
|
|
502
|
+
): ProtocolMessage | null => {
|
|
503
|
+
const buffer = createProtocolMessageBuffer(ownerId, {
|
|
504
|
+
messageType: MessageType.Request,
|
|
505
|
+
subscriptionFlag: subscriptionFlag ?? SubscriptionFlags.None,
|
|
506
|
+
});
|
|
507
|
+
const ownerIdBytes = ownerIdToOwnerIdBytes(ownerId);
|
|
485
508
|
|
|
486
|
-
const size = deps.storage.getSize(
|
|
509
|
+
const size = deps.storage.getSize(ownerIdBytes);
|
|
487
510
|
// Errors are handled by the storage.
|
|
488
511
|
if (size == null) return null;
|
|
489
512
|
|
|
490
513
|
splitRange(deps)(
|
|
491
|
-
|
|
514
|
+
ownerIdBytes,
|
|
492
515
|
0 as NonNegativeInt,
|
|
493
516
|
size,
|
|
494
517
|
InfiniteUpperBound,
|
|
@@ -498,6 +521,14 @@ export const createProtocolMessageForSync =
|
|
|
498
521
|
return buffer.unwrap();
|
|
499
522
|
};
|
|
500
523
|
|
|
524
|
+
export const createProtocolMessageForUnsubscribe = (
|
|
525
|
+
ownerId: OwnerId,
|
|
526
|
+
): ProtocolMessage =>
|
|
527
|
+
createProtocolMessageBuffer(ownerId, {
|
|
528
|
+
messageType: MessageType.Request,
|
|
529
|
+
subscriptionFlag: SubscriptionFlags.Unsubscribe,
|
|
530
|
+
}).unwrap();
|
|
531
|
+
|
|
501
532
|
/**
|
|
502
533
|
* Mutable builder for constructing {@link ProtocolMessage} respecting size
|
|
503
534
|
* limits.
|
|
@@ -525,18 +556,27 @@ export interface ProtocolMessageBuffer {
|
|
|
525
556
|
export const createProtocolMessageBuffer = (
|
|
526
557
|
ownerId: OwnerId,
|
|
527
558
|
options: {
|
|
528
|
-
readonly
|
|
529
|
-
readonly
|
|
530
|
-
readonly totalMaxSize?: PositiveInt | undefined;
|
|
531
|
-
readonly rangesMaxSize?: PositiveInt | undefined;
|
|
559
|
+
readonly totalMaxSize?: ProtocolMessageMaxSize | undefined;
|
|
560
|
+
readonly rangesMaxSize?: ProtocolMessageRangesMaxSize | undefined;
|
|
532
561
|
readonly version?: NonNegativeInt;
|
|
533
|
-
}
|
|
562
|
+
} & (
|
|
563
|
+
| {
|
|
564
|
+
readonly messageType: typeof MessageType.Request;
|
|
565
|
+
readonly writeKey?: OwnerWriteKey;
|
|
566
|
+
readonly subscriptionFlag?: SubscriptionFlag;
|
|
567
|
+
}
|
|
568
|
+
| {
|
|
569
|
+
readonly messageType: typeof MessageType.Response;
|
|
570
|
+
readonly errorCode: ProtocolErrorCode;
|
|
571
|
+
}
|
|
572
|
+
| {
|
|
573
|
+
readonly messageType: typeof MessageType.Broadcast;
|
|
574
|
+
}
|
|
575
|
+
),
|
|
534
576
|
): ProtocolMessageBuffer => {
|
|
535
577
|
const {
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
totalMaxSize = maxProtocolMessageSize,
|
|
539
|
-
rangesMaxSize = maxProtocolMessageRangesSize,
|
|
578
|
+
totalMaxSize = defaultProtocolMessageMaxSize,
|
|
579
|
+
rangesMaxSize = defaultProtocolMessageRangesMaxSize,
|
|
540
580
|
version = protocolVersion,
|
|
541
581
|
} = options;
|
|
542
582
|
|
|
@@ -554,8 +594,21 @@ export const createProtocolMessageBuffer = (
|
|
|
554
594
|
};
|
|
555
595
|
|
|
556
596
|
encodeNonNegativeInt(buffers.header, version);
|
|
557
|
-
buffers.header.extend(
|
|
558
|
-
|
|
597
|
+
buffers.header.extend(ownerIdToOwnerIdBytes(ownerId));
|
|
598
|
+
buffers.header.extend([options.messageType]);
|
|
599
|
+
|
|
600
|
+
if (options.messageType === MessageType.Request) {
|
|
601
|
+
if (!options.writeKey) {
|
|
602
|
+
buffers.header.extend([0]);
|
|
603
|
+
} else {
|
|
604
|
+
buffers.header.extend([1]);
|
|
605
|
+
buffers.header.extend(options.writeKey);
|
|
606
|
+
}
|
|
607
|
+
const subscriptionFlag = options.subscriptionFlag ?? SubscriptionFlags.None;
|
|
608
|
+
buffers.header.extend([subscriptionFlag]);
|
|
609
|
+
} else if (options.messageType === MessageType.Response) {
|
|
610
|
+
buffers.header.extend([options.errorCode]);
|
|
611
|
+
}
|
|
559
612
|
|
|
560
613
|
let isLastRangeInfinite = false;
|
|
561
614
|
|
|
@@ -567,10 +620,7 @@ export const createProtocolMessageBuffer = (
|
|
|
567
620
|
const getHeaderAndMessagesSize = () =>
|
|
568
621
|
buffers.header.getLength() +
|
|
569
622
|
buffers.messages.timestamps.getLength() +
|
|
570
|
-
buffers.messages.dbChanges.getLength()
|
|
571
|
-
(buffers.messages.timestamps.getCount() > 0 && writeKey
|
|
572
|
-
? writeKeyLength
|
|
573
|
-
: 0);
|
|
623
|
+
buffers.messages.dbChanges.getLength();
|
|
574
624
|
|
|
575
625
|
const getRangesSize = () =>
|
|
576
626
|
buffers.ranges.timestamps.getCount() > 0
|
|
@@ -639,10 +689,15 @@ export const createProtocolMessageBuffer = (
|
|
|
639
689
|
},
|
|
640
690
|
|
|
641
691
|
addRange: (range) => {
|
|
692
|
+
assert(
|
|
693
|
+
options.messageType !== MessageType.Broadcast,
|
|
694
|
+
"Cannot add a range into broadcast message",
|
|
695
|
+
);
|
|
642
696
|
assert(
|
|
643
697
|
!isLastRangeInfinite,
|
|
644
698
|
"Cannot add a range after an InfiniteUpperBound range",
|
|
645
699
|
);
|
|
700
|
+
|
|
646
701
|
isLastRangeInfinite = range.upperBound === InfiniteUpperBound;
|
|
647
702
|
|
|
648
703
|
/**
|
|
@@ -652,7 +707,7 @@ export const createProtocolMessageBuffer = (
|
|
|
652
707
|
*/
|
|
653
708
|
if (range.upperBound !== InfiniteUpperBound)
|
|
654
709
|
buffers.ranges.timestamps.add(
|
|
655
|
-
|
|
710
|
+
timestampBytesToTimestamp(range.upperBound),
|
|
656
711
|
);
|
|
657
712
|
else {
|
|
658
713
|
buffers.ranges.timestamps.addInfinite();
|
|
@@ -685,8 +740,6 @@ export const createProtocolMessageBuffer = (
|
|
|
685
740
|
|
|
686
741
|
buffers.messages.timestamps.append(buffers.header);
|
|
687
742
|
buffers.header.extend(buffers.messages.dbChanges.unwrap());
|
|
688
|
-
if (buffers.messages.timestamps.getCount() > 0 && writeKey)
|
|
689
|
-
buffers.header.extend(writeKey);
|
|
690
743
|
|
|
691
744
|
if (buffers.ranges.timestamps.getCount() > 0) {
|
|
692
745
|
buffers.ranges.timestamps.append(buffers.header);
|
|
@@ -701,6 +754,11 @@ export const createProtocolMessageBuffer = (
|
|
|
701
754
|
};
|
|
702
755
|
};
|
|
703
756
|
|
|
757
|
+
export interface TimestampsRangeWithTimestampsBuffer extends BaseRange {
|
|
758
|
+
readonly type: typeof RangeType.Timestamps;
|
|
759
|
+
readonly timestamps: TimestampsBuffer;
|
|
760
|
+
}
|
|
761
|
+
|
|
704
762
|
export interface TimestampsBuffer {
|
|
705
763
|
readonly add: (timestamp: Timestamp) => void;
|
|
706
764
|
readonly addInfinite: () => void;
|
|
@@ -802,42 +860,62 @@ const createRunLengthEncoder = <T>(
|
|
|
802
860
|
};
|
|
803
861
|
|
|
804
862
|
export interface ApplyProtocolMessageAsClientOptions {
|
|
805
|
-
getWriteKey?: (ownerId: OwnerId) =>
|
|
863
|
+
getWriteKey?: (ownerId: OwnerId) => OwnerWriteKey | null;
|
|
806
864
|
|
|
807
865
|
/** For testing purposes only; should not be used in production. */
|
|
808
866
|
version?: NonNegativeInt;
|
|
809
867
|
|
|
810
|
-
|
|
811
|
-
rangesMaxSize?: PositiveInt;
|
|
868
|
+
rangesMaxSize?: ProtocolMessageRangesMaxSize;
|
|
812
869
|
}
|
|
813
870
|
|
|
871
|
+
/**
|
|
872
|
+
* Result type for {@link applyProtocolMessageAsClient} that distinguishes
|
|
873
|
+
* between responses to client requests and broadcast messages.
|
|
874
|
+
*/
|
|
875
|
+
export type ApplyProtocolMessageAsClientResult =
|
|
876
|
+
| { readonly type: "response"; readonly message: ProtocolMessage }
|
|
877
|
+
| { readonly type: "no-response" }
|
|
878
|
+
| { readonly type: "broadcast" };
|
|
879
|
+
|
|
814
880
|
export const applyProtocolMessageAsClient =
|
|
815
881
|
(deps: StorageDep) =>
|
|
816
|
-
(
|
|
882
|
+
async (
|
|
817
883
|
inputMessage: Uint8Array,
|
|
818
|
-
{
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
884
|
+
options: ApplyProtocolMessageAsClientOptions = {},
|
|
885
|
+
): Promise<
|
|
886
|
+
Result<
|
|
887
|
+
ApplyProtocolMessageAsClientResult,
|
|
888
|
+
| ProtocolInvalidDataError
|
|
889
|
+
| ProtocolSyncError
|
|
890
|
+
| ProtocolUnsupportedVersionError
|
|
891
|
+
| ProtocolWriteError
|
|
892
|
+
| ProtocolWriteKeyError
|
|
893
|
+
| ProtocolQuotaExceededError
|
|
894
|
+
>
|
|
895
|
+
> => {
|
|
896
|
+
// try-catch instead of Result for performance and stacktraces
|
|
897
|
+
try {
|
|
898
|
+
const input = createBuffer(inputMessage);
|
|
899
|
+
const [requestedVersion, ownerId] = decodeVersionAndOwner(input);
|
|
900
|
+
const version = options.version ?? protocolVersion;
|
|
901
|
+
|
|
902
|
+
if (requestedVersion !== version) {
|
|
903
|
+
return err<ProtocolUnsupportedVersionError>({
|
|
904
|
+
type: "ProtocolUnsupportedVersionError",
|
|
905
|
+
unsupportedVersion: requestedVersion,
|
|
906
|
+
isInitiator: version < requestedVersion,
|
|
907
|
+
ownerId,
|
|
908
|
+
});
|
|
909
|
+
}
|
|
838
910
|
|
|
839
|
-
|
|
911
|
+
const messageType = input.shift() as MessageType;
|
|
912
|
+
assert(
|
|
913
|
+
messageType === MessageType.Response ||
|
|
914
|
+
messageType === MessageType.Broadcast,
|
|
915
|
+
"Invalid MessageType",
|
|
916
|
+
);
|
|
840
917
|
|
|
918
|
+
if (messageType === MessageType.Response) {
|
|
841
919
|
const errorCode = input.shift() as ProtocolErrorCode;
|
|
842
920
|
if (errorCode !== ProtocolErrorCode.NoError) {
|
|
843
921
|
switch (errorCode) {
|
|
@@ -856,147 +934,254 @@ export const applyProtocolMessageAsClient =
|
|
|
856
934
|
type: "ProtocolSyncError",
|
|
857
935
|
ownerId,
|
|
858
936
|
});
|
|
937
|
+
case ProtocolErrorCode.QuotaExceededError:
|
|
938
|
+
return err<ProtocolQuotaExceededError>({
|
|
939
|
+
type: "ProtocolQuotaExceededError",
|
|
940
|
+
ownerId,
|
|
941
|
+
});
|
|
859
942
|
default:
|
|
860
943
|
throw new ProtocolDecodeError(
|
|
861
944
|
`Invalid ProtocolErrorCode: ${errorCode}`,
|
|
862
945
|
);
|
|
863
946
|
}
|
|
864
947
|
}
|
|
948
|
+
}
|
|
865
949
|
|
|
866
|
-
|
|
950
|
+
const messages = decodeMessages(input);
|
|
951
|
+
const ownerIdBytes = ownerIdToOwnerIdBytes(ownerId);
|
|
867
952
|
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
953
|
+
if (
|
|
954
|
+
isNonEmptyReadonlyArray(messages) &&
|
|
955
|
+
!(await deps.storage.writeMessages(ownerIdBytes, messages))
|
|
956
|
+
) {
|
|
957
|
+
return ok({ type: "no-response" });
|
|
958
|
+
}
|
|
959
|
+
|
|
960
|
+
// Now: No writeKey, no sync.
|
|
961
|
+
// TODO: Allow to sync SharedReadonlyOwner
|
|
962
|
+
// Without local changes, writeKey will not be required.
|
|
963
|
+
// With local changes, writeKey will be required and if not provided,
|
|
964
|
+
// the sync will stop.
|
|
965
|
+
const writeKey = options.getWriteKey?.(ownerId);
|
|
966
|
+
if (writeKey == null) {
|
|
967
|
+
return ok({ type: "no-response" });
|
|
968
|
+
}
|
|
874
969
|
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
970
|
+
if (messageType === MessageType.Broadcast) {
|
|
971
|
+
return ok({ type: "broadcast" });
|
|
972
|
+
}
|
|
878
973
|
|
|
879
|
-
|
|
880
|
-
writeKey,
|
|
881
|
-
totalMaxSize,
|
|
882
|
-
rangesMaxSize,
|
|
883
|
-
});
|
|
974
|
+
const ranges = decodeRanges(input);
|
|
884
975
|
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
976
|
+
if (!isNonEmptyReadonlyArray(ranges)) {
|
|
977
|
+
return ok({ type: "no-response" });
|
|
978
|
+
}
|
|
979
|
+
|
|
980
|
+
const output = createProtocolMessageBuffer(ownerId, {
|
|
981
|
+
messageType: MessageType.Request,
|
|
982
|
+
writeKey,
|
|
983
|
+
rangesMaxSize: options.rangesMaxSize,
|
|
984
|
+
});
|
|
985
|
+
|
|
986
|
+
const syncResult = sync(deps)(ranges, output, ownerIdBytes);
|
|
987
|
+
|
|
988
|
+
// Client sync error (handled via Storage) or no changes.
|
|
989
|
+
if (!syncResult.ok || !syncResult.value) {
|
|
990
|
+
return ok({ type: "no-response" });
|
|
991
|
+
}
|
|
992
|
+
|
|
993
|
+
return ok({ type: "response", message: output.unwrap() });
|
|
994
|
+
} catch (error) {
|
|
995
|
+
return err<ProtocolInvalidDataError>({
|
|
996
|
+
type: "ProtocolInvalidDataError",
|
|
997
|
+
data: inputMessage,
|
|
998
|
+
error,
|
|
999
|
+
});
|
|
1000
|
+
}
|
|
1001
|
+
};
|
|
888
1002
|
|
|
889
1003
|
export interface ApplyProtocolMessageAsRelayOptions {
|
|
890
1004
|
/** To subscribe an owner for broadcasting. */
|
|
891
1005
|
subscribe?: (ownerId: OwnerId) => void;
|
|
892
1006
|
|
|
1007
|
+
/** To unsubscribe an owner from broadcasting. */
|
|
1008
|
+
unsubscribe?: (ownerId: OwnerId) => void;
|
|
1009
|
+
|
|
893
1010
|
/** To broadcast a protocol message to all subscribers. */
|
|
894
1011
|
broadcast?: (ownerId: OwnerId, message: ProtocolMessage) => void;
|
|
895
1012
|
|
|
896
|
-
totalMaxSize?:
|
|
897
|
-
rangesMaxSize?:
|
|
1013
|
+
totalMaxSize?: ProtocolMessageMaxSize;
|
|
1014
|
+
rangesMaxSize?: ProtocolMessageRangesMaxSize;
|
|
1015
|
+
}
|
|
1016
|
+
|
|
1017
|
+
/**
|
|
1018
|
+
* Result type for {@link applyProtocolMessageAsRelay}.
|
|
1019
|
+
*
|
|
1020
|
+
* Unlike {@link ApplyProtocolMessageAsClientResult}, relays always respond with
|
|
1021
|
+
* a message to provide sync completion feedback. This ensures the initiator can
|
|
1022
|
+
* reliably detect when synchronization is complete, even when there's nothing
|
|
1023
|
+
* to sync. Clients may choose not to respond in certain cases (like when they
|
|
1024
|
+
* receive broadcast messages or when they lack a write key for syncing).
|
|
1025
|
+
*/
|
|
1026
|
+
export interface ApplyProtocolMessageAsRelayResult {
|
|
1027
|
+
readonly type: "response";
|
|
1028
|
+
readonly message: ProtocolMessage;
|
|
898
1029
|
}
|
|
899
1030
|
|
|
900
1031
|
export const applyProtocolMessageAsRelay =
|
|
901
1032
|
(deps: StorageDep) =>
|
|
902
|
-
(
|
|
1033
|
+
async (
|
|
903
1034
|
inputMessage: Uint8Array,
|
|
904
|
-
{
|
|
905
|
-
subscribe,
|
|
906
|
-
broadcast,
|
|
907
|
-
totalMaxSize,
|
|
908
|
-
rangesMaxSize,
|
|
909
|
-
}: ApplyProtocolMessageAsRelayOptions = {},
|
|
1035
|
+
options: ApplyProtocolMessageAsRelayOptions = {},
|
|
910
1036
|
/** For testing purposes only; should not be used in production. */
|
|
911
1037
|
version = protocolVersion,
|
|
912
|
-
):
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
1038
|
+
): Promise<
|
|
1039
|
+
Result<ApplyProtocolMessageAsRelayResult, ProtocolInvalidDataError>
|
|
1040
|
+
> => {
|
|
1041
|
+
// try-catch instead of Result for performance and stacktraces
|
|
1042
|
+
try {
|
|
1043
|
+
const input = createBuffer(inputMessage);
|
|
1044
|
+
const [requestedVersion, ownerId] = decodeVersionAndOwner(input);
|
|
1045
|
+
const ownerIdBytes = ownerIdToOwnerIdBytes(ownerId);
|
|
917
1046
|
|
|
918
1047
|
if (requestedVersion !== version) {
|
|
919
1048
|
// Non-initiator responds with its version and ownerId.
|
|
920
1049
|
const output = createBuffer();
|
|
921
1050
|
encodeNonNegativeInt(output, version);
|
|
922
|
-
output.extend(
|
|
923
|
-
return ok(
|
|
1051
|
+
output.extend(ownerIdBytes);
|
|
1052
|
+
return ok({
|
|
1053
|
+
type: "response",
|
|
1054
|
+
message: output.unwrap() as ProtocolMessage,
|
|
1055
|
+
});
|
|
924
1056
|
}
|
|
925
1057
|
|
|
926
|
-
|
|
1058
|
+
const messageType = input.shift() as MessageType;
|
|
1059
|
+
assert(messageType === MessageType.Request, "Invalid MessageType");
|
|
927
1060
|
|
|
928
|
-
const
|
|
1061
|
+
const hasWriteKey = input.shift();
|
|
1062
|
+
let writeKey: OwnerWriteKey | undefined;
|
|
929
1063
|
|
|
930
|
-
if (
|
|
931
|
-
|
|
932
|
-
|
|
1064
|
+
if (hasWriteKey === 1) {
|
|
1065
|
+
writeKey = input.shiftN(ownerWriteKeyLength) as OwnerWriteKey;
|
|
1066
|
+
}
|
|
933
1067
|
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
1068
|
+
const subscriptionFlag = input.shift() as SubscriptionFlag;
|
|
1069
|
+
|
|
1070
|
+
switch (subscriptionFlag) {
|
|
1071
|
+
case SubscriptionFlags.Subscribe:
|
|
1072
|
+
options.subscribe?.(ownerId);
|
|
1073
|
+
break;
|
|
1074
|
+
case SubscriptionFlags.Unsubscribe:
|
|
1075
|
+
options.unsubscribe?.(ownerId);
|
|
1076
|
+
break;
|
|
1077
|
+
case SubscriptionFlags.None:
|
|
1078
|
+
break;
|
|
1079
|
+
}
|
|
938
1080
|
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
1081
|
+
if (writeKey) {
|
|
1082
|
+
const isValid = deps.storage.validateWriteKey(ownerIdBytes, writeKey);
|
|
1083
|
+
if (!isValid) {
|
|
1084
|
+
return ok({
|
|
1085
|
+
type: "response",
|
|
1086
|
+
message: createProtocolMessageBuffer(ownerId, {
|
|
1087
|
+
messageType: MessageType.Response,
|
|
942
1088
|
errorCode: ProtocolErrorCode.WriteKeyError,
|
|
943
1089
|
}).unwrap(),
|
|
944
|
-
);
|
|
1090
|
+
});
|
|
1091
|
+
}
|
|
1092
|
+
}
|
|
945
1093
|
|
|
946
|
-
|
|
947
|
-
// Instead of encoding a new protocol message, we reuse the inputMessage.
|
|
948
|
-
const broadcastMessage = concatBytes(
|
|
949
|
-
inputMessage.slice(0, 17),
|
|
950
|
-
new Uint8Array([ProtocolErrorCode.NoError]),
|
|
951
|
-
inputMessage.slice(17, messagesEnd),
|
|
952
|
-
) as ProtocolMessage;
|
|
1094
|
+
const messages = decodeMessages(input);
|
|
953
1095
|
|
|
954
|
-
|
|
1096
|
+
if (isNonEmptyReadonlyArray(messages)) {
|
|
1097
|
+
if (!writeKey) {
|
|
1098
|
+
return ok({
|
|
1099
|
+
type: "response",
|
|
1100
|
+
message: createProtocolMessageBuffer(ownerId, {
|
|
1101
|
+
messageType: MessageType.Response,
|
|
1102
|
+
errorCode: ProtocolErrorCode.WriteKeyError,
|
|
1103
|
+
}).unwrap(),
|
|
1104
|
+
});
|
|
955
1105
|
}
|
|
956
1106
|
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
1107
|
+
/**
|
|
1108
|
+
* Broadcast messages to all subscribed devices. This ensures real-time
|
|
1109
|
+
* synchronization between clients.
|
|
1110
|
+
*
|
|
1111
|
+
* When a relay's database is deleted or clients migrate to a new relay
|
|
1112
|
+
* (without data migration), clients will sync their data to the relay,
|
|
1113
|
+
* and the relay will broadcast those messages to other connected
|
|
1114
|
+
* clients. Those clients may receive messages they already have, but
|
|
1115
|
+
* this is safe because `applyMessages` is idempotent. As the relay
|
|
1116
|
+
* becomes more synchronized with clients over time, fewer duplicate
|
|
1117
|
+
* messages will be broadcasted.
|
|
1118
|
+
*/
|
|
1119
|
+
if (options.broadcast) {
|
|
1120
|
+
const broadcastBuffer = createProtocolMessageBuffer(ownerId, {
|
|
1121
|
+
messageType: MessageType.Broadcast,
|
|
1122
|
+
totalMaxSize: options.totalMaxSize,
|
|
1123
|
+
rangesMaxSize: options.rangesMaxSize,
|
|
1124
|
+
version,
|
|
1125
|
+
});
|
|
1126
|
+
for (const message of messages) {
|
|
1127
|
+
broadcastBuffer.addMessage(message);
|
|
1128
|
+
}
|
|
1129
|
+
options.broadcast(ownerId, broadcastBuffer.unwrap());
|
|
1130
|
+
}
|
|
1131
|
+
|
|
1132
|
+
if (!(await deps.storage.writeMessages(ownerIdBytes, messages))) {
|
|
1133
|
+
return ok({
|
|
1134
|
+
type: "response",
|
|
1135
|
+
message: createProtocolMessageBuffer(ownerId, {
|
|
1136
|
+
messageType: MessageType.Response,
|
|
960
1137
|
errorCode: ProtocolErrorCode.WriteError,
|
|
961
1138
|
}).unwrap(),
|
|
962
|
-
);
|
|
1139
|
+
});
|
|
1140
|
+
}
|
|
963
1141
|
}
|
|
964
1142
|
|
|
1143
|
+
const ranges = decodeRanges(input);
|
|
1144
|
+
|
|
965
1145
|
const output = createProtocolMessageBuffer(ownerId, {
|
|
1146
|
+
messageType: MessageType.Response,
|
|
966
1147
|
errorCode: ProtocolErrorCode.NoError,
|
|
967
|
-
totalMaxSize,
|
|
968
|
-
rangesMaxSize,
|
|
1148
|
+
totalMaxSize: options.totalMaxSize,
|
|
1149
|
+
rangesMaxSize: options.rangesMaxSize,
|
|
969
1150
|
});
|
|
970
1151
|
|
|
971
|
-
|
|
972
|
-
|
|
1152
|
+
// Non-initiators always respond to provide sync completion feedback,
|
|
1153
|
+
// even when there's nothing to sync.
|
|
1154
|
+
if (!isNonEmptyReadonlyArray(ranges)) {
|
|
1155
|
+
return ok({ type: "response", message: output.unwrap() });
|
|
1156
|
+
}
|
|
973
1157
|
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
if (error instanceof ProtocolDecodeError || error instanceof BufferError)
|
|
1158
|
+
const syncResult = sync(deps)(ranges, output, ownerIdBytes);
|
|
1159
|
+
|
|
1160
|
+
const message = syncResult.ok
|
|
1161
|
+
? output.unwrap()
|
|
1162
|
+
: createProtocolMessageBuffer(ownerId, {
|
|
1163
|
+
messageType: MessageType.Response,
|
|
1164
|
+
errorCode: syncResult.error,
|
|
1165
|
+
}).unwrap();
|
|
1166
|
+
|
|
1167
|
+
// Non-initiators always respond to provide sync completion feedback,
|
|
1168
|
+
return ok({ type: "response", message });
|
|
1169
|
+
} catch (error) {
|
|
987
1170
|
return err<ProtocolInvalidDataError>({
|
|
988
1171
|
type: "ProtocolInvalidDataError",
|
|
989
|
-
data,
|
|
1172
|
+
data: inputMessage,
|
|
990
1173
|
error,
|
|
991
1174
|
});
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
}
|
|
995
|
-
};
|
|
1175
|
+
}
|
|
1176
|
+
};
|
|
996
1177
|
|
|
997
1178
|
const decodeVersionAndOwner = (input: Buffer): [NonNegativeInt, OwnerId] => {
|
|
1179
|
+
// This structure must never change across protocol versions. The version
|
|
1180
|
+
// and owner ID must always be the first two fields in every protocol message
|
|
1181
|
+
// to enable version negotiation and owner identification before any other
|
|
1182
|
+
// processing occurs.
|
|
998
1183
|
const version = decodeNonNegativeInt(input);
|
|
999
|
-
const ownerId =
|
|
1184
|
+
const ownerId = decodeId(input) as OwnerId;
|
|
1000
1185
|
return [version, ownerId];
|
|
1001
1186
|
};
|
|
1002
1187
|
|
|
@@ -1031,34 +1216,14 @@ const decodeMessages = (
|
|
|
1031
1216
|
const sync =
|
|
1032
1217
|
(deps: StorageDep) =>
|
|
1033
1218
|
(
|
|
1034
|
-
|
|
1035
|
-
input: Buffer,
|
|
1219
|
+
ranges: NonEmptyReadonlyArray<Range>,
|
|
1036
1220
|
output: ProtocolMessageBuffer,
|
|
1037
|
-
|
|
1038
|
-
): Result<
|
|
1039
|
-
const ranges = decodeRanges(input);
|
|
1040
|
-
|
|
1041
|
-
if (!isNonEmptyReadonlyArray(ranges)) {
|
|
1042
|
-
// Nothing to sync.
|
|
1043
|
-
return ok(null);
|
|
1044
|
-
}
|
|
1045
|
-
|
|
1046
|
-
const binaryOwnerId = binaryOwnerIdToOwnerId(ownerId);
|
|
1221
|
+
ownerIdBytes: OwnerIdBytes,
|
|
1222
|
+
): Result<boolean, typeof ProtocolErrorCode.SyncError> => {
|
|
1047
1223
|
const outputInitialSize = output.getSize();
|
|
1048
1224
|
|
|
1049
|
-
const
|
|
1050
|
-
|
|
1051
|
-
if (role === "initiator") {
|
|
1052
|
-
return ok(null);
|
|
1053
|
-
}
|
|
1054
|
-
const message = createProtocolMessageBuffer(binaryOwnerId, {
|
|
1055
|
-
errorCode: ProtocolErrorCode.SyncError,
|
|
1056
|
-
});
|
|
1057
|
-
return ok(message.unwrap());
|
|
1058
|
-
};
|
|
1059
|
-
|
|
1060
|
-
const storageSize = deps.storage.getSize(ownerId);
|
|
1061
|
-
if (storageSize == null) return syncFail();
|
|
1225
|
+
const storageSize = deps.storage.getSize(ownerIdBytes);
|
|
1226
|
+
if (storageSize == null) return err(ProtocolErrorCode.SyncError);
|
|
1062
1227
|
|
|
1063
1228
|
let prevUpperBound: RangeUpperBound | null = null;
|
|
1064
1229
|
let prevIndex = 0 as NonNegativeInt;
|
|
@@ -1098,7 +1263,11 @@ const sync =
|
|
|
1098
1263
|
const addFingerprintForRemainingRange = (
|
|
1099
1264
|
begin: NonNegativeInt,
|
|
1100
1265
|
): boolean => {
|
|
1101
|
-
const fingerprint = deps.storage.fingerprint(
|
|
1266
|
+
const fingerprint = deps.storage.fingerprint(
|
|
1267
|
+
ownerIdBytes,
|
|
1268
|
+
begin,
|
|
1269
|
+
storageSize,
|
|
1270
|
+
);
|
|
1102
1271
|
if (!fingerprint) return false;
|
|
1103
1272
|
// There is always a space for a ramaining range.
|
|
1104
1273
|
output.addRange({
|
|
@@ -1114,12 +1283,12 @@ const sync =
|
|
|
1114
1283
|
|
|
1115
1284
|
const lower = prevIndex;
|
|
1116
1285
|
let upper = deps.storage.findLowerBound(
|
|
1117
|
-
|
|
1286
|
+
ownerIdBytes,
|
|
1118
1287
|
prevIndex,
|
|
1119
1288
|
storageSize,
|
|
1120
1289
|
currentUpperBound,
|
|
1121
1290
|
);
|
|
1122
|
-
if (upper == null) return
|
|
1291
|
+
if (upper == null) return err(ProtocolErrorCode.SyncError);
|
|
1123
1292
|
|
|
1124
1293
|
switch (range.type) {
|
|
1125
1294
|
case RangeType.Skip: {
|
|
@@ -1129,11 +1298,11 @@ const sync =
|
|
|
1129
1298
|
|
|
1130
1299
|
case RangeType.Fingerprint: {
|
|
1131
1300
|
const ourFingerprint = deps.storage.fingerprint(
|
|
1132
|
-
|
|
1301
|
+
ownerIdBytes,
|
|
1133
1302
|
lower,
|
|
1134
1303
|
upper,
|
|
1135
1304
|
);
|
|
1136
|
-
if (ourFingerprint == null) return
|
|
1305
|
+
if (ourFingerprint == null) return err(ProtocolErrorCode.SyncError);
|
|
1137
1306
|
|
|
1138
1307
|
if (eqArrayNumber(range.fingerprint, ourFingerprint)) {
|
|
1139
1308
|
skipRange(range);
|
|
@@ -1141,15 +1310,16 @@ const sync =
|
|
|
1141
1310
|
if (output.canSplitRange()) {
|
|
1142
1311
|
coalesceSkipsBeforeAdd();
|
|
1143
1312
|
splitRange(deps)(
|
|
1144
|
-
|
|
1313
|
+
ownerIdBytes,
|
|
1145
1314
|
lower,
|
|
1146
1315
|
upper,
|
|
1147
1316
|
currentUpperBound,
|
|
1148
1317
|
output,
|
|
1149
1318
|
);
|
|
1150
1319
|
} else {
|
|
1151
|
-
|
|
1152
|
-
|
|
1320
|
+
return addFingerprintForRemainingRange(upper)
|
|
1321
|
+
? ok(true)
|
|
1322
|
+
: err(ProtocolErrorCode.SyncError);
|
|
1153
1323
|
}
|
|
1154
1324
|
}
|
|
1155
1325
|
break;
|
|
@@ -1163,45 +1333,53 @@ const sync =
|
|
|
1163
1333
|
);
|
|
1164
1334
|
const ourTimestamps = createTimestampsBuffer();
|
|
1165
1335
|
|
|
1166
|
-
let
|
|
1336
|
+
let cantReadDbChange = false as boolean;
|
|
1167
1337
|
let exceeded = false as boolean;
|
|
1168
1338
|
|
|
1169
|
-
deps.storage.iterate(
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1339
|
+
deps.storage.iterate(
|
|
1340
|
+
ownerIdBytes,
|
|
1341
|
+
lower,
|
|
1342
|
+
upper,
|
|
1343
|
+
(timestamp, index) => {
|
|
1344
|
+
const timestampString = timestamp.join();
|
|
1345
|
+
const timestampBinary = timestampBytesToTimestamp(timestamp);
|
|
1346
|
+
|
|
1347
|
+
let message: EncryptedCrdtMessage | null = null;
|
|
1348
|
+
|
|
1349
|
+
if (timestampsWeNeed.has(timestampString)) {
|
|
1350
|
+
timestampsWeNeed.delete(timestampString);
|
|
1351
|
+
} else {
|
|
1352
|
+
const dbChange = deps.storage.readDbChange(
|
|
1353
|
+
ownerIdBytes,
|
|
1354
|
+
timestamp,
|
|
1355
|
+
);
|
|
1356
|
+
if (dbChange == null) {
|
|
1357
|
+
cantReadDbChange = true;
|
|
1358
|
+
return false;
|
|
1359
|
+
}
|
|
1360
|
+
message = {
|
|
1361
|
+
timestamp: timestampBinary,
|
|
1362
|
+
change: dbChange,
|
|
1363
|
+
};
|
|
1364
|
+
}
|
|
1174
1365
|
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1366
|
+
if (
|
|
1367
|
+
!output.canAddTimestampsRangeAndMessage(ourTimestamps, message)
|
|
1368
|
+
) {
|
|
1369
|
+
exceeded = true;
|
|
1370
|
+
endBound = timestamp;
|
|
1371
|
+
upper = index;
|
|
1181
1372
|
return false;
|
|
1182
1373
|
}
|
|
1183
|
-
message = {
|
|
1184
|
-
timestamp: timestampBinary,
|
|
1185
|
-
change: dbChange,
|
|
1186
|
-
};
|
|
1187
|
-
}
|
|
1188
1374
|
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
upper = index;
|
|
1195
|
-
return false;
|
|
1196
|
-
}
|
|
1197
|
-
|
|
1198
|
-
ourTimestamps.add(timestampBinary);
|
|
1199
|
-
if (message) output.addMessage(message);
|
|
1200
|
-
return true;
|
|
1201
|
-
});
|
|
1375
|
+
ourTimestamps.add(timestampBinary);
|
|
1376
|
+
if (message) output.addMessage(message);
|
|
1377
|
+
return true;
|
|
1378
|
+
},
|
|
1379
|
+
);
|
|
1202
1380
|
|
|
1203
|
-
if (
|
|
1204
|
-
return
|
|
1381
|
+
if (cantReadDbChange) {
|
|
1382
|
+
return err(ProtocolErrorCode.SyncError);
|
|
1205
1383
|
}
|
|
1206
1384
|
|
|
1207
1385
|
const addRange = () => {
|
|
@@ -1216,9 +1394,9 @@ const sync =
|
|
|
1216
1394
|
if (exceeded) {
|
|
1217
1395
|
addRange();
|
|
1218
1396
|
if (!addFingerprintForRemainingRange(upper)) {
|
|
1219
|
-
return
|
|
1397
|
+
return err(ProtocolErrorCode.SyncError);
|
|
1220
1398
|
}
|
|
1221
|
-
return ok(
|
|
1399
|
+
return ok(true);
|
|
1222
1400
|
}
|
|
1223
1401
|
|
|
1224
1402
|
// If we need something, we have to respond with our timestamps.
|
|
@@ -1238,13 +1416,14 @@ const sync =
|
|
|
1238
1416
|
|
|
1239
1417
|
// If all ranges were skipped, there are no changes and sync is complete.
|
|
1240
1418
|
const hasChange = output.getSize() > outputInitialSize;
|
|
1241
|
-
|
|
1419
|
+
|
|
1420
|
+
return ok(hasChange);
|
|
1242
1421
|
};
|
|
1243
1422
|
|
|
1244
1423
|
const splitRange =
|
|
1245
1424
|
(deps: StorageDep) =>
|
|
1246
1425
|
(
|
|
1247
|
-
ownerId:
|
|
1426
|
+
ownerId: OwnerIdBytes,
|
|
1248
1427
|
lower: NonNegativeInt,
|
|
1249
1428
|
upper: NonNegativeInt,
|
|
1250
1429
|
upperBound: RangeUpperBound,
|
|
@@ -1265,7 +1444,7 @@ const splitRange =
|
|
|
1265
1444
|
0 as NonNegativeInt,
|
|
1266
1445
|
itemCount,
|
|
1267
1446
|
(timestamp) => {
|
|
1268
|
-
range.timestamps.add(
|
|
1447
|
+
range.timestamps.add(timestampBytesToTimestamp(timestamp));
|
|
1269
1448
|
return true;
|
|
1270
1449
|
},
|
|
1271
1450
|
);
|
|
@@ -1324,7 +1503,7 @@ const decodeRanges = (buffer: Buffer): ReadonlyArray<Range> => {
|
|
|
1324
1503
|
for (let i = 0; i < rangesCount; i++) {
|
|
1325
1504
|
const upperBound =
|
|
1326
1505
|
i < timestampsCount
|
|
1327
|
-
?
|
|
1506
|
+
? timestampToTimestampBytes(timestamps[i])
|
|
1328
1507
|
: InfiniteUpperBound;
|
|
1329
1508
|
|
|
1330
1509
|
const rangeType = rangeTypes[i];
|
|
@@ -1346,7 +1525,7 @@ const decodeRanges = (buffer: Buffer): ReadonlyArray<Range> => {
|
|
|
1346
1525
|
|
|
1347
1526
|
case RangeType.Timestamps: {
|
|
1348
1527
|
const timestamps = decodeTimestamps(buffer).map(
|
|
1349
|
-
|
|
1528
|
+
timestampToTimestampBytes,
|
|
1350
1529
|
);
|
|
1351
1530
|
ranges.push({
|
|
1352
1531
|
type: RangeType.Timestamps,
|
|
@@ -1374,7 +1553,7 @@ const decodeTimestamps = (
|
|
|
1374
1553
|
for (let i = 0; i < length; i++) {
|
|
1375
1554
|
const deltaMillis = decodeNonNegativeInt(buffer);
|
|
1376
1555
|
const millis = Millis.from(previousMillis + deltaMillis);
|
|
1377
|
-
if (!millis.ok) throw new
|
|
1556
|
+
if (!millis.ok) throw new ProtocolDecodeError(millis.error.type);
|
|
1378
1557
|
millises.push(millis.value);
|
|
1379
1558
|
previousMillis = millis.value;
|
|
1380
1559
|
}
|
|
@@ -1383,7 +1562,7 @@ const decodeTimestamps = (
|
|
|
1383
1562
|
let counterIndex = 0;
|
|
1384
1563
|
while (counterIndex < length) {
|
|
1385
1564
|
const counter = Counter.from(decodeNonNegativeInt(buffer));
|
|
1386
|
-
if (!counter.ok) throw new
|
|
1565
|
+
if (!counter.ok) throw new ProtocolDecodeError(counter.error.type);
|
|
1387
1566
|
const runLength = decodeNonNegativeInt(buffer);
|
|
1388
1567
|
for (let i = 0; i < runLength; i++) {
|
|
1389
1568
|
counters.push(counter.value);
|
|
@@ -1414,134 +1593,31 @@ const decodeTimestamps = (
|
|
|
1414
1593
|
return timestamps;
|
|
1415
1594
|
};
|
|
1416
1595
|
|
|
1417
|
-
|
|
1418
|
-
|
|
1419
|
-
|
|
1420
|
-
export const binaryIdLength = 16 as NonNegativeInt;
|
|
1421
|
-
|
|
1422
|
-
export const idToBinaryId = (id: Id): BinaryId =>
|
|
1423
|
-
base64Url256ToBytes(id) as BinaryId;
|
|
1424
|
-
|
|
1425
|
-
export const binaryIdToId = (binaryId: BinaryId): Id =>
|
|
1426
|
-
decodeId(createBuffer(binaryId));
|
|
1427
|
-
|
|
1428
|
-
/** Binary representation of {@link OwnerId}. */
|
|
1429
|
-
export type BinaryOwnerId = Uint8Array & Brand<"BinaryOwnerId">;
|
|
1430
|
-
|
|
1431
|
-
export const ownerIdToBinaryOwnerId = (ownerId: OwnerId): BinaryOwnerId =>
|
|
1432
|
-
base64Url256ToBytes(ownerId) as BinaryOwnerId;
|
|
1433
|
-
|
|
1434
|
-
export const binaryOwnerIdToOwnerId = (binaryOwnerId: BinaryOwnerId): OwnerId =>
|
|
1435
|
-
decodeOwnerId(createBuffer(binaryOwnerId));
|
|
1436
|
-
|
|
1437
|
-
/**
|
|
1438
|
-
* Base64Url string with maximum length of 256 characters. Encoding strings as
|
|
1439
|
-
* Base64UrlString saves up to 25% in size compared to regular strings.
|
|
1440
|
-
*/
|
|
1441
|
-
export const Base64Url256 = maxLength(256)(Base64Url);
|
|
1442
|
-
export type Base64Url256 = typeof Base64Url256.Type;
|
|
1443
|
-
|
|
1444
|
-
/**
|
|
1445
|
-
* Union type for all variants of Base64Url strings with limited length. All
|
|
1446
|
-
* these types use Base64Url alphabet and are < 256 characters.
|
|
1447
|
-
*/
|
|
1448
|
-
export type Base64Url256Variant = Base64Url256 | Id | NanoId | OwnerId;
|
|
1449
|
-
|
|
1450
|
-
/**
|
|
1451
|
-
* Alphabet used for Base64Url encoding. This is copied from the `nanoid`
|
|
1452
|
-
* library to avoid dependency on a specific version of `nanoid`.
|
|
1453
|
-
*/
|
|
1454
|
-
const urlAlphabet =
|
|
1455
|
-
"useandom-26T198340PX75pxJACKVERYMINDBUSHWOLF_GQZbfghjklqvwyzrict";
|
|
1456
|
-
|
|
1457
|
-
/**
|
|
1458
|
-
* Converts a Base64Url string to a Uint8Array for binary storage. This encoding
|
|
1459
|
-
* is more space-efficient than UTF-8 for Base64Url strings.
|
|
1460
|
-
*/
|
|
1461
|
-
export const base64Url256ToBytes = (
|
|
1462
|
-
string: Base64Url256Variant,
|
|
1463
|
-
): globalThis.Uint8Array => {
|
|
1464
|
-
const totalBits = string.length * 6; // 6 bits per character
|
|
1465
|
-
const byteLength = Math.ceil(totalBits / 8);
|
|
1466
|
-
const value = new globalThis.Uint8Array(byteLength);
|
|
1467
|
-
|
|
1468
|
-
let bitBuffer = 0;
|
|
1469
|
-
let bitsInBuffer = 0;
|
|
1470
|
-
let byteIndex = 0;
|
|
1471
|
-
|
|
1472
|
-
for (const char of string) {
|
|
1473
|
-
const charValue = urlAlphabet.indexOf(char);
|
|
1474
|
-
bitBuffer = (bitBuffer << 6) | charValue;
|
|
1475
|
-
bitsInBuffer += 6;
|
|
1476
|
-
while (bitsInBuffer >= 8) {
|
|
1477
|
-
bitsInBuffer -= 8;
|
|
1478
|
-
value[byteIndex++] = (bitBuffer >> bitsInBuffer) & 0xff;
|
|
1479
|
-
}
|
|
1480
|
-
}
|
|
1481
|
-
|
|
1482
|
-
if (bitsInBuffer > 0 && byteIndex < byteLength) {
|
|
1483
|
-
value[byteIndex] = (bitBuffer << (8 - bitsInBuffer)) & 0xff;
|
|
1484
|
-
}
|
|
1485
|
-
|
|
1486
|
-
return value;
|
|
1487
|
-
};
|
|
1488
|
-
|
|
1489
|
-
export const decodeBase64Url256 = (
|
|
1490
|
-
buffer: Buffer,
|
|
1491
|
-
stringLength: number,
|
|
1492
|
-
): Base64Url256Variant => {
|
|
1493
|
-
const bytes = buffer.shiftN(
|
|
1494
|
-
Math.ceil((stringLength * 6) / 8) as NonNegativeInt,
|
|
1495
|
-
);
|
|
1496
|
-
|
|
1497
|
-
let bitBuffer = 0;
|
|
1498
|
-
let bitsInBuffer = 0;
|
|
1499
|
-
let string = "";
|
|
1500
|
-
|
|
1501
|
-
for (const byte of bytes) {
|
|
1502
|
-
bitBuffer = (bitBuffer << 8) | byte;
|
|
1503
|
-
bitsInBuffer += 8;
|
|
1504
|
-
while (bitsInBuffer >= 6) {
|
|
1505
|
-
bitsInBuffer -= 6;
|
|
1506
|
-
if (string.length < stringLength) {
|
|
1507
|
-
const charValue = (bitBuffer >> bitsInBuffer) & 0x3f;
|
|
1508
|
-
if (charValue < 0 || charValue >= urlAlphabet.length) {
|
|
1509
|
-
throw new ProtocolDecodeError("invalid charValue");
|
|
1510
|
-
}
|
|
1511
|
-
string += urlAlphabet[charValue];
|
|
1512
|
-
}
|
|
1513
|
-
}
|
|
1514
|
-
}
|
|
1515
|
-
|
|
1516
|
-
const result = Base64Url256.from(string);
|
|
1517
|
-
if (!result.ok) throw new ProtocolDecodeError(result.error.type);
|
|
1518
|
-
|
|
1519
|
-
return result.value;
|
|
1596
|
+
const decodeId = (buffer: Buffer): Id => {
|
|
1597
|
+
const bytes = buffer.shiftN(idBytesTypeValueLength);
|
|
1598
|
+
return idBytesToId(bytes as IdBytes);
|
|
1520
1599
|
};
|
|
1521
1600
|
|
|
1522
|
-
const decodeId = (buffer: Buffer): Id =>
|
|
1523
|
-
decodeBase64Url256(buffer, idTypeValueLength) as Id;
|
|
1524
|
-
|
|
1525
|
-
/** Not all 16 bytes are valid {@link OwnerId}. */
|
|
1526
|
-
const decodeOwnerId = (buffer: Buffer): OwnerId => decodeId(buffer) as OwnerId;
|
|
1527
|
-
|
|
1528
1601
|
/**
|
|
1529
1602
|
* Evolu uses MessagePack to handle all number variants except for
|
|
1530
1603
|
* NonNegativeInt. For NonNegativeInt, Evolu provides more efficient encoding.
|
|
1531
1604
|
*/
|
|
1532
1605
|
export const encodeNumber = (buffer: Buffer, number: number): void => {
|
|
1533
|
-
buffer.extend(pack(number));
|
|
1606
|
+
buffer.extend(packr.pack(number));
|
|
1534
1607
|
};
|
|
1535
1608
|
|
|
1536
1609
|
export const decodeNumber = (buffer: Buffer): number => {
|
|
1537
1610
|
let number: unknown;
|
|
1538
1611
|
let end: unknown;
|
|
1539
1612
|
|
|
1540
|
-
unpackMultiple(
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1613
|
+
packr.unpackMultiple(
|
|
1614
|
+
buffer.unwrap(),
|
|
1615
|
+
(n: unknown, _: unknown, e: unknown) => {
|
|
1616
|
+
number = n;
|
|
1617
|
+
end = e;
|
|
1618
|
+
return false;
|
|
1619
|
+
},
|
|
1620
|
+
);
|
|
1545
1621
|
|
|
1546
1622
|
const endResult = NonNegativeInt.fromUnknown(end);
|
|
1547
1623
|
if (!endResult.ok) throw new ProtocolDecodeError(endResult.error.type);
|
|
@@ -1553,28 +1629,33 @@ export const decodeNumber = (buffer: Buffer): number => {
|
|
|
1553
1629
|
return numberResult.value;
|
|
1554
1630
|
};
|
|
1555
1631
|
|
|
1556
|
-
export const binaryTimestampToFingerprint = (
|
|
1557
|
-
timestamp: BinaryTimestamp,
|
|
1558
|
-
): Fingerprint => {
|
|
1559
|
-
const hash = sha256(timestamp).slice(0, fingerprintSize);
|
|
1560
|
-
return hash as Fingerprint;
|
|
1561
|
-
};
|
|
1562
|
-
|
|
1563
1632
|
/**
|
|
1564
1633
|
* Encodes and encrypts a {@link DbChange} using the provided owner's encryption
|
|
1565
1634
|
* key. Returns an encrypted binary representation as {@link EncryptedDbChange}.
|
|
1635
|
+
*
|
|
1636
|
+
* The format includes the protocol version for backward compatibility and the
|
|
1637
|
+
* timestamp for tamper-proof verification that the timestamp matches the change
|
|
1638
|
+
* data.
|
|
1566
1639
|
*/
|
|
1567
1640
|
export const encodeAndEncryptDbChange =
|
|
1568
1641
|
(deps: SymmetricCryptoDep) =>
|
|
1569
|
-
(
|
|
1642
|
+
(message: CrdtMessage, key: EncryptionKey): EncryptedDbChange => {
|
|
1643
|
+
const change = message.change;
|
|
1570
1644
|
const buffer = createBuffer();
|
|
1571
1645
|
|
|
1572
|
-
|
|
1646
|
+
// Encode protocol version first for backward compatibility
|
|
1647
|
+
encodeNonNegativeInt(buffer, protocolVersion);
|
|
1648
|
+
|
|
1649
|
+
// Encode the timestamp (after version) for tamper verification
|
|
1650
|
+
const timestampBytes = timestampToTimestampBytes(message.timestamp);
|
|
1651
|
+
buffer.extend(timestampBytes);
|
|
1652
|
+
|
|
1653
|
+
encodeString(buffer, change.table);
|
|
1573
1654
|
|
|
1574
|
-
buffer.extend(
|
|
1655
|
+
buffer.extend(idToIdBytes(change.id));
|
|
1575
1656
|
|
|
1576
1657
|
const entries = objectToEntries(change.values).map(
|
|
1577
|
-
([column, value]): [
|
|
1658
|
+
([column, value]): [string, SqliteValue] => {
|
|
1578
1659
|
return [column, value];
|
|
1579
1660
|
},
|
|
1580
1661
|
);
|
|
@@ -1582,7 +1663,7 @@ export const encodeAndEncryptDbChange =
|
|
|
1582
1663
|
encodeLength(buffer, entries);
|
|
1583
1664
|
|
|
1584
1665
|
for (const [column, value] of entries) {
|
|
1585
|
-
|
|
1666
|
+
encodeString(buffer, column);
|
|
1586
1667
|
encodeSqliteValue(buffer, value);
|
|
1587
1668
|
}
|
|
1588
1669
|
|
|
@@ -1604,50 +1685,80 @@ export const encodeAndEncryptDbChange =
|
|
|
1604
1685
|
};
|
|
1605
1686
|
|
|
1606
1687
|
/**
|
|
1607
|
-
* Decrypts and decodes an {@link
|
|
1608
|
-
* encryption key.
|
|
1688
|
+
* Decrypts and decodes an {@link EncryptedCrdtMessage} using the provided
|
|
1689
|
+
* owner's encryption key. Verifies that the embedded timestamp matches the
|
|
1690
|
+
* expected timestamp to ensure message integrity.
|
|
1609
1691
|
*/
|
|
1610
1692
|
export const decryptAndDecodeDbChange =
|
|
1611
1693
|
(deps: SymmetricCryptoDep) =>
|
|
1612
1694
|
(
|
|
1613
|
-
|
|
1695
|
+
message: EncryptedCrdtMessage,
|
|
1614
1696
|
key: EncryptionKey,
|
|
1615
|
-
): Result<
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
1697
|
+
): Result<
|
|
1698
|
+
DbChange,
|
|
1699
|
+
| SymmetricCryptoDecryptError
|
|
1700
|
+
| ProtocolInvalidDataError
|
|
1701
|
+
| ProtocolTimestampMismatchError
|
|
1702
|
+
> => {
|
|
1703
|
+
// try-catch instead of Result for performance and stacktraces
|
|
1704
|
+
try {
|
|
1705
|
+
const buffer = createBuffer(message.change);
|
|
1706
|
+
const nonce = buffer.shiftN(deps.symmetricCrypto.nonceLength);
|
|
1707
|
+
|
|
1708
|
+
const ciphertextLength = decodeLength(buffer);
|
|
1709
|
+
const ciphertext = buffer.shiftN(ciphertextLength);
|
|
1710
|
+
|
|
1711
|
+
const plaintextBytes = deps.symmetricCrypto.decrypt(
|
|
1712
|
+
ciphertext,
|
|
1713
|
+
key,
|
|
1714
|
+
nonce,
|
|
1715
|
+
);
|
|
1716
|
+
if (!plaintextBytes.ok) return plaintextBytes;
|
|
1630
1717
|
|
|
1631
|
-
|
|
1632
|
-
|
|
1718
|
+
buffer.reset();
|
|
1719
|
+
buffer.extend(plaintextBytes.value);
|
|
1633
1720
|
|
|
1634
|
-
|
|
1635
|
-
|
|
1721
|
+
// Decode version (for future compatibility, no validation needed for now)
|
|
1722
|
+
decodeNonNegativeInt(buffer);
|
|
1636
1723
|
|
|
1637
|
-
|
|
1638
|
-
|
|
1724
|
+
// Decode and verify the embedded timestamp
|
|
1725
|
+
const embeddedTimestampBytes = buffer.shiftN(timestampBytesLength);
|
|
1726
|
+
const embeddedTimestamp = timestampBytesToTimestamp(
|
|
1727
|
+
embeddedTimestampBytes as TimestampBytes,
|
|
1728
|
+
);
|
|
1639
1729
|
|
|
1640
|
-
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
1730
|
+
// Verify timestamp integrity
|
|
1731
|
+
if (!eqTimestamp(embeddedTimestamp, message.timestamp)) {
|
|
1732
|
+
return err<ProtocolTimestampMismatchError>({
|
|
1733
|
+
type: "ProtocolTimestampMismatchError",
|
|
1734
|
+
expected: message.timestamp,
|
|
1735
|
+
embedded: embeddedTimestamp,
|
|
1736
|
+
});
|
|
1737
|
+
}
|
|
1645
1738
|
|
|
1646
|
-
|
|
1739
|
+
const table = decodeString(buffer);
|
|
1740
|
+
const id = decodeId(buffer);
|
|
1647
1741
|
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
1742
|
+
const length = decodeLength(buffer);
|
|
1743
|
+
const values = Object.create(null) as Record<string, SqliteValue>;
|
|
1744
|
+
|
|
1745
|
+
for (let i = 0; i < length; i++) {
|
|
1746
|
+
const column = decodeString(buffer);
|
|
1747
|
+
const value = decodeSqliteValue(buffer);
|
|
1748
|
+
values[column] = value;
|
|
1749
|
+
}
|
|
1750
|
+
|
|
1751
|
+
const dbChange = { table, id, values };
|
|
1752
|
+
|
|
1753
|
+
return ok(dbChange);
|
|
1754
|
+
} catch (error) {
|
|
1755
|
+
return err<ProtocolInvalidDataError>({
|
|
1756
|
+
type: "ProtocolInvalidDataError",
|
|
1757
|
+
data: message.change,
|
|
1758
|
+
error,
|
|
1759
|
+
});
|
|
1760
|
+
}
|
|
1761
|
+
};
|
|
1651
1762
|
|
|
1652
1763
|
/**
|
|
1653
1764
|
* Encodes a non-negative integer into a variable-length integer format. It's
|
|
@@ -1731,19 +1842,6 @@ export const decodeNodeId = (buffer: Buffer): NodeId => {
|
|
|
1731
1842
|
return bytesToHex(bytes) as NodeId;
|
|
1732
1843
|
};
|
|
1733
1844
|
|
|
1734
|
-
export const encodeBase64Url256 = (
|
|
1735
|
-
buffer: Buffer,
|
|
1736
|
-
string: Base64Url256Variant,
|
|
1737
|
-
): void => {
|
|
1738
|
-
encodeLength(buffer, string);
|
|
1739
|
-
buffer.extend(base64Url256ToBytes(string));
|
|
1740
|
-
};
|
|
1741
|
-
|
|
1742
|
-
export const decodeBase64Url256WithLength = (buffer: Buffer): Base64Url256 => {
|
|
1743
|
-
const length = decodeLength(buffer);
|
|
1744
|
-
return decodeBase64Url256(buffer, length) as Base64Url256;
|
|
1745
|
-
};
|
|
1746
|
-
|
|
1747
1845
|
// Small ints are encoded into ProtocolValueType, saving one byte per int.
|
|
1748
1846
|
const isSmallInt: Predicate<number> = (value: number) =>
|
|
1749
1847
|
value >= 0 && value < 20;
|
|
@@ -1755,20 +1853,23 @@ export const ProtocolValueType = {
|
|
|
1755
1853
|
String: 20 as NonNegativeInt,
|
|
1756
1854
|
Number: 21 as NonNegativeInt,
|
|
1757
1855
|
Null: 22 as NonNegativeInt,
|
|
1758
|
-
|
|
1856
|
+
Bytes: 23 as NonNegativeInt,
|
|
1759
1857
|
// We can add more types for other DBs or anything else later.
|
|
1760
1858
|
|
|
1761
1859
|
// Optimized types
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
|
|
1765
|
-
|
|
1860
|
+
NonNegativeInt: 30 as NonNegativeInt,
|
|
1861
|
+
|
|
1862
|
+
// String optimizations
|
|
1863
|
+
EmptyString: 31 as NonNegativeInt, // 1 byte vs 2 bytes (50% reduction)
|
|
1864
|
+
Base64Url: 32 as NonNegativeInt,
|
|
1865
|
+
Id: 33 as NonNegativeInt,
|
|
1866
|
+
Json: 34 as NonNegativeInt,
|
|
1766
1867
|
|
|
1767
1868
|
// new Date().toISOString() - 24 bytes
|
|
1768
1869
|
// encoded with fixed length - 8 bytes
|
|
1769
1870
|
// encode as NonNegativeInt - 6 bytes (additional 25% reduction)
|
|
1770
|
-
DateIsoWithNonNegativeTime:
|
|
1771
|
-
DateIsoWithNegativeTime:
|
|
1871
|
+
DateIsoWithNonNegativeTime: 35 as NonNegativeInt,
|
|
1872
|
+
DateIsoWithNegativeTime: 36 as NonNegativeInt, // 9 bytes
|
|
1772
1873
|
|
|
1773
1874
|
// TODO: Operations (from 40)
|
|
1774
1875
|
// Increment, Decrement, Patch, whatever.
|
|
@@ -1782,9 +1883,14 @@ export const encodeSqliteValue = (buffer: Buffer, value: SqliteValue): void => {
|
|
|
1782
1883
|
|
|
1783
1884
|
switch (typeof value) {
|
|
1784
1885
|
case "string": {
|
|
1785
|
-
|
|
1786
|
-
|
|
1787
|
-
|
|
1886
|
+
if (value === "") {
|
|
1887
|
+
encodeNonNegativeInt(buffer, ProtocolValueType.EmptyString);
|
|
1888
|
+
return;
|
|
1889
|
+
}
|
|
1890
|
+
|
|
1891
|
+
const dateIso = DateIso.fromParent(value);
|
|
1892
|
+
if (dateIso.ok) {
|
|
1893
|
+
const time = new Date(dateIso.value).getTime();
|
|
1788
1894
|
if (NonNegativeInt.is(time)) {
|
|
1789
1895
|
encodeNonNegativeInt(
|
|
1790
1896
|
buffer,
|
|
@@ -1801,27 +1907,34 @@ export const encodeSqliteValue = (buffer: Buffer, value: SqliteValue): void => {
|
|
|
1801
1907
|
return;
|
|
1802
1908
|
}
|
|
1803
1909
|
|
|
1804
|
-
const
|
|
1805
|
-
if (
|
|
1806
|
-
|
|
1807
|
-
|
|
1808
|
-
buffer.extend(base64Url256ToBytes(base64Url256.value));
|
|
1809
|
-
return;
|
|
1810
|
-
}
|
|
1811
|
-
encodeNonNegativeInt(buffer, ProtocolValueType.Base64Url256);
|
|
1812
|
-
encodeBase64Url256(buffer, base64Url256.value);
|
|
1910
|
+
const id = Id.fromParent(value);
|
|
1911
|
+
if (id.ok) {
|
|
1912
|
+
encodeNonNegativeInt(buffer, ProtocolValueType.Id);
|
|
1913
|
+
buffer.extend(idToIdBytes(id.value));
|
|
1813
1914
|
return;
|
|
1814
1915
|
}
|
|
1815
1916
|
|
|
1816
|
-
const
|
|
1817
|
-
if
|
|
1818
|
-
|
|
1917
|
+
const json = Json.fromParent(value);
|
|
1918
|
+
// Only encode as Json if it survives JSON.parse/JSON.stringify round-trip.
|
|
1919
|
+
// Some valid JSON strings like "-0E0" get normalized to "0" during parsing,
|
|
1920
|
+
// which would cause data corruption if we don't verify round-trip safety.
|
|
1921
|
+
if (json.ok && JSON.stringify(jsonToJsonValue(json.value)) === value) {
|
|
1922
|
+
const jsonBytes = packr.pack(jsonToJsonValue(json.value));
|
|
1819
1923
|
encodeNonNegativeInt(buffer, ProtocolValueType.Json);
|
|
1820
1924
|
encodeLength(buffer, jsonBytes);
|
|
1821
1925
|
buffer.extend(jsonBytes);
|
|
1822
1926
|
return;
|
|
1823
1927
|
}
|
|
1824
1928
|
|
|
1929
|
+
const base64Url = Base64Url.fromParent(value);
|
|
1930
|
+
if (base64Url.ok) {
|
|
1931
|
+
encodeNonNegativeInt(buffer, ProtocolValueType.Base64Url);
|
|
1932
|
+
const bytes = base64UrlToUint8Array(base64Url.value);
|
|
1933
|
+
encodeLength(buffer, bytes);
|
|
1934
|
+
buffer.extend(bytes);
|
|
1935
|
+
return;
|
|
1936
|
+
}
|
|
1937
|
+
|
|
1825
1938
|
encodeNonNegativeInt(buffer, ProtocolValueType.String);
|
|
1826
1939
|
encodeString(buffer, value);
|
|
1827
1940
|
return;
|
|
@@ -1843,7 +1956,7 @@ export const encodeSqliteValue = (buffer: Buffer, value: SqliteValue): void => {
|
|
|
1843
1956
|
}
|
|
1844
1957
|
}
|
|
1845
1958
|
|
|
1846
|
-
encodeNonNegativeInt(buffer, ProtocolValueType.
|
|
1959
|
+
encodeNonNegativeInt(buffer, ProtocolValueType.Bytes);
|
|
1847
1960
|
encodeLength(buffer, value);
|
|
1848
1961
|
buffer.extend(value);
|
|
1849
1962
|
};
|
|
@@ -1858,40 +1971,75 @@ export const decodeSqliteValue = (buffer: Buffer): SqliteValue => {
|
|
|
1858
1971
|
switch (type) {
|
|
1859
1972
|
case ProtocolValueType.String:
|
|
1860
1973
|
return decodeString(buffer);
|
|
1974
|
+
|
|
1861
1975
|
case ProtocolValueType.Number:
|
|
1862
1976
|
return decodeNumber(buffer);
|
|
1977
|
+
|
|
1863
1978
|
case ProtocolValueType.Null:
|
|
1864
1979
|
return null;
|
|
1865
|
-
|
|
1980
|
+
|
|
1981
|
+
case ProtocolValueType.Bytes: {
|
|
1866
1982
|
const length = decodeLength(buffer);
|
|
1867
1983
|
return buffer.shiftN(length);
|
|
1868
1984
|
}
|
|
1869
|
-
|
|
1985
|
+
|
|
1986
|
+
case ProtocolValueType.Id:
|
|
1870
1987
|
return decodeId(buffer);
|
|
1871
|
-
|
|
1872
|
-
case ProtocolValueType.Base64Url256:
|
|
1873
|
-
return decodeBase64Url256WithLength(buffer);
|
|
1988
|
+
|
|
1874
1989
|
case ProtocolValueType.NonNegativeInt:
|
|
1875
1990
|
return decodeNonNegativeInt(buffer);
|
|
1991
|
+
|
|
1876
1992
|
case ProtocolValueType.Json: {
|
|
1877
1993
|
const length = decodeLength(buffer);
|
|
1878
1994
|
const bytes = buffer.shiftN(length);
|
|
1879
|
-
return JSON.stringify(unpack(bytes));
|
|
1995
|
+
return JSON.stringify(packr.unpack(bytes));
|
|
1880
1996
|
}
|
|
1997
|
+
|
|
1881
1998
|
case ProtocolValueType.DateIsoWithNonNegativeTime:
|
|
1882
1999
|
case ProtocolValueType.DateIsoWithNegativeTime: {
|
|
1883
2000
|
const time =
|
|
1884
2001
|
type === ProtocolValueType.DateIsoWithNonNegativeTime
|
|
1885
2002
|
? decodeNonNegativeInt(buffer)
|
|
1886
2003
|
: decodeNumber(buffer);
|
|
1887
|
-
const
|
|
1888
|
-
|
|
1889
|
-
|
|
1890
|
-
|
|
1891
|
-
|
|
1892
|
-
|
|
2004
|
+
const dateIso = DateIso.fromParent(new Date(time).toISOString());
|
|
2005
|
+
if (!dateIso.ok) throw new ProtocolDecodeError(dateIso.error.type);
|
|
2006
|
+
return dateIso.value;
|
|
2007
|
+
}
|
|
2008
|
+
|
|
2009
|
+
case ProtocolValueType.EmptyString:
|
|
2010
|
+
return "";
|
|
2011
|
+
|
|
2012
|
+
case ProtocolValueType.Base64Url: {
|
|
2013
|
+
const length = decodeLength(buffer);
|
|
2014
|
+
const bytes = buffer.shiftN(length);
|
|
2015
|
+
return uint8ArrayToBase64Url(bytes);
|
|
1893
2016
|
}
|
|
2017
|
+
|
|
1894
2018
|
default:
|
|
1895
2019
|
throw new ProtocolDecodeError("invalid ProtocolValueType");
|
|
1896
2020
|
}
|
|
1897
2021
|
};
|
|
2022
|
+
|
|
2023
|
+
/**
|
|
2024
|
+
* Decodes a ProtocolMessage into a readable JSON object for debugging.
|
|
2025
|
+
*
|
|
2026
|
+
* Note: This is a stub for future implementation. It should use:
|
|
2027
|
+
*
|
|
2028
|
+
* - DecodeVersionAndOwner
|
|
2029
|
+
* - DecodeError or decodeWriteKeys (depending on context)
|
|
2030
|
+
* - DecodeMessages
|
|
2031
|
+
* - DecodeRanges
|
|
2032
|
+
*
|
|
2033
|
+
* If you want to help, please contribute to this function.
|
|
2034
|
+
*/
|
|
2035
|
+
export const decodeProtocolMessageToJson = (
|
|
2036
|
+
_protocolMessage: ProtocolMessage,
|
|
2037
|
+
_isInitiator: boolean,
|
|
2038
|
+
): unknown => {
|
|
2039
|
+
// TODO: Implement using
|
|
2040
|
+
// - decodeVersionAndOwner
|
|
2041
|
+
// -- decodeError or decodeWriteKeys (should be refactored out),
|
|
2042
|
+
// -- decodeMessages, and decodeRanges.
|
|
2043
|
+
// This is a stub for PRs and community contributions.
|
|
2044
|
+
throw new Error("decodeProtocolMessageToJson is not implemented yet.");
|
|
2045
|
+
};
|