@evolu/common 8.17.0 → 8.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/Bytes.d.ts +31 -14
- package/dist/src/Bytes.d.ts.map +1 -1
- package/dist/src/Bytes.js +48 -8
- package/dist/src/Error.d.ts.map +1 -1
- package/dist/src/Error.js +5 -1
- package/dist/src/Polyfills.d.ts.map +1 -1
- package/dist/src/Polyfills.js +3 -2
- package/dist/src/Sqlite.d.ts +1 -1
- package/dist/src/Task.d.ts +4 -3
- package/dist/src/Task.d.ts.map +1 -1
- package/dist/src/Task.js +4 -3
- package/dist/src/index.d.ts +1 -1
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/local-first/Db.d.ts +32 -3
- package/dist/src/local-first/Db.d.ts.map +1 -1
- package/dist/src/local-first/Db.js +30 -9
- package/dist/src/local-first/Evolu.d.ts +21 -6
- package/dist/src/local-first/Evolu.d.ts.map +1 -1
- package/dist/src/local-first/Evolu.js +5 -1
- package/dist/src/local-first/Protocol.d.ts +215 -93
- package/dist/src/local-first/Protocol.d.ts.map +1 -1
- package/dist/src/local-first/Protocol.js +799 -489
- package/dist/src/local-first/Relay.d.ts +9 -1
- package/dist/src/local-first/Relay.d.ts.map +1 -1
- package/dist/src/local-first/Relay.js +41 -36
- package/dist/src/local-first/Shared.d.ts +79 -53
- package/dist/src/local-first/Shared.d.ts.map +1 -1
- package/dist/src/local-first/Shared.js +67 -42
- package/dist/src/local-first/Storage.d.ts +80 -18
- package/dist/src/local-first/Storage.d.ts.map +1 -1
- package/dist/src/local-first/Storage.js +20 -4
- package/package.json +1 -1
- package/src/Bytes.test.ts +212 -0
- package/src/Bytes.ts +67 -17
- package/src/Error.ts +5 -1
- package/src/Polyfills.ts +3 -2
- package/src/Sqlite.ts +1 -1
- package/src/Task.ts +4 -3
- package/src/index.ts +4 -1
- package/src/local-first/Db.ts +76 -17
- package/src/local-first/Evolu.test.ts +43 -12
- package/src/local-first/Evolu.ts +34 -7
- package/src/local-first/Protocol.test.ts +1541 -44
- package/src/local-first/Protocol.ts +1070 -605
- package/src/local-first/Relay.ts +80 -69
- package/src/local-first/Shared.test.ts +37 -1
- package/src/local-first/Shared.ts +124 -73
- package/src/local-first/Storage.ts +100 -27
|
@@ -43,14 +43,24 @@
|
|
|
43
43
|
* | - {@link NonNegativeInt} | Number of ranges. |
|
|
44
44
|
* | - {@link Range} | |
|
|
45
45
|
*
|
|
46
|
+
* Range upper bounds must not decrease. The last range always has
|
|
47
|
+
* {@link InfiniteUpperBound}, which is not encoded. A message with decreasing
|
|
48
|
+
* bounds, or with a hasWriteKey or subscriptionFlag value not listed above, is
|
|
49
|
+
* rejected as {@link ProtocolInvalidDataError}. So is a request with a change
|
|
50
|
+
* shorter than 41 bytes, the smallest {@link EncryptedDbChange}. A client passes
|
|
51
|
+
* such a change to storage, which skips a change it cannot read.
|
|
52
|
+
*
|
|
46
53
|
* ## WriteKey validation
|
|
47
54
|
*
|
|
48
55
|
* The initiator sends a hasWriteKey flag and optionally a WriteKey. The
|
|
49
56
|
* WriteKey is required when sending messages as a secure token proving the
|
|
50
57
|
* initiator can write changes. It's ok to not send a WriteKey if the initiator
|
|
51
58
|
* is only syncing (read-only) and not sending messages. The non-initiator
|
|
52
|
-
*
|
|
53
|
-
*
|
|
59
|
+
* decodes the whole request first and then validates the WriteKey, before
|
|
60
|
+
* storing any messages or reconciling ranges. Only the subscriptionFlag takes
|
|
61
|
+
* effect before that, because subscribing needs no WriteKey: broadcasts carry
|
|
62
|
+
* only encrypted changes. So a request rejected with
|
|
63
|
+
* {@link ProtocolWriteKeyError} still changes its subscription.
|
|
54
64
|
*
|
|
55
65
|
* ## Synchronization
|
|
56
66
|
*
|
|
@@ -64,9 +74,14 @@
|
|
|
64
74
|
* if further sync is needed or possible, continuing until both sides are
|
|
65
75
|
* synchronized.
|
|
66
76
|
*
|
|
67
|
-
* The **non-initiator
|
|
68
|
-
* even with empty messages containing only the header and
|
|
69
|
-
* the initiator to detect when synchronization is
|
|
77
|
+
* The **non-initiator answers every request it can decode** to provide sync
|
|
78
|
+
* completion feedback, even with empty messages containing only the header and
|
|
79
|
+
* no error. This allows the initiator to detect when synchronization is
|
|
80
|
+
* complete. A request it cannot decode, or one larger than the relay's
|
|
81
|
+
* `totalMaxSize`, gets no answer and has no effect.
|
|
82
|
+
*
|
|
83
|
+
* Ranges are compared by their {@link Fingerprint}, which anyone who can write
|
|
84
|
+
* for an owner can make collide. Its documentation describes the consequences.
|
|
70
85
|
*
|
|
71
86
|
* Both **Messages** and **Ranges** are optional, allowing each side to send,
|
|
72
87
|
* sync, or only subscribe data as needed.
|
|
@@ -81,18 +96,35 @@
|
|
|
81
96
|
*
|
|
82
97
|
* ## Protocol errors
|
|
83
98
|
*
|
|
84
|
-
*
|
|
99
|
+
* A Response carries a {@link ProtocolErrorCode} in its header. The initiator
|
|
100
|
+
* reports every code except `NoError` as an error with the `OwnerId`:
|
|
85
101
|
*
|
|
86
|
-
* - {@link ProtocolWriteKeyError}: The
|
|
87
|
-
*
|
|
102
|
+
* - {@link ProtocolWriteKeyError}: The WriteKey is invalid, or missing from a
|
|
103
|
+
* request with messages.
|
|
104
|
+
* - {@link ProtocolWriteError}: The relay failed to store the messages or to
|
|
105
|
+
* validate the WriteKey.
|
|
88
106
|
* - {@link ProtocolQuotaError}: Storage or billing quota exceeded.
|
|
89
|
-
* - {@link ProtocolSyncError}:
|
|
90
|
-
*
|
|
91
|
-
*
|
|
92
|
-
*
|
|
107
|
+
* - {@link ProtocolSyncError}: The relay's storage failed while it reconciled the
|
|
108
|
+
* ranges.
|
|
109
|
+
*
|
|
110
|
+
* The initiator also reports {@link ProtocolVersionError}, with the `OwnerId`,
|
|
111
|
+
* for a reply of another version, and {@link ProtocolInvalidDataError}, without
|
|
112
|
+
* it, for a message it cannot decode, including one with an unknown code.
|
|
113
|
+
*
|
|
114
|
+
* A relay answers every request it can decode within its `totalMaxSize`. It
|
|
115
|
+
* decodes a whole request before acting on it, so for a malformed one it
|
|
116
|
+
* returns `ProtocolInvalidDataError` without subscribing, storing, or
|
|
117
|
+
* broadcasting anything, and sends no reply. When its storage throws while
|
|
118
|
+
* validating the WriteKey, for example because SQLite cannot store a new
|
|
119
|
+
* owner's key on a full disk, the relay logs the error and answers with
|
|
120
|
+
* `ProtocolWriteError`, like a failed write. Likewise, a client applies nothing
|
|
121
|
+
* from a message it cannot decode.
|
|
93
122
|
*
|
|
94
|
-
*
|
|
95
|
-
*
|
|
123
|
+
* {@link decryptAndDecodeDbChange} returns `ProtocolInvalidDataError`,
|
|
124
|
+
* {@link ProtocolTimestampMismatchError}, or
|
|
125
|
+
* {@link DecryptWithXChaCha20Poly1305Error} for a change it cannot read. These
|
|
126
|
+
* describe a change rather than a protocol message, and client storage skips
|
|
127
|
+
* such a change.
|
|
96
128
|
*
|
|
97
129
|
* ## Message size limit
|
|
98
130
|
*
|
|
@@ -105,7 +137,10 @@
|
|
|
105
137
|
*
|
|
106
138
|
* Each mutation is limited to {@link maxMutationSize}, so every change fits one
|
|
107
139
|
* message of {@link defaultProtocolMessageMaxSize} next to the largest ranges
|
|
108
|
-
* section.
|
|
140
|
+
* section. Changes saved before that limit existed, and crafted changes a relay
|
|
141
|
+
* stores, can be larger. Sync skips a stored change that cannot fit an empty
|
|
142
|
+
* message after a pending Skip range, which is the message a later round is
|
|
143
|
+
* guaranteed to reach, and reports it as a {@link ProtocolChangeTooLargeError}.
|
|
109
144
|
*
|
|
110
145
|
* ## Why Binary?
|
|
111
146
|
*
|
|
@@ -129,24 +164,18 @@
|
|
|
129
164
|
*
|
|
130
165
|
* ## Versioning
|
|
131
166
|
*
|
|
132
|
-
*
|
|
133
|
-
*
|
|
134
|
-
*
|
|
167
|
+
* Every message of every protocol version begins with the version and the
|
|
168
|
+
* `OwnerId`, so a peer can route and report a message of any version. Nothing
|
|
169
|
+
* negotiates a version, and no side falls back to the other's.
|
|
135
170
|
*
|
|
136
|
-
*
|
|
137
|
-
*
|
|
138
|
-
*
|
|
139
|
-
*
|
|
140
|
-
*
|
|
141
|
-
*
|
|
142
|
-
*
|
|
143
|
-
*
|
|
144
|
-
* containing **its own protocol version and the same `ownerId`**.
|
|
145
|
-
* - The initiator can then detect the version mismatch for that specific owner
|
|
146
|
-
* and handle it appropriately (e.g., prompt for an update or halt sync).
|
|
147
|
-
*
|
|
148
|
-
* Version negotiation is per-owner, allowing Evolu Protocol to evolve safely
|
|
149
|
-
* over time and provide clear feedback about version mismatches.
|
|
171
|
+
* A non-initiator answers a request of another version with only its own
|
|
172
|
+
* version and that `OwnerId`. The initiator reports it as
|
|
173
|
+
* {@link ProtocolVersionError} for that owner, whose `isInitiator` tells which
|
|
174
|
+
* side is older, and stops syncing the owner through that relay. Clients from
|
|
175
|
+
* `@evolu/common` 8.0.0 before 8.11.0 drop that reply and stop syncing the
|
|
176
|
+
* owner through that relay without reporting anything, while 7.x clients report
|
|
177
|
+
* it as a `ProtocolVersionError`. So a relay that moves to another version must
|
|
178
|
+
* keep answering version 1 while such clients remain.
|
|
150
179
|
*
|
|
151
180
|
* ## Credible exit
|
|
152
181
|
*
|
|
@@ -174,43 +203,50 @@
|
|
|
174
203
|
* indicators) to be forwarded by relays without storage overhead.
|
|
175
204
|
*/
|
|
176
205
|
import { createMutableArray, isNonEmptyArray, } from "../Array.js";
|
|
177
|
-
import { assert, assertNonEmptyArray, assertNonNullable
|
|
206
|
+
import { assert, assertNonEmptyArray, assertNonNullable } from "../Assert.js";
|
|
178
207
|
import { BufferError, createBuffer, createRunLengthEncoder, decodeFlags, decodeJsonValue, decodeLength, decodeNonNegativeInt, decodeNumber, decodeRle, decodeString, encodeFlags, encodeJsonValue, encodeLength, encodeNonNegativeInt, encodeNumber, encodeString, } from "../Bytes.js";
|
|
179
|
-
import { createPadmePadding, decryptWithXChaCha20Poly1305, EncryptionKey, encryptWithXChaCha20Poly1305,
|
|
208
|
+
import { createPadmePadding, decryptWithXChaCha20Poly1305, EncryptionKey, encryptWithXChaCha20Poly1305, xChaCha20Poly1305NonceLength, } from "../Crypto.js";
|
|
180
209
|
import { eqArrayNumber } from "../Eq.js";
|
|
210
|
+
import { exhaustiveCheck } from "../Function.js";
|
|
181
211
|
import { computeBalancedBuckets } from "../Number.js";
|
|
182
212
|
import { createMutableRecord, objectToEntries } from "../Object.js";
|
|
183
213
|
import { err, ok } from "../Result.js";
|
|
184
214
|
import { AbortError } from "../Task.js";
|
|
185
215
|
import { Millis } from "../Time.js";
|
|
186
|
-
import { assertType, Base64Url, base64UrlToUint8Array, between, DateIso,
|
|
216
|
+
import { assertType, Base64Url, base64UrlToUint8Array, between, DateIso, Id, IdBytes, idBytesToId, idBytesTypeValueLength, idToIdBytes, Int, Json, jsonToJsonValue, NonNegativeInt, onePositiveInt, uint8ArrayToBase64Url, zeroNonNegativeInt, } from "../Type.js";
|
|
187
217
|
import { OwnerId, OwnerIdBytes, ownerIdToOwnerIdBytes, OwnerWriteKey, ownerWriteKeyLength, } from "./Owner.js";
|
|
188
218
|
import { DbChange, fingerprintSize, InfiniteUpperBound, RangeType, } from "./Storage.js";
|
|
189
|
-
import { Counter, eqTimestamp, NodeId, nodeIdBytesLength, nodeIdBytesToNodeId, nodeIdToNodeIdBytes,
|
|
219
|
+
import { Counter, eqTimestamp, NodeId, nodeIdBytesLength, nodeIdBytesToNodeId, nodeIdToNodeIdBytes, orderTimestamp, Timestamp, timestampBytesLength, timestampBytesToTimestamp, timestampToTimestampBytes, } from "./Timestamp.js";
|
|
190
220
|
const minProtocolMessageMaxSize = 1_000_000;
|
|
191
221
|
const maxProtocolMessageMaxSize = 100_000_000;
|
|
222
|
+
const maxProtocolMessageRangesMaxSize = 100_000;
|
|
192
223
|
/**
|
|
193
|
-
* Protocol message maximum size.
|
|
224
|
+
* Protocol message maximum size, from 1MB to 100MB.
|
|
194
225
|
*
|
|
195
|
-
*
|
|
196
|
-
*
|
|
197
|
-
*
|
|
198
|
-
*
|
|
226
|
+
* A message never exceeds its maximum size, inclusive. Builders measure each
|
|
227
|
+
* write exactly and keep room to close the message (see
|
|
228
|
+
* {@link ProtocolMessageBuffer.tryWrite}), and sync sends what does not fit in
|
|
229
|
+
* later rounds.
|
|
199
230
|
*
|
|
200
|
-
*
|
|
201
|
-
*
|
|
202
|
-
*
|
|
231
|
+
* Clients send at most {@link defaultProtocolMessageMaxSize} and enforce no
|
|
232
|
+
* receive limit. A relay can answer with larger messages, configured with the
|
|
233
|
+
* `totalMaxSize` option of {@link applyProtocolMessageAsRelay}, to reduce
|
|
234
|
+
* roundtrips for large syncs. Clients must not send larger messages, because
|
|
235
|
+
* relays accept at most the default: the Node.js relay closes the connection on
|
|
236
|
+
* a larger one, and relays up to `@evolu/nodejs` 4.0.0 crash on it.
|
|
203
237
|
*
|
|
204
|
-
*
|
|
205
|
-
*
|
|
206
|
-
*
|
|
238
|
+
* The default cannot be smaller either. Deployed clients send messages of that
|
|
239
|
+
* size, a change within {@link maxMutationSize} fits one message next to the
|
|
240
|
+
* largest ranges section, and changes saved before that limit existed, which
|
|
241
|
+
* can be nearly as large as a message, must stay servable.
|
|
207
242
|
*/
|
|
208
243
|
export const ProtocolMessageMaxSize = /*#__PURE__*/ between(minProtocolMessageMaxSize, maxProtocolMessageMaxSize)(Int);
|
|
209
244
|
/**
|
|
210
245
|
* Default {@link ProtocolMessageMaxSize} (1MB).
|
|
211
246
|
*
|
|
212
|
-
* The standard size used across Evolu implementations.
|
|
213
|
-
*
|
|
247
|
+
* The standard size used across Evolu implementations. Clients send at most
|
|
248
|
+
* this size, and relays accept it. Relays with more resources can answer with
|
|
249
|
+
* larger messages to reduce roundtrips.
|
|
214
250
|
*/
|
|
215
251
|
export const defaultProtocolMessageMaxSize = minProtocolMessageMaxSize;
|
|
216
252
|
/**
|
|
@@ -222,9 +258,10 @@ export const defaultProtocolMessageMaxSize = minProtocolMessageMaxSize;
|
|
|
222
258
|
*
|
|
223
259
|
* The upper bound is set to ensure ranges fit within the default 1MB
|
|
224
260
|
* {@link defaultProtocolMessageMaxSize}, maintaining compatibility between all
|
|
225
|
-
* clients and relays.
|
|
261
|
+
* clients and relays. A received message whose ranges section exceeds twice the
|
|
262
|
+
* upper bound is rejected as {@link ProtocolInvalidDataError}.
|
|
226
263
|
*/
|
|
227
|
-
export const ProtocolMessageRangesMaxSize = /*#__PURE__*/ between(3_000,
|
|
264
|
+
export const ProtocolMessageRangesMaxSize = /*#__PURE__*/ between(3_000, maxProtocolMessageRangesMaxSize)(Int);
|
|
228
265
|
/**
|
|
229
266
|
* Default {@link ProtocolMessageRangesMaxSize} (30KB).
|
|
230
267
|
*
|
|
@@ -270,6 +307,14 @@ export const SubscriptionFlags = {
|
|
|
270
307
|
/** Unsubscribe from updates for this owner. */
|
|
271
308
|
Unsubscribe: 2,
|
|
272
309
|
};
|
|
310
|
+
/**
|
|
311
|
+
* The error code in the header of a Response.
|
|
312
|
+
*
|
|
313
|
+
* A client reports a code it does not know as {@link ProtocolInvalidDataError},
|
|
314
|
+
* which carries no `OwnerId`, and that relay's round for the owner ends. Every
|
|
315
|
+
* released client does this, so a relay can add a code under the same
|
|
316
|
+
* {@link protocolVersion} only where that outcome is acceptable.
|
|
317
|
+
*/
|
|
273
318
|
export const ProtocolErrorCode = {
|
|
274
319
|
NoError: 0,
|
|
275
320
|
/** A code for {@link ProtocolWriteKeyError}. */
|
|
@@ -284,9 +329,12 @@ export const ProtocolErrorCode = {
|
|
|
284
329
|
/**
|
|
285
330
|
* Creates a {@link ProtocolMessage} from CRDT messages.
|
|
286
331
|
*
|
|
287
|
-
*
|
|
288
|
-
*
|
|
289
|
-
*
|
|
332
|
+
* The message holds the leading CRDT messages that fit `maxSize`, by default
|
|
333
|
+
* {@link defaultProtocolMessageMaxSize}, measured exactly. When one does not
|
|
334
|
+
* fit, it and the rest are left out, and the message ends with a range that
|
|
335
|
+
* makes the non-initiator answer with ranges. Sync then sends them in later
|
|
336
|
+
* rounds, even over unidirectional and stateless transports, because every
|
|
337
|
+
* change within {@link maxMutationSize} fits an empty message.
|
|
290
338
|
*/
|
|
291
339
|
export const createProtocolMessageFromCrdtMessages = (deps) => (owner, messages, maxSize) => {
|
|
292
340
|
const buffer = createProtocolMessageBuffer(owner.id, {
|
|
@@ -294,42 +342,37 @@ export const createProtocolMessageFromCrdtMessages = (deps) => (owner, messages,
|
|
|
294
342
|
totalMaxSize: maxSize ?? defaultProtocolMessageMaxSize,
|
|
295
343
|
writeKey: owner.writeKey,
|
|
296
344
|
});
|
|
297
|
-
let notAllMessagesSent = false;
|
|
298
345
|
for (const message of messages) {
|
|
299
346
|
const change = encodeAndEncryptDbChange(deps)(message, owner.encryptionKey);
|
|
300
347
|
const encryptedCrdtMessage = { timestamp: message.timestamp, change };
|
|
301
|
-
if (buffer.
|
|
348
|
+
if (!buffer.tryWrite(() => {
|
|
302
349
|
buffer.addMessage(encryptedCrdtMessage);
|
|
303
|
-
}
|
|
304
|
-
|
|
305
|
-
|
|
350
|
+
})) {
|
|
351
|
+
/**
|
|
352
|
+
* DEV: If not all messages fit due to size limits, we trigger a sync
|
|
353
|
+
* continuation by appending a Range with a random fingerprint. This
|
|
354
|
+
* ensures the receiver always responds with ranges, prompting another
|
|
355
|
+
* sync round.
|
|
356
|
+
*
|
|
357
|
+
* The ideal approach would be to send three ranges (skip, fingerprint,
|
|
358
|
+
* skip) where the fingerprint of unsent messages would act as narrow
|
|
359
|
+
* sync probe. I think we can send `zeroFingerprint` which can be
|
|
360
|
+
* interpreted as an indication that the other side should reply with
|
|
361
|
+
* {@link TimestampsRange}, so no need to restart syncing.
|
|
362
|
+
*
|
|
363
|
+
* For now, using a random fingerprint avoids extra complexity and is
|
|
364
|
+
* good enough for this case.
|
|
365
|
+
*/
|
|
366
|
+
const randomFingerprint = deps.randomBytes.create(fingerprintSize);
|
|
367
|
+
// Every kept trial reserved space for this closing range.
|
|
368
|
+
buffer.addRange({
|
|
369
|
+
type: RangeType.Fingerprint,
|
|
370
|
+
upperBound: InfiniteUpperBound,
|
|
371
|
+
fingerprint: randomFingerprint,
|
|
372
|
+
});
|
|
306
373
|
break;
|
|
307
374
|
}
|
|
308
375
|
}
|
|
309
|
-
if (notAllMessagesSent) {
|
|
310
|
-
/**
|
|
311
|
-
* DEV: If not all messages fit due to size limits, we trigger a sync
|
|
312
|
-
* continuation by appending a Range with a random fingerprint. This
|
|
313
|
-
* ensures the receiver always responds with ranges, prompting another
|
|
314
|
-
* sync round.
|
|
315
|
-
*
|
|
316
|
-
* The ideal approach would be to send three ranges (skip, fingerprint,
|
|
317
|
-
* skip) where the fingerprint of unsent messages would act as narrow sync
|
|
318
|
-
* probe. I think we can send `zeroFingerprint` which can be interpreted
|
|
319
|
-
* as an indication that the other side should reply with
|
|
320
|
-
* {@link TimestampsRange}, so no need to restart syncing.
|
|
321
|
-
*
|
|
322
|
-
* For now, using a random fingerprint avoids extra complexity and is good
|
|
323
|
-
* enough for this case.
|
|
324
|
-
*/
|
|
325
|
-
const randomFingerprint = deps.randomBytes.create(fingerprintSize);
|
|
326
|
-
// There is always a space for Fingerprint with InfiniteUpperBound.
|
|
327
|
-
buffer.addRange({
|
|
328
|
-
type: RangeType.Fingerprint,
|
|
329
|
-
upperBound: InfiniteUpperBound,
|
|
330
|
-
fingerprint: randomFingerprint,
|
|
331
|
-
});
|
|
332
|
-
}
|
|
333
376
|
return buffer.unwrap();
|
|
334
377
|
};
|
|
335
378
|
/**
|
|
@@ -382,16 +425,19 @@ export const createProtocolBroadcastMessagesFromCrdtMessages = (deps) => (owner,
|
|
|
382
425
|
timestamp: message.timestamp,
|
|
383
426
|
change: encodeAndEncryptDbChange(deps)(message, owner.encryptionKey),
|
|
384
427
|
};
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
assert(nextBuffer.canAddMessage(encryptedMessage), "the message is too big");
|
|
391
|
-
broadcasts.push(buffer.unwrap());
|
|
392
|
-
buffer = nextBuffer;
|
|
428
|
+
const frame = buffer;
|
|
429
|
+
if (frame.tryWrite(() => {
|
|
430
|
+
frame.addMessage(encryptedMessage);
|
|
431
|
+
})) {
|
|
432
|
+
continue;
|
|
393
433
|
}
|
|
434
|
+
buffer = createProtocolMessageBuffer(owner.id, {
|
|
435
|
+
messageType: MessageType.Broadcast,
|
|
436
|
+
totalMaxSize: maxSize,
|
|
437
|
+
});
|
|
438
|
+
// Outside a trial, it asserts that the message fits an empty frame.
|
|
394
439
|
buffer.addMessage(encryptedMessage);
|
|
440
|
+
broadcasts.push(frame.unwrap());
|
|
395
441
|
}
|
|
396
442
|
broadcasts.push(buffer.unwrap());
|
|
397
443
|
assertNonEmptyArray(broadcasts);
|
|
@@ -405,7 +451,9 @@ export const createProtocolMessageForSync = (deps) => (ownerId, subscriptionFlag
|
|
|
405
451
|
});
|
|
406
452
|
const ownerIdBytes = ownerIdToOwnerIdBytes(ownerId);
|
|
407
453
|
const size = deps.storage.getSize(ownerIdBytes);
|
|
408
|
-
|
|
454
|
+
const ranges = readSplitRanges(deps)(ownerIdBytes, zeroNonNegativeInt, size, InfiniteUpperBound);
|
|
455
|
+
for (const range of ranges)
|
|
456
|
+
buffer.addRange(range);
|
|
409
457
|
return buffer.unwrap();
|
|
410
458
|
};
|
|
411
459
|
export const createProtocolMessageForUnsubscribe = (ownerId) => createProtocolMessageBuffer(ownerId, {
|
|
@@ -418,7 +466,7 @@ export const createProtocolMessageBuffer = (ownerId, options) => {
|
|
|
418
466
|
header: createBuffer(),
|
|
419
467
|
messages: {
|
|
420
468
|
timestamps: createTimestampsBuffer(),
|
|
421
|
-
|
|
469
|
+
dbChangeLengths: createBuffer(),
|
|
422
470
|
},
|
|
423
471
|
ranges: {
|
|
424
472
|
timestamps: createTimestampsBuffer(),
|
|
@@ -443,66 +491,38 @@ export const createProtocolMessageBuffer = (ownerId, options) => {
|
|
|
443
491
|
else if (options.messageType === MessageType.Response) {
|
|
444
492
|
buffers.header.extend([options.errorCode]);
|
|
445
493
|
}
|
|
494
|
+
// Changes are referenced, not copied, so a trial undoes messages by
|
|
495
|
+
// shortening this array, and unwrap copies each change once.
|
|
496
|
+
const dbChanges = [];
|
|
497
|
+
let dbChangesLength = 0;
|
|
446
498
|
let isLastRangeInfinite = false;
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
499
|
+
let isInTrial = false;
|
|
500
|
+
// Writes outside a trial assert this, so an overflow fails where it happens.
|
|
501
|
+
// A frame must never exceed totalMaxSize: relays at @evolu/nodejs 4.0.0 or
|
|
502
|
+
// older crash on a frame one byte larger than their 1,000,000-byte limit.
|
|
503
|
+
const isWithinSizeLimits = () => getFrameSize() <= totalMaxSize;
|
|
504
|
+
// Lengths after a header that is never empty.
|
|
505
|
+
const getFrameSize = () => (buffers.header.getLength() +
|
|
450
506
|
buffers.messages.timestamps.getLength() +
|
|
451
|
-
buffers.messages.
|
|
507
|
+
buffers.messages.dbChangeLengths.getLength() +
|
|
508
|
+
dbChangesLength +
|
|
509
|
+
getRangesSize());
|
|
510
|
+
// Without ranges, the ranges section is omitted, including its count.
|
|
452
511
|
const getRangesSize = () => buffers.ranges.timestamps.getCount() > 0
|
|
453
512
|
? buffers.ranges.timestamps.getLength() +
|
|
454
513
|
buffers.ranges.types.getLength() +
|
|
455
|
-
buffers.ranges.payloads.getLength()
|
|
456
|
-
safeMargins.remainingRange
|
|
514
|
+
buffers.ranges.payloads.getLength()
|
|
457
515
|
: 0;
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
* variable-length, run-length, and delta encoding.
|
|
462
|
-
*
|
|
463
|
-
* Runtime assertions (`assert`) are used to guarantee that size limits are
|
|
464
|
-
* never exceeded. If a limit is exceeded, the assertion will fail at the
|
|
465
|
-
* precise location, making it easy to identify and fix the issue.
|
|
466
|
-
*
|
|
467
|
-
* While it would be possible to avoid the safety margin by snapshotting
|
|
468
|
-
* buffer states and rolling back changes, this would likely impact
|
|
469
|
-
* performance. If someone has time and wants to experiment with this
|
|
470
|
-
* approach, contributions are welcome.
|
|
471
|
-
*/
|
|
472
|
-
const safeMargins = {
|
|
473
|
-
// bytes: range type + possible increased count varint
|
|
474
|
-
remainingRange: fingerprintSize + 10,
|
|
475
|
-
// bytes: max millis + max count + NodeId
|
|
476
|
-
timestamp: 30,
|
|
477
|
-
// bytes: maximum encoded DbChange length varint
|
|
478
|
-
dbChangeLength: 8,
|
|
479
|
-
// bytes: worst case is around 650 bytes
|
|
480
|
-
splitRange: 800,
|
|
481
|
-
// bytes: range type + its upperBound + possible increased count varint
|
|
482
|
-
timestampsRange: 50,
|
|
483
|
-
};
|
|
484
|
-
const addMessageSafeMargin = safeMargins.timestamp +
|
|
485
|
-
safeMargins.dbChangeLength +
|
|
486
|
-
safeMargins.remainingRange;
|
|
516
|
+
// A trial makes the write, measures the exact frame, and undoes the write in
|
|
517
|
+
// constant time when the frame no longer fits, so it needs no worst-case
|
|
518
|
+
// margins and costs about as much as the write itself.
|
|
487
519
|
return {
|
|
488
|
-
canAddMessage: (message) => getSize() + addMessageSafeMargin + message.change.length <= totalMaxSize,
|
|
489
520
|
addMessage: (message) => {
|
|
490
521
|
buffers.messages.timestamps.add(message.timestamp);
|
|
491
|
-
encodeLength(buffers.messages.
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
canSplitRange: () => getRangesSize() + safeMargins.splitRange <= rangesMaxSize,
|
|
496
|
-
canAddTimestampsRangeAndMessage: (timestamps, message) => {
|
|
497
|
-
const rangesNewSize = getRangesSize() + timestamps.getLength() + safeMargins.timestampsRange;
|
|
498
|
-
return (rangesNewSize <= rangesMaxSize &&
|
|
499
|
-
(message
|
|
500
|
-
? getHeaderAndMessagesSize() +
|
|
501
|
-
rangesNewSize +
|
|
502
|
-
addMessageSafeMargin +
|
|
503
|
-
message.change.length <=
|
|
504
|
-
totalMaxSize
|
|
505
|
-
: true));
|
|
522
|
+
encodeLength(buffers.messages.dbChangeLengths, message.change);
|
|
523
|
+
dbChanges.push(message.change);
|
|
524
|
+
dbChangesLength += message.change.length;
|
|
525
|
+
assert(isInTrial || isWithinSizeLimits(), "the message is too big");
|
|
506
526
|
},
|
|
507
527
|
addRange: (range) => {
|
|
508
528
|
assert(options.messageType !== MessageType.Broadcast, "Cannot add a range into broadcast message");
|
|
@@ -518,7 +538,7 @@ export const createProtocolMessageBuffer = (ownerId, options) => {
|
|
|
518
538
|
else {
|
|
519
539
|
buffers.ranges.timestamps.addInfinite();
|
|
520
540
|
}
|
|
521
|
-
encodeNonNegativeInt(buffers.ranges.types,
|
|
541
|
+
encodeNonNegativeInt(buffers.ranges.types, range.type);
|
|
522
542
|
switch (range.type) {
|
|
523
543
|
case RangeType.Skip:
|
|
524
544
|
break;
|
|
@@ -530,22 +550,89 @@ export const createProtocolMessageBuffer = (ownerId, options) => {
|
|
|
530
550
|
break;
|
|
531
551
|
}
|
|
532
552
|
}
|
|
533
|
-
assert(isWithinSizeLimits(), `the range ${range.type} is too big`);
|
|
553
|
+
assert(isInTrial || isWithinSizeLimits(), `the range ${range.type} is too big`);
|
|
554
|
+
},
|
|
555
|
+
tryWrite: (write, reserve = zeroNonNegativeInt) => {
|
|
556
|
+
const restoreMessageTimestamps = buffers.messages.timestamps.checkpoint();
|
|
557
|
+
const dbChangeLengthsLength = buffers.messages.dbChangeLengths.getLength();
|
|
558
|
+
const dbChangesCount = dbChanges.length;
|
|
559
|
+
const checkpointDbChangesLength = dbChangesLength;
|
|
560
|
+
const restoreRangeTimestamps = buffers.ranges.timestamps.checkpoint();
|
|
561
|
+
const typesLength = buffers.ranges.types.getLength();
|
|
562
|
+
const payloadsLength = buffers.ranges.payloads.getLength();
|
|
563
|
+
const wasLastRangeInfinite = isLastRangeInfinite;
|
|
564
|
+
const wasInTrial = isInTrial;
|
|
565
|
+
let isKept = false;
|
|
566
|
+
isInTrial = true;
|
|
567
|
+
try {
|
|
568
|
+
write();
|
|
569
|
+
// Fingerprint(InfiniteUpperBound) encodes no upper bound, only its
|
|
570
|
+
// type (1 byte) and fingerprint (12 bytes). The ranges count varint
|
|
571
|
+
// gains a byte when the new count is a power of 128. That includes
|
|
572
|
+
// 128^0 = 1, since the first range adds the count itself. So closing
|
|
573
|
+
// takes 13 or 14 bytes, and nothing for a Broadcast or a closed frame.
|
|
574
|
+
let newCount = buffers.ranges.timestamps.getCount() + 1;
|
|
575
|
+
while (newCount % 128 === 0)
|
|
576
|
+
newCount /= 128;
|
|
577
|
+
const closingReserve = options.messageType === MessageType.Broadcast || isLastRangeInfinite
|
|
578
|
+
? 0
|
|
579
|
+
: 1 + fingerprintSize + (newCount === 1 ? 1 : 0);
|
|
580
|
+
const reserves = closingReserve + reserve;
|
|
581
|
+
isKept =
|
|
582
|
+
getFrameSize() + reserves <= totalMaxSize &&
|
|
583
|
+
getRangesSize() + reserves <= rangesMaxSize;
|
|
584
|
+
}
|
|
585
|
+
finally {
|
|
586
|
+
isInTrial = wasInTrial;
|
|
587
|
+
if (!isKept) {
|
|
588
|
+
restoreMessageTimestamps();
|
|
589
|
+
buffers.messages.dbChangeLengths.truncate(dbChangeLengthsLength);
|
|
590
|
+
dbChanges.length = dbChangesCount;
|
|
591
|
+
dbChangesLength = checkpointDbChangesLength;
|
|
592
|
+
restoreRangeTimestamps();
|
|
593
|
+
buffers.ranges.types.truncate(typesLength);
|
|
594
|
+
buffers.ranges.payloads.truncate(payloadsLength);
|
|
595
|
+
isLastRangeInfinite = wasLastRangeInfinite;
|
|
596
|
+
}
|
|
597
|
+
}
|
|
598
|
+
return isKept;
|
|
534
599
|
},
|
|
535
600
|
unwrap: () => {
|
|
536
601
|
if (buffers.ranges.timestamps.getCount() > 0) {
|
|
537
602
|
assert(isLastRangeInfinite, "The last range's upperBound must be InfiniteUpperBound");
|
|
538
603
|
}
|
|
539
|
-
|
|
540
|
-
|
|
604
|
+
const frame = new Uint8Array(getFrameSize());
|
|
605
|
+
frame.set(buffers.header.unwrap());
|
|
606
|
+
let offset = buffers.header.getLength();
|
|
607
|
+
// Written to a new buffer, so the builder is unchanged and unwrap can be
|
|
608
|
+
// called again.
|
|
609
|
+
const messageTimestamps = createBuffer();
|
|
610
|
+
buffers.messages.timestamps.append(messageTimestamps);
|
|
611
|
+
frame.set(messageTimestamps.unwrap(), offset);
|
|
612
|
+
offset += messageTimestamps.getLength();
|
|
613
|
+
// Each change follows its length, whose last varint byte is below 128.
|
|
614
|
+
const lengths = buffers.messages.dbChangeLengths.unwrap();
|
|
615
|
+
let lengthsOffset = 0;
|
|
616
|
+
for (const change of dbChanges) {
|
|
617
|
+
do {
|
|
618
|
+
frame[offset++] = lengths[lengthsOffset];
|
|
619
|
+
} while (lengths[lengthsOffset++] >= 128);
|
|
620
|
+
frame.set(change, offset);
|
|
621
|
+
offset += change.length;
|
|
622
|
+
}
|
|
541
623
|
if (buffers.ranges.timestamps.getCount() > 0) {
|
|
542
|
-
|
|
543
|
-
buffers.
|
|
544
|
-
|
|
624
|
+
const ranges = createBuffer();
|
|
625
|
+
buffers.ranges.timestamps.append(ranges);
|
|
626
|
+
ranges.extend(buffers.ranges.types.unwrap());
|
|
627
|
+
ranges.extend(buffers.ranges.payloads.unwrap());
|
|
628
|
+
frame.set(ranges.unwrap(), offset);
|
|
629
|
+
offset += ranges.getLength();
|
|
545
630
|
}
|
|
546
|
-
|
|
631
|
+
// An overcounted size would end the frame with zero bytes peers accept.
|
|
632
|
+
assert(offset === frame.length, "the frame size is exact");
|
|
633
|
+
return frame;
|
|
547
634
|
},
|
|
548
|
-
getSize,
|
|
635
|
+
getSize: getFrameSize,
|
|
549
636
|
};
|
|
550
637
|
};
|
|
551
638
|
export const createTimestampsBuffer = () => {
|
|
@@ -590,252 +677,348 @@ export const createTimestampsBuffer = () => {
|
|
|
590
677
|
buffer.extend(counterEncoder.unwrap());
|
|
591
678
|
buffer.extend(nodeIdEncoder.unwrap());
|
|
592
679
|
},
|
|
680
|
+
checkpoint: () => {
|
|
681
|
+
const checkpointCount = count;
|
|
682
|
+
const millisLength = millisBuffer.getLength();
|
|
683
|
+
const checkpointPreviousMillis = previousMillis;
|
|
684
|
+
const restoreCounters = counterEncoder.checkpoint();
|
|
685
|
+
const restoreNodeIds = nodeIdEncoder.checkpoint();
|
|
686
|
+
return () => {
|
|
687
|
+
count = checkpointCount;
|
|
688
|
+
syncCount();
|
|
689
|
+
millisBuffer.truncate(millisLength);
|
|
690
|
+
previousMillis = checkpointPreviousMillis;
|
|
691
|
+
restoreCounters();
|
|
692
|
+
restoreNodeIds();
|
|
693
|
+
};
|
|
694
|
+
},
|
|
593
695
|
};
|
|
594
696
|
};
|
|
595
697
|
export const applyProtocolMessageAsClient = (inputMessage, options = {}) => async (run) => {
|
|
596
698
|
const { storage } = run.deps;
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
}
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
}
|
|
641
|
-
const messages = decodeMessages(input);
|
|
642
|
-
const ownerIdBytes = ownerIdToOwnerIdBytes(ownerId);
|
|
643
|
-
if (isNonEmptyArray(messages)) {
|
|
644
|
-
try {
|
|
645
|
-
const result = await run(storage.writeMessages(ownerIdBytes, messages));
|
|
646
|
-
if (!result.ok)
|
|
647
|
-
return result;
|
|
648
|
-
}
|
|
649
|
-
catch (error) {
|
|
650
|
-
if (AbortError.is(error))
|
|
651
|
-
throw error;
|
|
652
|
-
run.deps.console.error(error);
|
|
653
|
-
return ok({ type: "Failed", cause: "Write" });
|
|
654
|
-
}
|
|
655
|
-
}
|
|
656
|
-
if (messageType === MessageType.Broadcast) {
|
|
657
|
-
return ok({ type: "Broadcast" });
|
|
699
|
+
const version = options.version ?? protocolVersion;
|
|
700
|
+
const decoded = decodeProtocolMessage(inputMessage, version);
|
|
701
|
+
if (!decoded.ok)
|
|
702
|
+
return decoded;
|
|
703
|
+
const message = decoded.value;
|
|
704
|
+
if (message.type === "OtherVersion") {
|
|
705
|
+
return err({
|
|
706
|
+
type: "ProtocolVersionError",
|
|
707
|
+
version: message.version,
|
|
708
|
+
isInitiator: version < message.version,
|
|
709
|
+
ownerId: message.ownerId,
|
|
710
|
+
});
|
|
711
|
+
}
|
|
712
|
+
if (message.type === "Request") {
|
|
713
|
+
return err({
|
|
714
|
+
type: "ProtocolInvalidDataError",
|
|
715
|
+
data: inputMessage,
|
|
716
|
+
error: new ProtocolDecodeError("Expected a Response or a Broadcast"),
|
|
717
|
+
});
|
|
718
|
+
}
|
|
719
|
+
const { ownerId } = message;
|
|
720
|
+
if (message.type === "ErrorResponse") {
|
|
721
|
+
switch (message.errorCode) {
|
|
722
|
+
case ProtocolErrorCode.WriteKeyError:
|
|
723
|
+
return err({
|
|
724
|
+
type: "ProtocolWriteKeyError",
|
|
725
|
+
ownerId,
|
|
726
|
+
});
|
|
727
|
+
case ProtocolErrorCode.WriteError:
|
|
728
|
+
return err({
|
|
729
|
+
type: "ProtocolWriteError",
|
|
730
|
+
ownerId,
|
|
731
|
+
});
|
|
732
|
+
case ProtocolErrorCode.QuotaError:
|
|
733
|
+
return err({
|
|
734
|
+
type: "ProtocolQuotaError",
|
|
735
|
+
ownerId,
|
|
736
|
+
});
|
|
737
|
+
case ProtocolErrorCode.SyncError:
|
|
738
|
+
return err({
|
|
739
|
+
type: "ProtocolSyncError",
|
|
740
|
+
ownerId,
|
|
741
|
+
});
|
|
658
742
|
}
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
743
|
+
}
|
|
744
|
+
const { messages } = message;
|
|
745
|
+
const ownerIdBytes = ownerIdToOwnerIdBytes(ownerId);
|
|
746
|
+
if (isNonEmptyArray(messages)) {
|
|
747
|
+
try {
|
|
748
|
+
const result = await run(storage.writeMessages(ownerIdBytes, messages));
|
|
749
|
+
if (!result.ok)
|
|
750
|
+
return result;
|
|
667
751
|
}
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
752
|
+
catch (error) {
|
|
753
|
+
if (AbortError.is(error))
|
|
754
|
+
throw error;
|
|
755
|
+
run.deps.console.error(error);
|
|
756
|
+
return ok({ type: "Failed", cause: "Write" });
|
|
671
757
|
}
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
758
|
+
}
|
|
759
|
+
if (message.type === "Broadcast") {
|
|
760
|
+
return ok({ type: "Broadcast" });
|
|
761
|
+
}
|
|
762
|
+
// Now: No writeKey, no sync.
|
|
763
|
+
// TODO: Allow to sync SharedReadonlyOwner
|
|
764
|
+
// Without local changes, writeKey will not be required.
|
|
765
|
+
// With local changes, writeKey will be required and if not provided,
|
|
766
|
+
// the sync will stop.
|
|
767
|
+
const writeKey = options.writeKey;
|
|
768
|
+
if (writeKey == null) {
|
|
769
|
+
return ok({ type: "Readonly" });
|
|
770
|
+
}
|
|
771
|
+
const { ranges } = message;
|
|
772
|
+
if (!isNonEmptyArray(ranges)) {
|
|
773
|
+
return ok({ type: "Converged" });
|
|
774
|
+
}
|
|
775
|
+
const createOutput = () => createProtocolMessageBuffer(ownerId, {
|
|
776
|
+
messageType: MessageType.Request,
|
|
777
|
+
writeKey,
|
|
778
|
+
rangesMaxSize: options.rangesMaxSize,
|
|
779
|
+
});
|
|
780
|
+
const output = createOutput();
|
|
781
|
+
let broadcast;
|
|
782
|
+
const result = sync(run.deps)(ranges, output, ownerIdBytes, {
|
|
783
|
+
createEmptyOutput: createOutput,
|
|
784
|
+
onChangeTooLarge: options.onChangeTooLarge ?? run.deps.console.warn,
|
|
785
|
+
onMessage: (message) => {
|
|
679
786
|
broadcast ??= createProtocolMessageBuffer(ownerId, {
|
|
680
787
|
messageType: MessageType.Broadcast,
|
|
681
788
|
});
|
|
789
|
+
// The request holds the same messages after a larger header.
|
|
682
790
|
broadcast.addMessage(message);
|
|
791
|
+
},
|
|
792
|
+
});
|
|
793
|
+
// A failure was logged by sync.
|
|
794
|
+
if (!result.ok)
|
|
795
|
+
return ok({ type: "Failed", cause: "Sync" });
|
|
796
|
+
if (!result.value)
|
|
797
|
+
return ok({ type: "Converged" });
|
|
798
|
+
return ok({
|
|
799
|
+
type: "Response",
|
|
800
|
+
message: output.unwrap(),
|
|
801
|
+
...(broadcast && { broadcast: broadcast.unwrap() }),
|
|
802
|
+
});
|
|
803
|
+
};
|
|
804
|
+
export const applyProtocolMessageAsRelay = (inputMessage, options = {},
|
|
805
|
+
/** For tests only. */
|
|
806
|
+
version = protocolVersion) => async (run) => {
|
|
807
|
+
const { storage } = run.deps;
|
|
808
|
+
// The relay broadcasts a request's messages in a frame of totalMaxSize.
|
|
809
|
+
// That frame omits the request's write key, subscription flag, and ranges,
|
|
810
|
+
// so it holds the messages of any request up to that size.
|
|
811
|
+
if (inputMessage.length >
|
|
812
|
+
(options.totalMaxSize ?? defaultProtocolMessageMaxSize))
|
|
813
|
+
return err({
|
|
814
|
+
type: "ProtocolInvalidDataError",
|
|
815
|
+
data: inputMessage,
|
|
816
|
+
error: new ProtocolDecodeError("Request is too large"),
|
|
683
817
|
});
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
818
|
+
const decoded = decodeProtocolMessage(inputMessage, version);
|
|
819
|
+
if (!decoded.ok)
|
|
820
|
+
return decoded;
|
|
821
|
+
const request = decoded.value;
|
|
822
|
+
if (request.type === "OtherVersion") {
|
|
823
|
+
// Non-initiator responds with its version and ownerId.
|
|
824
|
+
const output = createBuffer();
|
|
825
|
+
encodeNonNegativeInt(output, version);
|
|
826
|
+
output.extend(ownerIdToOwnerIdBytes(request.ownerId));
|
|
689
827
|
return ok({
|
|
690
828
|
type: "Response",
|
|
691
829
|
message: output.unwrap(),
|
|
692
|
-
...(broadcast && { broadcast: broadcast.unwrap() }),
|
|
693
830
|
});
|
|
694
831
|
}
|
|
695
|
-
|
|
696
|
-
if (AbortError.is(error))
|
|
697
|
-
throw error;
|
|
832
|
+
if (request.type !== "Request") {
|
|
698
833
|
return err({
|
|
699
834
|
type: "ProtocolInvalidDataError",
|
|
700
835
|
data: inputMessage,
|
|
701
|
-
error,
|
|
836
|
+
error: new ProtocolDecodeError("Expected a Request"),
|
|
702
837
|
});
|
|
703
838
|
}
|
|
704
|
-
};
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
839
|
+
const { ownerId, writeKey, messages, ranges } = request;
|
|
840
|
+
const ownerIdBytes = ownerIdToOwnerIdBytes(ownerId);
|
|
841
|
+
const createErrorResponse = (errorCode) => ({
|
|
842
|
+
type: "Response",
|
|
843
|
+
message: createProtocolMessageBuffer(ownerId, {
|
|
844
|
+
messageType: MessageType.Response,
|
|
845
|
+
errorCode,
|
|
846
|
+
}).unwrap(),
|
|
847
|
+
});
|
|
848
|
+
switch (request.subscriptionFlag) {
|
|
849
|
+
case SubscriptionFlags.Subscribe:
|
|
850
|
+
options.subscribe?.(ownerId);
|
|
851
|
+
break;
|
|
852
|
+
case SubscriptionFlags.Unsubscribe:
|
|
853
|
+
options.unsubscribe?.(ownerId);
|
|
854
|
+
break;
|
|
855
|
+
case SubscriptionFlags.None:
|
|
856
|
+
break;
|
|
857
|
+
default:
|
|
858
|
+
exhaustiveCheck(request.subscriptionFlag);
|
|
859
|
+
}
|
|
860
|
+
if (writeKey) {
|
|
861
|
+
let isValid;
|
|
862
|
+
try {
|
|
863
|
+
isValid = storage.validateWriteKey(ownerIdBytes, writeKey);
|
|
722
864
|
}
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
865
|
+
catch (error) {
|
|
866
|
+
// A relay storage stores the write key of a new owner, which SQLite
|
|
867
|
+
// can fail, for example on a full disk. A boolean has no room for the
|
|
868
|
+
// error, so it is answered like a failed write.
|
|
869
|
+
run.deps.console.error(error);
|
|
870
|
+
return ok(createErrorResponse(ProtocolErrorCode.WriteError));
|
|
729
871
|
}
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
case SubscriptionFlags.Subscribe:
|
|
733
|
-
options.subscribe?.(ownerId);
|
|
734
|
-
break;
|
|
735
|
-
case SubscriptionFlags.Unsubscribe:
|
|
736
|
-
options.unsubscribe?.(ownerId);
|
|
737
|
-
break;
|
|
738
|
-
case SubscriptionFlags.None:
|
|
739
|
-
break;
|
|
872
|
+
if (!isValid) {
|
|
873
|
+
return ok(createErrorResponse(ProtocolErrorCode.WriteKeyError));
|
|
740
874
|
}
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
875
|
+
}
|
|
876
|
+
if (isNonEmptyArray(messages)) {
|
|
877
|
+
if (!writeKey) {
|
|
878
|
+
return ok(createErrorResponse(ProtocolErrorCode.WriteKeyError));
|
|
879
|
+
}
|
|
880
|
+
try {
|
|
881
|
+
const result = await run(storage.writeMessages(ownerIdBytes, messages));
|
|
882
|
+
if (!result.ok) {
|
|
883
|
+
// A storage returns a failed write without reporting it.
|
|
884
|
+
if (result.error.type === "UnknownError")
|
|
885
|
+
run.deps.console.error(result.error);
|
|
886
|
+
return ok(createErrorResponse(result.error.type === "StorageQuotaError"
|
|
887
|
+
? ProtocolErrorCode.QuotaError
|
|
888
|
+
: ProtocolErrorCode.WriteError));
|
|
751
889
|
}
|
|
752
890
|
}
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
891
|
+
catch (error) {
|
|
892
|
+
if (AbortError.is(error))
|
|
893
|
+
throw error;
|
|
894
|
+
run.deps.console.error(error);
|
|
895
|
+
return ok(createErrorResponse(ProtocolErrorCode.WriteError));
|
|
896
|
+
}
|
|
897
|
+
/**
|
|
898
|
+
* Broadcast messages to all subscribed owners for real-time
|
|
899
|
+
* synchronization between clients.
|
|
900
|
+
*
|
|
901
|
+
* Messages are only broadcasted after successful write to ensure devices
|
|
902
|
+
* that can still sync aren't affected by quota errors, and to prevent
|
|
903
|
+
* using a half-working relay service (broadcasting without persistence).
|
|
904
|
+
*
|
|
905
|
+
* When a relay's database is deleted or clients migrate to a new relay
|
|
906
|
+
* (without data migration), clients will sync their data to the relay,
|
|
907
|
+
* and the relay will broadcast those messages to other connected clients.
|
|
908
|
+
* Those clients may receive messages they already have, but this is safe
|
|
909
|
+
* because Evolu sync is idempotent. As the relay becomes more
|
|
910
|
+
* synchronized with clients over time, fewer duplicate messages will be
|
|
911
|
+
* broadcasted.
|
|
912
|
+
*/
|
|
913
|
+
if (options.broadcast) {
|
|
914
|
+
const broadcastBuffer = createProtocolMessageBuffer(ownerId, {
|
|
915
|
+
messageType: MessageType.Broadcast,
|
|
916
|
+
totalMaxSize: options.totalMaxSize,
|
|
917
|
+
rangesMaxSize: options.rangesMaxSize,
|
|
918
|
+
version,
|
|
919
|
+
});
|
|
920
|
+
for (const message of messages) {
|
|
921
|
+
broadcastBuffer.addMessage(message);
|
|
922
|
+
}
|
|
923
|
+
options.broadcast(ownerId, broadcastBuffer.unwrap());
|
|
924
|
+
}
|
|
925
|
+
}
|
|
926
|
+
const createOutput = () => createProtocolMessageBuffer(ownerId, {
|
|
927
|
+
messageType: MessageType.Response,
|
|
928
|
+
errorCode: ProtocolErrorCode.NoError,
|
|
929
|
+
totalMaxSize: options.totalMaxSize,
|
|
930
|
+
rangesMaxSize: options.rangesMaxSize,
|
|
931
|
+
});
|
|
932
|
+
const output = createOutput();
|
|
933
|
+
// A relay answers every request it decodes, even with nothing to sync, so
|
|
934
|
+
// the initiator knows the sync is complete.
|
|
935
|
+
if (!isNonEmptyArray(ranges)) {
|
|
936
|
+
return ok({ type: "Response", message: output.unwrap() });
|
|
937
|
+
}
|
|
938
|
+
const result = sync(run.deps)(ranges, output, ownerIdBytes, {
|
|
939
|
+
createEmptyOutput: createOutput,
|
|
940
|
+
onChangeTooLarge: run.deps.console.warn,
|
|
941
|
+
});
|
|
942
|
+
// A relay answers every request it decodes, a failed reconciliation with
|
|
943
|
+
// its error code.
|
|
944
|
+
return ok(result.ok
|
|
945
|
+
? { type: "Response", message: output.unwrap() }
|
|
946
|
+
: createErrorResponse(result.error));
|
|
947
|
+
};
|
|
948
|
+
/**
|
|
949
|
+
* Decodes a whole {@link ProtocolMessage}, so a malformed one is rejected before
|
|
950
|
+
* anything it carries is applied.
|
|
951
|
+
*
|
|
952
|
+
* Applying a message turns only a throw from here into
|
|
953
|
+
* {@link ProtocolInvalidDataError}. A throw after decoding is a defect.
|
|
954
|
+
*/
|
|
955
|
+
const decodeProtocolMessage = (inputMessage, version) => {
|
|
956
|
+
try {
|
|
957
|
+
const input = createBuffer(inputMessage);
|
|
958
|
+
const [messageVersion, ownerId] = decodeVersionAndOwner(input);
|
|
959
|
+
if (messageVersion !== version)
|
|
960
|
+
return ok({ type: "OtherVersion", version: messageVersion, ownerId });
|
|
961
|
+
switch (decodeMessageType(input)) {
|
|
962
|
+
case MessageType.Request: {
|
|
963
|
+
const hasWriteKey = input.shift();
|
|
964
|
+
if (hasWriteKey > 1)
|
|
965
|
+
throw new ProtocolDecodeError(`Invalid hasWriteKey: ${hasWriteKey}`);
|
|
966
|
+
const writeKey = hasWriteKey === 1
|
|
967
|
+
? input.shiftN(ownerWriteKeyLength)
|
|
968
|
+
: null;
|
|
969
|
+
const subscriptionFlag = input.shift();
|
|
970
|
+
switch (subscriptionFlag) {
|
|
971
|
+
case SubscriptionFlags.None:
|
|
972
|
+
case SubscriptionFlags.Subscribe:
|
|
973
|
+
case SubscriptionFlags.Unsubscribe:
|
|
974
|
+
break;
|
|
975
|
+
default:
|
|
976
|
+
throw new ProtocolDecodeError(`Invalid SubscriptionFlag: ${subscriptionFlag}`);
|
|
977
|
+
}
|
|
978
|
+
const messages = decodeMessages(input);
|
|
979
|
+
// Only a relay accepts a Request, so only the relay checks this.
|
|
980
|
+
// Deployed relays already store shorter changes, and a client skips a
|
|
981
|
+
// change it cannot read, whereas a check in decodeMessages would make
|
|
982
|
+
// it reject every response holding one.
|
|
983
|
+
for (const { change } of messages)
|
|
984
|
+
if (change.length < minEncryptedDbChangeLength)
|
|
985
|
+
throw new ProtocolDecodeError("EncryptedDbChange is too short");
|
|
986
|
+
const ranges = decodeRanges(input);
|
|
756
987
|
return ok({
|
|
757
|
-
type: "
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
988
|
+
type: "Request",
|
|
989
|
+
ownerId,
|
|
990
|
+
writeKey,
|
|
991
|
+
subscriptionFlag,
|
|
992
|
+
messages,
|
|
993
|
+
ranges,
|
|
762
994
|
});
|
|
763
995
|
}
|
|
764
|
-
|
|
765
|
-
const
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
996
|
+
case MessageType.Response: {
|
|
997
|
+
const errorCode = input.shift();
|
|
998
|
+
switch (errorCode) {
|
|
999
|
+
case ProtocolErrorCode.NoError: {
|
|
1000
|
+
const messages = decodeMessages(input);
|
|
1001
|
+
const ranges = decodeRanges(input);
|
|
1002
|
+
return ok({ type: "Response", ownerId, messages, ranges });
|
|
1003
|
+
}
|
|
1004
|
+
case ProtocolErrorCode.WriteKeyError:
|
|
1005
|
+
case ProtocolErrorCode.WriteError:
|
|
1006
|
+
case ProtocolErrorCode.QuotaError:
|
|
1007
|
+
case ProtocolErrorCode.SyncError:
|
|
1008
|
+
return ok({ type: "ErrorResponse", ownerId, errorCode });
|
|
1009
|
+
default:
|
|
1010
|
+
throw new ProtocolDecodeError(`Invalid ProtocolErrorCode: ${errorCode}`);
|
|
772
1011
|
}
|
|
773
1012
|
}
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
messageType: MessageType.Response,
|
|
780
|
-
errorCode: ProtocolErrorCode.WriteError,
|
|
781
|
-
}).unwrap();
|
|
782
|
-
return ok({ type: "Response", message });
|
|
783
|
-
}
|
|
784
|
-
/**
|
|
785
|
-
* Broadcast messages to all subscribed owners for real-time
|
|
786
|
-
* synchronization between clients.
|
|
787
|
-
*
|
|
788
|
-
* Messages are only broadcasted after successful write to ensure
|
|
789
|
-
* devices that can still sync aren't affected by quota errors, and to
|
|
790
|
-
* prevent using a half-working relay service (broadcasting without
|
|
791
|
-
* persistence).
|
|
792
|
-
*
|
|
793
|
-
* When a relay's database is deleted or clients migrate to a new relay
|
|
794
|
-
* (without data migration), clients will sync their data to the relay,
|
|
795
|
-
* and the relay will broadcast those messages to other connected
|
|
796
|
-
* clients. Those clients may receive messages they already have, but
|
|
797
|
-
* this is safe because Evolu sync is idempotent. As the relay becomes
|
|
798
|
-
* more synchronized with clients over time, fewer duplicate messages
|
|
799
|
-
* will be broadcasted.
|
|
800
|
-
*/
|
|
801
|
-
if (options.broadcast) {
|
|
802
|
-
const broadcastBuffer = createProtocolMessageBuffer(ownerId, {
|
|
803
|
-
messageType: MessageType.Broadcast,
|
|
804
|
-
totalMaxSize: options.totalMaxSize,
|
|
805
|
-
rangesMaxSize: options.rangesMaxSize,
|
|
806
|
-
version,
|
|
1013
|
+
case MessageType.Broadcast:
|
|
1014
|
+
return ok({
|
|
1015
|
+
type: "Broadcast",
|
|
1016
|
+
ownerId,
|
|
1017
|
+
messages: decodeMessages(input),
|
|
807
1018
|
});
|
|
808
|
-
for (const message of messages) {
|
|
809
|
-
broadcastBuffer.addMessage(message);
|
|
810
|
-
}
|
|
811
|
-
options.broadcast(ownerId, broadcastBuffer.unwrap());
|
|
812
|
-
}
|
|
813
|
-
}
|
|
814
|
-
const ranges = decodeRanges(input);
|
|
815
|
-
const output = createProtocolMessageBuffer(ownerId, {
|
|
816
|
-
messageType: MessageType.Response,
|
|
817
|
-
errorCode: ProtocolErrorCode.NoError,
|
|
818
|
-
totalMaxSize: options.totalMaxSize,
|
|
819
|
-
rangesMaxSize: options.rangesMaxSize,
|
|
820
|
-
});
|
|
821
|
-
// Non-initiators always respond to provide sync completion feedback,
|
|
822
|
-
// even when there's nothing to sync.
|
|
823
|
-
if (!isNonEmptyArray(ranges)) {
|
|
824
|
-
return ok({ type: "Response", message: output.unwrap() });
|
|
825
1019
|
}
|
|
826
|
-
const result = sync(run.deps)(ranges, output, ownerIdBytes);
|
|
827
|
-
const message = result.ok
|
|
828
|
-
? output.unwrap()
|
|
829
|
-
: createProtocolMessageBuffer(ownerId, {
|
|
830
|
-
messageType: MessageType.Response,
|
|
831
|
-
errorCode: result.error,
|
|
832
|
-
}).unwrap();
|
|
833
|
-
// Non-initiators always respond to provide sync completion feedback,
|
|
834
|
-
return ok({ type: "Response", message });
|
|
835
1020
|
}
|
|
836
1021
|
catch (error) {
|
|
837
|
-
if (AbortError.is(error))
|
|
838
|
-
throw error;
|
|
839
1022
|
return err({
|
|
840
1023
|
type: "ProtocolInvalidDataError",
|
|
841
1024
|
data: inputMessage,
|
|
@@ -846,8 +1029,7 @@ version = protocolVersion) => async (run) => {
|
|
|
846
1029
|
const decodeVersionAndOwner = (input) => {
|
|
847
1030
|
// This structure must never change across protocol versions. The version
|
|
848
1031
|
// and owner ID must always be the first two fields in every protocol message
|
|
849
|
-
// to
|
|
850
|
-
// processing occurs.
|
|
1032
|
+
// to route and report a version mismatch before any other processing occurs.
|
|
851
1033
|
const version = decodeNonNegativeInt(input);
|
|
852
1034
|
const ownerId = decodeId(input);
|
|
853
1035
|
return [version, ownerId];
|
|
@@ -857,22 +1039,19 @@ const parseProtocolHeaderFromBuffer = (input) => {
|
|
|
857
1039
|
if (version !== protocolVersion) {
|
|
858
1040
|
return { type: "ProtocolHeader", version, ownerId };
|
|
859
1041
|
}
|
|
860
|
-
const
|
|
861
|
-
|
|
862
|
-
|
|
1042
|
+
const messageType = decodeMessageType(input);
|
|
1043
|
+
return { type: "ProtocolHeader", version, ownerId, messageType };
|
|
1044
|
+
};
|
|
1045
|
+
const decodeMessageType = (input) => {
|
|
1046
|
+
const messageType = input.shift();
|
|
1047
|
+
switch (messageType) {
|
|
863
1048
|
case MessageType.Request:
|
|
864
|
-
messageType = MessageType.Request;
|
|
865
|
-
break;
|
|
866
1049
|
case MessageType.Response:
|
|
867
|
-
messageType = MessageType.Response;
|
|
868
|
-
break;
|
|
869
1050
|
case MessageType.Broadcast:
|
|
870
|
-
messageType
|
|
871
|
-
break;
|
|
1051
|
+
return messageType;
|
|
872
1052
|
default:
|
|
873
1053
|
throw new ProtocolDecodeError("Invalid MessageType");
|
|
874
1054
|
}
|
|
875
|
-
return { type: "ProtocolHeader", version, ownerId, messageType };
|
|
876
1055
|
};
|
|
877
1056
|
/**
|
|
878
1057
|
* Error thrown for internal protocol validation failures, such as invalid data
|
|
@@ -882,7 +1061,6 @@ class ProtocolDecodeError extends Error {
|
|
|
882
1061
|
constructor(message) {
|
|
883
1062
|
super(message);
|
|
884
1063
|
this.name = this.constructor.name;
|
|
885
|
-
Error.captureStackTrace(this, this.constructor);
|
|
886
1064
|
}
|
|
887
1065
|
}
|
|
888
1066
|
const decodeMessages = (buffer) => {
|
|
@@ -896,7 +1074,39 @@ const decodeMessages = (buffer) => {
|
|
|
896
1074
|
}
|
|
897
1075
|
return messages;
|
|
898
1076
|
};
|
|
899
|
-
|
|
1077
|
+
// The smallest envelope encodeAndEncryptDbChange can produce: the nonce, a
|
|
1078
|
+
// 1-byte ciphertext length, and the 16-byte Poly1305 tag. Every v1 client sends
|
|
1079
|
+
// at least 77 bytes. The relay quota counts change bytes, so a relay storing
|
|
1080
|
+
// shorter changes, zero-length ones above all, would store rows the quota does
|
|
1081
|
+
// not count.
|
|
1082
|
+
const minEncryptedDbChangeLength = xChaCha20Poly1305NonceLength + 1 + 16;
|
|
1083
|
+
/**
|
|
1084
|
+
* Answers the ranges into `output`, returning whether it has anything to send.
|
|
1085
|
+
*
|
|
1086
|
+
* Every write except a closing one is a trial (see
|
|
1087
|
+
* {@link ProtocolMessageBuffer.tryWrite}), so the frame always has room to close
|
|
1088
|
+
* with one Fingerprint range with {@link InfiniteUpperBound}. When a write does
|
|
1089
|
+
* not fit, that range closes the frame. Its fingerprint covers everything from
|
|
1090
|
+
* the last upper bound the frame states, which every v1 peer assumes, and the
|
|
1091
|
+
* peer reconciles it in the next round.
|
|
1092
|
+
*
|
|
1093
|
+
* A stored change that cannot fit an empty frame with this header, with only
|
|
1094
|
+
* its own timestamp listed after a pending skip, is skipped, and
|
|
1095
|
+
* `onChangeTooLarge` reports it as a {@link ProtocolChangeTooLargeError}. A
|
|
1096
|
+
* later round is guaranteed to reach exactly that frame, so a change that fits
|
|
1097
|
+
* it is sent eventually.
|
|
1098
|
+
*
|
|
1099
|
+
* An answer to a Timestamps range neither sends nor lists a skipped change, but
|
|
1100
|
+
* a request or a split lists its timestamp, so a peer that lacks it may ask for
|
|
1101
|
+
* it once per sync and gets an answer without it. Fingerprints keep disagreeing
|
|
1102
|
+
* about it, so every sync narrows them to it, skips it again, and ends.
|
|
1103
|
+
*
|
|
1104
|
+
* A throw from storage, or from a split's checks of what storage returned, is
|
|
1105
|
+
* logged and returns `SyncError`, except an AbortError from a split's reads,
|
|
1106
|
+
* which is rethrown. Any other throw, such as from writing the frame,
|
|
1107
|
+
* `onChangeTooLarge`, or `onMessage`, is a defect.
|
|
1108
|
+
*/
|
|
1109
|
+
const sync = (deps) => (ranges, output, ownerIdBytes, { createEmptyOutput, onChangeTooLarge, onMessage, }) => {
|
|
900
1110
|
const outputInitialSize = output.getSize();
|
|
901
1111
|
let storageSize;
|
|
902
1112
|
try {
|
|
@@ -908,11 +1118,18 @@ const sync = (deps) => (ranges, output, ownerIdBytes, onMessage) => {
|
|
|
908
1118
|
}
|
|
909
1119
|
let prevUpperBound = null;
|
|
910
1120
|
let prevIndex = zeroNonNegativeInt;
|
|
1121
|
+
// The index of the last upper bound the frame states, 0 before any.
|
|
1122
|
+
let statedIndex = zeroNonNegativeInt;
|
|
1123
|
+
// Consecutive skipped ranges become one Skip range, written only before
|
|
1124
|
+
// the next non-skip range.
|
|
911
1125
|
let skip = false;
|
|
912
1126
|
let nonSkipRangeAdded = false;
|
|
913
1127
|
const skipRange = (range) => {
|
|
914
1128
|
// The last range, if any non skip was added, must have InfiniteUpperBound.
|
|
915
1129
|
if (nonSkipRangeAdded && range.upperBound === InfiniteUpperBound) {
|
|
1130
|
+
// A closing write. Like the closing Fingerprint range, it encodes no
|
|
1131
|
+
// upper bound, and its type takes 1 byte of the 13 that every kept
|
|
1132
|
+
// trial reserved beyond the ranges count.
|
|
916
1133
|
output.addRange({
|
|
917
1134
|
type: RangeType.Skip,
|
|
918
1135
|
upperBound: InfiniteUpperBound,
|
|
@@ -922,36 +1139,50 @@ const sync = (deps) => (ranges, output, ownerIdBytes, onMessage) => {
|
|
|
922
1139
|
skip = true;
|
|
923
1140
|
}
|
|
924
1141
|
};
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
1142
|
+
/**
|
|
1143
|
+
* Tries to write a non-skip range ending at the index `upper` after the
|
|
1144
|
+
* pending skip, if any.
|
|
1145
|
+
*/
|
|
1146
|
+
const tryWriteRange = (upper, write) => {
|
|
1147
|
+
const isKept = output.tryWrite(() => {
|
|
1148
|
+
if (skip) {
|
|
1149
|
+
assertNonNullable(prevUpperBound, "prevUpperBound is null");
|
|
1150
|
+
output.addRange({
|
|
1151
|
+
type: RangeType.Skip,
|
|
1152
|
+
upperBound: prevUpperBound,
|
|
1153
|
+
});
|
|
1154
|
+
}
|
|
1155
|
+
write();
|
|
1156
|
+
});
|
|
1157
|
+
if (isKept) {
|
|
929
1158
|
skip = false;
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
output.addRange({
|
|
933
|
-
type: RangeType.Skip,
|
|
934
|
-
upperBound: prevUpperBound,
|
|
935
|
-
});
|
|
1159
|
+
nonSkipRangeAdded = true;
|
|
1160
|
+
statedIndex = upper;
|
|
936
1161
|
}
|
|
1162
|
+
return isKept;
|
|
937
1163
|
};
|
|
938
|
-
|
|
939
|
-
|
|
1164
|
+
/**
|
|
1165
|
+
* Closes the frame with a Fingerprint range with InfiniteUpperBound over
|
|
1166
|
+
* the items from the last upper bound the frame states. A frame that cannot
|
|
1167
|
+
* fit a range closes without writing the pending Skip range, because only
|
|
1168
|
+
* this range has room reserved, so it covers the skipped items too.
|
|
1169
|
+
*/
|
|
1170
|
+
const closeWithFingerprint = () => {
|
|
940
1171
|
let fingerprint;
|
|
941
1172
|
try {
|
|
942
|
-
fingerprint = deps.storage.fingerprint(ownerIdBytes,
|
|
1173
|
+
fingerprint = deps.storage.fingerprint(ownerIdBytes, statedIndex, storageSize);
|
|
943
1174
|
}
|
|
944
1175
|
catch (error) {
|
|
945
1176
|
deps.console.error(error);
|
|
946
|
-
return
|
|
1177
|
+
return err(ProtocolErrorCode.SyncError);
|
|
947
1178
|
}
|
|
948
|
-
//
|
|
1179
|
+
// A closing write. Every kept trial reserved room for it.
|
|
949
1180
|
output.addRange({
|
|
950
1181
|
type: RangeType.Fingerprint,
|
|
951
1182
|
upperBound: InfiniteUpperBound,
|
|
952
1183
|
fingerprint,
|
|
953
1184
|
});
|
|
954
|
-
return true;
|
|
1185
|
+
return ok(true);
|
|
955
1186
|
};
|
|
956
1187
|
for (const range of ranges) {
|
|
957
1188
|
const currentUpperBound = range.upperBound;
|
|
@@ -980,24 +1211,24 @@ const sync = (deps) => (ranges, output, ownerIdBytes, onMessage) => {
|
|
|
980
1211
|
}
|
|
981
1212
|
if (eqArrayNumber(range.fingerprint, ourFingerprint)) {
|
|
982
1213
|
skipRange(range);
|
|
1214
|
+
break;
|
|
983
1215
|
}
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
splitRange(deps)(ownerIdBytes, lower, upper, currentUpperBound, output);
|
|
988
|
-
}
|
|
989
|
-
catch (error) {
|
|
990
|
-
if (AbortError.is(error))
|
|
991
|
-
throw error;
|
|
992
|
-
deps.console.error(error);
|
|
993
|
-
return err(ProtocolErrorCode.SyncError);
|
|
994
|
-
}
|
|
1216
|
+
let splitRanges;
|
|
1217
|
+
try {
|
|
1218
|
+
splitRanges = readSplitRanges(deps)(ownerIdBytes, lower, upper, currentUpperBound);
|
|
995
1219
|
}
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1220
|
+
catch (error) {
|
|
1221
|
+
if (AbortError.is(error))
|
|
1222
|
+
throw error;
|
|
1223
|
+
deps.console.error(error);
|
|
1224
|
+
return err(ProtocolErrorCode.SyncError);
|
|
1000
1225
|
}
|
|
1226
|
+
const isSplit = tryWriteRange(upper, () => {
|
|
1227
|
+
for (const splitRange of splitRanges)
|
|
1228
|
+
output.addRange(splitRange);
|
|
1229
|
+
});
|
|
1230
|
+
if (!isSplit)
|
|
1231
|
+
return closeWithFingerprint();
|
|
1001
1232
|
break;
|
|
1002
1233
|
}
|
|
1003
1234
|
case RangeType.Timestamps: {
|
|
@@ -1006,66 +1237,109 @@ const sync = (deps) => (ranges, output, ownerIdBytes, onMessage) => {
|
|
|
1006
1237
|
const ourTimestamps = createTimestampsBuffer();
|
|
1007
1238
|
let exceeded = false;
|
|
1008
1239
|
let iterateFailed = false;
|
|
1240
|
+
// A pending skip stays pending until the range is written.
|
|
1241
|
+
const isSkipPending = skip;
|
|
1242
|
+
// Only a throw from the storage itself is a storage failure. A throw
|
|
1243
|
+
// from the callback, such as from onChangeTooLarge or onMessage, is
|
|
1244
|
+
// a defect, rethrown after iterating.
|
|
1245
|
+
let callbackError = null;
|
|
1009
1246
|
try {
|
|
1010
1247
|
deps.storage.iterate(ownerIdBytes, lower, upper, (timestamp, index) => {
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
timestampsWeNeed.
|
|
1016
|
-
|
|
1017
|
-
else {
|
|
1018
|
-
try {
|
|
1019
|
-
message = {
|
|
1020
|
-
timestamp: timestampBinary,
|
|
1021
|
-
change: deps.storage.readDbChange(ownerIdBytes, timestamp),
|
|
1022
|
-
};
|
|
1248
|
+
try {
|
|
1249
|
+
const timestampString = timestamp.join();
|
|
1250
|
+
const timestampBinary = timestampBytesToTimestamp(timestamp);
|
|
1251
|
+
let message = null;
|
|
1252
|
+
if (timestampsWeNeed.has(timestampString)) {
|
|
1253
|
+
timestampsWeNeed.delete(timestampString);
|
|
1023
1254
|
}
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1255
|
+
else {
|
|
1256
|
+
try {
|
|
1257
|
+
message = {
|
|
1258
|
+
timestamp: timestampBinary,
|
|
1259
|
+
change: deps.storage.readDbChange(ownerIdBytes, timestamp),
|
|
1260
|
+
};
|
|
1261
|
+
}
|
|
1262
|
+
catch (error) {
|
|
1263
|
+
deps.console.error(error);
|
|
1264
|
+
iterateFailed = true;
|
|
1265
|
+
return false;
|
|
1266
|
+
}
|
|
1267
|
+
}
|
|
1268
|
+
// One trial per timestamp: its entry in ourTimestamps and its
|
|
1269
|
+
// message if the peer lacks it, keeping room to write
|
|
1270
|
+
// ourTimestamps as a range after the pending skip.
|
|
1271
|
+
const restoreOurTimestamps = ourTimestamps.checkpoint();
|
|
1272
|
+
ourTimestamps.add(timestampBinary);
|
|
1273
|
+
if (output.tryWrite(() => {
|
|
1274
|
+
if (message)
|
|
1275
|
+
output.addMessage(message);
|
|
1276
|
+
}, getTimestampsRangeReserve(ourTimestamps, isSkipPending))) {
|
|
1277
|
+
if (message)
|
|
1278
|
+
onMessage?.(message);
|
|
1279
|
+
return true;
|
|
1280
|
+
}
|
|
1281
|
+
restoreOurTimestamps();
|
|
1282
|
+
if (message) {
|
|
1283
|
+
// Whether an empty frame can hold the message in the trial
|
|
1284
|
+
// a Timestamps range makes for it, with only its own
|
|
1285
|
+
// timestamp listed and a skip pending. A later round is
|
|
1286
|
+
// guaranteed to reach exactly that frame, so a message that
|
|
1287
|
+
// fits is sent eventually. A trial without a pending skip
|
|
1288
|
+
// reserves 22 bytes less, so a message up to 22 bytes too
|
|
1289
|
+
// large for this check may still be sent in such a frame.
|
|
1290
|
+
const emptyOutput = createEmptyOutput();
|
|
1291
|
+
const emptyOutputTimestamps = createTimestampsBuffer();
|
|
1292
|
+
emptyOutputTimestamps.add(message.timestamp);
|
|
1293
|
+
const fitsEmptyOutput = emptyOutput.tryWrite(() => {
|
|
1294
|
+
emptyOutput.addMessage(message);
|
|
1295
|
+
}, getTimestampsRangeReserve(emptyOutputTimestamps, true));
|
|
1296
|
+
if (!fitsEmptyOutput) {
|
|
1297
|
+
onChangeTooLarge({
|
|
1298
|
+
type: "ProtocolChangeTooLargeError",
|
|
1299
|
+
timestamp: timestampBinary,
|
|
1300
|
+
size: message.change.length,
|
|
1301
|
+
});
|
|
1302
|
+
return true;
|
|
1303
|
+
}
|
|
1028
1304
|
}
|
|
1029
|
-
}
|
|
1030
|
-
if (!output.canAddTimestampsRangeAndMessage(ourTimestamps, message)) {
|
|
1031
1305
|
exceeded = true;
|
|
1032
1306
|
endBound = timestamp;
|
|
1033
1307
|
upper = index;
|
|
1034
1308
|
return false;
|
|
1035
1309
|
}
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
onMessage?.(message);
|
|
1310
|
+
catch (error) {
|
|
1311
|
+
callbackError = { error };
|
|
1312
|
+
return false;
|
|
1040
1313
|
}
|
|
1041
|
-
return true;
|
|
1042
1314
|
});
|
|
1043
1315
|
}
|
|
1044
1316
|
catch (error) {
|
|
1045
1317
|
deps.console.error(error);
|
|
1046
|
-
|
|
1318
|
+
iterateFailed = true;
|
|
1047
1319
|
}
|
|
1320
|
+
if (callbackError)
|
|
1321
|
+
throw callbackError.error;
|
|
1048
1322
|
if (iterateFailed) {
|
|
1049
1323
|
return err(ProtocolErrorCode.SyncError);
|
|
1050
1324
|
}
|
|
1051
|
-
|
|
1052
|
-
|
|
1325
|
+
// When any timestamp was kept, its trial reserved room for this
|
|
1326
|
+
// write. Otherwise the range may not fit, and the frame closes
|
|
1327
|
+
// without it.
|
|
1328
|
+
const tryWriteTimestampsRange = () => tryWriteRange(upper, () => {
|
|
1053
1329
|
output.addRange({
|
|
1054
1330
|
type: RangeType.Timestamps,
|
|
1055
1331
|
upperBound: endBound,
|
|
1056
1332
|
timestamps: ourTimestamps,
|
|
1057
1333
|
});
|
|
1058
|
-
};
|
|
1334
|
+
});
|
|
1059
1335
|
if (exceeded) {
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
return err(ProtocolErrorCode.SyncError);
|
|
1063
|
-
}
|
|
1064
|
-
return ok(true);
|
|
1336
|
+
tryWriteTimestampsRange();
|
|
1337
|
+
return closeWithFingerprint();
|
|
1065
1338
|
}
|
|
1066
1339
|
// If we need something, we have to respond with our timestamps.
|
|
1067
1340
|
if (timestampsWeNeed.size > 0) {
|
|
1068
|
-
|
|
1341
|
+
if (!tryWriteTimestampsRange())
|
|
1342
|
+
return closeWithFingerprint();
|
|
1069
1343
|
}
|
|
1070
1344
|
else {
|
|
1071
1345
|
skipRange(range);
|
|
@@ -1080,7 +1354,29 @@ const sync = (deps) => (ranges, output, ownerIdBytes, onMessage) => {
|
|
|
1080
1354
|
const hasChange = output.getSize() > outputInitialSize;
|
|
1081
1355
|
return ok(hasChange);
|
|
1082
1356
|
};
|
|
1083
|
-
|
|
1357
|
+
// The most a range adds to a frame beyond its payload. Its upper bound adds at
|
|
1358
|
+
// most 20 bytes to the ranges' timestamps: a millis delta varint of up to 7
|
|
1359
|
+
// bytes, as maxMillis has 48 bits; up to 4 for the counter, as a new run is a
|
|
1360
|
+
// varint of up to 3 bytes, Counter being at most 65,535, and a 1-byte run
|
|
1361
|
+
// length, while extending a run adds at most 1 byte; and up to 9 for the
|
|
1362
|
+
// NodeId, as a new run is 8 bytes and a 1-byte run length. Its type takes 1
|
|
1363
|
+
// byte, and the ranges count gains at most 1 byte.
|
|
1364
|
+
const maxRangeOverhead = 22;
|
|
1365
|
+
/**
|
|
1366
|
+
* Returns the bytes a frame must keep to write `timestamps` as a Timestamps
|
|
1367
|
+
* range, after a Skip range when one is pending. A Skip range has no payload,
|
|
1368
|
+
* and the length of `timestamps` is exact.
|
|
1369
|
+
*/
|
|
1370
|
+
const getTimestampsRangeReserve = (timestamps, isSkipPending) => (timestamps.getLength() +
|
|
1371
|
+
maxRangeOverhead +
|
|
1372
|
+
(isSkipPending ? maxRangeOverhead : 0));
|
|
1373
|
+
/**
|
|
1374
|
+
* Reads the ranges that split the items from `lower` to `upper`: one Timestamps
|
|
1375
|
+
* range listing them when they are too few for buckets, otherwise Fingerprint
|
|
1376
|
+
* ranges over the buckets. It only reads storage, so a throw is a storage
|
|
1377
|
+
* failure.
|
|
1378
|
+
*/
|
|
1379
|
+
const readSplitRanges = (deps) => (ownerId, lower, upper, upperBound) => {
|
|
1084
1380
|
const itemCount = NonNegativeInt.orThrow(upper - lower);
|
|
1085
1381
|
const buckets = computeBalancedBuckets(itemCount);
|
|
1086
1382
|
if (!buckets.ok) {
|
|
@@ -1093,30 +1389,43 @@ const splitRange = (deps) => (ownerId, lower, upper, upperBound, buffer) => {
|
|
|
1093
1389
|
range.timestamps.add(timestampBytesToTimestamp(timestamp));
|
|
1094
1390
|
return true;
|
|
1095
1391
|
});
|
|
1096
|
-
|
|
1097
|
-
return;
|
|
1392
|
+
return [range];
|
|
1098
1393
|
}
|
|
1099
1394
|
// Check Storage.ts `fingerprint` and `fingerprintRanges` docs.
|
|
1100
1395
|
const fingerprintRangesBuckets = lower === 0
|
|
1101
1396
|
? buckets.value
|
|
1102
|
-
: [
|
|
1103
|
-
lower,
|
|
1104
|
-
...buckets.value.map((b) => NonNegativeInt.orThrow(b + lower)),
|
|
1105
|
-
];
|
|
1397
|
+
: [lower, ...buckets.value.map((b) => (b + lower))];
|
|
1106
1398
|
const fingerprintRanges = deps.storage.fingerprintRanges(ownerId, fingerprintRangesBuckets, upperBound);
|
|
1107
|
-
|
|
1108
|
-
for (const range of rangesToUse) {
|
|
1109
|
-
buffer.addRange(range);
|
|
1110
|
-
}
|
|
1399
|
+
return lower > 0 ? fingerprintRanges.slice(1) : fingerprintRanges;
|
|
1111
1400
|
};
|
|
1401
|
+
// Twice the largest rangesMaxSize rather than the receiver's own, because
|
|
1402
|
+
// deployed senders exceed theirs. Up to @evolu/common 8.17, sync answered each
|
|
1403
|
+
// Timestamps range that listed timestamps it lacked, while it held none in that
|
|
1404
|
+
// range, with an empty Timestamps range, without checking the size. Such an
|
|
1405
|
+
// echo reuses the peer's bounds with a 1-byte payload, so the echoes take less
|
|
1406
|
+
// than the peer's ranges that list timestamps, which the peer's size checks
|
|
1407
|
+
// kept within its rangesMaxSize. An empty list asks for nothing, so an echo is
|
|
1408
|
+
// never echoed again.
|
|
1409
|
+
const maxRangesSectionSize = 2 * maxProtocolMessageRangesMaxSize;
|
|
1112
1410
|
const decodeRanges = (buffer) => {
|
|
1411
|
+
// The ranges section ends the frame, so it is checked before any allocation.
|
|
1412
|
+
if (buffer.getLength() > maxRangesSectionSize)
|
|
1413
|
+
throw new ProtocolDecodeError(`Ranges section exceeds ${maxRangesSectionSize} bytes`);
|
|
1113
1414
|
if (buffer.getLength() === 0)
|
|
1114
1415
|
return [];
|
|
1115
1416
|
const rangesCount = decodeNonNegativeInt(buffer);
|
|
1116
1417
|
if (rangesCount === 0)
|
|
1117
1418
|
return [];
|
|
1118
|
-
const timestampsCount =
|
|
1419
|
+
const timestampsCount = (rangesCount - 1);
|
|
1119
1420
|
const timestamps = decodeTimestamps(buffer, timestampsCount);
|
|
1421
|
+
// Storage resolves each bound from the owner's first timestamp, so a lower
|
|
1422
|
+
// bound would move back over ranges already answered. Equal bounds are
|
|
1423
|
+
// valid: sync ends a range at the change that did not fit, which can be
|
|
1424
|
+
// where the previous range ended. Timestamps listed in a range are not
|
|
1425
|
+
// checked, because peers before @evolu/common 8.11 list some outside it.
|
|
1426
|
+
for (let i = 1; i < timestamps.length; i++)
|
|
1427
|
+
if (orderTimestamp(timestamps[i - 1], timestamps[i]) > 0)
|
|
1428
|
+
throw new ProtocolDecodeError("Range upper bounds must be non-decreasing");
|
|
1120
1429
|
const rangeTypes = createMutableArray(rangesCount);
|
|
1121
1430
|
for (let i = 0; i < rangesCount; i++) {
|
|
1122
1431
|
const rangeType = decodeNonNegativeInt(buffer);
|
|
@@ -1164,6 +1473,9 @@ const decodeRanges = (buffer) => {
|
|
|
1164
1473
|
};
|
|
1165
1474
|
const decodeTimestamps = (buffer, length) => {
|
|
1166
1475
|
length ??= decodeNonNegativeInt(buffer);
|
|
1476
|
+
// Every timestamp takes at least its 1-byte millis delta.
|
|
1477
|
+
if (length > buffer.getLength())
|
|
1478
|
+
throw new ProtocolDecodeError("Invalid timestamps count");
|
|
1167
1479
|
let previousMillis = 0;
|
|
1168
1480
|
const millises = createMutableArray(length);
|
|
1169
1481
|
for (let i = 0; i < length; i++) {
|
|
@@ -1195,13 +1507,27 @@ const decodeId = (buffer) => {
|
|
|
1195
1507
|
const bytes = buffer.shiftN(idBytesTypeValueLength);
|
|
1196
1508
|
return idBytesToId(bytes);
|
|
1197
1509
|
};
|
|
1510
|
+
/**
|
|
1511
|
+
* The format version that starts every {@link EncryptedDbChange} plaintext.
|
|
1512
|
+
*
|
|
1513
|
+
* It is independent of {@link protocolVersion}, so a new protocol version that
|
|
1514
|
+
* keeps this layout does not make decoders reject changes. 6.0.1-preview.35
|
|
1515
|
+
* wrote 0 with this layout, so decoding accepts every version up to this one.
|
|
1516
|
+
*
|
|
1517
|
+
* Decoders in `@evolu/common` 8.17 and earlier ignore the version. A future
|
|
1518
|
+
* layout must still make them fail, for example with a value they cannot
|
|
1519
|
+
* decode, rather than let them decode wrong values.
|
|
1520
|
+
*/
|
|
1521
|
+
const encryptedDbChangeVersion = onePositiveInt;
|
|
1198
1522
|
/**
|
|
1199
1523
|
* Encodes and encrypts a {@link DbChange} using the provided owner's encryption
|
|
1200
1524
|
* key. Returns an encrypted binary representation as {@link EncryptedDbChange}.
|
|
1201
1525
|
*
|
|
1202
|
-
* The
|
|
1203
|
-
*
|
|
1204
|
-
*
|
|
1526
|
+
* The plaintext starts with the format version of the change, which is
|
|
1527
|
+
* independent of {@link protocolVersion}, and the timestamp, which proves that
|
|
1528
|
+
* the change belongs to the timestamp it is sent with.
|
|
1529
|
+
* {@link decryptAndDecodeDbChange} rejects a newer format version and accepts
|
|
1530
|
+
* older ones.
|
|
1205
1531
|
*/
|
|
1206
1532
|
export const encodeAndEncryptDbChange = (deps) => (message, key) => {
|
|
1207
1533
|
const buffer = createBuffer();
|
|
@@ -1224,7 +1550,7 @@ export const encodeAndEncryptDbChange = (deps) => (message, key) => {
|
|
|
1224
1550
|
* within it fits one protocol message.
|
|
1225
1551
|
*/
|
|
1226
1552
|
export const encodeDbChange = (buffer, message) => {
|
|
1227
|
-
encodeNonNegativeInt(buffer,
|
|
1553
|
+
encodeNonNegativeInt(buffer, encryptedDbChangeVersion);
|
|
1228
1554
|
// Encode the timestamp to prevent tampering (e.g., a malicious relay
|
|
1229
1555
|
// assigning this EncryptedDbChange to a different EncryptedCrdtMessage)
|
|
1230
1556
|
buffer.extend(timestampToTimestampBytes(message.timestamp));
|
|
@@ -1239,8 +1565,9 @@ export const encodeDbChange = (buffer, message) => {
|
|
|
1239
1565
|
const entries = objectToEntries(message.change.values);
|
|
1240
1566
|
encodeLength(buffer, entries);
|
|
1241
1567
|
for (const [column, value] of entries) {
|
|
1242
|
-
assertNotUndefined(value);
|
|
1243
1568
|
encodeString(buffer, column);
|
|
1569
|
+
// DbChange validated every value as a SqliteValue; only its Partial record
|
|
1570
|
+
// type admits undefined.
|
|
1244
1571
|
encodeSqliteValue(buffer, value);
|
|
1245
1572
|
}
|
|
1246
1573
|
};
|
|
@@ -1248,20 +1575,23 @@ export const encodeDbChange = (buffer, message) => {
|
|
|
1248
1575
|
* Decrypts and decodes an {@link EncryptedCrdtMessage} using the provided
|
|
1249
1576
|
* owner's encryption key. Verifies that the embedded timestamp matches the
|
|
1250
1577
|
* expected timestamp to ensure message integrity.
|
|
1578
|
+
*
|
|
1579
|
+
* A change with a newer format version than {@link encodeAndEncryptDbChange}
|
|
1580
|
+
* writes is a {@link ProtocolInvalidDataError}.
|
|
1251
1581
|
*/
|
|
1252
1582
|
export const decryptAndDecodeDbChange = (message, key) => {
|
|
1253
1583
|
try {
|
|
1254
1584
|
const buffer = createBuffer(message.change);
|
|
1255
1585
|
const nonce = buffer.shiftN(xChaCha20Poly1305NonceLength);
|
|
1256
1586
|
const ciphertext = buffer.shiftN(decodeLength(buffer));
|
|
1257
|
-
const plaintextBytes = decryptWithXChaCha20Poly1305(
|
|
1587
|
+
const plaintextBytes = decryptWithXChaCha20Poly1305(ciphertext, nonce, key);
|
|
1258
1588
|
if (!plaintextBytes.ok)
|
|
1259
1589
|
return plaintextBytes;
|
|
1260
1590
|
buffer.reset();
|
|
1261
1591
|
buffer.extend(plaintextBytes.value);
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
const timestamp = timestampBytesToTimestamp(
|
|
1592
|
+
if (decodeNonNegativeInt(buffer) > encryptedDbChangeVersion)
|
|
1593
|
+
throw new ProtocolDecodeError("Unsupported EncryptedDbChange version");
|
|
1594
|
+
const timestamp = timestampBytesToTimestamp(buffer.shiftN(timestampBytesLength));
|
|
1265
1595
|
if (!eqTimestamp(timestamp, message.timestamp)) {
|
|
1266
1596
|
return err({
|
|
1267
1597
|
type: "ProtocolTimestampMismatchError",
|
|
@@ -1269,7 +1599,7 @@ export const decryptAndDecodeDbChange = (message, key) => {
|
|
|
1269
1599
|
timestamp,
|
|
1270
1600
|
});
|
|
1271
1601
|
}
|
|
1272
|
-
const flags = decodeFlags(buffer,
|
|
1602
|
+
const flags = decodeFlags(buffer, 3);
|
|
1273
1603
|
const table = decodeString(buffer);
|
|
1274
1604
|
const id = decodeId(buffer);
|
|
1275
1605
|
const length = decodeLength(buffer);
|
|
@@ -1296,26 +1626,6 @@ export const decryptAndDecodeDbChange = (message, key) => {
|
|
|
1296
1626
|
});
|
|
1297
1627
|
}
|
|
1298
1628
|
};
|
|
1299
|
-
/**
|
|
1300
|
-
* Decodes a ProtocolMessage into a readable JSON object for debugging.
|
|
1301
|
-
*
|
|
1302
|
-
* Note: This is a stub for future implementation. It should use:
|
|
1303
|
-
*
|
|
1304
|
-
* - DecodeVersionAndOwner
|
|
1305
|
-
* - DecodeError or decodeWriteKeys (depending on context)
|
|
1306
|
-
* - DecodeMessages
|
|
1307
|
-
* - DecodeRanges
|
|
1308
|
-
*
|
|
1309
|
-
* If you want to help, please contribute to this function.
|
|
1310
|
-
*/
|
|
1311
|
-
export const decodeProtocolMessageToJson = (_protocolMessage, _isInitiator) => {
|
|
1312
|
-
// TODO: Implement using
|
|
1313
|
-
// - decodeVersionAndOwner
|
|
1314
|
-
// -- decodeError or decodeWriteKeys (should be refactored out),
|
|
1315
|
-
// -- decodeMessages, and decodeRanges.
|
|
1316
|
-
// This is a stub for PRs and community contributions.
|
|
1317
|
-
throw new Error("decodeProtocolMessageToJson is not implemented yet.");
|
|
1318
|
-
};
|
|
1319
1629
|
// Small ints are encoded into ProtocolValueType, saving one byte per int.
|
|
1320
1630
|
export const ProtocolValueType = {
|
|
1321
1631
|
// 0-19 small ints
|
|
@@ -1363,7 +1673,7 @@ export const encodeSqliteValue = (buffer, value) => {
|
|
|
1363
1673
|
}
|
|
1364
1674
|
else {
|
|
1365
1675
|
encodeNonNegativeInt(buffer, ProtocolValueType.DateIsoWithNegativeTime);
|
|
1366
|
-
encodeNumber(buffer,
|
|
1676
|
+
encodeNumber(buffer, time);
|
|
1367
1677
|
}
|
|
1368
1678
|
return;
|
|
1369
1679
|
}
|
|
@@ -1376,7 +1686,7 @@ export const encodeSqliteValue = (buffer, value) => {
|
|
|
1376
1686
|
const json = Json.from.parent(value);
|
|
1377
1687
|
if (json.ok) {
|
|
1378
1688
|
const jsonValue = jsonToJsonValue(json.value);
|
|
1379
|
-
jsonBuffer
|
|
1689
|
+
const jsonBuffer = createBuffer();
|
|
1380
1690
|
try {
|
|
1381
1691
|
// Encoding first rejects nesting deeper than decoding allows, before
|
|
1382
1692
|
// the recursive JSON.stringify below could overflow the stack. Such
|
|
@@ -1398,9 +1708,6 @@ export const encodeSqliteValue = (buffer, value) => {
|
|
|
1398
1708
|
if (!(error instanceof BufferError))
|
|
1399
1709
|
throw error;
|
|
1400
1710
|
}
|
|
1401
|
-
finally {
|
|
1402
|
-
jsonBuffer.reset();
|
|
1403
|
-
}
|
|
1404
1711
|
}
|
|
1405
1712
|
const base64Url = Base64Url.from.parent(value);
|
|
1406
1713
|
if (base64Url.ok) {
|
|
@@ -1410,6 +1717,10 @@ export const encodeSqliteValue = (buffer, value) => {
|
|
|
1410
1717
|
buffer.extend(bytes);
|
|
1411
1718
|
return;
|
|
1412
1719
|
}
|
|
1720
|
+
// encodeString replaces a lone surrogate with U+FFFD, so the bytes are
|
|
1721
|
+
// always valid UTF-8. Encoding WTF-8 instead would make peers disagree:
|
|
1722
|
+
// deployed decoders read a lone surrogate in WTF-8 as three U+FFFD,
|
|
1723
|
+
// while a decoder that kept it would read the surrogate.
|
|
1413
1724
|
encodeNonNegativeInt(buffer, ProtocolValueType.String);
|
|
1414
1725
|
encodeString(buffer, value);
|
|
1415
1726
|
return;
|
|
@@ -1485,5 +1796,4 @@ export const decodeSqliteValue = (buffer) => {
|
|
|
1485
1796
|
throw new ProtocolDecodeError("invalid ProtocolValueType");
|
|
1486
1797
|
}
|
|
1487
1798
|
};
|
|
1488
|
-
const jsonBuffer = createBuffer();
|
|
1489
1799
|
const isSmallInt = (value) => value >= 0 && value < 20;
|