@evolu/common 8.17.0 → 8.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/Bytes.d.ts +31 -14
- package/dist/src/Bytes.d.ts.map +1 -1
- package/dist/src/Bytes.js +48 -8
- package/dist/src/Error.d.ts.map +1 -1
- package/dist/src/Error.js +5 -1
- package/dist/src/Polyfills.d.ts.map +1 -1
- package/dist/src/Polyfills.js +3 -2
- package/dist/src/Sqlite.d.ts +1 -1
- package/dist/src/Task.d.ts +4 -3
- package/dist/src/Task.d.ts.map +1 -1
- package/dist/src/Task.js +4 -3
- package/dist/src/index.d.ts +1 -1
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/local-first/Db.d.ts +32 -3
- package/dist/src/local-first/Db.d.ts.map +1 -1
- package/dist/src/local-first/Db.js +30 -9
- package/dist/src/local-first/Evolu.d.ts +21 -6
- package/dist/src/local-first/Evolu.d.ts.map +1 -1
- package/dist/src/local-first/Evolu.js +5 -1
- package/dist/src/local-first/Protocol.d.ts +215 -93
- package/dist/src/local-first/Protocol.d.ts.map +1 -1
- package/dist/src/local-first/Protocol.js +799 -489
- package/dist/src/local-first/Relay.d.ts +9 -1
- package/dist/src/local-first/Relay.d.ts.map +1 -1
- package/dist/src/local-first/Relay.js +41 -36
- package/dist/src/local-first/Shared.d.ts +79 -53
- package/dist/src/local-first/Shared.d.ts.map +1 -1
- package/dist/src/local-first/Shared.js +67 -42
- package/dist/src/local-first/Storage.d.ts +80 -18
- package/dist/src/local-first/Storage.d.ts.map +1 -1
- package/dist/src/local-first/Storage.js +20 -4
- package/package.json +1 -1
- package/src/Bytes.test.ts +212 -0
- package/src/Bytes.ts +67 -17
- package/src/Error.ts +5 -1
- package/src/Polyfills.ts +3 -2
- package/src/Sqlite.ts +1 -1
- package/src/Task.ts +4 -3
- package/src/index.ts +4 -1
- package/src/local-first/Db.ts +76 -17
- package/src/local-first/Evolu.test.ts +43 -12
- package/src/local-first/Evolu.ts +34 -7
- package/src/local-first/Protocol.test.ts +1541 -44
- package/src/local-first/Protocol.ts +1070 -605
- package/src/local-first/Relay.ts +80 -69
- package/src/local-first/Shared.test.ts +37 -1
- package/src/local-first/Shared.ts +124 -73
- package/src/local-first/Storage.ts +100 -27
package/src/Bytes.test.ts
CHANGED
|
@@ -1,10 +1,15 @@
|
|
|
1
|
+
import * as fc from "fast-check";
|
|
1
2
|
import { describe, it, mock } from "node:test";
|
|
3
|
+
import { setFlagsFromString } from "node:v8";
|
|
4
|
+
import { runInNewContext } from "node:vm";
|
|
2
5
|
import {
|
|
6
|
+
assert,
|
|
3
7
|
assertEqual,
|
|
4
8
|
assertEqualBytes,
|
|
5
9
|
assertErr,
|
|
6
10
|
assertFalse,
|
|
7
11
|
assertInstanceOf,
|
|
12
|
+
assertNonNullable,
|
|
8
13
|
assertOk,
|
|
9
14
|
assertSame,
|
|
10
15
|
assertThrowsInstanceOf,
|
|
@@ -95,6 +100,23 @@ describe("BufferError", () => {
|
|
|
95
100
|
assertEqual(error.name, "BufferError");
|
|
96
101
|
assertEqual(error.message, "test error");
|
|
97
102
|
});
|
|
103
|
+
|
|
104
|
+
it("constructs without Error.captureStackTrace", () => {
|
|
105
|
+
const descriptor = globalThis.Object.getOwnPropertyDescriptor(
|
|
106
|
+
Error,
|
|
107
|
+
"captureStackTrace",
|
|
108
|
+
);
|
|
109
|
+
assertNonNullable(descriptor);
|
|
110
|
+
Reflect.deleteProperty(Error, "captureStackTrace");
|
|
111
|
+
try {
|
|
112
|
+
const error = new BufferError("x");
|
|
113
|
+
assertInstanceOf(error, BufferError);
|
|
114
|
+
assertEqual(error.message, "x");
|
|
115
|
+
assertThrowsInstanceOf(() => decodeJson([0xc1]), BufferError);
|
|
116
|
+
} finally {
|
|
117
|
+
globalThis.Object.defineProperty(Error, "captureStackTrace", descriptor);
|
|
118
|
+
}
|
|
119
|
+
});
|
|
98
120
|
});
|
|
99
121
|
|
|
100
122
|
describe("concatByteArrays", () => {
|
|
@@ -279,6 +301,16 @@ describe("Buffer", () => {
|
|
|
279
301
|
assertEqual(buffer.getLength(), 2);
|
|
280
302
|
assertEqualBytes(buffer.unwrap(), [6, 7]);
|
|
281
303
|
});
|
|
304
|
+
|
|
305
|
+
it("has readonly members", () => {
|
|
306
|
+
const buffer = createBuffer();
|
|
307
|
+
const { shift } = buffer;
|
|
308
|
+
|
|
309
|
+
// @ts-expect-error Cannot assign to 'shift' because it is a read-only property.
|
|
310
|
+
buffer.shift = shift;
|
|
311
|
+
|
|
312
|
+
assertSame(buffer.shift, shift);
|
|
313
|
+
});
|
|
282
314
|
});
|
|
283
315
|
|
|
284
316
|
describe("number encoding", () => {
|
|
@@ -482,6 +514,31 @@ describe("string encoding", () => {
|
|
|
482
514
|
);
|
|
483
515
|
assertEqual(decodeString(buffer), value);
|
|
484
516
|
});
|
|
517
|
+
|
|
518
|
+
// Unlike the JSON codec, the string codec keeps the wire valid UTF-8, because
|
|
519
|
+
// deployed decoders read WTF-8 as three U+FFFD per lone surrogate.
|
|
520
|
+
it("replaces a lone surrogate with U+FFFD", () => {
|
|
521
|
+
const buffer = createBuffer();
|
|
522
|
+
encodeString(buffer, "a\uD800b");
|
|
523
|
+
|
|
524
|
+
assertEqualBytes(buffer.unwrap(), [5, 97, 0xef, 0xbf, 0xbd, 98]);
|
|
525
|
+
assertSame(decodeString(buffer), "a�b");
|
|
526
|
+
});
|
|
527
|
+
|
|
528
|
+
it("decodes invalid UTF-8 as U+FFFD", () => {
|
|
529
|
+
assertSame(
|
|
530
|
+
decodeString(createBuffer([5, 97, 0xed, 0xa0, 0x80, 98])),
|
|
531
|
+
"a���b",
|
|
532
|
+
);
|
|
533
|
+
});
|
|
534
|
+
|
|
535
|
+
it("keeps a byte order mark", () => {
|
|
536
|
+
for (const value of ["\uFEFFabc", "\uFEFF", "a\uFEFFb"]) {
|
|
537
|
+
const buffer = createBuffer();
|
|
538
|
+
encodeString(buffer, value);
|
|
539
|
+
assertSame(decodeString(buffer), value);
|
|
540
|
+
}
|
|
541
|
+
});
|
|
485
542
|
});
|
|
486
543
|
|
|
487
544
|
describe("run-length encoding", () => {
|
|
@@ -574,6 +631,140 @@ describe("run-length encoding", () => {
|
|
|
574
631
|
});
|
|
575
632
|
});
|
|
576
633
|
|
|
634
|
+
describe("run-length encoder checkpoint", () => {
|
|
635
|
+
const setupEncoder = (values: ReadonlyArray<number> = []) => {
|
|
636
|
+
const encoder =
|
|
637
|
+
createRunLengthEncoder<NonNegativeInt>(encodeNonNegativeInt);
|
|
638
|
+
for (const value of values) encoder.add(NonNegativeInt.orThrow(value));
|
|
639
|
+
return encoder;
|
|
640
|
+
};
|
|
641
|
+
|
|
642
|
+
const decodeEncoder = (
|
|
643
|
+
encoder: ReturnType<typeof setupEncoder>,
|
|
644
|
+
length: number,
|
|
645
|
+
) => {
|
|
646
|
+
const buffer = createBuffer(encoder.unwrap());
|
|
647
|
+
const values = decodeRle(buffer, NonNegativeInt.orThrow(length), () =>
|
|
648
|
+
decodeNonNegativeInt(buffer),
|
|
649
|
+
);
|
|
650
|
+
assertSame(buffer.getLength(), 0);
|
|
651
|
+
return values;
|
|
652
|
+
};
|
|
653
|
+
|
|
654
|
+
it("restores a run rewritten in place", () => {
|
|
655
|
+
const encoder = setupEncoder([7, 5]);
|
|
656
|
+
const restore = encoder.checkpoint();
|
|
657
|
+
encoder.add(NonNegativeInt.orThrow(5));
|
|
658
|
+
encoder.add(NonNegativeInt.orThrow(5));
|
|
659
|
+
assertEqualBytes(encoder.unwrap(), [7, 1, 5, 3]);
|
|
660
|
+
|
|
661
|
+
restore();
|
|
662
|
+
assertEqualBytes(encoder.unwrap(), [7, 1, 5, 1]);
|
|
663
|
+
|
|
664
|
+
encoder.add(NonNegativeInt.orThrow(5));
|
|
665
|
+
assertEqualBytes(encoder.unwrap(), setupEncoder([7, 5, 5]).unwrap());
|
|
666
|
+
assertEqual(decodeEncoder(encoder, 3), [7, 5, 5]);
|
|
667
|
+
});
|
|
668
|
+
|
|
669
|
+
it("restores a run length across the 127/128 varint boundary", () => {
|
|
670
|
+
const run127 = Array.from({ length: 127 }, () => 5);
|
|
671
|
+
const encoder = setupEncoder(run127);
|
|
672
|
+
const restoreAt127 = encoder.checkpoint();
|
|
673
|
+
encoder.add(NonNegativeInt.orThrow(5));
|
|
674
|
+
assertEqualBytes(encoder.unwrap(), [5, 128, 1]);
|
|
675
|
+
const restoreAt128 = encoder.checkpoint();
|
|
676
|
+
encoder.add(NonNegativeInt.orThrow(5));
|
|
677
|
+
assertEqualBytes(encoder.unwrap(), [5, 129, 1]);
|
|
678
|
+
|
|
679
|
+
restoreAt128();
|
|
680
|
+
assertEqualBytes(encoder.unwrap(), [5, 128, 1]);
|
|
681
|
+
|
|
682
|
+
restoreAt127();
|
|
683
|
+
assertEqualBytes(encoder.unwrap(), [5, 127]);
|
|
684
|
+
assertEqual(encoder.getLength(), 2);
|
|
685
|
+
|
|
686
|
+
encoder.add(NonNegativeInt.orThrow(6));
|
|
687
|
+
assertEqualBytes(encoder.unwrap(), [5, 127, 6, 1]);
|
|
688
|
+
assertEqual(decodeEncoder(encoder, 128), [...run127, 6]);
|
|
689
|
+
});
|
|
690
|
+
|
|
691
|
+
it("restores the previous value after a different value", () => {
|
|
692
|
+
const encoder = setupEncoder([5]);
|
|
693
|
+
const restore = encoder.checkpoint();
|
|
694
|
+
encoder.add(NonNegativeInt.orThrow(7));
|
|
695
|
+
encoder.add(NonNegativeInt.orThrow(7));
|
|
696
|
+
assertEqualBytes(encoder.unwrap(), [5, 1, 7, 2]);
|
|
697
|
+
|
|
698
|
+
restore();
|
|
699
|
+
assertEqualBytes(encoder.unwrap(), [5, 1]);
|
|
700
|
+
|
|
701
|
+
// Without the previous value, the next 5 would start a new run.
|
|
702
|
+
encoder.add(NonNegativeInt.orThrow(5));
|
|
703
|
+
assertEqualBytes(encoder.unwrap(), [5, 2]);
|
|
704
|
+
});
|
|
705
|
+
|
|
706
|
+
it("restores an empty encoder", () => {
|
|
707
|
+
const encoder = setupEncoder();
|
|
708
|
+
const restore = encoder.checkpoint();
|
|
709
|
+
encoder.add(NonNegativeInt.orThrow(5));
|
|
710
|
+
|
|
711
|
+
restore();
|
|
712
|
+
assertEqual(encoder.getLength(), 0);
|
|
713
|
+
assertEqualBytes(encoder.unwrap(), []);
|
|
714
|
+
|
|
715
|
+
encoder.add(NonNegativeInt.orThrow(5));
|
|
716
|
+
assertEqualBytes(encoder.unwrap(), [5, 1]);
|
|
717
|
+
});
|
|
718
|
+
|
|
719
|
+
it("encodes only the committed values after restores", () => {
|
|
720
|
+
fc.assert(
|
|
721
|
+
fc.property(
|
|
722
|
+
fc.array(
|
|
723
|
+
fc.oneof(
|
|
724
|
+
fc.record({
|
|
725
|
+
value: fc.constantFrom(0, 5, 300),
|
|
726
|
+
count: fc.integer({ min: 1, max: 200 }),
|
|
727
|
+
}),
|
|
728
|
+
fc.constant("checkpoint" as const),
|
|
729
|
+
fc.constant("restore" as const),
|
|
730
|
+
),
|
|
731
|
+
{ maxLength: 30 },
|
|
732
|
+
),
|
|
733
|
+
(operations) => {
|
|
734
|
+
const encoder = setupEncoder();
|
|
735
|
+
const committed: Array<number> = [];
|
|
736
|
+
const checkpoints: Array<{
|
|
737
|
+
readonly restore: () => void;
|
|
738
|
+
readonly length: number;
|
|
739
|
+
}> = [];
|
|
740
|
+
|
|
741
|
+
for (const operation of operations) {
|
|
742
|
+
if (operation === "checkpoint") {
|
|
743
|
+
checkpoints.push({
|
|
744
|
+
restore: encoder.checkpoint(),
|
|
745
|
+
length: committed.length,
|
|
746
|
+
});
|
|
747
|
+
} else if (operation === "restore") {
|
|
748
|
+
const checkpoint = checkpoints.pop();
|
|
749
|
+
if (!checkpoint) continue;
|
|
750
|
+
checkpoint.restore();
|
|
751
|
+
committed.length = checkpoint.length;
|
|
752
|
+
} else {
|
|
753
|
+
for (let i = 0; i < operation.count; i++) {
|
|
754
|
+
encoder.add(NonNegativeInt.orThrow(operation.value));
|
|
755
|
+
committed.push(operation.value);
|
|
756
|
+
}
|
|
757
|
+
}
|
|
758
|
+
}
|
|
759
|
+
|
|
760
|
+
assertEqualBytes(encoder.unwrap(), setupEncoder(committed).unwrap());
|
|
761
|
+
assertEqual(decodeEncoder(encoder, committed.length), committed);
|
|
762
|
+
},
|
|
763
|
+
),
|
|
764
|
+
);
|
|
765
|
+
});
|
|
766
|
+
});
|
|
767
|
+
|
|
577
768
|
describe("JSON binary codec", () => {
|
|
578
769
|
it("matches fixed msgpackr 2.0.5 fixtures", () => {
|
|
579
770
|
const fixtures: ReadonlyArray<readonly [unknown, ReadonlyArray<number>]> = [
|
|
@@ -1164,6 +1355,27 @@ describe("JSON binary codec", () => {
|
|
|
1164
1355
|
}
|
|
1165
1356
|
});
|
|
1166
1357
|
|
|
1358
|
+
it("releases an oversized encoder arena", () => {
|
|
1359
|
+
setFlagsFromString("--expose-gc");
|
|
1360
|
+
const gc = runInNewContext("gc") as () => void;
|
|
1361
|
+
gc();
|
|
1362
|
+
gc();
|
|
1363
|
+
const before = process.memoryUsage().arrayBuffers;
|
|
1364
|
+
const buffer = createBuffer();
|
|
1365
|
+
|
|
1366
|
+
// A string reserves 3 bytes per UTF-16 code unit, so this one grows the
|
|
1367
|
+
// module-scope arena to 6 MB.
|
|
1368
|
+
encodeJsonValue(buffer, JsonValue.orThrow("x".repeat(2_000_000)));
|
|
1369
|
+
gc();
|
|
1370
|
+
gc();
|
|
1371
|
+
|
|
1372
|
+
const retained =
|
|
1373
|
+
process.memoryUsage().arrayBuffers - before - buffer.getCapacity();
|
|
1374
|
+
assert(retained < 1_000_000, "Expected the arena to be released.", {
|
|
1375
|
+
actual: retained,
|
|
1376
|
+
});
|
|
1377
|
+
});
|
|
1378
|
+
|
|
1167
1379
|
it("converts unexpected decoder errors to BufferError", () => {
|
|
1168
1380
|
const errorBuffer = {
|
|
1169
1381
|
...createBuffer([0xc0]),
|
package/src/Bytes.ts
CHANGED
|
@@ -103,16 +103,11 @@ export const concatByteArrays = (
|
|
|
103
103
|
return result;
|
|
104
104
|
};
|
|
105
105
|
|
|
106
|
-
/**
|
|
107
|
-
* Custom error for {@link Buffer}-related failures like premature end of data.
|
|
108
|
-
* Provides better stack traces for debugging binary protocol issues.
|
|
109
|
-
*/
|
|
106
|
+
/** Custom error for {@link Buffer}-related failures like premature end of data. */
|
|
110
107
|
export class BufferError extends Error {
|
|
111
108
|
constructor(message: string) {
|
|
112
109
|
super(message);
|
|
113
110
|
this.name = this.constructor.name;
|
|
114
|
-
|
|
115
|
-
Error.captureStackTrace(this, this.constructor);
|
|
116
111
|
}
|
|
117
112
|
}
|
|
118
113
|
|
|
@@ -173,34 +168,34 @@ export class BufferError extends Error {
|
|
|
173
168
|
*/
|
|
174
169
|
export interface Buffer {
|
|
175
170
|
/** Returns the current capacity of the buffer. */
|
|
176
|
-
getCapacity: () => NonNegativeInt;
|
|
171
|
+
readonly getCapacity: () => NonNegativeInt;
|
|
177
172
|
|
|
178
173
|
/** Returns the current number of bytes stored in the buffer. */
|
|
179
|
-
getLength: () => NonNegativeInt;
|
|
174
|
+
readonly getLength: () => NonNegativeInt;
|
|
180
175
|
|
|
181
176
|
/**
|
|
182
177
|
* Appends binary data to the buffer, resizing if necessary. Throws if
|
|
183
178
|
* `arg.length` is not a non-negative safe integer.
|
|
184
179
|
*/
|
|
185
|
-
extend: (arg: Uint8Array | ArrayLike<number>) => void;
|
|
180
|
+
readonly extend: (arg: Uint8Array | ArrayLike<number>) => void;
|
|
186
181
|
|
|
187
182
|
/**
|
|
188
183
|
* Removes and returns the first byte. Throws an `Error` with message "Buffer
|
|
189
184
|
* parse ended prematurely" if the buffer is empty.
|
|
190
185
|
*/
|
|
191
|
-
shift: () => NonNegativeInt;
|
|
186
|
+
readonly shift: () => NonNegativeInt;
|
|
192
187
|
|
|
193
188
|
/**
|
|
194
189
|
* Removes and returns the first `n` bytes. Throws an `Error` with message
|
|
195
190
|
* "Buffer parse ended prematurely" if fewer than `n` bytes remain.
|
|
196
191
|
*/
|
|
197
|
-
shiftN: (n: NonNegativeInt) => Uint8Array;
|
|
192
|
+
readonly shiftN: (n: NonNegativeInt) => Uint8Array;
|
|
198
193
|
|
|
199
194
|
/**
|
|
200
195
|
* Truncates the buffer to the specified length, discarding data from the end.
|
|
201
196
|
* Throws if the new length is greater than the current length.
|
|
202
197
|
*/
|
|
203
|
-
truncate: (length: NonNegativeInt) => void;
|
|
198
|
+
readonly truncate: (length: NonNegativeInt) => void;
|
|
204
199
|
|
|
205
200
|
/**
|
|
206
201
|
* Resets the buffer to its initial empty state, preserving its capacity.
|
|
@@ -209,14 +204,14 @@ export interface Buffer {
|
|
|
209
204
|
* when you want to clear the buffer and write new data, avoiding unnecessary
|
|
210
205
|
* allocations.
|
|
211
206
|
*/
|
|
212
|
-
reset: () => void;
|
|
207
|
+
readonly reset: () => void;
|
|
213
208
|
|
|
214
209
|
/**
|
|
215
210
|
* Returns a view of the buffer’s current data. Do not modify this array, as
|
|
216
211
|
* it directly alters the buffer’s internal state, potentially breaking
|
|
217
212
|
* subsequent operations.
|
|
218
213
|
*/
|
|
219
|
-
unwrap: () => Uint8Array;
|
|
214
|
+
readonly unwrap: () => Uint8Array;
|
|
220
215
|
}
|
|
221
216
|
|
|
222
217
|
/** Creates a {@link Buffer} for efficient byte operations. */
|
|
@@ -438,18 +433,37 @@ export const encodeLength = (
|
|
|
438
433
|
/** Decodes an array-like value length. */
|
|
439
434
|
export const decodeLength = decodeNonNegativeInt;
|
|
440
435
|
|
|
441
|
-
/**
|
|
436
|
+
/**
|
|
437
|
+
* Encodes a length-prefixed UTF-8 string.
|
|
438
|
+
*
|
|
439
|
+
* A lone surrogate becomes U+FFFD, so the bytes are always valid UTF-8. Unlike
|
|
440
|
+
* {@link encodeJsonValue}, which keeps lone surrogates, this does not round-trip
|
|
441
|
+
* every JavaScript string.
|
|
442
|
+
*/
|
|
442
443
|
export const encodeString = (buffer: Buffer, value: string): void => {
|
|
443
444
|
const bytes = utf8ToBytes(value);
|
|
444
445
|
encodeLength(buffer, bytes);
|
|
445
446
|
buffer.extend(bytes);
|
|
446
447
|
};
|
|
447
448
|
|
|
448
|
-
/**
|
|
449
|
+
/**
|
|
450
|
+
* Decodes a length-prefixed UTF-8 string.
|
|
451
|
+
*
|
|
452
|
+
* Invalid UTF-8 decodes as U+FFFD instead of throwing. A leading U+FEFF is
|
|
453
|
+
* kept. Decoders in `@evolu/common` 8.17 and earlier drop it, so such peers
|
|
454
|
+
* still store a string that starts with it without the mark.
|
|
455
|
+
*/
|
|
449
456
|
export const decodeString = (buffer: Buffer): string => {
|
|
450
457
|
const length = decodeLength(buffer);
|
|
451
458
|
const bytes = buffer.shiftN(length);
|
|
452
|
-
|
|
459
|
+
const string = bytesToUtf8(bytes);
|
|
460
|
+
// bytesToUtf8 uses the default TextDecoder, which drops a leading U+FEFF
|
|
461
|
+
// (EF BB BF in UTF-8), so it is restored here. Passing { ignoreBOM: true }
|
|
462
|
+
// to a TextDecoder would also keep it, but costs more compiler types in the
|
|
463
|
+
// type benchmark.
|
|
464
|
+
return bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf
|
|
465
|
+
? `\uFEFF${string}`
|
|
466
|
+
: string;
|
|
453
467
|
};
|
|
454
468
|
|
|
455
469
|
/** Incrementally encodes consecutive equal values using run-length encoding. */
|
|
@@ -457,6 +471,15 @@ export interface RunLengthEncoder<T> {
|
|
|
457
471
|
readonly add: (value: T) => void;
|
|
458
472
|
readonly getLength: () => NonNegativeInt;
|
|
459
473
|
readonly unwrap: () => Uint8Array;
|
|
474
|
+
|
|
475
|
+
/**
|
|
476
|
+
* Returns a function that restores the encoder, in constant time, to the
|
|
477
|
+
* state it had when this was called, discarding every value added since.
|
|
478
|
+
*
|
|
479
|
+
* A restore function can be called repeatedly. It is valid until the encoder
|
|
480
|
+
* is restored to an earlier checkpoint.
|
|
481
|
+
*/
|
|
482
|
+
readonly checkpoint: () => () => void;
|
|
460
483
|
}
|
|
461
484
|
|
|
462
485
|
/** Creates an incremental run-length encoder. */
|
|
@@ -485,6 +508,26 @@ export const createRunLengthEncoder = <T>(
|
|
|
485
508
|
getLength: () => buffer.getLength(),
|
|
486
509
|
|
|
487
510
|
unwrap: () => buffer.unwrap(),
|
|
511
|
+
|
|
512
|
+
checkpoint: () => {
|
|
513
|
+
const checkpointLength = previousLength;
|
|
514
|
+
const checkpointValue = previousValue;
|
|
515
|
+
const checkpointRunLength = runLength;
|
|
516
|
+
|
|
517
|
+
return () => {
|
|
518
|
+
// `add` rewrites the last run in place, so the bytes after its start
|
|
519
|
+
// may differ from the checkpoint's even at the same length. Bytes
|
|
520
|
+
// before it never change, so the last run is encoded again.
|
|
521
|
+
buffer.truncate(checkpointLength);
|
|
522
|
+
previousLength = checkpointLength;
|
|
523
|
+
previousValue = checkpointValue;
|
|
524
|
+
runLength = checkpointRunLength;
|
|
525
|
+
if (runLength > 0) {
|
|
526
|
+
encodeValue(buffer, previousValue as T);
|
|
527
|
+
encodeNonNegativeInt(buffer, runLength);
|
|
528
|
+
}
|
|
529
|
+
};
|
|
530
|
+
},
|
|
488
531
|
};
|
|
489
532
|
};
|
|
490
533
|
|
|
@@ -557,6 +600,9 @@ interface JsonKeyCacheEntry {
|
|
|
557
600
|
const jsonKeyCacheSize = 4096;
|
|
558
601
|
const maxCachedJsonKeyByteLength = 32;
|
|
559
602
|
const maxJsonNestingDepth = 1_000;
|
|
603
|
+
// A string reserves 3 bytes per UTF-16 code unit, so one large value would
|
|
604
|
+
// otherwise keep several times its size allocated after encoding.
|
|
605
|
+
const maxRetainedJsonEncoderLength = 1024 * 1024;
|
|
560
606
|
let jsonEncoderTarget = new Uint8Array(8192);
|
|
561
607
|
let jsonEncoderTargetView = new DataView(jsonEncoderTarget.buffer);
|
|
562
608
|
let jsonEncoderPosition = 0;
|
|
@@ -622,6 +668,10 @@ export const encodeJsonValue = (buffer: Buffer, value: JsonValue): void => {
|
|
|
622
668
|
jsonEncoderPosition = 0;
|
|
623
669
|
jsonEncoderDepth = 0;
|
|
624
670
|
jsonEncoderIsActive = false;
|
|
671
|
+
if (jsonEncoderTarget.length > maxRetainedJsonEncoderLength) {
|
|
672
|
+
jsonEncoderTarget = new Uint8Array(8192);
|
|
673
|
+
jsonEncoderTargetView = new DataView(jsonEncoderTarget.buffer);
|
|
674
|
+
}
|
|
625
675
|
}
|
|
626
676
|
};
|
|
627
677
|
|
package/src/Error.ts
CHANGED
|
@@ -172,7 +172,11 @@ export const defectToError = (reported: unknown): Error => {
|
|
|
172
172
|
// instanceof fails.
|
|
173
173
|
const tag = Object.prototype.toString.call(defect);
|
|
174
174
|
// Chromium reports a DOMException from a worker without its name or message,
|
|
175
|
-
// so it is described in an Error.
|
|
175
|
+
// so it is described in an Error. Its fix covered only exceptions thrown in
|
|
176
|
+
// classic workers (https://issues.chromium.org/issues/41241359); Chrome 154
|
|
177
|
+
// still drops both for reportError
|
|
178
|
+
// (https://issues.chromium.org/issues/568850673) and for a module worker's
|
|
179
|
+
// evaluation.
|
|
176
180
|
if (tag === "[object DOMException]") {
|
|
177
181
|
const { name, message } = defect as Error;
|
|
178
182
|
return new Error(`${name}: ${message}`, { cause: defect });
|
package/src/Polyfills.ts
CHANGED
|
@@ -37,8 +37,9 @@ export const installPolyfills = (): void => {
|
|
|
37
37
|
* This module intentionally owns `DisposableStack` and `AsyncDisposableStack`
|
|
38
38
|
* polyfills instead of depending on `es-shims/DisposableStack` at runtime.
|
|
39
39
|
*
|
|
40
|
-
* Evolu originally used the upstream package, but WebKit
|
|
41
|
-
* disposal completion bug (`completion["?"]` crash,
|
|
40
|
+
* Evolu originally used the upstream package, but WebKit, which needs the
|
|
41
|
+
* polyfill, hit its async disposal completion bug (`completion["?"]` crash,
|
|
42
|
+
* https://github.com/es-shims/DisposableStack/issues/9). The local
|
|
42
43
|
* implementation applies the fix and keeps behavior deterministic across
|
|
43
44
|
* runtimes used by Evolu.
|
|
44
45
|
*
|
package/src/Sqlite.ts
CHANGED
|
@@ -188,7 +188,7 @@ export interface SqliteQueryOptions {
|
|
|
188
188
|
* statements can improve performance for repeated queries by reusing the
|
|
189
189
|
* compiled query.
|
|
190
190
|
*
|
|
191
|
-
* See: {@link https://sqlite.org/
|
|
191
|
+
* See: {@link https://sqlite.org/c3ref/prepare.html}.
|
|
192
192
|
*/
|
|
193
193
|
readonly prepare?: boolean;
|
|
194
194
|
}
|
package/src/Task.ts
CHANGED
|
@@ -1798,9 +1798,10 @@ export interface AbortReason extends InferType<typeof AbortReason> {}
|
|
|
1798
1798
|
* Helpers that abort their own child Tasks should catch or normalize AbortError
|
|
1799
1799
|
* before it escapes the helper boundary. The reason carries typed domain data.
|
|
1800
1800
|
*
|
|
1801
|
-
* WebKit fetch rejects with its own abort error instead of `signal.reason
|
|
1802
|
-
*
|
|
1803
|
-
*
|
|
1801
|
+
* WebKit fetch rejects with its own abort error instead of `signal.reason`
|
|
1802
|
+
* ([WebKit bug 246069](https://bugs.webkit.org/show_bug.cgi?id=246069)). Native
|
|
1803
|
+
* wrappers should treat `signal.reason` as the source of truth and normalize
|
|
1804
|
+
* aborts to AbortError.
|
|
1804
1805
|
*
|
|
1805
1806
|
* @group Core
|
|
1806
1807
|
*/
|
package/src/index.ts
CHANGED
|
@@ -51,7 +51,10 @@ export * from "./WebSocket.ts";
|
|
|
51
51
|
export * from "./Worker.ts";
|
|
52
52
|
|
|
53
53
|
// Local-first essentials.
|
|
54
|
-
export type {
|
|
54
|
+
export type {
|
|
55
|
+
DatabaseHeldError,
|
|
56
|
+
UnsupportedDbVersionError,
|
|
57
|
+
} from "./local-first/Db.ts";
|
|
55
58
|
export {
|
|
56
59
|
AppName,
|
|
57
60
|
createEvolu,
|
package/src/local-first/Db.ts
CHANGED
|
@@ -23,8 +23,10 @@
|
|
|
23
23
|
* version 1. A newer stored version refuses startup with
|
|
24
24
|
* {@link UnsupportedDbVersionError}. The refusal returns from the startup
|
|
25
25
|
* transaction before anything is written and is posted to the SharedWorker; the
|
|
26
|
-
* worker then exits and releases its resources.
|
|
27
|
-
*
|
|
26
|
+
* worker then exits and releases its resources. A database whose files another
|
|
27
|
+
* context keeps holding refuses startup the same way, before it is opened, with
|
|
28
|
+
* {@link DatabaseHeldError}; see {@link WaitForDatabaseRelease}. The single
|
|
29
|
+
* version row is created in the same transaction as the other system tables.
|
|
28
30
|
*
|
|
29
31
|
* Code released before the version record existed never reads it. It recognizes
|
|
30
32
|
* an initialized database by the `evolu_version` table alone, so it opens a
|
|
@@ -57,7 +59,7 @@ import {
|
|
|
57
59
|
type RandomBytesDep,
|
|
58
60
|
} from "../Crypto.ts";
|
|
59
61
|
import { createUnknownError } from "../Error.ts";
|
|
60
|
-
import { constFalse
|
|
62
|
+
import { constFalse } from "../Function.ts";
|
|
61
63
|
import type { LockManagerDep } from "../LockManager.ts";
|
|
62
64
|
import {
|
|
63
65
|
acquireLeaderLock,
|
|
@@ -116,6 +118,7 @@ import {
|
|
|
116
118
|
decryptAndDecodeDbChange,
|
|
117
119
|
encodeAndEncryptDbChange,
|
|
118
120
|
SubscriptionFlags,
|
|
121
|
+
type ProtocolChangeTooLargeError,
|
|
119
122
|
type ProtocolInvalidDataError,
|
|
120
123
|
type ProtocolMessage,
|
|
121
124
|
type ProtocolTimestampMismatchError,
|
|
@@ -194,7 +197,28 @@ export interface CreateDbWorkerDep {
|
|
|
194
197
|
export type DbWorkerDeps = WorkerDeps &
|
|
195
198
|
CreateBroadcastChannelDep &
|
|
196
199
|
LockManagerDep &
|
|
197
|
-
CreateSqliteDriverDep
|
|
200
|
+
CreateSqliteDriverDep &
|
|
201
|
+
Partial<WaitForDatabaseReleaseDep>;
|
|
202
|
+
|
|
203
|
+
/**
|
|
204
|
+
* Waits until no other context holds the files of the persistent database
|
|
205
|
+
* `name`, and fails with {@link DatabaseHeldError} when they stay held.
|
|
206
|
+
*
|
|
207
|
+
* {@link startDbWorker} calls it, except for a memory-only database, while it
|
|
208
|
+
* holds the database's leader lock and before it opens the database. Only a
|
|
209
|
+
* context that ended without closing the database can then hold its files, and
|
|
210
|
+
* such a context can only release them, so the files stay free until the
|
|
211
|
+
* database opens. Platforms whose databases cannot be held this way, such as
|
|
212
|
+
* Node.js and React Native, omit it.
|
|
213
|
+
*/
|
|
214
|
+
export type WaitForDatabaseRelease = (
|
|
215
|
+
name: Name,
|
|
216
|
+
) => Task<void, DatabaseHeldError>;
|
|
217
|
+
|
|
218
|
+
/** Dependency wrapper for {@link WaitForDatabaseRelease}. */
|
|
219
|
+
export interface WaitForDatabaseReleaseDep {
|
|
220
|
+
readonly waitForDatabaseRelease: WaitForDatabaseRelease;
|
|
221
|
+
}
|
|
198
222
|
|
|
199
223
|
/** The database version this code creates and supports; see the module doc. */
|
|
200
224
|
const dbVersion = PositiveInt.orThrow(2);
|
|
@@ -215,6 +239,18 @@ export interface UnsupportedDbVersionError extends Typed<"UnsupportedDbVersionEr
|
|
|
215
239
|
readonly supportedVersion: PositiveInt;
|
|
216
240
|
}
|
|
217
241
|
|
|
242
|
+
/**
|
|
243
|
+
* Another context kept holding the database's files, so the database did not
|
|
244
|
+
* open; see {@link WaitForDatabaseRelease}.
|
|
245
|
+
*
|
|
246
|
+
* A browser can keep the files of a worker that ended without closing them
|
|
247
|
+
* until the browser restarts. Ask the user to close the app's other tabs or to
|
|
248
|
+
* restart the browser, then to reload the app.
|
|
249
|
+
*/
|
|
250
|
+
export interface DatabaseHeldError extends Typed<"DatabaseHeldError"> {
|
|
251
|
+
readonly name: Name;
|
|
252
|
+
}
|
|
253
|
+
|
|
218
254
|
/**
|
|
219
255
|
* Starts the platform-agnostic Evolu DbWorker and owns its resources until
|
|
220
256
|
* startup is refused, the worker receives a dispose message, the SharedWorker
|
|
@@ -258,6 +294,15 @@ export const startDbWorker =
|
|
|
258
294
|
|
|
259
295
|
disposer.use(await run.ok(acquireLeaderLock(initMessage.name)));
|
|
260
296
|
|
|
297
|
+
if (!initMessage.memoryOnly && deps.waitForDatabaseRelease) {
|
|
298
|
+
const released = await run(deps.waitForDatabaseRelease(initMessage.name));
|
|
299
|
+
if (!released.ok) {
|
|
300
|
+
// Returning lets the disposer release the database lock.
|
|
301
|
+
port.postMessage({ type: "LeaderRefused", error: released.error });
|
|
302
|
+
return ok();
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
|
|
261
306
|
const sqlite = disposer.use(
|
|
262
307
|
await run.ok(
|
|
263
308
|
createSqlite(
|
|
@@ -374,6 +419,7 @@ export const startDbWorker =
|
|
|
374
419
|
const result = await run.abortable(
|
|
375
420
|
applyProtocolMessageAsClient(inputMessage, {
|
|
376
421
|
writeKey: owner.writeKey,
|
|
422
|
+
onChangeTooLarge: storage.onChangeTooLarge,
|
|
377
423
|
}),
|
|
378
424
|
);
|
|
379
425
|
postQueuedResponse({
|
|
@@ -475,7 +521,8 @@ export const startDbWorker =
|
|
|
475
521
|
|
|
476
522
|
// Stops once the SharedWorker no longer leads for its ID, which the
|
|
477
523
|
// platform releases when it closes, because Firefox can lose a Dispose
|
|
478
|
-
// posted right before that
|
|
524
|
+
// posted right before that
|
|
525
|
+
// (https://bugzilla.mozilla.org/show_bug.cgi?id=2077609).
|
|
479
526
|
const sharedWorkerEnded = acquireLeaderLockCallback(deps)(
|
|
480
527
|
initMessage.sharedWorkerId,
|
|
481
528
|
() => resolve(ok()),
|
|
@@ -917,14 +964,18 @@ interface ClientStorage extends Storage, BaseSqliteStorage {
|
|
|
917
964
|
) => void;
|
|
918
965
|
readonly didWriteMessages: () => boolean;
|
|
919
966
|
/**
|
|
920
|
-
* The first error of a message
|
|
921
|
-
*
|
|
967
|
+
* The first error of a message skipped in this request, or null: one that
|
|
968
|
+
* {@link Storage.writeMessages} could not store, or a stored change that sync
|
|
969
|
+
* could not send.
|
|
922
970
|
*/
|
|
923
971
|
readonly skippedError: () =>
|
|
924
972
|
| DecryptWithXChaCha20Poly1305Error
|
|
925
973
|
| ProtocolInvalidDataError
|
|
926
974
|
| ProtocolTimestampMismatchError
|
|
975
|
+
| ProtocolChangeTooLargeError
|
|
927
976
|
| null;
|
|
977
|
+
/** Records a stored change that sync skipped, unless an error came first. */
|
|
978
|
+
readonly onChangeTooLarge: (error: ProtocolChangeTooLargeError) => void;
|
|
928
979
|
}
|
|
929
980
|
|
|
930
981
|
const createClientStorage = (
|
|
@@ -961,10 +1012,12 @@ const createClientStorage = (
|
|
|
961
1012
|
|
|
962
1013
|
didWriteMessages: () => didWriteMessages,
|
|
963
1014
|
skippedError: () => skippedError,
|
|
1015
|
+
onChangeTooLarge: (error) => {
|
|
1016
|
+
skippedError ??= error;
|
|
1017
|
+
},
|
|
964
1018
|
|
|
965
1019
|
// Not implemented yet.
|
|
966
1020
|
validateWriteKey: constFalse,
|
|
967
|
-
setWriteKey: constVoid,
|
|
968
1021
|
|
|
969
1022
|
writeMessages: (ownerIdBytes, encryptedMessages) => () => {
|
|
970
1023
|
// TODO: Add quota checking for collaborative scenarios.
|
|
@@ -1020,15 +1073,21 @@ const createClientStorage = (
|
|
|
1020
1073
|
}
|
|
1021
1074
|
|
|
1022
1075
|
let wroteNewMessages = false;
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1076
|
+
// SQLite can fail the write, for example on a full disk. The transaction
|
|
1077
|
+
// has rolled back, so the route reports the error instead of a throw
|
|
1078
|
+
// panicking the DbWorker.
|
|
1079
|
+
const written = trySync(() => {
|
|
1080
|
+
deps.sqlite.transaction(() => {
|
|
1081
|
+
wroteNewMessages = applyMessages(deps)(
|
|
1082
|
+
ownerIdBytesToOwnerId(ownerIdBytes),
|
|
1083
|
+
messages,
|
|
1084
|
+
QuarantineOrigin.ReceivedMessage,
|
|
1085
|
+
now,
|
|
1086
|
+
);
|
|
1087
|
+
saveClock(deps)(clockTimestamp);
|
|
1088
|
+
});
|
|
1089
|
+
}, createUnknownError);
|
|
1090
|
+
if (!written.ok) return written;
|
|
1032
1091
|
clock.set(clockTimestamp);
|
|
1033
1092
|
// A batch of duplicates changes no table, so queries need no refresh.
|
|
1034
1093
|
if (wroteNewMessages) didWriteMessages = true;
|