@evolu/common 6.0.1-preview.3 → 6.0.1-preview.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/Array.d.ts +69 -5
- package/dist/src/Array.d.ts.map +1 -1
- package/dist/src/Array.js +64 -5
- package/dist/src/Assert.d.ts +6 -16
- package/dist/src/Assert.d.ts.map +1 -1
- package/dist/src/Assert.js +6 -18
- package/dist/src/Brand.d.ts +75 -0
- package/dist/src/Brand.d.ts.map +1 -0
- package/dist/src/Brand.js +1 -0
- package/dist/src/Buffer.d.ts +1 -1
- package/dist/src/Buffer.d.ts.map +1 -1
- package/dist/src/Buffer.js +8 -7
- package/dist/src/Cache.d.ts +44 -0
- package/dist/src/Cache.d.ts.map +1 -0
- package/dist/src/Cache.js +52 -0
- package/dist/src/Callbacks.d.ts +45 -12
- package/dist/src/Callbacks.d.ts.map +1 -1
- package/dist/src/Callbacks.js +14 -7
- package/dist/src/Console.d.ts +31 -6
- package/dist/src/Console.d.ts.map +1 -1
- package/dist/src/Console.js +72 -9
- package/dist/src/Crypto.d.ts +61 -34
- package/dist/src/Crypto.d.ts.map +1 -1
- package/dist/src/Crypto.js +32 -45
- package/dist/src/Evolu/Db.d.ts +161 -65
- package/dist/src/Evolu/Db.d.ts.map +1 -1
- package/dist/src/Evolu/Db.js +286 -694
- package/dist/src/Evolu/Diff.d.ts +3 -3
- package/dist/src/Evolu/Diff.d.ts.map +1 -1
- package/dist/src/Evolu/Diff.js +7 -5
- package/dist/src/Evolu/Evolu.d.ts +208 -133
- package/dist/src/Evolu/Evolu.d.ts.map +1 -1
- package/dist/src/Evolu/Evolu.js +188 -183
- package/dist/src/Evolu/Internal.d.ts +0 -2
- package/dist/src/Evolu/Internal.d.ts.map +1 -1
- package/dist/src/Evolu/Internal.js +0 -2
- package/dist/src/Evolu/LocalAuth.d.ts +150 -0
- package/dist/src/Evolu/LocalAuth.d.ts.map +1 -0
- package/dist/src/Evolu/LocalAuth.js +174 -0
- package/dist/src/Evolu/Owner.d.ts +273 -120
- package/dist/src/Evolu/Owner.d.ts.map +1 -1
- package/dist/src/Evolu/Owner.js +130 -104
- package/dist/src/Evolu/Platform.d.ts +9 -7
- package/dist/src/Evolu/Platform.d.ts.map +1 -1
- package/dist/src/Evolu/Protocol.d.ts +277 -232
- package/dist/src/Evolu/Protocol.d.ts.map +1 -1
- package/dist/src/Evolu/Protocol.js +603 -378
- package/dist/src/Evolu/Public.d.ts +6 -8
- package/dist/src/Evolu/Public.d.ts.map +1 -1
- package/dist/src/Evolu/Public.js +2 -3
- package/dist/src/Evolu/PublicKysely.js +3 -3
- package/dist/src/Evolu/Query.d.ts +2 -1
- package/dist/src/Evolu/Query.d.ts.map +1 -1
- package/dist/src/Evolu/Relay.d.ts +92 -7
- package/dist/src/Evolu/Relay.d.ts.map +1 -1
- package/dist/src/Evolu/Relay.js +238 -76
- package/dist/src/Evolu/Schema.d.ts +129 -73
- package/dist/src/Evolu/Schema.d.ts.map +1 -1
- package/dist/src/Evolu/Schema.js +169 -89
- package/dist/src/Evolu/Storage.d.ts +240 -26
- package/dist/src/Evolu/Storage.d.ts.map +1 -1
- package/dist/src/Evolu/Storage.js +189 -91
- package/dist/src/Evolu/Sync.d.ts +67 -13
- package/dist/src/Evolu/Sync.d.ts.map +1 -1
- package/dist/src/Evolu/Sync.js +441 -20
- package/dist/src/Evolu/Timestamp.d.ts +85 -27
- package/dist/src/Evolu/Timestamp.d.ts.map +1 -1
- package/dist/src/Evolu/Timestamp.js +77 -18
- package/dist/src/Identicon.d.ts +35 -0
- package/dist/src/Identicon.d.ts.map +1 -0
- package/dist/src/Identicon.js +143 -0
- package/dist/src/Instances.d.ts +34 -0
- package/dist/src/Instances.d.ts.map +1 -0
- package/dist/src/Instances.js +44 -0
- package/dist/src/ManyToManyMap.d.ts +71 -10
- package/dist/src/ManyToManyMap.d.ts.map +1 -1
- package/dist/src/ManyToManyMap.js +41 -6
- package/dist/src/Number.d.ts +4 -3
- package/dist/src/Number.d.ts.map +1 -1
- package/dist/src/Number.js +5 -4
- package/dist/src/Platform.d.ts +20 -0
- package/dist/src/Platform.d.ts.map +1 -0
- package/dist/src/Platform.js +22 -0
- package/dist/src/Random.d.ts +3 -2
- package/dist/src/Random.d.ts.map +1 -1
- package/dist/src/Resources.d.ts +118 -0
- package/dist/src/Resources.d.ts.map +1 -0
- package/dist/src/Resources.js +197 -0
- package/dist/src/Result.d.ts +184 -52
- package/dist/src/Result.d.ts.map +1 -1
- package/dist/src/Result.js +30 -241
- package/dist/src/Skiplist.js +2 -1
- package/dist/src/Sqlite.d.ts +63 -5
- package/dist/src/Sqlite.d.ts.map +1 -1
- package/dist/src/Sqlite.js +110 -9
- package/dist/src/Task.d.ts +586 -0
- package/dist/src/Task.d.ts.map +1 -0
- package/dist/src/Task.js +469 -0
- package/dist/src/Time.d.ts +66 -1
- package/dist/src/Time.d.ts.map +1 -1
- package/dist/src/Time.js +99 -5
- package/dist/src/Type.d.ts +622 -340
- package/dist/src/Type.d.ts.map +1 -1
- package/dist/src/Type.js +666 -464
- package/dist/src/Types.d.ts +1 -75
- package/dist/src/Types.d.ts.map +1 -1
- package/dist/src/WebSocket.d.ts +5 -2
- package/dist/src/WebSocket.d.ts.map +1 -1
- package/dist/src/WebSocket.js +12 -18
- package/dist/src/Worker.d.ts +39 -11
- package/dist/src/Worker.d.ts.map +1 -1
- package/dist/src/Worker.js +22 -4
- package/dist/src/index.d.ts +7 -2
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +7 -2
- package/package.json +14 -13
- package/src/Array.ts +90 -11
- package/src/Assert.ts +6 -24
- package/src/Brand.ts +75 -0
- package/src/Buffer.ts +7 -7
- package/src/Cache.ts +85 -0
- package/src/Callbacks.ts +62 -22
- package/src/Console.ts +91 -11
- package/src/Crypto.ts +97 -82
- package/src/Evolu/Db.ts +517 -1020
- package/src/Evolu/Diff.ts +7 -5
- package/src/Evolu/Evolu.ts +464 -355
- package/src/Evolu/Internal.ts +0 -2
- package/src/Evolu/LocalAuth.ts +463 -0
- package/src/Evolu/Owner.ts +355 -228
- package/src/Evolu/Platform.ts +9 -9
- package/src/Evolu/Protocol.ts +859 -676
- package/src/Evolu/Public.ts +7 -14
- package/src/Evolu/PublicKysely.ts +3 -3
- package/src/Evolu/Query.ts +2 -1
- package/src/Evolu/Relay.ts +437 -93
- package/src/Evolu/Schema.ts +391 -191
- package/src/Evolu/Storage.ts +532 -135
- package/src/Evolu/Sync.ts +766 -37
- package/src/Evolu/Timestamp.ts +88 -35
- package/src/Identicon.ts +197 -0
- package/src/Instances.ts +90 -0
- package/src/ManyToManyMap.ts +124 -24
- package/src/Number.ts +6 -10
- package/src/Platform.ts +26 -0
- package/src/Random.ts +3 -2
- package/src/Resources.ts +367 -0
- package/src/Result.ts +191 -54
- package/src/Skiplist.ts +1 -1
- package/src/Sqlite.ts +122 -17
- package/src/Task.ts +901 -0
- package/src/Time.ts +180 -5
- package/src/Type.ts +1084 -727
- package/src/Types.ts +1 -77
- package/src/WebSocket.ts +27 -25
- package/src/Worker.ts +72 -23
- package/src/index.ts +7 -2
- package/dist/src/Evolu/Config.d.ts +0 -69
- package/dist/src/Evolu/Config.d.ts.map +0 -1
- package/dist/src/Evolu/Config.js +0 -9
- package/dist/src/Evolu/Kysely.d.ts +0 -6
- package/dist/src/Evolu/Kysely.d.ts.map +0 -1
- package/dist/src/Evolu/Kysely.js +0 -21
- package/dist/src/NanoId.d.ts +0 -27
- package/dist/src/NanoId.d.ts.map +0 -1
- package/dist/src/NanoId.js +0 -6
- package/dist/src/Promise.d.ts +0 -180
- package/dist/src/Promise.d.ts.map +0 -1
- package/dist/src/Promise.js +0 -176
- package/src/Evolu/Config.ts +0 -83
- package/src/Evolu/Kysely.ts +0 -38
- package/src/NanoId.ts +0 -39
- package/src/Promise.ts +0 -295
|
@@ -1,75 +1,78 @@
|
|
|
1
|
+
import { sha256 } from "@noble/hashes/sha2.js";
|
|
2
|
+
import { firstInArray, isNonEmptyReadonlyArray, } from "../Array.js";
|
|
3
|
+
import { assert } from "../Assert.js";
|
|
4
|
+
import { concatBytes } from "../Buffer.js";
|
|
5
|
+
import { decrement } from "../Number.js";
|
|
6
|
+
import { ok } from "../Result.js";
|
|
7
|
+
import { sql, SqliteValue } from "../Sqlite.js";
|
|
8
|
+
import { Id, NonNegativeInt, object, PositiveInt, record, String, } from "../Type.js";
|
|
9
|
+
import { orderTimestampBytes } from "./Timestamp.js";
|
|
10
|
+
export const fingerprintSize = NonNegativeInt.orThrow(12);
|
|
11
|
+
/** A fingerprint of an empty range. */
|
|
12
|
+
export const zeroFingerprint = new Uint8Array(fingerprintSize);
|
|
13
|
+
export const InfiniteUpperBound = Symbol("InfiniteUpperBound");
|
|
14
|
+
export const RangeType = {
|
|
15
|
+
Fingerprint: 1,
|
|
16
|
+
Skip: 0,
|
|
17
|
+
Timestamps: 2,
|
|
18
|
+
};
|
|
1
19
|
/**
|
|
2
|
-
*
|
|
3
|
-
*
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
*
|
|
12
|
-
* The ideal client implementation should probably use the SQLite extension
|
|
13
|
-
* instead of SQL or even a KV storage, when such a thing for browsers/native
|
|
14
|
-
* will exist and will be faster than SQLite.
|
|
15
|
-
*
|
|
16
|
-
* # Scaling
|
|
20
|
+
* A DbChange is a change to a table row. Together with a unique
|
|
21
|
+
* {@link Timestamp}, it forms a {@link CrdtMessage}.
|
|
22
|
+
*/
|
|
23
|
+
export const DbChange = object({
|
|
24
|
+
table: String,
|
|
25
|
+
id: Id,
|
|
26
|
+
values: record(String, SqliteValue),
|
|
27
|
+
});
|
|
28
|
+
/**
|
|
29
|
+
* Creates a {@link BaseSqliteStorage} implementation.
|
|
17
30
|
*
|
|
18
|
-
*
|
|
19
|
-
* each other, if necessary. One relay should handle hundreds of thousands of
|
|
20
|
-
* users, and when it goes down, nothing happens, because it will be
|
|
21
|
-
* synchronized later.
|
|
31
|
+
* # Stateless Design
|
|
22
32
|
*
|
|
23
|
-
*
|
|
33
|
+
* This implementation is fully stateless - it requires no in-memory state
|
|
34
|
+
* between invocations. All necessary metadata (timestamp bounds for insertion
|
|
35
|
+
* strategy optimization) is persisted in the evolu_usage table. This makes
|
|
36
|
+
* Evolu Relay suitable for stateless serverless environments like AWS Lambda,
|
|
37
|
+
* Cloudflare Workers with Durable Objects, and other platforms where memory
|
|
38
|
+
* doesn't persist between requests. While not extensively tested in all these
|
|
39
|
+
* environments yet, the stateless design should work well across them.
|
|
24
40
|
*/
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
import { sql } from "../Sqlite.js";
|
|
29
|
-
import { binaryOwnerIdToOwnerId, binaryTimestampToFingerprint, InfiniteUpperBound, RangeType, zeroFingerprint, } from "./Protocol.js";
|
|
30
|
-
import { orderBinaryTimestamp } from "./Timestamp.js";
|
|
31
|
-
export const createSqliteStorageBase = (deps) => (options) => {
|
|
32
|
-
const createTablesResult = createTables(deps);
|
|
33
|
-
if (!createTablesResult.ok)
|
|
34
|
-
return createTablesResult;
|
|
35
|
-
const ownerStats = new Map();
|
|
36
|
-
return ok({
|
|
37
|
-
insertTimestamp: (ownerId, timestamp) => {
|
|
38
|
-
const ownerIdString = binaryOwnerIdToOwnerId(ownerId);
|
|
41
|
+
export const createBaseSqliteStorage = (deps) => (config) => {
|
|
42
|
+
return {
|
|
43
|
+
insertTimestamp: (ownerId, timestamp, strategy) => {
|
|
39
44
|
const level = randomSkiplistLevel(deps);
|
|
40
|
-
let stats = ownerStats.get(ownerIdString);
|
|
41
|
-
if (!stats) {
|
|
42
|
-
const result = deps.sqlite.exec(sql.prepared `
|
|
43
|
-
select min(t) as minT, max(t) as maxT
|
|
44
|
-
from evolu_timestamp
|
|
45
|
-
where ownerId = ${ownerId};
|
|
46
|
-
`);
|
|
47
|
-
if (!result.ok)
|
|
48
|
-
return result;
|
|
49
|
-
stats = {
|
|
50
|
-
minT: result.value.rows[0].minT ?? timestamp,
|
|
51
|
-
maxT: result.value.rows[0].maxT ?? timestamp,
|
|
52
|
-
};
|
|
53
|
-
ownerStats.set(ownerIdString, stats);
|
|
54
|
-
}
|
|
55
|
-
let strategy;
|
|
56
|
-
if (orderBinaryTimestamp(timestamp, stats.maxT) === 1) {
|
|
57
|
-
strategy = "append";
|
|
58
|
-
stats.maxT = timestamp;
|
|
59
|
-
}
|
|
60
|
-
else if (orderBinaryTimestamp(timestamp, stats.minT) === -1) {
|
|
61
|
-
strategy = "prepend";
|
|
62
|
-
stats.minT = timestamp;
|
|
63
|
-
}
|
|
64
|
-
else {
|
|
65
|
-
strategy = "insert";
|
|
66
|
-
}
|
|
67
45
|
return insertTimestamp(deps)(ownerId, timestamp, level, strategy);
|
|
68
46
|
},
|
|
47
|
+
getExistingTimestamps: (ownerIdBytes, timestampsBytes) => {
|
|
48
|
+
const concatenatedTimestamps = concatBytes(...timestampsBytes);
|
|
49
|
+
const result = deps.sqlite.exec(sql `
|
|
50
|
+
with recursive
|
|
51
|
+
split_timestamps(timestampBytes, pos) as (
|
|
52
|
+
select
|
|
53
|
+
substr(${concatenatedTimestamps}, 1, 16),
|
|
54
|
+
17 as pos
|
|
55
|
+
union all
|
|
56
|
+
select
|
|
57
|
+
substr(${concatenatedTimestamps}, pos, 16),
|
|
58
|
+
pos + 16
|
|
59
|
+
from split_timestamps
|
|
60
|
+
where pos <= length(${concatenatedTimestamps})
|
|
61
|
+
)
|
|
62
|
+
select s.timestampBytes
|
|
63
|
+
from
|
|
64
|
+
split_timestamps s
|
|
65
|
+
join evolu_timestamp t
|
|
66
|
+
on t.ownerId = ${ownerIdBytes} and s.timestampBytes = t.t;
|
|
67
|
+
`);
|
|
68
|
+
if (!result.ok)
|
|
69
|
+
return result;
|
|
70
|
+
return ok(result.value.rows.map((row) => row.timestampBytes));
|
|
71
|
+
},
|
|
69
72
|
getSize: (ownerId) => {
|
|
70
73
|
const size = getSize(deps)(ownerId);
|
|
71
74
|
if (!size.ok) {
|
|
72
|
-
|
|
75
|
+
config.onStorageError(size.error);
|
|
73
76
|
return null;
|
|
74
77
|
}
|
|
75
78
|
return size.value;
|
|
@@ -78,7 +81,7 @@ export const createSqliteStorageBase = (deps) => (options) => {
|
|
|
78
81
|
assertBeginEnd(begin, end);
|
|
79
82
|
const result = fingerprint(deps)(ownerId, begin, end);
|
|
80
83
|
if (!result.ok) {
|
|
81
|
-
|
|
84
|
+
config.onStorageError(result.error);
|
|
82
85
|
return null;
|
|
83
86
|
}
|
|
84
87
|
return result.value;
|
|
@@ -86,7 +89,7 @@ export const createSqliteStorageBase = (deps) => (options) => {
|
|
|
86
89
|
fingerprintRanges: (ownerId, buckets, upperBound) => {
|
|
87
90
|
const ranges = fingerprintRanges(deps)(ownerId, buckets, upperBound);
|
|
88
91
|
if (!ranges.ok) {
|
|
89
|
-
|
|
92
|
+
config.onStorageError(ranges.error);
|
|
90
93
|
return null;
|
|
91
94
|
}
|
|
92
95
|
return ranges.value;
|
|
@@ -94,7 +97,7 @@ export const createSqliteStorageBase = (deps) => (options) => {
|
|
|
94
97
|
findLowerBound: (ownerId, begin, end, upperBound) => {
|
|
95
98
|
const lowerBound = findLowerBound(deps)(ownerId, begin, end, upperBound);
|
|
96
99
|
if (!lowerBound.ok) {
|
|
97
|
-
|
|
100
|
+
config.onStorageError(lowerBound.error);
|
|
98
101
|
return null;
|
|
99
102
|
}
|
|
100
103
|
return lowerBound.value;
|
|
@@ -107,7 +110,7 @@ export const createSqliteStorageBase = (deps) => (options) => {
|
|
|
107
110
|
// This is much faster than SQL limit with offset.
|
|
108
111
|
const first = getTimestampByIndex(deps)(ownerId, begin);
|
|
109
112
|
if (!first.ok) {
|
|
110
|
-
|
|
113
|
+
config.onStorageError(first.error);
|
|
111
114
|
return;
|
|
112
115
|
}
|
|
113
116
|
if (!callback(first.value, begin))
|
|
@@ -132,21 +135,31 @@ export const createSqliteStorageBase = (deps) => (options) => {
|
|
|
132
135
|
limit ${length - 1};
|
|
133
136
|
`);
|
|
134
137
|
if (!result.ok) {
|
|
135
|
-
|
|
138
|
+
config.onStorageError(result.error);
|
|
136
139
|
return;
|
|
137
140
|
}
|
|
138
141
|
for (let i = 0; i < result.value.rows.length; i++) {
|
|
139
|
-
const index = (begin + 1 + i);
|
|
142
|
+
const index = NonNegativeInt.orThrow(begin + 1 + i);
|
|
140
143
|
if (!callback(result.value.rows[i].t, index))
|
|
141
144
|
return;
|
|
142
145
|
}
|
|
143
146
|
},
|
|
144
|
-
|
|
147
|
+
deleteOwner: (ownerId) => {
|
|
148
|
+
const result = deps.sqlite.exec(sql `
|
|
149
|
+
delete from evolu_timestamp where ownerId = ${ownerId};
|
|
150
|
+
`);
|
|
151
|
+
if (!result.ok) {
|
|
152
|
+
config.onStorageError(result.error);
|
|
153
|
+
return false;
|
|
154
|
+
}
|
|
155
|
+
return true;
|
|
156
|
+
},
|
|
157
|
+
};
|
|
145
158
|
};
|
|
146
159
|
const assertBeginEnd = (begin, end) => {
|
|
147
160
|
assert(begin <= end, "invalid begin or end");
|
|
148
161
|
};
|
|
149
|
-
const
|
|
162
|
+
export const createBaseSqliteStorageTables = (deps) => {
|
|
150
163
|
for (const query of [
|
|
151
164
|
/**
|
|
152
165
|
* Creates the `evolu_timestamp` table for storing timestamps of multiple
|
|
@@ -159,19 +172,13 @@ const createTables = (deps) => {
|
|
|
159
172
|
*
|
|
160
173
|
* Columns:
|
|
161
174
|
*
|
|
162
|
-
* - `t` –
|
|
175
|
+
* - `t` – TimestampBytes
|
|
163
176
|
* - `h1`/`h2` – 12-byte fingerprint split into two integers for fast XOR
|
|
164
177
|
* - `c` – incremental count
|
|
165
|
-
* - `l` – Skiplist level (1 to
|
|
166
|
-
*
|
|
167
|
-
* For scaling or isolation, sharding is possible—each owner can have a
|
|
168
|
-
* separate SQLite database.
|
|
169
|
-
*
|
|
170
|
-
* Maybe we could use an integer surrogate key for ownerId, but it's fast
|
|
171
|
-
* enough even without it.
|
|
178
|
+
* - `l` – Skiplist level (1 to 10)
|
|
172
179
|
*/
|
|
173
180
|
sql `
|
|
174
|
-
create table
|
|
181
|
+
create table evolu_timestamp (
|
|
175
182
|
"ownerId" blob not null,
|
|
176
183
|
"t" blob not null,
|
|
177
184
|
"h1" integer,
|
|
@@ -183,7 +190,7 @@ const createTables = (deps) => {
|
|
|
183
190
|
strict;
|
|
184
191
|
`,
|
|
185
192
|
sql `
|
|
186
|
-
create index
|
|
193
|
+
create index evolu_timestamp_index on evolu_timestamp (
|
|
187
194
|
"ownerId",
|
|
188
195
|
"l",
|
|
189
196
|
"t",
|
|
@@ -191,6 +198,25 @@ const createTables = (deps) => {
|
|
|
191
198
|
"h2",
|
|
192
199
|
"c"
|
|
193
200
|
);
|
|
201
|
+
`,
|
|
202
|
+
/**
|
|
203
|
+
* Creates the `evolu_usage` table for tracking data consumption per owner.
|
|
204
|
+
*
|
|
205
|
+
* Columns:
|
|
206
|
+
*
|
|
207
|
+
* - `ownerId` – OwnerIdBytes (primary key)
|
|
208
|
+
* - `storedBytes` – total bytes stored in database
|
|
209
|
+
* - `firstTimestamp` – for timestamp insertion strategies
|
|
210
|
+
* - `lastTimestamp` – for timestamp insertion strategies
|
|
211
|
+
*/
|
|
212
|
+
sql `
|
|
213
|
+
create table evolu_usage (
|
|
214
|
+
"ownerId" blob primary key,
|
|
215
|
+
"storedBytes" integer not null,
|
|
216
|
+
"firstTimestamp" blob,
|
|
217
|
+
"lastTimestamp" blob
|
|
218
|
+
)
|
|
219
|
+
strict;
|
|
194
220
|
`,
|
|
195
221
|
]) {
|
|
196
222
|
const result = deps.sqlite.exec(query);
|
|
@@ -199,16 +225,37 @@ const createTables = (deps) => {
|
|
|
199
225
|
}
|
|
200
226
|
return ok();
|
|
201
227
|
};
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
228
|
+
/**
|
|
229
|
+
* Determines the insertion strategy for a timestamp based on its position
|
|
230
|
+
* relative to the current first and last timestamps.
|
|
231
|
+
*
|
|
232
|
+
* Returns a tuple with the strategy and updated timestamp bounds.
|
|
233
|
+
*/
|
|
234
|
+
export const getTimestampInsertStrategy = (timestamp, firstTimestamp, lastTimestamp) => {
|
|
235
|
+
if (orderTimestampBytes(timestamp, lastTimestamp) === 1) {
|
|
236
|
+
return ["append", firstTimestamp, timestamp];
|
|
237
|
+
}
|
|
238
|
+
if (orderTimestampBytes(timestamp, firstTimestamp) === -1) {
|
|
239
|
+
return ["prepend", timestamp, lastTimestamp];
|
|
240
|
+
}
|
|
241
|
+
return ["insert", firstTimestamp, lastTimestamp];
|
|
242
|
+
};
|
|
243
|
+
/**
|
|
244
|
+
* AFAIK, we can't do both insert and update in one query, and that's probably
|
|
245
|
+
* why append is 2x faster than insert. Prepend also has to update parents, but
|
|
246
|
+
* it's constantly fast. Insert degrades for reversed (yet LIMIT X magically
|
|
247
|
+
* fixes that) but it's OK for append.
|
|
248
|
+
*
|
|
249
|
+
* Note: SQL operations are idempotent (using `on conflict do nothing` and
|
|
250
|
+
* `changes() > 0`), but this is no longer required here since we use
|
|
251
|
+
* {@link BaseSqliteStorage.getExistingTimestamps} to filter out duplicates
|
|
252
|
+
* before insertion, which we need for quota checks anyway.
|
|
253
|
+
*
|
|
254
|
+
* TODO: Remove idempotency (`on conflict do nothing` and `changes() > 0`) since
|
|
255
|
+
* duplicates are now filtered before insertion.
|
|
256
|
+
*/
|
|
210
257
|
const insertTimestamp = (deps) => (ownerId, timestamp, level, strategy) => {
|
|
211
|
-
const [h1, h2] = fingerprintToSqliteFingerprint(
|
|
258
|
+
const [h1, h2] = fingerprintToSqliteFingerprint(timestampBytesToFingerprint(timestamp));
|
|
212
259
|
let queries = [];
|
|
213
260
|
switch (strategy) {
|
|
214
261
|
case "append":
|
|
@@ -717,6 +764,10 @@ const insertTimestamp = (deps) => (ownerId, timestamp, level, strategy) => {
|
|
|
717
764
|
}
|
|
718
765
|
return ok();
|
|
719
766
|
};
|
|
767
|
+
export const timestampBytesToFingerprint = (timestamp) => {
|
|
768
|
+
const hash = sha256(timestamp).slice(0, fingerprintSize);
|
|
769
|
+
return hash;
|
|
770
|
+
};
|
|
720
771
|
/**
|
|
721
772
|
* Generates a random skiplist level in the range [1, skiplistMaxLevel].
|
|
722
773
|
* Probabilistic approach avoids the need for explicit tree balancing.
|
|
@@ -727,7 +778,7 @@ const randomSkiplistLevel = (deps) => {
|
|
|
727
778
|
level < skiplistMaxLevel) {
|
|
728
779
|
level += 1;
|
|
729
780
|
}
|
|
730
|
-
return level;
|
|
781
|
+
return PositiveInt.orThrow(level);
|
|
731
782
|
};
|
|
732
783
|
/**
|
|
733
784
|
* Probability used for generating Skiplist levels.
|
|
@@ -830,7 +881,7 @@ const findLowerBound = (deps) => (ownerId, begin, end, upperBound) => {
|
|
|
830
881
|
if (!count.ok)
|
|
831
882
|
return count;
|
|
832
883
|
// `decrement` converts a count to an index.
|
|
833
|
-
return ok(decrement(count.value));
|
|
884
|
+
return ok(NonNegativeInt.orThrow(decrement(count.value)));
|
|
834
885
|
};
|
|
835
886
|
const getTimestampCount = (deps) => (ownerId, timestamp) => {
|
|
836
887
|
const result = deps.sqlite.exec(sql.prepared `
|
|
@@ -1109,3 +1160,50 @@ export const getTimestampByIndex = (deps) => (ownerId, index) => {
|
|
|
1109
1160
|
return result;
|
|
1110
1161
|
return ok(result.value.rows[0].pt);
|
|
1111
1162
|
};
|
|
1163
|
+
/** Retrieves usage information for an owner from the evolu_usage table. */
|
|
1164
|
+
export const getOwnerUsage = (deps) => (ownerIdBytes, initialTimestamp) => {
|
|
1165
|
+
const result = deps.sqlite.exec(sql `
|
|
1166
|
+
select storedBytes, firstTimestamp, lastTimestamp
|
|
1167
|
+
from evolu_usage
|
|
1168
|
+
where ownerId = ${ownerIdBytes};
|
|
1169
|
+
`);
|
|
1170
|
+
if (!result.ok)
|
|
1171
|
+
return result;
|
|
1172
|
+
if (!isNonEmptyReadonlyArray(result.value.rows)) {
|
|
1173
|
+
return ok({
|
|
1174
|
+
storedBytes: null,
|
|
1175
|
+
firstTimestamp: initialTimestamp,
|
|
1176
|
+
lastTimestamp: initialTimestamp,
|
|
1177
|
+
});
|
|
1178
|
+
}
|
|
1179
|
+
const row = firstInArray(result.value.rows);
|
|
1180
|
+
assert(row.firstTimestamp, "not null");
|
|
1181
|
+
assert(row.lastTimestamp, "not null");
|
|
1182
|
+
return ok({
|
|
1183
|
+
storedBytes: row.storedBytes,
|
|
1184
|
+
firstTimestamp: row.firstTimestamp,
|
|
1185
|
+
lastTimestamp: row.lastTimestamp,
|
|
1186
|
+
});
|
|
1187
|
+
};
|
|
1188
|
+
/**
|
|
1189
|
+
* Updates timestamp bounds in evolu_usage table.
|
|
1190
|
+
*
|
|
1191
|
+
* Used by both relay and client to maintain firstTimestamp/lastTimestamp after
|
|
1192
|
+
* processing messages.
|
|
1193
|
+
*/
|
|
1194
|
+
export const updateOwnerUsage = (deps) => (ownerIdBytes, storedBytes, firstTimestamp, lastTimestamp) => {
|
|
1195
|
+
const result = deps.sqlite.exec(sql `
|
|
1196
|
+
insert into evolu_usage
|
|
1197
|
+
("ownerId", "storedBytes", "firstTimestamp", "lastTimestamp")
|
|
1198
|
+
values
|
|
1199
|
+
(${ownerIdBytes}, ${storedBytes}, ${firstTimestamp}, ${lastTimestamp})
|
|
1200
|
+
on conflict (ownerId) do update
|
|
1201
|
+
set
|
|
1202
|
+
storedBytes = ${storedBytes},
|
|
1203
|
+
firstTimestamp = ${firstTimestamp},
|
|
1204
|
+
lastTimestamp = ${lastTimestamp};
|
|
1205
|
+
`);
|
|
1206
|
+
if (!result.ok)
|
|
1207
|
+
return result;
|
|
1208
|
+
return ok();
|
|
1209
|
+
};
|
package/dist/src/Evolu/Sync.d.ts
CHANGED
|
@@ -1,23 +1,77 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
|
|
5
|
-
|
|
1
|
+
import { NonEmptyReadonlyArray } from "../Array.js";
|
|
2
|
+
import { ConsoleDep } from "../Console.js";
|
|
3
|
+
import { RandomBytesDep, SymmetricCryptoDecryptError, SymmetricCryptoDep } from "../Crypto.js";
|
|
4
|
+
import { TransferableError } from "../Error.js";
|
|
5
|
+
import { RandomDep } from "../Random.js";
|
|
6
|
+
import { Result } from "../Result.js";
|
|
7
|
+
import { SqliteDep, SqliteError } from "../Sqlite.js";
|
|
8
|
+
import { TimeDep } from "../Time.js";
|
|
9
|
+
import { CreateWebSocketDep } from "../WebSocket.js";
|
|
10
|
+
import type { PostMessageDep } from "./Db.js";
|
|
11
|
+
import { AppOwner, OwnerEncryptionKey, OwnerId, OwnerIdBytes, OwnerTransport, OwnerWriteKey } from "./Owner.js";
|
|
12
|
+
import { ProtocolError, ProtocolInvalidDataError, ProtocolTimestampMismatchError } from "./Protocol.js";
|
|
13
|
+
import { MutationChange } from "./Schema.js";
|
|
14
|
+
import { BaseSqliteStorage, CrdtMessage, Storage, StorageInsertTimestampStrategy } from "./Storage.js";
|
|
15
|
+
import { Millis, Timestamp, TimestampConfigDep, TimestampCounterOverflowError, TimestampDriftError, TimestampTimeOutOfRangeError } from "./Timestamp.js";
|
|
16
|
+
export interface Sync extends Disposable {
|
|
17
|
+
/**
|
|
18
|
+
* Assigns or removes an owner to/from transports with reference counting.
|
|
19
|
+
*
|
|
20
|
+
* Owners are only "active" if assigned to at least one transport. Uses
|
|
21
|
+
* `owner.transports` or falls back to config transports. Multiple calls
|
|
22
|
+
* increment/decrement reference counts (useful for React Hooks).
|
|
23
|
+
*/
|
|
24
|
+
readonly useOwner: (use: boolean, owner: SyncOwner) => void;
|
|
25
|
+
readonly applyChanges: (changes: NonEmptyReadonlyArray<MutationChange>) => Result<void, SqliteError | TimestampCounterOverflowError | TimestampDriftError | TimestampTimeOutOfRangeError>;
|
|
6
26
|
}
|
|
7
27
|
export interface SyncDep {
|
|
8
28
|
readonly sync: Sync;
|
|
9
29
|
}
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
30
|
+
/**
|
|
31
|
+
* Represents an owner for sync operations. This is a unified interface that
|
|
32
|
+
* abstracts over the specific owner types ({@link ShardOwner},
|
|
33
|
+
* {@link SharedOwner}, {@link SharedReadonlyOwner}) for the sync layer.
|
|
34
|
+
*
|
|
35
|
+
* The sync layer only needs the essential data for synchronization and doesn't
|
|
36
|
+
* need to distinguish between different owner types.
|
|
37
|
+
*/
|
|
38
|
+
export interface SyncOwner {
|
|
39
|
+
readonly id: OwnerId;
|
|
40
|
+
readonly encryptionKey: OwnerEncryptionKey;
|
|
41
|
+
/** Optional for read-only owners like {@link SharedReadonlyOwner}. */
|
|
42
|
+
readonly writeKey?: OwnerWriteKey;
|
|
43
|
+
readonly transports?: ReadonlyArray<OwnerTransport>;
|
|
44
|
+
}
|
|
45
|
+
export interface SyncConfig {
|
|
46
|
+
readonly appOwner: AppOwner;
|
|
47
|
+
readonly transports: ReadonlyArray<OwnerTransport>;
|
|
48
|
+
/**
|
|
49
|
+
* Delay in milliseconds before disposing unused WebSocket connections.
|
|
50
|
+
* Defaults to 100ms.
|
|
51
|
+
*/
|
|
52
|
+
readonly disposalDelayMs?: number;
|
|
53
|
+
readonly onError: (error: ProtocolError | ProtocolInvalidDataError | ProtocolTimestampMismatchError | SqliteError | SymmetricCryptoDecryptError | TimestampCounterOverflowError | TimestampDriftError | TimestampTimeOutOfRangeError | TransferableError) => void;
|
|
54
|
+
readonly onReceive: () => void;
|
|
13
55
|
}
|
|
14
|
-
export
|
|
15
|
-
|
|
16
|
-
readonly
|
|
17
|
-
readonly onMessage: (message: Uint8Array, send: Sync["send"]) => void;
|
|
56
|
+
export declare const createSync: (deps: ClockDep & ConsoleDep & CreateWebSocketDep & PostMessageDep & RandomBytesDep & RandomDep & SqliteDep & SymmetricCryptoDep & TimeDep & TimestampConfigDep) => (config: SyncConfig) => Result<Sync, SqliteError>;
|
|
57
|
+
export interface ClockDep {
|
|
58
|
+
readonly clock: Clock;
|
|
18
59
|
}
|
|
19
|
-
export
|
|
60
|
+
export interface Clock {
|
|
61
|
+
readonly get: () => Timestamp;
|
|
62
|
+
readonly save: (timestamp: Timestamp) => Result<void, SqliteError>;
|
|
63
|
+
}
|
|
64
|
+
export declare const createClock: (deps: RandomBytesDep & SqliteDep) => (initialTimestamp?: Timestamp) => Clock;
|
|
65
|
+
export interface ClientStorage extends Storage, BaseSqliteStorage {
|
|
66
|
+
}
|
|
67
|
+
export interface ClientStorageDep {
|
|
68
|
+
readonly storage: ClientStorage;
|
|
69
|
+
}
|
|
70
|
+
export declare const applyLocalOnlyChange: (deps: SqliteDep & TimeDep) => (change: MutationChange) => Result<void, SqliteError>;
|
|
71
|
+
export declare const applyMessageToTimestampAndHistoryTables: (deps: ClientStorageDep & SqliteDep) => (ownerId: OwnerIdBytes, message: CrdtMessage, strategy: StorageInsertTimestampStrategy) => Result<void, SqliteError>;
|
|
20
72
|
/**
|
|
73
|
+
* TODO: Rework for the new owners API.
|
|
74
|
+
*
|
|
21
75
|
* The possible states of a synchronization process. The `SyncState` can be one
|
|
22
76
|
* of the following:
|
|
23
77
|
*
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Sync.d.ts","sourceRoot":"","sources":["../../../src/Evolu/Sync.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"Sync.d.ts","sourceRoot":"","sources":["../../../src/Evolu/Sync.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,qBAAqB,EACtB,MAAM,aAAa,CAAC;AAGrB,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC3C,OAAO,EACL,cAAc,EACd,2BAA2B,EAC3B,kBAAkB,EACnB,MAAM,cAAc,CAAC;AAEtB,OAAO,EAA2B,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAGzE,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAEzC,OAAO,EAAW,MAAM,EAAE,MAAM,cAAc,CAAC;AAC/C,OAAO,EAAO,SAAS,EAAE,WAAW,EAAe,MAAM,cAAc,CAAC;AAExE,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAErC,OAAO,EAAE,kBAAkB,EAAa,MAAM,iBAAiB,CAAC;AAChE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAC9C,OAAO,EACL,QAAQ,EACR,kBAAkB,EAClB,OAAO,EACP,YAAY,EAGZ,cAAc,EACd,aAAa,EAId,MAAM,YAAY,CAAC;AACpB,OAAO,EAOL,aAAa,EACb,wBAAwB,EACxB,8BAA8B,EAE/B,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EACL,iBAAiB,EACjB,WAAW,EAKX,OAAO,EACP,8BAA8B,EAG/B,MAAM,cAAc,CAAC;AACtB,OAAO,EAEL,MAAM,EAGN,SAAS,EAET,kBAAkB,EAClB,6BAA6B,EAC7B,mBAAmB,EACnB,4BAA4B,EAG7B,MAAM,gBAAgB,CAAC;AAExB,MAAM,WAAW,IAAK,SAAQ,UAAU;IACtC;;;;;;OAMG;IACH,QAAQ,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,SAAS,KAAK,IAAI,CAAC;IAE5D,QAAQ,CAAC,YAAY,EAAE,CACrB,OAAO,EAAE,qBAAqB,CAAC,cAAc,CAAC,KAC3C,MAAM,CACT,IAAI,EACF,WAAW,GACX,6BAA6B,GAC7B,mBAAmB,GACnB,4BAA4B,CAC/B,CAAC;CACH;AAED,MAAM,WAAW,OAAO;IACtB,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;CACrB;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,EAAE,EAAE,OAAO,CAAC;IACrB,QAAQ,CAAC,aAAa,EAAE,kBAAkB,CAAC;IAC3C,sEAAsE;IACtE,QAAQ,CAAC,QAAQ,CAAC,EAAE,aAAa,CAAC;IAClC,QAAQ,CAAC,UAAU,CAAC,EAAE,aAAa,CAAC,cAAc,CAAC,CAAC;CACrD;AAED,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC;IAE5B,QAAQ,CAAC,UAAU,EAAE,aAAa,CAAC,cAAc,CAAC,CAAC;IAEnD;;;OAGG;IACH,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAC;IAElC,QAAQ,CAAC,OAAO,EAAE,CAChB,KAAK,EACD,aAAa,GACb,wBAAwB,GACxB,8BAA8B,GAC9B,WAAW,GACX,2BAA2B,GAC3B,6BAA6B,GAC7B,mBAAmB,GACnB,4BAA4B,GAC5B,iBAAiB,KAClB,IAAI,CAAC;IAEV,QAAQ,CAAC,SAAS,EAAE,MAAM,IAAI,CAAC;CAChC;AAED,eAAO,MAAM,UAAU,GAEnB,MAAM,QAAQ,GACZ,UAAU,GACV,kBAAkB,GAClB,cAAc,GACd,cAAc,GACd,SAAS,GACT,SAAS,GACT,kBAAkB,GAClB,OAAO,GACP,kBAAkB,MAErB,QAAQ,UAAU,KAAG,MAAM,CAAC,IAAI,EAAE,WAAW,CAwO7C,CAAC;AAEJ,MAAM,WAAW,QAAQ;IACvB,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC;CACvB;AAGD,MAAM,WAAW,KAAK;IACpB,QAAQ,CAAC,GAAG,EAAE,MAAM,SAAS,CAAC;IAC9B,QAAQ,CAAC,IAAI,EAAE,CAAC,SAAS,EAAE,SAAS,KAAK,MAAM,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;CACpE;AAED,eAAO,MAAM,WAAW,GACrB,MAAM,cAAc,GAAG,SAAS,MAChC,4BAA+C,KAAG,KAiBlD,CAAC;AAMJ,MAAM,WAAW,aAAc,SAAQ,OAAO,EAAE,iBAAiB;CAAG;AAEpE,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC;CACjC;AAyLD,eAAO,MAAM,oBAAoB,GAC9B,MAAM,SAAS,GAAG,OAAO,MACzB,QAAQ,cAAc,KAAG,MAAM,CAAC,IAAI,EAAE,WAAW,CAkCjD,CAAC;AA4FJ,eAAO,MAAM,uCAAuC,GACjD,MAAM,gBAAgB,GAAG,SAAS,MAEjC,SAAS,YAAY,EACrB,SAAS,WAAW,EACpB,UAAU,8BAA8B,KACvC,MAAM,CAAC,IAAI,EAAE,WAAW,CA0B1B,CAAC;AAEJ;;;;;;;;;;GAUG;AACH,MAAM,MAAM,SAAS,GACjB,gBAAgB,GAChB,kBAAkB,GAClB,iBAAiB,GACjB,oBAAoB,CAAC;AAEzB;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,IAAI,EAAE,kBAAkB,CAAC;CACnC;AAED,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,IAAI,EAAE,oBAAoB,CAAC;CACrC;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,IAAI,EAAE,mBAAmB,CAAC;IACnC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,IAAI,EAAE,sBAAsB,CAAC;IACtC,QAAQ,CAAC,KAAK,EAAE,YAAY,GAAG,WAAW,GAAG,oBAAoB,CAAC;CACnE;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC;CAC/B;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAC;IAC7B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,IAAI,EAAE,sBAAsB,CAAC;CACvC;AAED,eAAO,MAAM,gBAAgB,EAAE,gBAA+C,CAAC"}
|