@evolu/common 6.0.1-preview.8 → 7.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/src/Array.d.ts +256 -12
- package/dist/src/Array.d.ts.map +1 -1
- package/dist/src/Array.js +215 -9
- package/dist/src/Assert.d.ts +0 -13
- package/dist/src/Assert.d.ts.map +1 -1
- package/dist/src/Assert.js +0 -15
- package/dist/src/Brand.d.ts +75 -0
- package/dist/src/Brand.d.ts.map +1 -0
- package/dist/src/Brand.js +1 -0
- package/dist/src/Buffer.d.ts +1 -1
- package/dist/src/Buffer.d.ts.map +1 -1
- package/dist/src/Buffer.js +8 -7
- package/dist/src/Cache.d.ts +44 -0
- package/dist/src/Cache.d.ts.map +1 -0
- package/dist/src/Cache.js +52 -0
- package/dist/src/Callbacks.d.ts +45 -12
- package/dist/src/Callbacks.d.ts.map +1 -1
- package/dist/src/Callbacks.js +14 -7
- package/dist/src/Console.d.ts +31 -6
- package/dist/src/Console.d.ts.map +1 -1
- package/dist/src/Console.js +72 -9
- package/dist/src/Crypto.d.ts +56 -42
- package/dist/src/Crypto.d.ts.map +1 -1
- package/dist/src/Crypto.js +40 -53
- package/dist/src/Evolu/Db.d.ts +162 -74
- package/dist/src/Evolu/Db.d.ts.map +1 -1
- package/dist/src/Evolu/Db.js +284 -702
- package/dist/src/Evolu/Diff.d.ts +3 -3
- package/dist/src/Evolu/Diff.d.ts.map +1 -1
- package/dist/src/Evolu/Diff.js +7 -5
- package/dist/src/Evolu/Evolu.d.ts +214 -134
- package/dist/src/Evolu/Evolu.d.ts.map +1 -1
- package/dist/src/Evolu/Evolu.js +189 -180
- package/dist/src/Evolu/Internal.d.ts +0 -2
- package/dist/src/Evolu/Internal.d.ts.map +1 -1
- package/dist/src/Evolu/Internal.js +0 -2
- package/dist/src/Evolu/LocalAuth.d.ts +150 -0
- package/dist/src/Evolu/LocalAuth.d.ts.map +1 -0
- package/dist/src/Evolu/LocalAuth.js +174 -0
- package/dist/src/Evolu/Owner.d.ts +273 -120
- package/dist/src/Evolu/Owner.d.ts.map +1 -1
- package/dist/src/Evolu/Owner.js +130 -104
- package/dist/src/Evolu/Platform.d.ts +9 -7
- package/dist/src/Evolu/Platform.d.ts.map +1 -1
- package/dist/src/Evolu/Protocol.d.ts +268 -240
- package/dist/src/Evolu/Protocol.d.ts.map +1 -1
- package/dist/src/Evolu/Protocol.js +600 -454
- package/dist/src/Evolu/Public.d.ts +6 -8
- package/dist/src/Evolu/Public.d.ts.map +1 -1
- package/dist/src/Evolu/Public.js +2 -3
- package/dist/src/Evolu/PublicKysely.d.ts.map +1 -1
- package/dist/src/Evolu/PublicKysely.js +3 -4
- package/dist/src/Evolu/Query.d.ts +2 -1
- package/dist/src/Evolu/Query.d.ts.map +1 -1
- package/dist/src/Evolu/Query.js +1 -1
- package/dist/src/Evolu/Relay.d.ts +91 -8
- package/dist/src/Evolu/Relay.d.ts.map +1 -1
- package/dist/src/Evolu/Relay.js +214 -88
- package/dist/src/Evolu/Schema.d.ts +125 -47
- package/dist/src/Evolu/Schema.d.ts.map +1 -1
- package/dist/src/Evolu/Schema.js +175 -31
- package/dist/src/Evolu/Storage.d.ts +249 -27
- package/dist/src/Evolu/Storage.d.ts.map +1 -1
- package/dist/src/Evolu/Storage.js +198 -92
- package/dist/src/Evolu/Sync.d.ts +68 -13
- package/dist/src/Evolu/Sync.d.ts.map +1 -1
- package/dist/src/Evolu/Sync.js +469 -20
- package/dist/src/Evolu/Timestamp.d.ts +83 -30
- package/dist/src/Evolu/Timestamp.d.ts.map +1 -1
- package/dist/src/Evolu/Timestamp.js +79 -34
- package/dist/src/Identicon.d.ts +35 -0
- package/dist/src/Identicon.d.ts.map +1 -0
- package/dist/src/Identicon.js +143 -0
- package/dist/src/Instances.d.ts +34 -0
- package/dist/src/Instances.d.ts.map +1 -0
- package/dist/src/Instances.js +44 -0
- package/dist/src/Number.d.ts +4 -3
- package/dist/src/Number.d.ts.map +1 -1
- package/dist/src/Number.js +5 -4
- package/dist/src/Object.d.ts +10 -4
- package/dist/src/Object.d.ts.map +1 -1
- package/dist/src/Object.js +9 -3
- package/dist/src/Platform.d.ts +20 -0
- package/dist/src/Platform.d.ts.map +1 -0
- package/dist/src/Platform.js +22 -0
- package/dist/src/Random.d.ts +3 -2
- package/dist/src/Random.d.ts.map +1 -1
- package/dist/src/Relation.d.ts +79 -0
- package/dist/src/Relation.d.ts.map +1 -0
- package/dist/src/Relation.js +127 -0
- package/dist/src/Resources.d.ts +118 -0
- package/dist/src/Resources.d.ts.map +1 -0
- package/dist/src/Resources.js +197 -0
- package/dist/src/Result.d.ts +184 -52
- package/dist/src/Result.d.ts.map +1 -1
- package/dist/src/Result.js +30 -241
- package/dist/src/Skiplist.js +2 -1
- package/dist/src/Sqlite.d.ts +89 -5
- package/dist/src/Sqlite.d.ts.map +1 -1
- package/dist/src/Sqlite.js +136 -9
- package/dist/src/Task.d.ts +586 -0
- package/dist/src/Task.d.ts.map +1 -0
- package/dist/src/Task.js +469 -0
- package/dist/src/Time.d.ts +66 -1
- package/dist/src/Time.d.ts.map +1 -1
- package/dist/src/Time.js +99 -5
- package/dist/src/Type.d.ts +676 -343
- package/dist/src/Type.d.ts.map +1 -1
- package/dist/src/Type.js +718 -467
- package/dist/src/Types.d.ts +1 -75
- package/dist/src/Types.d.ts.map +1 -1
- package/dist/src/WebSocket.d.ts +5 -2
- package/dist/src/WebSocket.d.ts.map +1 -1
- package/dist/src/WebSocket.js +12 -18
- package/dist/src/Worker.d.ts +39 -11
- package/dist/src/Worker.d.ts.map +1 -1
- package/dist/src/Worker.js +22 -4
- package/dist/src/index.d.ts +8 -3
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +8 -3
- package/package.json +15 -14
- package/src/Array.ts +301 -19
- package/src/Assert.ts +0 -21
- package/src/Brand.ts +75 -0
- package/src/Buffer.ts +7 -7
- package/src/Cache.ts +85 -0
- package/src/Callbacks.ts +62 -22
- package/src/Console.ts +91 -11
- package/src/Crypto.ts +94 -90
- package/src/Evolu/Db.ts +519 -1026
- package/src/Evolu/Diff.ts +7 -5
- package/src/Evolu/Evolu.ts +477 -361
- package/src/Evolu/Internal.ts +0 -2
- package/src/Evolu/LocalAuth.ts +463 -0
- package/src/Evolu/Owner.ts +355 -228
- package/src/Evolu/Platform.ts +9 -9
- package/src/Evolu/Protocol.ts +842 -742
- package/src/Evolu/Public.ts +7 -14
- package/src/Evolu/PublicKysely.ts +4 -5
- package/src/Evolu/Query.ts +3 -2
- package/src/Evolu/Relay.ts +406 -103
- package/src/Evolu/Schema.ts +323 -91
- package/src/Evolu/Storage.ts +559 -137
- package/src/Evolu/Sync.ts +819 -36
- package/src/Evolu/Timestamp.ts +90 -58
- package/src/Identicon.ts +197 -0
- package/src/Instances.ts +90 -0
- package/src/Number.ts +6 -10
- package/src/Object.ts +13 -5
- package/src/Platform.ts +26 -0
- package/src/Random.ts +3 -2
- package/src/Relation.ts +234 -0
- package/src/Resources.ts +367 -0
- package/src/Result.ts +191 -54
- package/src/Skiplist.ts +1 -1
- package/src/Sqlite.ts +152 -17
- package/src/Task.ts +901 -0
- package/src/Time.ts +180 -5
- package/src/Type.ts +1135 -730
- package/src/Types.ts +1 -77
- package/src/WebSocket.ts +27 -25
- package/src/Worker.ts +72 -23
- package/src/index.ts +8 -3
- package/dist/src/Evolu/Config.d.ts +0 -69
- package/dist/src/Evolu/Config.d.ts.map +0 -1
- package/dist/src/Evolu/Config.js +0 -9
- package/dist/src/Evolu/Kysely.d.ts +0 -6
- package/dist/src/Evolu/Kysely.d.ts.map +0 -1
- package/dist/src/Evolu/Kysely.js +0 -21
- package/dist/src/ManyToManyMap.d.ts +0 -26
- package/dist/src/ManyToManyMap.d.ts.map +0 -1
- package/dist/src/ManyToManyMap.js +0 -92
- package/dist/src/NanoId.d.ts +0 -27
- package/dist/src/NanoId.d.ts.map +0 -1
- package/dist/src/NanoId.js +0 -6
- package/dist/src/Promise.d.ts +0 -180
- package/dist/src/Promise.d.ts.map +0 -1
- package/dist/src/Promise.js +0 -176
- package/src/Evolu/Config.ts +0 -83
- package/src/Evolu/Kysely.ts +0 -38
- package/src/ManyToManyMap.ts +0 -140
- package/src/NanoId.ts +0 -39
- package/src/Promise.ts +0 -295
|
@@ -1,75 +1,96 @@
|
|
|
1
|
+
import { sha256 } from "@noble/hashes/sha2.js";
|
|
2
|
+
import { firstInArray, isNonEmptyReadonlyArray, } from "../Array.js";
|
|
3
|
+
import { assert } from "../Assert.js";
|
|
4
|
+
import { concatBytes } from "../Buffer.js";
|
|
5
|
+
import { decrement } from "../Number.js";
|
|
6
|
+
import { err, ok } from "../Result.js";
|
|
7
|
+
import { sql, SqliteValue } from "../Sqlite.js";
|
|
8
|
+
import { Boolean, brand, Id, NonNegativeInt, nullOr, object, PositiveInt, record, String, } from "../Type.js";
|
|
9
|
+
import { systemColumns } from "./Schema.js";
|
|
10
|
+
import { orderTimestampBytes } from "./Timestamp.js";
|
|
11
|
+
export const fingerprintSize = NonNegativeInt.orThrow(12);
|
|
12
|
+
/** A fingerprint of an empty range. */
|
|
13
|
+
export const zeroFingerprint = new Uint8Array(fingerprintSize);
|
|
14
|
+
export const InfiniteUpperBound = Symbol("InfiniteUpperBound");
|
|
15
|
+
export const RangeType = {
|
|
16
|
+
Fingerprint: 1,
|
|
17
|
+
Skip: 0,
|
|
18
|
+
Timestamps: 2,
|
|
19
|
+
};
|
|
20
|
+
export const DbChangeValues = record(String, SqliteValue);
|
|
21
|
+
// "createdAt" and "updatedAt" are derived from Timestamp.
|
|
22
|
+
// "id" and "isDeleted" are encoded separately to save bytes
|
|
23
|
+
// "ownerId" is part of the protocol message
|
|
24
|
+
const forbiddenSystemColumns = systemColumns.concat("id");
|
|
25
|
+
export const ValidDbChangeValues = brand("ValidDbChangeValues", DbChangeValues, (value) => {
|
|
26
|
+
const invalidColumns = forbiddenSystemColumns.filter((key) => key in value);
|
|
27
|
+
if (invalidColumns.length > 0)
|
|
28
|
+
return err({
|
|
29
|
+
type: "ValidDbChangeValues",
|
|
30
|
+
value,
|
|
31
|
+
invalidColumns,
|
|
32
|
+
});
|
|
33
|
+
return ok(value);
|
|
34
|
+
});
|
|
1
35
|
/**
|
|
2
|
-
*
|
|
3
|
-
*
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
*
|
|
14
|
-
* will exist and will be faster than SQLite.
|
|
15
|
-
*
|
|
16
|
-
* # Scaling
|
|
36
|
+
* A DbChange is a change to a table row. Together with a unique
|
|
37
|
+
* {@link Timestamp}, it forms a {@link CrdtMessage}.
|
|
38
|
+
*/
|
|
39
|
+
export const DbChange = object({
|
|
40
|
+
table: String,
|
|
41
|
+
id: Id,
|
|
42
|
+
values: ValidDbChangeValues,
|
|
43
|
+
isInsert: Boolean,
|
|
44
|
+
isDelete: nullOr(Boolean),
|
|
45
|
+
});
|
|
46
|
+
/**
|
|
47
|
+
* Creates a {@link BaseSqliteStorage} implementation.
|
|
17
48
|
*
|
|
18
|
-
*
|
|
19
|
-
* each other, if necessary. One relay should handle hundreds of thousands of
|
|
20
|
-
* users, and when it goes down, nothing happens, because it will be
|
|
21
|
-
* synchronized later.
|
|
49
|
+
* # Stateless Design
|
|
22
50
|
*
|
|
23
|
-
*
|
|
51
|
+
* This implementation is fully stateless - it requires no in-memory state
|
|
52
|
+
* between invocations. All necessary metadata (timestamp bounds for insertion
|
|
53
|
+
* strategy optimization) is persisted in the evolu_usage table. This makes
|
|
54
|
+
* Evolu Relay suitable for stateless serverless environments like AWS Lambda,
|
|
55
|
+
* Cloudflare Workers with Durable Objects, and other platforms where memory
|
|
56
|
+
* doesn't persist between requests. While not extensively tested in all these
|
|
57
|
+
* environments yet, the stateless design should work well across them.
|
|
24
58
|
*/
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
import { sql } from "../Sqlite.js";
|
|
29
|
-
import { binaryOwnerIdToOwnerId, binaryTimestampToFingerprint, InfiniteUpperBound, RangeType, zeroFingerprint, } from "./Protocol.js";
|
|
30
|
-
import { orderBinaryTimestamp } from "./Timestamp.js";
|
|
31
|
-
export const createSqliteStorageBase = (deps) => (options) => {
|
|
32
|
-
const createTablesResult = createTables(deps);
|
|
33
|
-
if (!createTablesResult.ok)
|
|
34
|
-
return createTablesResult;
|
|
35
|
-
const ownerStats = new Map();
|
|
36
|
-
return ok({
|
|
37
|
-
insertTimestamp: (ownerId, timestamp) => {
|
|
38
|
-
const ownerIdString = binaryOwnerIdToOwnerId(ownerId);
|
|
59
|
+
export const createBaseSqliteStorage = (deps) => (config) => {
|
|
60
|
+
return {
|
|
61
|
+
insertTimestamp: (ownerId, timestamp, strategy) => {
|
|
39
62
|
const level = randomSkiplistLevel(deps);
|
|
40
|
-
let stats = ownerStats.get(ownerIdString);
|
|
41
|
-
if (!stats) {
|
|
42
|
-
const result = deps.sqlite.exec(sql.prepared `
|
|
43
|
-
select min(t) as minT, max(t) as maxT
|
|
44
|
-
from evolu_timestamp
|
|
45
|
-
where ownerId = ${ownerId};
|
|
46
|
-
`);
|
|
47
|
-
if (!result.ok)
|
|
48
|
-
return result;
|
|
49
|
-
stats = {
|
|
50
|
-
minT: result.value.rows[0].minT ?? timestamp,
|
|
51
|
-
maxT: result.value.rows[0].maxT ?? timestamp,
|
|
52
|
-
};
|
|
53
|
-
ownerStats.set(ownerIdString, stats);
|
|
54
|
-
}
|
|
55
|
-
let strategy;
|
|
56
|
-
if (orderBinaryTimestamp(timestamp, stats.maxT) === 1) {
|
|
57
|
-
strategy = "append";
|
|
58
|
-
stats.maxT = timestamp;
|
|
59
|
-
}
|
|
60
|
-
else if (orderBinaryTimestamp(timestamp, stats.minT) === -1) {
|
|
61
|
-
strategy = "prepend";
|
|
62
|
-
stats.minT = timestamp;
|
|
63
|
-
}
|
|
64
|
-
else {
|
|
65
|
-
strategy = "insert";
|
|
66
|
-
}
|
|
67
63
|
return insertTimestamp(deps)(ownerId, timestamp, level, strategy);
|
|
68
64
|
},
|
|
65
|
+
getExistingTimestamps: (ownerIdBytes, timestampsBytes) => {
|
|
66
|
+
const concatenatedTimestamps = concatBytes(...timestampsBytes);
|
|
67
|
+
const result = deps.sqlite.exec(sql `
|
|
68
|
+
with recursive
|
|
69
|
+
split_timestamps(timestampBytes, pos) as (
|
|
70
|
+
select
|
|
71
|
+
substr(${concatenatedTimestamps}, 1, 16),
|
|
72
|
+
17 as pos
|
|
73
|
+
union all
|
|
74
|
+
select
|
|
75
|
+
substr(${concatenatedTimestamps}, pos, 16),
|
|
76
|
+
pos + 16
|
|
77
|
+
from split_timestamps
|
|
78
|
+
where pos <= length(${concatenatedTimestamps})
|
|
79
|
+
)
|
|
80
|
+
select s.timestampBytes
|
|
81
|
+
from
|
|
82
|
+
split_timestamps s
|
|
83
|
+
join evolu_timestamp t
|
|
84
|
+
on t.ownerId = ${ownerIdBytes} and s.timestampBytes = t.t;
|
|
85
|
+
`);
|
|
86
|
+
if (!result.ok)
|
|
87
|
+
return result;
|
|
88
|
+
return ok(result.value.rows.map((row) => row.timestampBytes));
|
|
89
|
+
},
|
|
69
90
|
getSize: (ownerId) => {
|
|
70
91
|
const size = getSize(deps)(ownerId);
|
|
71
92
|
if (!size.ok) {
|
|
72
|
-
|
|
93
|
+
config.onStorageError(size.error);
|
|
73
94
|
return null;
|
|
74
95
|
}
|
|
75
96
|
return size.value;
|
|
@@ -78,7 +99,7 @@ export const createSqliteStorageBase = (deps) => (options) => {
|
|
|
78
99
|
assertBeginEnd(begin, end);
|
|
79
100
|
const result = fingerprint(deps)(ownerId, begin, end);
|
|
80
101
|
if (!result.ok) {
|
|
81
|
-
|
|
102
|
+
config.onStorageError(result.error);
|
|
82
103
|
return null;
|
|
83
104
|
}
|
|
84
105
|
return result.value;
|
|
@@ -86,7 +107,7 @@ export const createSqliteStorageBase = (deps) => (options) => {
|
|
|
86
107
|
fingerprintRanges: (ownerId, buckets, upperBound) => {
|
|
87
108
|
const ranges = fingerprintRanges(deps)(ownerId, buckets, upperBound);
|
|
88
109
|
if (!ranges.ok) {
|
|
89
|
-
|
|
110
|
+
config.onStorageError(ranges.error);
|
|
90
111
|
return null;
|
|
91
112
|
}
|
|
92
113
|
return ranges.value;
|
|
@@ -94,7 +115,7 @@ export const createSqliteStorageBase = (deps) => (options) => {
|
|
|
94
115
|
findLowerBound: (ownerId, begin, end, upperBound) => {
|
|
95
116
|
const lowerBound = findLowerBound(deps)(ownerId, begin, end, upperBound);
|
|
96
117
|
if (!lowerBound.ok) {
|
|
97
|
-
|
|
118
|
+
config.onStorageError(lowerBound.error);
|
|
98
119
|
return null;
|
|
99
120
|
}
|
|
100
121
|
return lowerBound.value;
|
|
@@ -107,7 +128,7 @@ export const createSqliteStorageBase = (deps) => (options) => {
|
|
|
107
128
|
// This is much faster than SQL limit with offset.
|
|
108
129
|
const first = getTimestampByIndex(deps)(ownerId, begin);
|
|
109
130
|
if (!first.ok) {
|
|
110
|
-
|
|
131
|
+
config.onStorageError(first.error);
|
|
111
132
|
return;
|
|
112
133
|
}
|
|
113
134
|
if (!callback(first.value, begin))
|
|
@@ -132,11 +153,11 @@ export const createSqliteStorageBase = (deps) => (options) => {
|
|
|
132
153
|
limit ${length - 1};
|
|
133
154
|
`);
|
|
134
155
|
if (!result.ok) {
|
|
135
|
-
|
|
156
|
+
config.onStorageError(result.error);
|
|
136
157
|
return;
|
|
137
158
|
}
|
|
138
159
|
for (let i = 0; i < result.value.rows.length; i++) {
|
|
139
|
-
const index = (begin + 1 + i);
|
|
160
|
+
const index = NonNegativeInt.orThrow(begin + 1 + i);
|
|
140
161
|
if (!callback(result.value.rows[i].t, index))
|
|
141
162
|
return;
|
|
142
163
|
}
|
|
@@ -146,17 +167,17 @@ export const createSqliteStorageBase = (deps) => (options) => {
|
|
|
146
167
|
delete from evolu_timestamp where ownerId = ${ownerId};
|
|
147
168
|
`);
|
|
148
169
|
if (!result.ok) {
|
|
149
|
-
|
|
170
|
+
config.onStorageError(result.error);
|
|
150
171
|
return false;
|
|
151
172
|
}
|
|
152
173
|
return true;
|
|
153
174
|
},
|
|
154
|
-
}
|
|
175
|
+
};
|
|
155
176
|
};
|
|
156
177
|
const assertBeginEnd = (begin, end) => {
|
|
157
178
|
assert(begin <= end, "invalid begin or end");
|
|
158
179
|
};
|
|
159
|
-
const
|
|
180
|
+
export const createBaseSqliteStorageTables = (deps) => {
|
|
160
181
|
for (const query of [
|
|
161
182
|
/**
|
|
162
183
|
* Creates the `evolu_timestamp` table for storing timestamps of multiple
|
|
@@ -169,19 +190,13 @@ const createTables = (deps) => {
|
|
|
169
190
|
*
|
|
170
191
|
* Columns:
|
|
171
192
|
*
|
|
172
|
-
* - `t` –
|
|
193
|
+
* - `t` – TimestampBytes
|
|
173
194
|
* - `h1`/`h2` – 12-byte fingerprint split into two integers for fast XOR
|
|
174
195
|
* - `c` – incremental count
|
|
175
|
-
* - `l` – Skiplist level (1 to
|
|
176
|
-
*
|
|
177
|
-
* For scaling or isolation, sharding is possible—each owner can have a
|
|
178
|
-
* separate SQLite database.
|
|
179
|
-
*
|
|
180
|
-
* Maybe we could use an integer surrogate key for ownerId, but it's fast
|
|
181
|
-
* enough even without it.
|
|
196
|
+
* - `l` – Skiplist level (1 to 10)
|
|
182
197
|
*/
|
|
183
198
|
sql `
|
|
184
|
-
create table
|
|
199
|
+
create table evolu_timestamp (
|
|
185
200
|
"ownerId" blob not null,
|
|
186
201
|
"t" blob not null,
|
|
187
202
|
"h1" integer,
|
|
@@ -193,7 +208,7 @@ const createTables = (deps) => {
|
|
|
193
208
|
strict;
|
|
194
209
|
`,
|
|
195
210
|
sql `
|
|
196
|
-
create index
|
|
211
|
+
create index evolu_timestamp_index on evolu_timestamp (
|
|
197
212
|
"ownerId",
|
|
198
213
|
"l",
|
|
199
214
|
"t",
|
|
@@ -201,6 +216,25 @@ const createTables = (deps) => {
|
|
|
201
216
|
"h2",
|
|
202
217
|
"c"
|
|
203
218
|
);
|
|
219
|
+
`,
|
|
220
|
+
/**
|
|
221
|
+
* Creates the `evolu_usage` table for tracking data consumption per owner.
|
|
222
|
+
*
|
|
223
|
+
* Columns:
|
|
224
|
+
*
|
|
225
|
+
* - `ownerId` – OwnerIdBytes (primary key)
|
|
226
|
+
* - `storedBytes` – total bytes stored in database
|
|
227
|
+
* - `firstTimestamp` – for timestamp insertion strategies
|
|
228
|
+
* - `lastTimestamp` – for timestamp insertion strategies
|
|
229
|
+
*/
|
|
230
|
+
sql `
|
|
231
|
+
create table evolu_usage (
|
|
232
|
+
"ownerId" blob primary key,
|
|
233
|
+
"storedBytes" integer not null,
|
|
234
|
+
"firstTimestamp" blob,
|
|
235
|
+
"lastTimestamp" blob
|
|
236
|
+
)
|
|
237
|
+
strict;
|
|
204
238
|
`,
|
|
205
239
|
]) {
|
|
206
240
|
const result = deps.sqlite.exec(query);
|
|
@@ -209,16 +243,37 @@ const createTables = (deps) => {
|
|
|
209
243
|
}
|
|
210
244
|
return ok();
|
|
211
245
|
};
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
246
|
+
/**
|
|
247
|
+
* Determines the insertion strategy for a timestamp based on its position
|
|
248
|
+
* relative to the current first and last timestamps.
|
|
249
|
+
*
|
|
250
|
+
* Returns a tuple with the strategy and updated timestamp bounds.
|
|
251
|
+
*/
|
|
252
|
+
export const getTimestampInsertStrategy = (timestamp, firstTimestamp, lastTimestamp) => {
|
|
253
|
+
if (orderTimestampBytes(timestamp, lastTimestamp) === 1) {
|
|
254
|
+
return ["append", firstTimestamp, timestamp];
|
|
255
|
+
}
|
|
256
|
+
if (orderTimestampBytes(timestamp, firstTimestamp) === -1) {
|
|
257
|
+
return ["prepend", timestamp, lastTimestamp];
|
|
258
|
+
}
|
|
259
|
+
return ["insert", firstTimestamp, lastTimestamp];
|
|
260
|
+
};
|
|
261
|
+
/**
|
|
262
|
+
* AFAIK, we can't do both insert and update in one query, and that's probably
|
|
263
|
+
* why append is 2x faster than insert. Prepend also has to update parents, but
|
|
264
|
+
* it's constantly fast. Insert degrades for reversed (yet LIMIT X magically
|
|
265
|
+
* fixes that) but it's OK for append.
|
|
266
|
+
*
|
|
267
|
+
* Note: SQL operations are idempotent (using `on conflict do nothing` and
|
|
268
|
+
* `changes() > 0`), but this is no longer required here since we use
|
|
269
|
+
* {@link BaseSqliteStorage.getExistingTimestamps} to filter out duplicates
|
|
270
|
+
* before insertion, which we need for quota checks anyway.
|
|
271
|
+
*
|
|
272
|
+
* TODO: Remove idempotency (`on conflict do nothing` and `changes() > 0`) since
|
|
273
|
+
* duplicates are now filtered before insertion.
|
|
274
|
+
*/
|
|
220
275
|
const insertTimestamp = (deps) => (ownerId, timestamp, level, strategy) => {
|
|
221
|
-
const [h1, h2] = fingerprintToSqliteFingerprint(
|
|
276
|
+
const [h1, h2] = fingerprintToSqliteFingerprint(timestampBytesToFingerprint(timestamp));
|
|
222
277
|
let queries = [];
|
|
223
278
|
switch (strategy) {
|
|
224
279
|
case "append":
|
|
@@ -727,6 +782,10 @@ const insertTimestamp = (deps) => (ownerId, timestamp, level, strategy) => {
|
|
|
727
782
|
}
|
|
728
783
|
return ok();
|
|
729
784
|
};
|
|
785
|
+
export const timestampBytesToFingerprint = (timestamp) => {
|
|
786
|
+
const hash = sha256(timestamp).slice(0, fingerprintSize);
|
|
787
|
+
return hash;
|
|
788
|
+
};
|
|
730
789
|
/**
|
|
731
790
|
* Generates a random skiplist level in the range [1, skiplistMaxLevel].
|
|
732
791
|
* Probabilistic approach avoids the need for explicit tree balancing.
|
|
@@ -737,7 +796,7 @@ const randomSkiplistLevel = (deps) => {
|
|
|
737
796
|
level < skiplistMaxLevel) {
|
|
738
797
|
level += 1;
|
|
739
798
|
}
|
|
740
|
-
return level;
|
|
799
|
+
return PositiveInt.orThrow(level);
|
|
741
800
|
};
|
|
742
801
|
/**
|
|
743
802
|
* Probability used for generating Skiplist levels.
|
|
@@ -840,7 +899,7 @@ const findLowerBound = (deps) => (ownerId, begin, end, upperBound) => {
|
|
|
840
899
|
if (!count.ok)
|
|
841
900
|
return count;
|
|
842
901
|
// `decrement` converts a count to an index.
|
|
843
|
-
return ok(decrement(count.value));
|
|
902
|
+
return ok(NonNegativeInt.orThrow(decrement(count.value)));
|
|
844
903
|
};
|
|
845
904
|
const getTimestampCount = (deps) => (ownerId, timestamp) => {
|
|
846
905
|
const result = deps.sqlite.exec(sql.prepared `
|
|
@@ -1119,3 +1178,50 @@ export const getTimestampByIndex = (deps) => (ownerId, index) => {
|
|
|
1119
1178
|
return result;
|
|
1120
1179
|
return ok(result.value.rows[0].pt);
|
|
1121
1180
|
};
|
|
1181
|
+
/** Retrieves usage information for an owner from the evolu_usage table. */
|
|
1182
|
+
export const getOwnerUsage = (deps) => (ownerIdBytes, initialTimestamp) => {
|
|
1183
|
+
const result = deps.sqlite.exec(sql `
|
|
1184
|
+
select storedBytes, firstTimestamp, lastTimestamp
|
|
1185
|
+
from evolu_usage
|
|
1186
|
+
where ownerId = ${ownerIdBytes};
|
|
1187
|
+
`);
|
|
1188
|
+
if (!result.ok)
|
|
1189
|
+
return result;
|
|
1190
|
+
if (!isNonEmptyReadonlyArray(result.value.rows)) {
|
|
1191
|
+
return ok({
|
|
1192
|
+
storedBytes: null,
|
|
1193
|
+
firstTimestamp: initialTimestamp,
|
|
1194
|
+
lastTimestamp: initialTimestamp,
|
|
1195
|
+
});
|
|
1196
|
+
}
|
|
1197
|
+
const row = firstInArray(result.value.rows);
|
|
1198
|
+
assert(row.firstTimestamp, "not null");
|
|
1199
|
+
assert(row.lastTimestamp, "not null");
|
|
1200
|
+
return ok({
|
|
1201
|
+
storedBytes: row.storedBytes,
|
|
1202
|
+
firstTimestamp: row.firstTimestamp,
|
|
1203
|
+
lastTimestamp: row.lastTimestamp,
|
|
1204
|
+
});
|
|
1205
|
+
};
|
|
1206
|
+
/**
|
|
1207
|
+
* Updates timestamp bounds in evolu_usage table.
|
|
1208
|
+
*
|
|
1209
|
+
* Used by both relay and client to maintain firstTimestamp/lastTimestamp after
|
|
1210
|
+
* processing messages.
|
|
1211
|
+
*/
|
|
1212
|
+
export const updateOwnerUsage = (deps) => (ownerIdBytes, storedBytes, firstTimestamp, lastTimestamp) => {
|
|
1213
|
+
const result = deps.sqlite.exec(sql `
|
|
1214
|
+
insert into evolu_usage
|
|
1215
|
+
("ownerId", "storedBytes", "firstTimestamp", "lastTimestamp")
|
|
1216
|
+
values
|
|
1217
|
+
(${ownerIdBytes}, ${storedBytes}, ${firstTimestamp}, ${lastTimestamp})
|
|
1218
|
+
on conflict (ownerId) do update
|
|
1219
|
+
set
|
|
1220
|
+
storedBytes = ${storedBytes},
|
|
1221
|
+
firstTimestamp = ${firstTimestamp},
|
|
1222
|
+
lastTimestamp = ${lastTimestamp};
|
|
1223
|
+
`);
|
|
1224
|
+
if (!result.ok)
|
|
1225
|
+
return result;
|
|
1226
|
+
return ok();
|
|
1227
|
+
};
|
package/dist/src/Evolu/Sync.d.ts
CHANGED
|
@@ -1,23 +1,78 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
|
|
5
|
-
|
|
1
|
+
import { NonEmptyReadonlyArray } from "../Array.js";
|
|
2
|
+
import { ConsoleDep } from "../Console.js";
|
|
3
|
+
import { RandomBytesDep, SymmetricCryptoDecryptError, SymmetricCryptoDep } from "../Crypto.js";
|
|
4
|
+
import { TransferableError } from "../Error.js";
|
|
5
|
+
import { RandomDep } from "../Random.js";
|
|
6
|
+
import { Result } from "../Result.js";
|
|
7
|
+
import { SqliteDep, SqliteError } from "../Sqlite.js";
|
|
8
|
+
import { TimeDep } from "../Time.js";
|
|
9
|
+
import { DateIso } from "../Type.js";
|
|
10
|
+
import { CreateWebSocketDep } from "../WebSocket.js";
|
|
11
|
+
import type { AppOwnerDep, PostMessageDep } from "./Db.js";
|
|
12
|
+
import { AppOwner, OwnerEncryptionKey, OwnerId, OwnerIdBytes, OwnerTransport, OwnerWriteKey } from "./Owner.js";
|
|
13
|
+
import { ProtocolError, ProtocolInvalidDataError, ProtocolTimestampMismatchError } from "./Protocol.js";
|
|
14
|
+
import { MutationChange } from "./Schema.js";
|
|
15
|
+
import { BaseSqliteStorage, CrdtMessage, Storage, StorageInsertTimestampStrategy } from "./Storage.js";
|
|
16
|
+
import { Millis, Timestamp, TimestampConfigDep, TimestampCounterOverflowError, TimestampDriftError, TimestampTimeOutOfRangeError } from "./Timestamp.js";
|
|
17
|
+
export interface Sync extends Disposable {
|
|
18
|
+
/**
|
|
19
|
+
* Assigns or removes an owner to/from transports with reference counting.
|
|
20
|
+
*
|
|
21
|
+
* Owners are only "active" if assigned to at least one transport. Uses
|
|
22
|
+
* `owner.transports` or falls back to config transports. Multiple calls
|
|
23
|
+
* increment/decrement reference counts (useful for React Hooks).
|
|
24
|
+
*/
|
|
25
|
+
readonly useOwner: (use: boolean, owner: SyncOwner) => void;
|
|
26
|
+
readonly applyChanges: (changes: NonEmptyReadonlyArray<MutationChange>) => Result<void, SqliteError | TimestampCounterOverflowError | TimestampDriftError | TimestampTimeOutOfRangeError>;
|
|
6
27
|
}
|
|
7
28
|
export interface SyncDep {
|
|
8
29
|
readonly sync: Sync;
|
|
9
30
|
}
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
31
|
+
/**
|
|
32
|
+
* Represents an owner for sync operations. This is a unified interface that
|
|
33
|
+
* abstracts over the specific owner types ({@link ShardOwner},
|
|
34
|
+
* {@link SharedOwner}, {@link SharedReadonlyOwner}) for the sync layer.
|
|
35
|
+
*
|
|
36
|
+
* The sync layer only needs the essential data for synchronization and doesn't
|
|
37
|
+
* need to distinguish between different owner types.
|
|
38
|
+
*/
|
|
39
|
+
export interface SyncOwner {
|
|
40
|
+
readonly id: OwnerId;
|
|
41
|
+
readonly encryptionKey: OwnerEncryptionKey;
|
|
42
|
+
/** Optional for read-only owners like {@link SharedReadonlyOwner}. */
|
|
43
|
+
readonly writeKey?: OwnerWriteKey;
|
|
44
|
+
readonly transports?: ReadonlyArray<OwnerTransport>;
|
|
45
|
+
}
|
|
46
|
+
export interface SyncConfig {
|
|
47
|
+
readonly appOwner: AppOwner;
|
|
48
|
+
readonly transports: ReadonlyArray<OwnerTransport>;
|
|
49
|
+
/**
|
|
50
|
+
* Delay in milliseconds before disposing unused WebSocket connections.
|
|
51
|
+
* Defaults to 100ms.
|
|
52
|
+
*/
|
|
53
|
+
readonly disposalDelayMs?: number;
|
|
54
|
+
readonly onError: (error: ProtocolError | ProtocolInvalidDataError | ProtocolTimestampMismatchError | SqliteError | SymmetricCryptoDecryptError | TimestampCounterOverflowError | TimestampDriftError | TimestampTimeOutOfRangeError | TransferableError) => void;
|
|
55
|
+
readonly onReceive: () => void;
|
|
13
56
|
}
|
|
14
|
-
export
|
|
15
|
-
|
|
16
|
-
readonly
|
|
17
|
-
readonly onMessage: (message: Uint8Array, send: Sync["send"]) => void;
|
|
57
|
+
export declare const createSync: (deps: ClockDep & ConsoleDep & CreateWebSocketDep & PostMessageDep & RandomBytesDep & RandomDep & SqliteDep & SymmetricCryptoDep & TimeDep & TimestampConfigDep) => (config: SyncConfig) => Result<Sync, SqliteError>;
|
|
58
|
+
export interface ClockDep {
|
|
59
|
+
readonly clock: Clock;
|
|
18
60
|
}
|
|
19
|
-
export
|
|
61
|
+
export interface Clock {
|
|
62
|
+
readonly get: () => Timestamp;
|
|
63
|
+
readonly save: (timestamp: Timestamp) => Result<void, SqliteError>;
|
|
64
|
+
}
|
|
65
|
+
export declare const createClock: (deps: RandomBytesDep & SqliteDep) => (initialTimestamp?: Timestamp) => Clock;
|
|
66
|
+
export interface ClientStorage extends Storage, BaseSqliteStorage {
|
|
67
|
+
}
|
|
68
|
+
export interface ClientStorageDep {
|
|
69
|
+
readonly storage: ClientStorage;
|
|
70
|
+
}
|
|
71
|
+
export declare const applyLocalOnlyChange: (deps: SqliteDep & TimeDep & AppOwnerDep) => (change: MutationChange) => Result<void, SqliteError>;
|
|
72
|
+
export declare const applyMessageToTimestampAndHistoryTables: (deps: ClientStorageDep & SqliteDep) => (ownerId: OwnerIdBytes, message: CrdtMessage, strategy: StorageInsertTimestampStrategy, date: DateIso) => Result<void, SqliteError>;
|
|
20
73
|
/**
|
|
74
|
+
* TODO: Rework for the new owners API.
|
|
75
|
+
*
|
|
21
76
|
* The possible states of a synchronization process. The `SyncState` can be one
|
|
22
77
|
* of the following:
|
|
23
78
|
*
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Sync.d.ts","sourceRoot":"","sources":["../../../src/Evolu/Sync.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"Sync.d.ts","sourceRoot":"","sources":["../../../src/Evolu/Sync.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,qBAAqB,EACtB,MAAM,aAAa,CAAC;AAGrB,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC3C,OAAO,EACL,cAAc,EACd,2BAA2B,EAC3B,kBAAkB,EACnB,MAAM,cAAc,CAAC;AAEtB,OAAO,EAA2B,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAGzE,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAEzC,OAAO,EAAW,MAAM,EAAE,MAAM,cAAc,CAAC;AAC/C,OAAO,EAKL,SAAS,EACT,WAAW,EAGZ,MAAM,cAAc,CAAC;AAEtB,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EACL,OAAO,EAKR,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,kBAAkB,EAAa,MAAM,iBAAiB,CAAC;AAChE,OAAO,KAAK,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAC3D,OAAO,EACL,QAAQ,EACR,kBAAkB,EAClB,OAAO,EACP,YAAY,EAGZ,cAAc,EACd,aAAa,EAId,MAAM,YAAY,CAAC;AACpB,OAAO,EAOL,aAAa,EACb,wBAAwB,EACxB,8BAA8B,EAE/B,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EACL,iBAAiB,EACjB,WAAW,EAKX,OAAO,EACP,8BAA8B,EAG/B,MAAM,cAAc,CAAC;AACtB,OAAO,EAEL,MAAM,EAGN,SAAS,EAET,kBAAkB,EAClB,6BAA6B,EAC7B,mBAAmB,EACnB,4BAA4B,EAG7B,MAAM,gBAAgB,CAAC;AAExB,MAAM,WAAW,IAAK,SAAQ,UAAU;IACtC;;;;;;OAMG;IACH,QAAQ,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,SAAS,KAAK,IAAI,CAAC;IAE5D,QAAQ,CAAC,YAAY,EAAE,CACrB,OAAO,EAAE,qBAAqB,CAAC,cAAc,CAAC,KAC3C,MAAM,CACT,IAAI,EACF,WAAW,GACX,6BAA6B,GAC7B,mBAAmB,GACnB,4BAA4B,CAC/B,CAAC;CACH;AAED,MAAM,WAAW,OAAO;IACtB,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;CACrB;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,EAAE,EAAE,OAAO,CAAC;IACrB,QAAQ,CAAC,aAAa,EAAE,kBAAkB,CAAC;IAC3C,sEAAsE;IACtE,QAAQ,CAAC,QAAQ,CAAC,EAAE,aAAa,CAAC;IAClC,QAAQ,CAAC,UAAU,CAAC,EAAE,aAAa,CAAC,cAAc,CAAC,CAAC;CACrD;AAED,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC;IAE5B,QAAQ,CAAC,UAAU,EAAE,aAAa,CAAC,cAAc,CAAC,CAAC;IAEnD;;;OAGG;IACH,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAC;IAElC,QAAQ,CAAC,OAAO,EAAE,CAChB,KAAK,EACD,aAAa,GACb,wBAAwB,GACxB,8BAA8B,GAC9B,WAAW,GACX,2BAA2B,GAC3B,6BAA6B,GAC7B,mBAAmB,GACnB,4BAA4B,GAC5B,iBAAiB,KAClB,IAAI,CAAC;IAEV,QAAQ,CAAC,SAAS,EAAE,MAAM,IAAI,CAAC;CAChC;AAED,eAAO,MAAM,UAAU,GAEnB,MAAM,QAAQ,GACZ,UAAU,GACV,kBAAkB,GAClB,cAAc,GACd,cAAc,GACd,SAAS,GACT,SAAS,GACT,kBAAkB,GAClB,OAAO,GACP,kBAAkB,MAErB,QAAQ,UAAU,KAAG,MAAM,CAAC,IAAI,EAAE,WAAW,CA2O7C,CAAC;AAEJ,MAAM,WAAW,QAAQ;IACvB,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC;CACvB;AAED,MAAM,WAAW,KAAK;IACpB,QAAQ,CAAC,GAAG,EAAE,MAAM,SAAS,CAAC;IAC9B,QAAQ,CAAC,IAAI,EAAE,CAAC,SAAS,EAAE,SAAS,KAAK,MAAM,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;CACpE;AAED,eAAO,MAAM,WAAW,GACrB,MAAM,cAAc,GAAG,SAAS,MAChC,4BAA+C,KAAG,KAiBlD,CAAC;AAMJ,MAAM,WAAW,aAAc,SAAQ,OAAO,EAAE,iBAAiB;CAAG;AAEpE,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC;CACjC;AA0MD,eAAO,MAAM,oBAAoB,GAC9B,MAAM,SAAS,GAAG,OAAO,GAAG,WAAW,MACvC,QAAQ,cAAc,KAAG,MAAM,CAAC,IAAI,EAAE,WAAW,CA+BjD,CAAC;AAkGJ,eAAO,MAAM,uCAAuC,GACjD,MAAM,gBAAgB,GAAG,SAAS,MAEjC,SAAS,YAAY,EACrB,SAAS,WAAW,EACpB,UAAU,8BAA8B,EACxC,MAAM,OAAO,KACZ,MAAM,CAAC,IAAI,EAAE,WAAW,CA4B1B,CAAC;AAgBJ;;;;;;;;;;GAUG;AACH,MAAM,MAAM,SAAS,GACjB,gBAAgB,GAChB,kBAAkB,GAClB,iBAAiB,GACjB,oBAAoB,CAAC;AAEzB;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,IAAI,EAAE,kBAAkB,CAAC;CACnC;AAED,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,IAAI,EAAE,oBAAoB,CAAC;CACrC;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,IAAI,EAAE,mBAAmB,CAAC;IACnC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,IAAI,EAAE,sBAAsB,CAAC;IACtC,QAAQ,CAAC,KAAK,EAAE,YAAY,GAAG,WAAW,GAAG,oBAAoB,CAAC;CACnE;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC;CAC/B;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAC;IAC7B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,IAAI,EAAE,sBAAsB,CAAC;CACvC;AAED,eAAO,MAAM,gBAAgB,EAAE,gBAA+C,CAAC"}
|