@evolu/common 6.0.1-preview.30 → 6.0.1-preview.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/Array.d.ts +27 -3
- package/dist/src/Array.d.ts.map +1 -1
- package/dist/src/Array.js +39 -3
- package/dist/src/Evolu/Db.d.ts +3 -0
- package/dist/src/Evolu/Db.d.ts.map +1 -1
- package/dist/src/Evolu/Evolu.d.ts.map +1 -1
- package/dist/src/Evolu/Evolu.js +8 -9
- package/dist/src/Evolu/Owner.d.ts +13 -4
- package/dist/src/Evolu/Owner.d.ts.map +1 -1
- package/dist/src/Evolu/Relay.d.ts.map +1 -1
- package/dist/src/Evolu/Relay.js +13 -18
- package/dist/src/Evolu/Storage.d.ts +42 -14
- package/dist/src/Evolu/Storage.d.ts.map +1 -1
- package/dist/src/Evolu/Storage.js +125 -85
- package/dist/src/Evolu/Sync.d.ts +2 -3
- package/dist/src/Evolu/Sync.d.ts.map +1 -1
- package/dist/src/Evolu/Sync.js +27 -8
- package/dist/src/Type.d.ts +3 -2
- package/dist/src/Type.d.ts.map +1 -1
- package/dist/src/Type.js +3 -2
- package/package.json +1 -1
- package/src/Array.ts +47 -3
- package/src/Evolu/Db.ts +3 -0
- package/src/Evolu/Evolu.ts +12 -9
- package/src/Evolu/Owner.ts +15 -29
- package/src/Evolu/Relay.ts +36 -20
- package/src/Evolu/Storage.ts +184 -120
- package/src/Evolu/Sync.ts +57 -11
- package/src/Type.ts +3 -2
package/src/Array.ts
CHANGED
|
@@ -20,9 +20,8 @@
|
|
|
20
20
|
* const filtered = filterArray(readonly, (x) => x > 1); // ReadonlyArray<number>
|
|
21
21
|
*
|
|
22
22
|
* // ✅ NonEmptyArray enforces non-emptiness
|
|
23
|
-
* const
|
|
24
|
-
*
|
|
25
|
-
* first(["a"]); // ✅ Works
|
|
23
|
+
* const value = firstInArray(["a", "b"]); // "a"
|
|
24
|
+
* firstInArray([]); // ❌ Compiler error
|
|
26
25
|
* ```
|
|
27
26
|
*
|
|
28
27
|
* @module
|
|
@@ -102,3 +101,48 @@ export const filterArray = <T>(
|
|
|
102
101
|
* **Mutates** the original array. Use only with mutable arrays.
|
|
103
102
|
*/
|
|
104
103
|
export const shiftArray = <T>(array: NonEmptyArray<T>): T => array.shift() as T;
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* Returns the first element of a non-empty readonly array.
|
|
107
|
+
*
|
|
108
|
+
* Does not mutate the original array.
|
|
109
|
+
*/
|
|
110
|
+
export const firstInArray = <T>(array: NonEmptyReadonlyArray<T>): T => array[0];
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* Returns the last element of a non-empty readonly array.
|
|
114
|
+
*
|
|
115
|
+
* Does not mutate the original array.
|
|
116
|
+
*/
|
|
117
|
+
export const lastInArray = <T>(array: NonEmptyReadonlyArray<T>): T =>
|
|
118
|
+
array[array.length - 1];
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* Returns a new readonly array with duplicate items removed based on a key
|
|
122
|
+
* extractor function. Preserves the first occurrence of each distinct key.
|
|
123
|
+
*
|
|
124
|
+
* Accepts both mutable and readonly arrays. Does not mutate the original array.
|
|
125
|
+
*/
|
|
126
|
+
|
|
127
|
+
/**
|
|
128
|
+
* Deduplicates items in an array. If `by` is provided, it will be used to
|
|
129
|
+
* derive the key for uniqueness; otherwise values are used directly.
|
|
130
|
+
*
|
|
131
|
+
* Returns a new readonly array and does not mutate the input.
|
|
132
|
+
*/
|
|
133
|
+
export const dedupeArray = <T>(
|
|
134
|
+
array: ReadonlyArray<T>,
|
|
135
|
+
by?: (item: T) => unknown,
|
|
136
|
+
): ReadonlyArray<T> => {
|
|
137
|
+
if (by == null) {
|
|
138
|
+
return Array.from(new Set(array)) as ReadonlyArray<T>;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
const seen = new Set<unknown>();
|
|
142
|
+
return array.filter((item) => {
|
|
143
|
+
const key = by(item);
|
|
144
|
+
if (seen.has(key)) return false;
|
|
145
|
+
seen.add(key);
|
|
146
|
+
return true;
|
|
147
|
+
}) as ReadonlyArray<T>;
|
|
148
|
+
};
|
package/src/Evolu/Db.ts
CHANGED
|
@@ -198,6 +198,9 @@ export interface DbConfig extends ConsoleConfig, TimestampConfig {
|
|
|
198
198
|
/**
|
|
199
199
|
* Encryption key for the SQLite database.
|
|
200
200
|
*
|
|
201
|
+
* Note: If an unencrypted SQLite database already exists and you provide an
|
|
202
|
+
* encryptionKey, SQLite will throw an error.
|
|
203
|
+
*
|
|
201
204
|
* @experimental
|
|
202
205
|
*/
|
|
203
206
|
readonly encryptionKey?: EncryptionKey;
|
package/src/Evolu/Evolu.ts
CHANGED
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
import { pack } from "msgpackr";
|
|
2
|
-
import {
|
|
3
|
-
|
|
2
|
+
import {
|
|
3
|
+
dedupeArray,
|
|
4
|
+
isNonEmptyArray,
|
|
5
|
+
isNonEmptyReadonlyArray,
|
|
6
|
+
} from "../Array.js";
|
|
7
|
+
import { assert, assertNonEmptyReadonlyArray } from "../Assert.js";
|
|
4
8
|
import { createCallbacks } from "../Callbacks.js";
|
|
5
9
|
import { ConsoleDep } from "../Console.js";
|
|
6
10
|
import { RandomBytesDep, SymmetricCryptoDecryptError } from "../Crypto.js";
|
|
@@ -611,10 +615,10 @@ const createEvoluInstance =
|
|
|
611
615
|
const loadingPromisesQueries = loadingPromises.getQueries();
|
|
612
616
|
loadingPromises.releaseUnsubscribedOnMutation();
|
|
613
617
|
|
|
614
|
-
const queries = [
|
|
615
|
-
|
|
616
|
-
...
|
|
617
|
-
];
|
|
618
|
+
const queries = dedupeArray([
|
|
619
|
+
...loadingPromisesQueries,
|
|
620
|
+
...subscribedQueries.get(),
|
|
621
|
+
]);
|
|
618
622
|
|
|
619
623
|
if (isNonEmptyReadonlyArray(queries)) {
|
|
620
624
|
dbWorker.postMessage({ type: "query", tabId: getTabId(), queries });
|
|
@@ -790,10 +794,9 @@ const createEvoluInstance =
|
|
|
790
794
|
loadQueryMicrotaskQueue.push(query);
|
|
791
795
|
if (loadQueryMicrotaskQueue.length === 1) {
|
|
792
796
|
queueMicrotask(() => {
|
|
793
|
-
|
|
794
|
-
const queries = [...new Set(loadQueryMicrotaskQueue)];
|
|
797
|
+
const queries = dedupeArray(loadQueryMicrotaskQueue);
|
|
795
798
|
loadQueryMicrotaskQueue.length = 0;
|
|
796
|
-
|
|
799
|
+
assertNonEmptyReadonlyArray(queries);
|
|
797
800
|
deps.console.log("[evolu]", "loadQuery", { queries });
|
|
798
801
|
dbWorker.postMessage({
|
|
799
802
|
type: "query",
|
package/src/Evolu/Owner.ts
CHANGED
|
@@ -18,7 +18,8 @@ import {
|
|
|
18
18
|
Mnemonic,
|
|
19
19
|
NonNegativeInt,
|
|
20
20
|
} from "../Type.js";
|
|
21
|
-
import type {
|
|
21
|
+
import type { EncryptedDbChange, Storage } from "./Storage.js";
|
|
22
|
+
import { TimestampBytes } from "./Timestamp.js";
|
|
22
23
|
|
|
23
24
|
/**
|
|
24
25
|
* The Owner represents ownership of data in Evolu. Every database change is
|
|
@@ -390,11 +391,11 @@ export interface BaseOwnerError {
|
|
|
390
391
|
/**
|
|
391
392
|
* Usage data for an {@link OwnerId}.
|
|
392
393
|
*
|
|
393
|
-
* Tracks
|
|
394
|
-
* needed. Used by both relays and clients.
|
|
394
|
+
* Tracks storage usage to enforce quotas if needed, and some other stuff.
|
|
395
395
|
*
|
|
396
|
-
*
|
|
397
|
-
*
|
|
396
|
+
* TODO:
|
|
397
|
+
*
|
|
398
|
+
* - Add transferredBytes for billing and monitoring network usage.
|
|
398
399
|
*/
|
|
399
400
|
export interface OwnerUsage {
|
|
400
401
|
/** The {@link Owner} this usage data belongs to. */
|
|
@@ -416,28 +417,13 @@ export interface OwnerUsage {
|
|
|
416
417
|
*/
|
|
417
418
|
readonly storedBytes: NonNegativeInt;
|
|
418
419
|
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
// /**
|
|
429
|
-
// * The minimum {@link Timestamp}.
|
|
430
|
-
// *
|
|
431
|
-
// * Helps {@link Storage} choose faster algorithms.
|
|
432
|
-
// */
|
|
433
|
-
// readonly firstTimestamp: TimestampBytes | null;
|
|
434
|
-
|
|
435
|
-
// TODO: Decide how to use lastTimestamp.
|
|
436
|
-
// /**
|
|
437
|
-
// * The maximum {@link Timestamp}.
|
|
438
|
-
// *
|
|
439
|
-
// * Helps {@link Storage} choose faster algorithms. Free relays can use it to
|
|
440
|
-
// * identify inactive accounts for cleanup or archival.
|
|
441
|
-
// */
|
|
442
|
-
// readonly lastTimestamp: TimestampBytes | null;
|
|
420
|
+
/** Tracks the earliest timestamp for timestamp insertion strategies. */
|
|
421
|
+
readonly firstTimestamp: TimestampBytes | null;
|
|
422
|
+
|
|
423
|
+
/**
|
|
424
|
+
* Tracks the latest timestamp for timestamp insertion strategies.
|
|
425
|
+
*
|
|
426
|
+
* Free relays can use it to identify inactive accounts for cleanup.
|
|
427
|
+
*/
|
|
428
|
+
readonly lastTimestamp: TimestampBytes | null;
|
|
443
429
|
}
|
package/src/Evolu/Relay.ts
CHANGED
|
@@ -1,4 +1,9 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {
|
|
2
|
+
filterArray,
|
|
3
|
+
firstInArray,
|
|
4
|
+
isNonEmptyReadonlyArray,
|
|
5
|
+
mapArray,
|
|
6
|
+
} from "../Array.js";
|
|
2
7
|
import { ConsoleConfig, ConsoleDep } from "../Console.js";
|
|
3
8
|
import { TimingSafeEqualDep } from "../Crypto.js";
|
|
4
9
|
import { LazyValue } from "../Function.js";
|
|
@@ -6,7 +11,7 @@ import { createInstances } from "../Instances.js";
|
|
|
6
11
|
import { err, ok, Result } from "../Result.js";
|
|
7
12
|
import { sql, SqliteDep, SqliteError } from "../Sqlite.js";
|
|
8
13
|
import { createMutex, isAsync, MaybeAsync, Mutex } from "../Task.js";
|
|
9
|
-
import {
|
|
14
|
+
import { PositiveInt, SimpleName } from "../Type.js";
|
|
10
15
|
import {
|
|
11
16
|
OwnerId,
|
|
12
17
|
ownerIdBytesToOwnerId,
|
|
@@ -18,10 +23,13 @@ import {
|
|
|
18
23
|
createBaseSqliteStorage,
|
|
19
24
|
CreateBaseSqliteStorageConfig,
|
|
20
25
|
EncryptedDbChange,
|
|
26
|
+
getOwnerUsage,
|
|
27
|
+
getTimestampInsertStrategy,
|
|
21
28
|
SqliteStorageDeps,
|
|
22
29
|
Storage,
|
|
23
30
|
StorageConfig,
|
|
24
31
|
StorageQuotaError,
|
|
32
|
+
updateOwnerUsage,
|
|
25
33
|
} from "./Storage.js";
|
|
26
34
|
import { timestampToTimestampBytes } from "./Timestamp.js";
|
|
27
35
|
|
|
@@ -193,23 +201,20 @@ export const createRelaySqliteStorage =
|
|
|
193
201
|
return ok();
|
|
194
202
|
}
|
|
195
203
|
|
|
196
|
-
const
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
const storedBytes =
|
|
206
|
-
storedBytesResult.value.rows[0]?.storedBytes ?? 0;
|
|
204
|
+
const usage = getOwnerUsage(deps)(
|
|
205
|
+
ownerIdBytes,
|
|
206
|
+
firstInArray(newMessages).timestamp,
|
|
207
|
+
);
|
|
208
|
+
if (!usage.ok) return usage;
|
|
209
|
+
|
|
210
|
+
const { storedBytes } = usage.value;
|
|
211
|
+
|
|
207
212
|
const incomingBytes = newMessages.reduce(
|
|
208
213
|
(sum, m) => sum + m.change.length,
|
|
209
214
|
0,
|
|
210
215
|
);
|
|
211
216
|
const newStoredBytes = PositiveInt.orThrow(
|
|
212
|
-
storedBytes + incomingBytes,
|
|
217
|
+
(storedBytes ?? 0) + incomingBytes,
|
|
213
218
|
);
|
|
214
219
|
|
|
215
220
|
const withinQuotaResult = config.isOwnerWithinQuota(
|
|
@@ -223,11 +228,22 @@ export const createRelaySqliteStorage =
|
|
|
223
228
|
return err({ type: "StorageQuotaError", ownerId });
|
|
224
229
|
}
|
|
225
230
|
|
|
231
|
+
let { firstTimestamp, lastTimestamp } = usage.value;
|
|
232
|
+
|
|
226
233
|
return deps.sqlite.transaction(() => {
|
|
227
234
|
for (const { timestamp, change } of newMessages) {
|
|
235
|
+
let strategy;
|
|
236
|
+
[strategy, firstTimestamp, lastTimestamp] =
|
|
237
|
+
getTimestampInsertStrategy(
|
|
238
|
+
timestamp,
|
|
239
|
+
firstTimestamp,
|
|
240
|
+
lastTimestamp,
|
|
241
|
+
);
|
|
242
|
+
|
|
228
243
|
const insertTimestampResult = sqliteStorageBase.insertTimestamp(
|
|
229
244
|
ownerIdBytes,
|
|
230
245
|
timestamp,
|
|
246
|
+
strategy,
|
|
231
247
|
);
|
|
232
248
|
if (!insertTimestampResult.ok) return insertTimestampResult;
|
|
233
249
|
|
|
@@ -239,12 +255,12 @@ export const createRelaySqliteStorage =
|
|
|
239
255
|
if (!insertMessage.ok) return insertMessage;
|
|
240
256
|
}
|
|
241
257
|
|
|
242
|
-
const updateUsage = deps
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
258
|
+
const updateUsage = updateOwnerUsage(deps)(
|
|
259
|
+
ownerIdBytes,
|
|
260
|
+
newStoredBytes,
|
|
261
|
+
firstTimestamp,
|
|
262
|
+
lastTimestamp,
|
|
263
|
+
);
|
|
248
264
|
if (!updateUsage.ok) return updateUsage;
|
|
249
265
|
|
|
250
266
|
return ok();
|
package/src/Evolu/Storage.ts
CHANGED
|
@@ -1,5 +1,9 @@
|
|
|
1
1
|
import { sha256 } from "@noble/hashes/sha2.js";
|
|
2
|
-
import {
|
|
2
|
+
import {
|
|
3
|
+
firstInArray,
|
|
4
|
+
isNonEmptyReadonlyArray,
|
|
5
|
+
NonEmptyReadonlyArray,
|
|
6
|
+
} from "../Array.js";
|
|
3
7
|
import { assert } from "../Assert.js";
|
|
4
8
|
import { Brand } from "../Brand.js";
|
|
5
9
|
import { concatBytes } from "../Buffer.js";
|
|
@@ -22,7 +26,6 @@ import {
|
|
|
22
26
|
Owner,
|
|
23
27
|
OwnerId,
|
|
24
28
|
OwnerIdBytes,
|
|
25
|
-
ownerIdBytesToOwnerId,
|
|
26
29
|
OwnerWriteKey,
|
|
27
30
|
} from "./Owner.js";
|
|
28
31
|
import { orderTimestampBytes, Timestamp, TimestampBytes } from "./Timestamp.js";
|
|
@@ -32,16 +35,16 @@ export interface StorageConfig {
|
|
|
32
35
|
* Callback called before an attempt to write, to check if an {@link OwnerId}
|
|
33
36
|
* has sufficient quota for the write.
|
|
34
37
|
*
|
|
35
|
-
* The callback receives the {@link OwnerId} and the
|
|
36
|
-
*
|
|
37
|
-
*
|
|
38
|
+
* The callback receives the {@link OwnerId} and the total bytes that would be
|
|
39
|
+
* stored after the write (current stored bytes plus incoming bytes), and
|
|
40
|
+
* returns a {@link MaybeAsync} boolean: `true` to allow the write, or `false`
|
|
41
|
+
* to deny it due to quota limits.
|
|
38
42
|
*
|
|
39
43
|
* The callback can be synchronous (for SQLite or in-memory checks) or
|
|
40
44
|
* asynchronous (for calling remote APIs).
|
|
41
45
|
*
|
|
42
|
-
* The callback returns a boolean rather than an error
|
|
43
|
-
*
|
|
44
|
-
* implementation.
|
|
46
|
+
* The callback returns a boolean rather than an error because error handling
|
|
47
|
+
* and logging are the responsibility of the callback implementation.
|
|
45
48
|
*
|
|
46
49
|
* ### Example
|
|
47
50
|
*
|
|
@@ -299,16 +302,11 @@ export interface BaseSqliteStorage
|
|
|
299
302
|
| "iterate"
|
|
300
303
|
| "deleteOwner"
|
|
301
304
|
> {
|
|
302
|
-
/**
|
|
303
|
-
* Inserts a timestamp for an owner into the skiplist-based storage.
|
|
304
|
-
*
|
|
305
|
-
* Must be idempotent - inserting the same timestamp multiple times has no
|
|
306
|
-
* effect after the first insertion. This is crucial for sync reliability as
|
|
307
|
-
* messages may be received and processed multiple times.
|
|
308
|
-
*/
|
|
305
|
+
/** Inserts a timestamp for an owner into the skiplist-based storage. */
|
|
309
306
|
readonly insertTimestamp: (
|
|
310
307
|
ownerId: OwnerIdBytes,
|
|
311
308
|
timestamp: TimestampBytes,
|
|
309
|
+
strategy: StorageInsertTimestampStrategy,
|
|
312
310
|
) => Result<void, SqliteError>;
|
|
313
311
|
|
|
314
312
|
/**
|
|
@@ -331,19 +329,62 @@ export interface CreateBaseSqliteStorageConfig extends StorageConfig {
|
|
|
331
329
|
onStorageError: (error: SqliteError) => void;
|
|
332
330
|
}
|
|
333
331
|
|
|
332
|
+
/**
|
|
333
|
+
* Creates a {@link BaseSqliteStorage} implementation.
|
|
334
|
+
*
|
|
335
|
+
* # Stateless Design
|
|
336
|
+
*
|
|
337
|
+
* This implementation is fully stateless - it requires no in-memory state
|
|
338
|
+
* between invocations. All necessary metadata (timestamp bounds for insertion
|
|
339
|
+
* strategy optimization) is persisted in the evolu_usage table. This makes
|
|
340
|
+
* Evolu Relay suitable for stateless serverless environments like AWS Lambda,
|
|
341
|
+
* Cloudflare Workers with Durable Objects, and other platforms where memory
|
|
342
|
+
* doesn't persist between requests. While not extensively tested in all these
|
|
343
|
+
* environments yet, the stateless design should work well across them.
|
|
344
|
+
*/
|
|
334
345
|
export const createBaseSqliteStorage =
|
|
335
346
|
(deps: SqliteStorageDeps) =>
|
|
336
347
|
(config: CreateBaseSqliteStorageConfig): BaseSqliteStorage => {
|
|
337
|
-
// TODO: Use evolu_usage table.
|
|
338
|
-
const ownerStats = new Map<
|
|
339
|
-
OwnerId,
|
|
340
|
-
{
|
|
341
|
-
minT: TimestampBytes;
|
|
342
|
-
maxT: TimestampBytes;
|
|
343
|
-
}
|
|
344
|
-
>();
|
|
345
|
-
|
|
346
348
|
return {
|
|
349
|
+
insertTimestamp: (
|
|
350
|
+
ownerId: OwnerIdBytes,
|
|
351
|
+
timestamp: TimestampBytes,
|
|
352
|
+
strategy: StorageInsertTimestampStrategy,
|
|
353
|
+
) => {
|
|
354
|
+
const level = randomSkiplistLevel(deps);
|
|
355
|
+
return insertTimestamp(deps)(ownerId, timestamp, level, strategy);
|
|
356
|
+
},
|
|
357
|
+
|
|
358
|
+
getExistingTimestamps: (ownerIdBytes, timestampsBytes) => {
|
|
359
|
+
const concatenatedTimestamps = concatBytes(...timestampsBytes);
|
|
360
|
+
|
|
361
|
+
const result = deps.sqlite.exec<{
|
|
362
|
+
timestampBytes: TimestampBytes;
|
|
363
|
+
}>(sql`
|
|
364
|
+
with recursive
|
|
365
|
+
split_timestamps(timestampBytes, pos) as (
|
|
366
|
+
select
|
|
367
|
+
substr(${concatenatedTimestamps}, 1, 16),
|
|
368
|
+
17 as pos
|
|
369
|
+
union all
|
|
370
|
+
select
|
|
371
|
+
substr(${concatenatedTimestamps}, pos, 16),
|
|
372
|
+
pos + 16
|
|
373
|
+
from split_timestamps
|
|
374
|
+
where pos <= length(${concatenatedTimestamps})
|
|
375
|
+
)
|
|
376
|
+
select s.timestampBytes
|
|
377
|
+
from
|
|
378
|
+
split_timestamps s
|
|
379
|
+
join evolu_timestamp t
|
|
380
|
+
on t.ownerId = ${ownerIdBytes} and s.timestampBytes = t.t;
|
|
381
|
+
`);
|
|
382
|
+
|
|
383
|
+
if (!result.ok) return result;
|
|
384
|
+
|
|
385
|
+
return ok(result.value.rows.map((row) => row.timestampBytes));
|
|
386
|
+
},
|
|
387
|
+
|
|
347
388
|
getSize: (ownerId) => {
|
|
348
389
|
const size = getSize(deps)(ownerId);
|
|
349
390
|
if (!size.ok) {
|
|
@@ -439,75 +480,6 @@ export const createBaseSqliteStorage =
|
|
|
439
480
|
}
|
|
440
481
|
return true;
|
|
441
482
|
},
|
|
442
|
-
|
|
443
|
-
insertTimestamp: (ownerId: OwnerIdBytes, timestamp: TimestampBytes) => {
|
|
444
|
-
const ownerIdString = ownerIdBytesToOwnerId(ownerId);
|
|
445
|
-
const level = randomSkiplistLevel(deps);
|
|
446
|
-
|
|
447
|
-
let stats = ownerStats.get(ownerIdString);
|
|
448
|
-
|
|
449
|
-
if (!stats) {
|
|
450
|
-
const result = deps.sqlite.exec<{
|
|
451
|
-
maxT: TimestampBytes | null;
|
|
452
|
-
minT: TimestampBytes | null;
|
|
453
|
-
}>(sql.prepared`
|
|
454
|
-
select min(t) as minT, max(t) as maxT
|
|
455
|
-
from evolu_timestamp
|
|
456
|
-
where ownerId = ${ownerId};
|
|
457
|
-
`);
|
|
458
|
-
if (!result.ok) return result;
|
|
459
|
-
|
|
460
|
-
stats = {
|
|
461
|
-
minT: result.value.rows[0].minT ?? timestamp,
|
|
462
|
-
maxT: result.value.rows[0].maxT ?? timestamp,
|
|
463
|
-
};
|
|
464
|
-
ownerStats.set(ownerIdString, stats);
|
|
465
|
-
}
|
|
466
|
-
|
|
467
|
-
let strategy: InsertTimestampStrategy;
|
|
468
|
-
|
|
469
|
-
if (orderTimestampBytes(timestamp, stats.maxT) === 1) {
|
|
470
|
-
strategy = "append";
|
|
471
|
-
stats.maxT = timestamp;
|
|
472
|
-
} else if (orderTimestampBytes(timestamp, stats.minT) === -1) {
|
|
473
|
-
strategy = "prepend";
|
|
474
|
-
stats.minT = timestamp;
|
|
475
|
-
} else {
|
|
476
|
-
strategy = "insert";
|
|
477
|
-
}
|
|
478
|
-
|
|
479
|
-
return insertTimestamp(deps)(ownerId, timestamp, level, strategy);
|
|
480
|
-
},
|
|
481
|
-
|
|
482
|
-
getExistingTimestamps: (ownerIdBytes, timestampsBytes) => {
|
|
483
|
-
const concatenatedTimestamps = concatBytes(...timestampsBytes);
|
|
484
|
-
|
|
485
|
-
const result = deps.sqlite.exec<{
|
|
486
|
-
timestampBytes: TimestampBytes;
|
|
487
|
-
}>(sql`
|
|
488
|
-
with recursive
|
|
489
|
-
split_timestamps(timestampBytes, pos) as (
|
|
490
|
-
select
|
|
491
|
-
substr(${concatenatedTimestamps}, 1, 16),
|
|
492
|
-
17 as pos
|
|
493
|
-
union all
|
|
494
|
-
select
|
|
495
|
-
substr(${concatenatedTimestamps}, pos, 16),
|
|
496
|
-
pos + 16
|
|
497
|
-
from split_timestamps
|
|
498
|
-
where pos <= length(${concatenatedTimestamps})
|
|
499
|
-
)
|
|
500
|
-
select s.timestampBytes
|
|
501
|
-
from
|
|
502
|
-
split_timestamps s
|
|
503
|
-
join evolu_timestamp t
|
|
504
|
-
on t.ownerId = ${ownerIdBytes} and s.timestampBytes = t.t;
|
|
505
|
-
`);
|
|
506
|
-
|
|
507
|
-
if (!result.ok) return result;
|
|
508
|
-
|
|
509
|
-
return ok(result.value.rows.map((row) => row.timestampBytes));
|
|
510
|
-
},
|
|
511
483
|
};
|
|
512
484
|
};
|
|
513
485
|
|
|
@@ -533,13 +505,7 @@ export const createBaseSqliteStorageTables = (
|
|
|
533
505
|
* - `t` – TimestampBytes
|
|
534
506
|
* - `h1`/`h2` – 12-byte fingerprint split into two integers for fast XOR
|
|
535
507
|
* - `c` – incremental count
|
|
536
|
-
* - `l` – Skiplist level (1 to
|
|
537
|
-
*
|
|
538
|
-
* For scaling or isolation, sharding is possible—each owner can have a
|
|
539
|
-
* separate SQLite database.
|
|
540
|
-
*
|
|
541
|
-
* Maybe we could use an integer surrogate key for ownerId, but it's fast
|
|
542
|
-
* enough even without it.
|
|
508
|
+
* - `l` – Skiplist level (1 to 10)
|
|
543
509
|
*/
|
|
544
510
|
sql`
|
|
545
511
|
create table evolu_timestamp (
|
|
@@ -572,20 +538,15 @@ export const createBaseSqliteStorageTables = (
|
|
|
572
538
|
*
|
|
573
539
|
* - `ownerId` – OwnerIdBytes (primary key)
|
|
574
540
|
* - `storedBytes` – total bytes stored in database
|
|
575
|
-
* - `
|
|
576
|
-
* - `
|
|
577
|
-
* - `firstTimestamp` – TODO: Decide how to use (nullable)
|
|
578
|
-
* - `lastTimestamp` – TODO: Decide how to use (nullable)
|
|
541
|
+
* - `firstTimestamp` – for timestamp insertion strategies
|
|
542
|
+
* - `lastTimestamp` – for timestamp insertion strategies
|
|
579
543
|
*/
|
|
580
544
|
sql`
|
|
581
545
|
create table evolu_usage (
|
|
582
546
|
"ownerId" blob primary key,
|
|
583
|
-
"storedBytes" integer not null
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
-- "sentBytes" integer not null,
|
|
587
|
-
-- "firstTimestamp" blob,
|
|
588
|
-
-- "lastTimestamp" blob
|
|
547
|
+
"storedBytes" integer not null,
|
|
548
|
+
"firstTimestamp" blob,
|
|
549
|
+
"lastTimestamp" blob
|
|
589
550
|
)
|
|
590
551
|
strict;
|
|
591
552
|
`,
|
|
@@ -596,23 +557,53 @@ export const createBaseSqliteStorageTables = (
|
|
|
596
557
|
return ok();
|
|
597
558
|
};
|
|
598
559
|
|
|
599
|
-
type
|
|
560
|
+
export type StorageInsertTimestampStrategy = "append" | "prepend" | "insert";
|
|
561
|
+
|
|
562
|
+
/**
|
|
563
|
+
* Determines the insertion strategy for a timestamp based on its position
|
|
564
|
+
* relative to the current first and last timestamps.
|
|
565
|
+
*
|
|
566
|
+
* Returns a tuple with the strategy and updated timestamp bounds.
|
|
567
|
+
*/
|
|
568
|
+
export const getTimestampInsertStrategy = (
|
|
569
|
+
timestamp: TimestampBytes,
|
|
570
|
+
firstTimestamp: TimestampBytes,
|
|
571
|
+
lastTimestamp: TimestampBytes,
|
|
572
|
+
): [
|
|
573
|
+
strategy: StorageInsertTimestampStrategy,
|
|
574
|
+
firstTimestamp: TimestampBytes,
|
|
575
|
+
lastTimestamp: TimestampBytes,
|
|
576
|
+
] => {
|
|
577
|
+
if (orderTimestampBytes(timestamp, lastTimestamp) === 1) {
|
|
578
|
+
return ["append", firstTimestamp, timestamp];
|
|
579
|
+
}
|
|
580
|
+
if (orderTimestampBytes(timestamp, firstTimestamp) === -1) {
|
|
581
|
+
return ["prepend", timestamp, lastTimestamp];
|
|
582
|
+
}
|
|
583
|
+
return ["insert", firstTimestamp, lastTimestamp];
|
|
584
|
+
};
|
|
600
585
|
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
586
|
+
/**
|
|
587
|
+
* AFAIK, we can't do both insert and update in one query, and that's probably
|
|
588
|
+
* why append is 2x faster than insert. Prepend also has to update parents, but
|
|
589
|
+
* it's constantly fast. Insert degrades for reversed (yet LIMIT X magically
|
|
590
|
+
* fixes that) but it's OK for append.
|
|
591
|
+
*
|
|
592
|
+
* Note: SQL operations are idempotent (using `on conflict do nothing` and
|
|
593
|
+
* `changes() > 0`), but this is no longer required here since we use
|
|
594
|
+
* {@link BaseSqliteStorage.getExistingTimestamps} to filter out duplicates
|
|
595
|
+
* before insertion, which we need for quota checks anyway.
|
|
596
|
+
*
|
|
597
|
+
* TODO: Remove idempotency (`on conflict do nothing` and `changes() > 0`) since
|
|
598
|
+
* duplicates are now filtered before insertion.
|
|
599
|
+
*/
|
|
609
600
|
const insertTimestamp =
|
|
610
601
|
(deps: SqliteDep) =>
|
|
611
602
|
(
|
|
612
603
|
ownerId: OwnerIdBytes,
|
|
613
604
|
timestamp: TimestampBytes,
|
|
614
605
|
level: PositiveInt,
|
|
615
|
-
strategy:
|
|
606
|
+
strategy: StorageInsertTimestampStrategy,
|
|
616
607
|
): Result<void, SqliteError> => {
|
|
617
608
|
const [h1, h2] = fingerprintToSqliteFingerprint(
|
|
618
609
|
timestampBytesToFingerprint(timestamp),
|
|
@@ -1612,3 +1603,76 @@ export const getTimestampByIndex =
|
|
|
1612
1603
|
if (!result.ok) return result;
|
|
1613
1604
|
return ok(result.value.rows[0].pt);
|
|
1614
1605
|
};
|
|
1606
|
+
|
|
1607
|
+
/** Retrieves usage information for an owner from the evolu_usage table. */
|
|
1608
|
+
export const getOwnerUsage =
|
|
1609
|
+
(deps: SqliteDep) =>
|
|
1610
|
+
(
|
|
1611
|
+
ownerIdBytes: OwnerIdBytes,
|
|
1612
|
+
initialTimestamp: TimestampBytes,
|
|
1613
|
+
): Result<
|
|
1614
|
+
{
|
|
1615
|
+
storedBytes: NonNegativeInt | null;
|
|
1616
|
+
firstTimestamp: TimestampBytes;
|
|
1617
|
+
lastTimestamp: TimestampBytes;
|
|
1618
|
+
},
|
|
1619
|
+
SqliteError
|
|
1620
|
+
> => {
|
|
1621
|
+
const result = deps.sqlite.exec<{
|
|
1622
|
+
storedBytes: NonNegativeInt;
|
|
1623
|
+
firstTimestamp: TimestampBytes | null;
|
|
1624
|
+
lastTimestamp: TimestampBytes | null;
|
|
1625
|
+
}>(sql`
|
|
1626
|
+
select storedBytes, firstTimestamp, lastTimestamp
|
|
1627
|
+
from evolu_usage
|
|
1628
|
+
where ownerId = ${ownerIdBytes};
|
|
1629
|
+
`);
|
|
1630
|
+
if (!result.ok) return result;
|
|
1631
|
+
|
|
1632
|
+
if (!isNonEmptyReadonlyArray(result.value.rows)) {
|
|
1633
|
+
return ok({
|
|
1634
|
+
storedBytes: null,
|
|
1635
|
+
firstTimestamp: initialTimestamp,
|
|
1636
|
+
lastTimestamp: initialTimestamp,
|
|
1637
|
+
});
|
|
1638
|
+
}
|
|
1639
|
+
|
|
1640
|
+
const row = firstInArray(result.value.rows);
|
|
1641
|
+
assert(row.firstTimestamp, "not null");
|
|
1642
|
+
assert(row.lastTimestamp, "not null");
|
|
1643
|
+
|
|
1644
|
+
return ok({
|
|
1645
|
+
storedBytes: row.storedBytes,
|
|
1646
|
+
firstTimestamp: row.firstTimestamp,
|
|
1647
|
+
lastTimestamp: row.lastTimestamp,
|
|
1648
|
+
});
|
|
1649
|
+
};
|
|
1650
|
+
|
|
1651
|
+
/**
|
|
1652
|
+
* Updates timestamp bounds in evolu_usage table.
|
|
1653
|
+
*
|
|
1654
|
+
* Used by both relay and client to maintain firstTimestamp/lastTimestamp after
|
|
1655
|
+
* processing messages.
|
|
1656
|
+
*/
|
|
1657
|
+
export const updateOwnerUsage =
|
|
1658
|
+
(deps: SqliteDep) =>
|
|
1659
|
+
(
|
|
1660
|
+
ownerIdBytes: OwnerIdBytes,
|
|
1661
|
+
storedBytes: PositiveInt,
|
|
1662
|
+
firstTimestamp: TimestampBytes,
|
|
1663
|
+
lastTimestamp: TimestampBytes,
|
|
1664
|
+
): Result<void, SqliteError> => {
|
|
1665
|
+
const result = deps.sqlite.exec(sql`
|
|
1666
|
+
insert into evolu_usage
|
|
1667
|
+
("ownerId", "storedBytes", "firstTimestamp", "lastTimestamp")
|
|
1668
|
+
values
|
|
1669
|
+
(${ownerIdBytes}, ${storedBytes}, ${firstTimestamp}, ${lastTimestamp})
|
|
1670
|
+
on conflict (ownerId) do update
|
|
1671
|
+
set
|
|
1672
|
+
storedBytes = ${storedBytes},
|
|
1673
|
+
firstTimestamp = ${firstTimestamp},
|
|
1674
|
+
lastTimestamp = ${lastTimestamp};
|
|
1675
|
+
`);
|
|
1676
|
+
if (!result.ok) return result;
|
|
1677
|
+
return ok();
|
|
1678
|
+
};
|