@sqlite-sync/core 0.1.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-UGF5IU53.js → chunk-NHT3ELMN.js} +34 -6
- package/dist/chunk-NHT3ELMN.js.map +1 -0
- package/dist/{chunk-627DSM2Q.js → chunk-RKBTBPNC.js} +566 -177
- package/dist/chunk-RKBTBPNC.js.map +1 -0
- package/dist/{crdt-sync-remote-source-idoIjMcs.d.ts → crdt-sync-remote-source-Da77s4k0.d.ts} +131 -46
- package/dist/index.d.ts +66 -34
- package/dist/index.js +158 -98
- package/dist/index.js.map +1 -1
- package/dist/{crdt-schema-DQ1cYsFE.d.ts → reset-state-0LGwO78x.d.ts} +20 -2
- package/dist/server.d.ts +9 -2
- package/dist/server.js +2 -2
- package/dist/server.js.map +1 -1
- package/dist/worker.d.ts +11 -3
- package/dist/worker.js +119 -89
- package/dist/worker.js.map +1 -1
- package/package.json +7 -3
- package/dist/chunk-627DSM2Q.js.map +0 -1
- package/dist/chunk-UGF5IU53.js.map +0 -1
|
@@ -2,9 +2,11 @@ import {
|
|
|
2
2
|
TypedBroadcastChannel,
|
|
3
3
|
createTypedEventTarget,
|
|
4
4
|
ensureSingletonExecution,
|
|
5
|
+
generateId,
|
|
6
|
+
parseTableName,
|
|
5
7
|
quoteId,
|
|
6
8
|
tryCatchAsync
|
|
7
|
-
} from "./chunk-
|
|
9
|
+
} from "./chunk-NHT3ELMN.js";
|
|
8
10
|
|
|
9
11
|
// src/dummy-kysely.ts
|
|
10
12
|
import { DummyDriver, Kysely, SqliteAdapter, SqliteIntrospector, SqliteQueryCompiler } from "kysely";
|
|
@@ -17,6 +19,29 @@ var dummyKysely = new Kysely({
|
|
|
17
19
|
}
|
|
18
20
|
});
|
|
19
21
|
|
|
22
|
+
// src/hash.ts
|
|
23
|
+
import initXxhash from "xxhash-wasm";
|
|
24
|
+
var loadPromise = null;
|
|
25
|
+
var api = null;
|
|
26
|
+
function ensureLoaded() {
|
|
27
|
+
if (!loadPromise) {
|
|
28
|
+
loadPromise = initXxhash().then((hasher) => {
|
|
29
|
+
api = hasher;
|
|
30
|
+
});
|
|
31
|
+
}
|
|
32
|
+
return loadPromise;
|
|
33
|
+
}
|
|
34
|
+
function h64(input, seed = 0n) {
|
|
35
|
+
if (!api) {
|
|
36
|
+
throw new Error("xxhash is not initialized; call xxhash.ensureLoaded() first");
|
|
37
|
+
}
|
|
38
|
+
return api.h64(input, seed);
|
|
39
|
+
}
|
|
40
|
+
var xxhash = {
|
|
41
|
+
ensureLoaded,
|
|
42
|
+
h64
|
|
43
|
+
};
|
|
44
|
+
|
|
20
45
|
// src/hlc.ts
|
|
21
46
|
var MAX_COUNTER = 36 ** 5 - 1;
|
|
22
47
|
var DEFAULT_MAX_DRIFT_MS = 6 * 60 * 60 * 1e3;
|
|
@@ -126,8 +151,8 @@ function introspectDb(_db) {
|
|
|
126
151
|
columnsByTable[row.table].push(row);
|
|
127
152
|
}
|
|
128
153
|
return Object.fromEntries(
|
|
129
|
-
tables.map(({ name, sql:
|
|
130
|
-
let autoIncrementCol =
|
|
154
|
+
tables.map(({ name, sql: sql4, type }) => {
|
|
155
|
+
let autoIncrementCol = sql4?.split(/[(),]/)?.find((it) => it.toLowerCase().includes("autoincrement"))?.trimStart()?.split(/\s+/)?.[0]?.replace(/["`]/g, "");
|
|
131
156
|
const columns = columnsByTable[name] ?? [];
|
|
132
157
|
if (!autoIncrementCol) {
|
|
133
158
|
const pkCols = columns.filter((r) => r.pk > 0);
|
|
@@ -198,16 +223,16 @@ var SQLiteDbWrapper = class {
|
|
|
198
223
|
return this.loadedDbSchema;
|
|
199
224
|
}
|
|
200
225
|
execute(opts, meta) {
|
|
201
|
-
const
|
|
226
|
+
const sql4 = typeof opts === "string" ? opts : opts.sql;
|
|
202
227
|
const bind = typeof opts === "string" ? void 0 : opts.parameters;
|
|
203
228
|
const perf = this.logger ? startPerformanceLogger(this.logger) : void 0;
|
|
204
229
|
const rows = this.ensureDb.exec({
|
|
205
|
-
sql:
|
|
230
|
+
sql: sql4,
|
|
206
231
|
bind,
|
|
207
232
|
returnValue: "resultRows",
|
|
208
233
|
rowMode: "object"
|
|
209
234
|
});
|
|
210
|
-
perf?.logEnd(`${this.loggerPrefix ?? ""}:query`,
|
|
235
|
+
perf?.logEnd(`${this.loggerPrefix ?? ""}:query`, sql4, meta?.loggerLevel);
|
|
211
236
|
return { rows };
|
|
212
237
|
}
|
|
213
238
|
executeTransaction(callback) {
|
|
@@ -253,26 +278,29 @@ var SQLiteDbWrapper = class {
|
|
|
253
278
|
}
|
|
254
279
|
};
|
|
255
280
|
}
|
|
256
|
-
prepare(
|
|
281
|
+
prepare(sql4, opts) {
|
|
257
282
|
const perf = this.logger ? startPerformanceLogger(this.logger) : void 0;
|
|
258
|
-
const stmt = this.ensureDb.prepare(
|
|
259
|
-
perf?.logEnd(`${this.loggerPrefix ?? ""}:prepare`,
|
|
283
|
+
const stmt = this.ensureDb.prepare(sql4);
|
|
284
|
+
perf?.logEnd(`${this.loggerPrefix ?? ""}:prepare`, sql4, opts?.loggerLevel);
|
|
260
285
|
let isFinalized = false;
|
|
261
286
|
const execute = (params) => {
|
|
262
287
|
if (isFinalized) {
|
|
263
288
|
throw new Error("Statement is finalized");
|
|
264
289
|
}
|
|
265
290
|
const perf2 = this.logger ? startPerformanceLogger(this.logger) : void 0;
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
291
|
+
try {
|
|
292
|
+
if (params.length > 0) {
|
|
293
|
+
stmt.bind(params);
|
|
294
|
+
}
|
|
295
|
+
const results = [];
|
|
296
|
+
while (stmt.step()) {
|
|
297
|
+
results.push(stmt.get({}));
|
|
298
|
+
}
|
|
299
|
+
return results;
|
|
300
|
+
} finally {
|
|
301
|
+
stmt.reset(true);
|
|
302
|
+
perf2?.logEnd(`${this.loggerPrefix ?? ""}:prepare-execute`, sql4, opts?.loggerLevel);
|
|
272
303
|
}
|
|
273
|
-
stmt.reset(true);
|
|
274
|
-
perf2?.logEnd(`${this.loggerPrefix ?? ""}:prepare-execute`, sql2, opts?.loggerLevel);
|
|
275
|
-
return results;
|
|
276
304
|
};
|
|
277
305
|
const finalize = () => {
|
|
278
306
|
isFinalized = true;
|
|
@@ -315,13 +343,13 @@ var SQLiteDbWrapper = class {
|
|
|
315
343
|
}
|
|
316
344
|
executePreparedRaw({
|
|
317
345
|
key,
|
|
318
|
-
sql:
|
|
346
|
+
sql: sql4,
|
|
319
347
|
params,
|
|
320
348
|
meta
|
|
321
349
|
}) {
|
|
322
350
|
let statement = this.preparedRawStatementsMap.get(key);
|
|
323
351
|
if (!statement) {
|
|
324
|
-
statement = this.prepare(
|
|
352
|
+
statement = this.prepare(sql4, meta);
|
|
325
353
|
this.preparedRawStatementsMap.set(key, statement);
|
|
326
354
|
}
|
|
327
355
|
return statement.execute(params ?? []);
|
|
@@ -394,6 +422,12 @@ var SQLiteDbWrapper = class {
|
|
|
394
422
|
}
|
|
395
423
|
};
|
|
396
424
|
|
|
425
|
+
// src/sqlite-crdt/crdt-table-schema.ts
|
|
426
|
+
var CRDT_EVENT_NO_OP_PAYLOAD = "no-op";
|
|
427
|
+
function isNoOpCrdtEventPayload(payload) {
|
|
428
|
+
return payload === CRDT_EVENT_NO_OP_PAYLOAD;
|
|
429
|
+
}
|
|
430
|
+
|
|
397
431
|
// src/migrations/migrator.ts
|
|
398
432
|
var protectedColumns = ["id", "tombstone"];
|
|
399
433
|
function assertColumnNotProtected(column, operation) {
|
|
@@ -488,21 +522,21 @@ var migrationSteps = {
|
|
|
488
522
|
}
|
|
489
523
|
};
|
|
490
524
|
function buildMigrationSql(steps) {
|
|
491
|
-
return steps.flatMap((step) => Array.isArray(step.sql) ? step.sql : [step.sql]).map((
|
|
492
|
-
if (typeof
|
|
493
|
-
return { sql:
|
|
525
|
+
return steps.flatMap((step) => Array.isArray(step.sql) ? step.sql : [step.sql]).map((sql4) => {
|
|
526
|
+
if (typeof sql4 === "string") {
|
|
527
|
+
return { sql: sql4, parameters: [] };
|
|
494
528
|
}
|
|
495
|
-
if (typeof
|
|
496
|
-
const query =
|
|
529
|
+
if (typeof sql4 === "function") {
|
|
530
|
+
const query = sql4(dummyKysely).compile();
|
|
497
531
|
return { sql: query.sql, parameters: query.parameters };
|
|
498
532
|
}
|
|
499
|
-
if ("compile" in
|
|
500
|
-
const query =
|
|
533
|
+
if ("compile" in sql4) {
|
|
534
|
+
const query = sql4.compile();
|
|
501
535
|
return { sql: query.sql, parameters: query.parameters };
|
|
502
536
|
}
|
|
503
537
|
return {
|
|
504
|
-
sql:
|
|
505
|
-
parameters:
|
|
538
|
+
sql: sql4.sql,
|
|
539
|
+
parameters: sql4.parameters ?? []
|
|
506
540
|
};
|
|
507
541
|
});
|
|
508
542
|
}
|
|
@@ -595,6 +629,12 @@ function createMigrator({
|
|
|
595
629
|
`Target schema version ${targetVersion} is greater than current schema version ${schemaVersion.current}`
|
|
596
630
|
);
|
|
597
631
|
}
|
|
632
|
+
if (isNoOpCrdtEventPayload(event.payload)) {
|
|
633
|
+
if (event.schema_version < targetVersion) {
|
|
634
|
+
event.schema_version = targetVersion;
|
|
635
|
+
}
|
|
636
|
+
return event;
|
|
637
|
+
}
|
|
598
638
|
if (event.schema_version >= targetVersion) {
|
|
599
639
|
return event;
|
|
600
640
|
}
|
|
@@ -667,15 +707,13 @@ function createStoredValue({
|
|
|
667
707
|
}
|
|
668
708
|
|
|
669
709
|
// src/sqlite-kv-store.ts
|
|
710
|
+
import { sql as sql2 } from "kysely";
|
|
670
711
|
function createKvStoreTableQuery(schema, tableName) {
|
|
671
|
-
return schema.createTable(tableName).ifNotExists().addColumn("key", "text", (col) => col.notNull().primaryKey()).addColumn("value", "text", (col) => col.notNull());
|
|
712
|
+
return schema.createTable(tableName).ifNotExists().addColumn("key", "text", (col) => col.notNull().primaryKey()).addColumn("value", "text", (col) => col.notNull()).modifyEnd(sql2`without rowid`);
|
|
672
713
|
}
|
|
673
|
-
function createSQLiteKvStore({
|
|
674
|
-
db,
|
|
675
|
-
metaTableName
|
|
676
|
-
}) {
|
|
714
|
+
function createSQLiteKvStore({ db, metaTableName }) {
|
|
677
715
|
const metaDb = db;
|
|
678
|
-
const
|
|
716
|
+
const get2 = (key) => {
|
|
679
717
|
const [result] = metaDb.executePrepared(
|
|
680
718
|
"get-meta-value",
|
|
681
719
|
{ key },
|
|
@@ -684,7 +722,7 @@ function createSQLiteKvStore({
|
|
|
684
722
|
);
|
|
685
723
|
return result?.value ?? null;
|
|
686
724
|
};
|
|
687
|
-
const
|
|
725
|
+
const set2 = (key, value) => {
|
|
688
726
|
metaDb.executePrepared(
|
|
689
727
|
"set-meta-value",
|
|
690
728
|
{ key, value },
|
|
@@ -701,32 +739,49 @@ function createSQLiteKvStore({
|
|
|
701
739
|
);
|
|
702
740
|
};
|
|
703
741
|
const getNumberOrDefault = (key, defaultValue) => {
|
|
704
|
-
const value =
|
|
742
|
+
const value = get2(key);
|
|
705
743
|
if (!value) return defaultValue;
|
|
706
744
|
const parsedValue = Number.parseInt(value, 10);
|
|
707
745
|
return Number.isNaN(parsedValue) ? defaultValue : parsedValue;
|
|
708
746
|
};
|
|
709
747
|
return {
|
|
710
|
-
get,
|
|
711
|
-
set,
|
|
748
|
+
get: get2,
|
|
749
|
+
set: set2,
|
|
712
750
|
remove,
|
|
713
751
|
createStringStoredValue: (key, defaultValue) => createStoredValue({
|
|
714
|
-
initialValue:
|
|
715
|
-
saveToStorage: (val) =>
|
|
752
|
+
initialValue: get2(key) ?? defaultValue,
|
|
753
|
+
saveToStorage: (val) => set2(key, val)
|
|
716
754
|
}),
|
|
717
755
|
createNumberStoredValue: (key, defaultValue) => createStoredValue({
|
|
718
756
|
initialValue: getNumberOrDefault(key, defaultValue),
|
|
719
|
-
saveToStorage: (val) =>
|
|
757
|
+
saveToStorage: (val) => set2(key, val.toString())
|
|
720
758
|
})
|
|
721
759
|
};
|
|
722
760
|
}
|
|
723
761
|
|
|
724
762
|
// src/migrations/system-schema.ts
|
|
763
|
+
function createSystemDbConfig({
|
|
764
|
+
eventsTableName,
|
|
765
|
+
updateLogTableName
|
|
766
|
+
}) {
|
|
767
|
+
return {
|
|
768
|
+
eventsTable: parseTableName(eventsTableName),
|
|
769
|
+
updateLogTable: parseTableName(updateLogTableName)
|
|
770
|
+
};
|
|
771
|
+
}
|
|
772
|
+
var workerDbConfig = createSystemDbConfig({
|
|
773
|
+
eventsTableName: "worker.crdt_events",
|
|
774
|
+
updateLogTableName: "crdt_update_log"
|
|
775
|
+
});
|
|
776
|
+
var memoryDbConfig = createSystemDbConfig({
|
|
777
|
+
eventsTableName: "persisted_crdt_events",
|
|
778
|
+
updateLogTableName: "crdt_update_log"
|
|
779
|
+
});
|
|
725
780
|
var baseSystemMigrations = [
|
|
726
781
|
{
|
|
727
782
|
version: 0,
|
|
728
783
|
up: (ctx) => {
|
|
729
|
-
ctx.execute(`CREATE TABLE IF NOT EXISTS ${ctx.
|
|
784
|
+
ctx.execute(`CREATE TABLE IF NOT EXISTS ${ctx.eventsTable.fullIdentifier} (
|
|
730
785
|
"sync_id" integer NOT NULL PRIMARY KEY,
|
|
731
786
|
"schema_version" integer NOT NULL,
|
|
732
787
|
"status" text NOT NULL,
|
|
@@ -737,7 +792,7 @@ var baseSystemMigrations = [
|
|
|
737
792
|
"item_id" text NOT NULL,
|
|
738
793
|
"payload" text NOT NULL
|
|
739
794
|
)`);
|
|
740
|
-
ctx.execute(`CREATE TABLE IF NOT EXISTS ${ctx.
|
|
795
|
+
ctx.execute(`CREATE TABLE IF NOT EXISTS ${ctx.updateLogTable.fullIdentifier} (
|
|
741
796
|
"dataset" text NOT NULL,
|
|
742
797
|
"item_id" text NOT NULL,
|
|
743
798
|
"payload" text NOT NULL,
|
|
@@ -748,14 +803,31 @@ var baseSystemMigrations = [
|
|
|
748
803
|
{
|
|
749
804
|
version: 1,
|
|
750
805
|
up: (ctx) => {
|
|
751
|
-
ctx.execute(`ALTER TABLE ${ctx.
|
|
806
|
+
ctx.execute(`ALTER TABLE ${ctx.eventsTable.fullIdentifier} ADD COLUMN "source_node_id" TEXT NOT NULL DEFAULT ''`);
|
|
807
|
+
}
|
|
808
|
+
},
|
|
809
|
+
{
|
|
810
|
+
version: 2,
|
|
811
|
+
up: (ctx) => {
|
|
812
|
+
const indexName = `${ctx.eventsTable.table}_status_sync_id_idx`;
|
|
813
|
+
ctx.execute(
|
|
814
|
+
`CREATE INDEX IF NOT EXISTS ${ctx.eventsTable.schema}.${indexName} ON ${ctx.eventsTable.table} ("status", "sync_id")`
|
|
815
|
+
);
|
|
816
|
+
}
|
|
817
|
+
},
|
|
818
|
+
{
|
|
819
|
+
version: 3,
|
|
820
|
+
up: (ctx) => {
|
|
821
|
+
const indexName = `${ctx.eventsTable.table}_timestamp_status_sync_id_idx`;
|
|
822
|
+
ctx.execute(
|
|
823
|
+
`CREATE INDEX IF NOT EXISTS ${ctx.eventsTable.schema}.${indexName} ON ${ctx.eventsTable.table} ("timestamp", "status", "sync_id")`
|
|
824
|
+
);
|
|
752
825
|
}
|
|
753
826
|
}
|
|
754
827
|
];
|
|
755
828
|
function runSystemMigrations(opts) {
|
|
756
829
|
const ctx = {
|
|
757
|
-
|
|
758
|
-
updateLogTableName: opts.updateLogTableName,
|
|
830
|
+
...opts.dbConfig,
|
|
759
831
|
execute: opts.execute
|
|
760
832
|
};
|
|
761
833
|
for (const migration of opts.migrations) {
|
|
@@ -773,15 +845,15 @@ function applyWorkerDbSchema(db) {
|
|
|
773
845
|
runSystemMigrations({
|
|
774
846
|
migrations: baseSystemMigrations,
|
|
775
847
|
version: kvStore.createNumberStoredValue("internal-schema-version", -1),
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
execute: (sql2) => db.execute(sql2, { loggerLevel: "system" }),
|
|
848
|
+
dbConfig: workerDbConfig,
|
|
849
|
+
execute: (sql4) => db.execute(sql4, { loggerLevel: "system" }),
|
|
779
850
|
transaction: (callback) => db.executeTransaction(callback)
|
|
780
851
|
});
|
|
852
|
+
return { kvStore };
|
|
781
853
|
}
|
|
782
854
|
function applyMemoryDbSchema(db) {
|
|
783
855
|
db.execute(
|
|
784
|
-
`CREATE TABLE
|
|
856
|
+
`CREATE TABLE IF NOT EXISTS ${memoryDbConfig.eventsTable.fullIdentifier} (
|
|
785
857
|
"sync_id" integer NOT NULL PRIMARY KEY,
|
|
786
858
|
"schema_version" integer NOT NULL,
|
|
787
859
|
"status" text NOT NULL,
|
|
@@ -792,6 +864,27 @@ function applyMemoryDbSchema(db) {
|
|
|
792
864
|
"dataset" text NOT NULL,
|
|
793
865
|
"item_id" text NOT NULL,
|
|
794
866
|
"payload" text NOT NULL
|
|
867
|
+
)`,
|
|
868
|
+
{ loggerLevel: "system" }
|
|
869
|
+
);
|
|
870
|
+
db.execute(
|
|
871
|
+
`CREATE INDEX IF NOT EXISTS ${memoryDbConfig.eventsTable.table}_status_sync_id_idx ON ${memoryDbConfig.eventsTable.table} ("status", "sync_id")`,
|
|
872
|
+
{
|
|
873
|
+
loggerLevel: "system"
|
|
874
|
+
}
|
|
875
|
+
);
|
|
876
|
+
db.execute(
|
|
877
|
+
`CREATE INDEX IF NOT EXISTS ${memoryDbConfig.eventsTable.table}_timestamp_status_sync_id_idx ON ${memoryDbConfig.eventsTable.table} ("timestamp", "status", "sync_id")`,
|
|
878
|
+
{
|
|
879
|
+
loggerLevel: "system"
|
|
880
|
+
}
|
|
881
|
+
);
|
|
882
|
+
db.execute(
|
|
883
|
+
`CREATE TABLE IF NOT EXISTS ${memoryDbConfig.updateLogTable.fullIdentifier} (
|
|
884
|
+
"dataset" text NOT NULL,
|
|
885
|
+
"item_id" text NOT NULL,
|
|
886
|
+
"payload" text NOT NULL,
|
|
887
|
+
PRIMARY KEY ("item_id", "dataset")
|
|
795
888
|
)`,
|
|
796
889
|
{ loggerLevel: "system" }
|
|
797
890
|
);
|
|
@@ -800,7 +893,7 @@ function applyMemoryDbSchema(db) {
|
|
|
800
893
|
// src/sqlite-crdt/apply-crdt-event.ts
|
|
801
894
|
var createSQLiteCrdtApplyFunction = ({
|
|
802
895
|
db,
|
|
803
|
-
|
|
896
|
+
dbConfig
|
|
804
897
|
}) => {
|
|
805
898
|
const applyCrdtEvent = createCrdtApplyFunction({
|
|
806
899
|
getCrdtUpdateLog(opts) {
|
|
@@ -811,7 +904,7 @@ var createSQLiteCrdtApplyFunction = ({
|
|
|
811
904
|
dataset: opts.dataset
|
|
812
905
|
},
|
|
813
906
|
(db2, params) => {
|
|
814
|
-
return db2.selectFrom(
|
|
907
|
+
return db2.selectFrom(dbConfig.updateLogTable.fullIdentifier).select("payload").where("item_id", "=", params("item_id")).where("dataset", "=", params("dataset"));
|
|
815
908
|
},
|
|
816
909
|
{ loggerLevel: "system" }
|
|
817
910
|
);
|
|
@@ -826,7 +919,7 @@ var createSQLiteCrdtApplyFunction = ({
|
|
|
826
919
|
dataset: opts.dataset,
|
|
827
920
|
payload: opts.payload
|
|
828
921
|
},
|
|
829
|
-
(db2, params) => db2.insertInto(
|
|
922
|
+
(db2, params) => db2.insertInto(dbConfig.updateLogTable.fullIdentifier).values({
|
|
830
923
|
item_id: params("item_id"),
|
|
831
924
|
dataset: params("dataset"),
|
|
832
925
|
payload: params("payload")
|
|
@@ -842,7 +935,7 @@ var createSQLiteCrdtApplyFunction = ({
|
|
|
842
935
|
dataset: opts.dataset,
|
|
843
936
|
payload: opts.payload
|
|
844
937
|
},
|
|
845
|
-
(db2, params) => db2.updateTable(
|
|
938
|
+
(db2, params) => db2.updateTable(dbConfig.updateLogTable.fullIdentifier).set({
|
|
846
939
|
payload: params("payload")
|
|
847
940
|
}).where("item_id", "=", params("item_id")).where("dataset", "=", params("dataset")),
|
|
848
941
|
{ loggerLevel: "system" }
|
|
@@ -862,13 +955,13 @@ var createSQLiteCrdtApplyFunction = ({
|
|
|
862
955
|
},
|
|
863
956
|
updateItem(opts) {
|
|
864
957
|
const keys = Array.from(Object.keys(opts.payload));
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
{ loggerLevel: "system" }
|
|
871
|
-
);
|
|
958
|
+
keys.sort();
|
|
959
|
+
db.executePreparedRaw({
|
|
960
|
+
key: `update-item-${opts.dataset}-${keys.join("-")}`,
|
|
961
|
+
sql: `update ${quoteId(opts.dataset)} set ${keys.map((key) => `${quoteId(key)} = ?`).join(",")} where id = ?`,
|
|
962
|
+
params: [...keys.map((key) => opts.payload[key]), opts.itemId],
|
|
963
|
+
meta: { loggerLevel: "system" }
|
|
964
|
+
});
|
|
872
965
|
}
|
|
873
966
|
});
|
|
874
967
|
return applyCrdtEvent;
|
|
@@ -880,11 +973,7 @@ function createCrdtApplyFunction({
|
|
|
880
973
|
updateItem,
|
|
881
974
|
updateCrdtUpdateLog
|
|
882
975
|
}) {
|
|
883
|
-
const applyItemCreated = ({ event
|
|
884
|
-
if (meta) {
|
|
885
|
-
applyItemUpdated({ event, meta });
|
|
886
|
-
return;
|
|
887
|
-
}
|
|
976
|
+
const applyItemCreated = ({ event }) => {
|
|
888
977
|
const eventPayload = JSON.parse(event.payload);
|
|
889
978
|
eventPayload.tombstone = false;
|
|
890
979
|
insertItem({ dataset: event.dataset, payload: eventPayload });
|
|
@@ -902,10 +991,13 @@ function createCrdtApplyFunction({
|
|
|
902
991
|
if (!meta) {
|
|
903
992
|
throw new Error(`Item ${event.item_id} in dataset ${event.dataset} not found`);
|
|
904
993
|
}
|
|
905
|
-
const eventPayload = JSON.parse(event.payload);
|
|
994
|
+
const eventPayload = event.type === "item-deleted" ? { tombstone: 1 } : JSON.parse(event.payload);
|
|
906
995
|
const updatePayload = {};
|
|
907
996
|
let hasUpdates = false;
|
|
908
997
|
for (const [key, value] of Object.entries(eventPayload)) {
|
|
998
|
+
if (key === "id") {
|
|
999
|
+
continue;
|
|
1000
|
+
}
|
|
909
1001
|
const lastUpdateTimestamp = meta[key];
|
|
910
1002
|
const currentUpdateTimestamp = event.timestamp;
|
|
911
1003
|
if (!lastUpdateTimestamp || !currentUpdateTimestamp || currentUpdateTimestamp > lastUpdateTimestamp) {
|
|
@@ -929,45 +1021,95 @@ function createCrdtApplyFunction({
|
|
|
929
1021
|
});
|
|
930
1022
|
};
|
|
931
1023
|
return (event) => {
|
|
1024
|
+
if (isNoOpCrdtEventPayload(event.payload)) {
|
|
1025
|
+
return;
|
|
1026
|
+
}
|
|
932
1027
|
const meta = getCrdtUpdateLog({
|
|
933
1028
|
itemId: event.item_id,
|
|
934
1029
|
dataset: event.dataset
|
|
935
1030
|
});
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
throw new Error(`Item ${event.item_id} in dataset ${event.dataset} not found`);
|
|
947
|
-
}
|
|
948
|
-
applyItemUpdated({
|
|
949
|
-
event,
|
|
950
|
-
meta
|
|
951
|
-
});
|
|
952
|
-
break;
|
|
953
|
-
}
|
|
954
|
-
default:
|
|
955
|
-
event.type;
|
|
1031
|
+
if (event.type !== "item-created" && event.type !== "item-updated" && event.type !== "item-deleted") {
|
|
1032
|
+
throw new Error(`Unknown event type: ${event.type}`);
|
|
1033
|
+
}
|
|
1034
|
+
if (meta) {
|
|
1035
|
+
applyItemUpdated({ event, meta });
|
|
1036
|
+
return;
|
|
1037
|
+
}
|
|
1038
|
+
if (event.type === "item-created") {
|
|
1039
|
+
applyItemCreated({ event });
|
|
1040
|
+
return;
|
|
956
1041
|
}
|
|
1042
|
+
throw new Error(`Item ${event.item_id} in dataset ${event.dataset} not found`);
|
|
957
1043
|
};
|
|
958
1044
|
}
|
|
959
1045
|
|
|
1046
|
+
// src/sqlite-crdt/crdt-storage.ts
|
|
1047
|
+
import { sql as sql3 } from "kysely";
|
|
1048
|
+
|
|
1049
|
+
// src/sqlite-crdt/event-consistency.ts
|
|
1050
|
+
var MASK_128 = (1n << 128n) - 1n;
|
|
1051
|
+
var HEX_128_PATTERN = /^[0-9a-f]{32}$/;
|
|
1052
|
+
function createEventHlcAccumulator(initialValue) {
|
|
1053
|
+
let current = parseHex128(initialValue);
|
|
1054
|
+
return {
|
|
1055
|
+
add(timestamp) {
|
|
1056
|
+
current = current + hash128BigInt(timestamp) & MASK_128;
|
|
1057
|
+
},
|
|
1058
|
+
get current() {
|
|
1059
|
+
return toHex128(current);
|
|
1060
|
+
}
|
|
1061
|
+
};
|
|
1062
|
+
}
|
|
1063
|
+
function parseHex128(value) {
|
|
1064
|
+
if (value === "") {
|
|
1065
|
+
return 0n;
|
|
1066
|
+
}
|
|
1067
|
+
const normalized = value.toLowerCase();
|
|
1068
|
+
if (!HEX_128_PATTERN.test(normalized)) {
|
|
1069
|
+
throw new Error(`Invalid event HLC accumulator value: ${value}`);
|
|
1070
|
+
}
|
|
1071
|
+
return BigInt(`0x${normalized}`);
|
|
1072
|
+
}
|
|
1073
|
+
function toHex128(value) {
|
|
1074
|
+
return value.toString(16).padStart(32, "0");
|
|
1075
|
+
}
|
|
1076
|
+
function hash128BigInt(value) {
|
|
1077
|
+
return xxhash.h64(value, 0n) | xxhash.h64(value, 1n) << 64n;
|
|
1078
|
+
}
|
|
1079
|
+
|
|
960
1080
|
// src/sqlite-crdt/crdt-storage.ts
|
|
961
1081
|
function createCrdtStorage(storage) {
|
|
962
|
-
|
|
1082
|
+
let localSyncId = storage.initialLocalSyncId;
|
|
1083
|
+
const db = storage.db;
|
|
1084
|
+
const crdtEventsTable = storage.dbConfig.eventsTable.fullIdentifier;
|
|
963
1085
|
const eventTarget = createTypedEventTarget();
|
|
1086
|
+
const persistEvent = (tx, event) => {
|
|
1087
|
+
tx.executePrepared(
|
|
1088
|
+
"persist-crdt-event",
|
|
1089
|
+
event,
|
|
1090
|
+
(db2, params) => db2.insertInto(crdtEventsTable).values({
|
|
1091
|
+
type: params("type"),
|
|
1092
|
+
dataset: params("dataset"),
|
|
1093
|
+
item_id: params("item_id"),
|
|
1094
|
+
payload: params("payload"),
|
|
1095
|
+
schema_version: params("schema_version"),
|
|
1096
|
+
sync_id: params("sync_id"),
|
|
1097
|
+
status: params("status"),
|
|
1098
|
+
timestamp: params("timestamp"),
|
|
1099
|
+
origin: params("origin"),
|
|
1100
|
+
source_node_id: params("source_node_id")
|
|
1101
|
+
}),
|
|
1102
|
+
{ loggerLevel: "system" }
|
|
1103
|
+
);
|
|
1104
|
+
};
|
|
964
1105
|
const enqueueEvents = (origin, sourceNodeId, events) => {
|
|
1106
|
+
const beforeSyncId = localSyncId;
|
|
965
1107
|
if (events.length === 0) {
|
|
966
|
-
return;
|
|
1108
|
+
return { beforeSyncId, afterSyncId: beforeSyncId, processed: Promise.resolve() };
|
|
967
1109
|
}
|
|
968
|
-
|
|
1110
|
+
db.executeTransaction((tx) => {
|
|
969
1111
|
for (const event of events) {
|
|
970
|
-
|
|
1112
|
+
persistEvent(tx, {
|
|
971
1113
|
schema_version: event.schema_version ?? storage.migrator.currentSchemaVersion,
|
|
972
1114
|
timestamp: event.timestamp ?? serializeHLC(storage.hlc.getNextHLC()),
|
|
973
1115
|
type: event.type,
|
|
@@ -976,21 +1118,26 @@ function createCrdtStorage(storage) {
|
|
|
976
1118
|
origin,
|
|
977
1119
|
source_node_id: sourceNodeId,
|
|
978
1120
|
payload: event.payload,
|
|
979
|
-
sync_id: ++
|
|
1121
|
+
sync_id: ++localSyncId,
|
|
980
1122
|
status: "pending"
|
|
981
1123
|
});
|
|
982
1124
|
}
|
|
983
1125
|
});
|
|
984
|
-
processEnqueuedEvents();
|
|
1126
|
+
return { beforeSyncId, afterSyncId: localSyncId, processed: processEnqueuedEvents() };
|
|
985
1127
|
};
|
|
986
1128
|
const enqueueLocalEvents = (events, sourceNodeId) => {
|
|
987
|
-
enqueueEvents("local", sourceNodeId, events);
|
|
1129
|
+
return enqueueEvents("local", sourceNodeId, events);
|
|
988
1130
|
};
|
|
989
1131
|
const enqueueOwnEvents = (events) => {
|
|
990
|
-
enqueueEvents("own", storage.nodeId, events);
|
|
1132
|
+
return enqueueEvents("own", storage.nodeId, events);
|
|
991
1133
|
};
|
|
992
1134
|
const enqueueRemoteEvents = (events) => {
|
|
993
|
-
enqueueEvents("remote", "", events);
|
|
1135
|
+
return enqueueEvents("remote", "", events);
|
|
1136
|
+
};
|
|
1137
|
+
const notifyEventApplied = (event) => {
|
|
1138
|
+
if (event.status === "applied") {
|
|
1139
|
+
storage.onEventApplied?.(event);
|
|
1140
|
+
}
|
|
994
1141
|
};
|
|
995
1142
|
const applyOwnEvent = (event, { wrapInTransaction } = {}) => {
|
|
996
1143
|
const persistedEvent = {
|
|
@@ -1002,30 +1149,54 @@ function createCrdtStorage(storage) {
|
|
|
1002
1149
|
origin: "own",
|
|
1003
1150
|
source_node_id: storage.nodeId,
|
|
1004
1151
|
payload: event.payload,
|
|
1005
|
-
sync_id: ++
|
|
1152
|
+
sync_id: ++localSyncId,
|
|
1006
1153
|
status: "pending"
|
|
1007
1154
|
};
|
|
1008
1155
|
if (wrapInTransaction) {
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
processPersistedEvent(persistedEvent);
|
|
1156
|
+
db.executeTransaction((tx) => {
|
|
1157
|
+
persistEvent(tx, persistedEvent);
|
|
1158
|
+
processPersistedEvent(tx, persistedEvent);
|
|
1159
|
+
persistEventHlcAccumulator();
|
|
1012
1160
|
});
|
|
1013
1161
|
} else {
|
|
1014
|
-
|
|
1015
|
-
processPersistedEvent(persistedEvent);
|
|
1162
|
+
persistEvent(db, persistedEvent);
|
|
1163
|
+
processPersistedEvent(db, persistedEvent);
|
|
1164
|
+
persistEventHlcAccumulator();
|
|
1016
1165
|
}
|
|
1017
1166
|
};
|
|
1018
|
-
const dispatchEventsApplied = () => {
|
|
1167
|
+
const dispatchEventsApplied = (syncId = localSyncId) => {
|
|
1019
1168
|
eventTarget.dispatchEvent("events-applied", {
|
|
1020
|
-
syncId
|
|
1169
|
+
syncId,
|
|
1170
|
+
eventHlcSum: eventHlcAccumulator?.current ?? null
|
|
1021
1171
|
});
|
|
1022
1172
|
};
|
|
1173
|
+
const hasPendingEvents = () => {
|
|
1174
|
+
const events = db.executePrepared(
|
|
1175
|
+
"has-pending-events",
|
|
1176
|
+
{ status: "pending" },
|
|
1177
|
+
(db2, params) => db2.selectFrom(crdtEventsTable).select("sync_id").where("status", "=", params("status")).limit(sql3.lit(1)),
|
|
1178
|
+
{ loggerLevel: "system" }
|
|
1179
|
+
);
|
|
1180
|
+
return events.length > 0;
|
|
1181
|
+
};
|
|
1023
1182
|
const getEventsBatch = (options) => {
|
|
1024
1183
|
const limit = options.limit ?? 50;
|
|
1025
|
-
const
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1184
|
+
const queryParams = {
|
|
1185
|
+
limit: limit + 1,
|
|
1186
|
+
status: options.status ?? null,
|
|
1187
|
+
afterSyncId: options.afterSyncId ?? null,
|
|
1188
|
+
excludeOrigin: options.excludeOrigin ?? null,
|
|
1189
|
+
excludeNodeId: options.excludeNodeId ?? null
|
|
1190
|
+
};
|
|
1191
|
+
const filterKeys = [
|
|
1192
|
+
queryParams.excludeNodeId ? "nodeid" : "no-nodeid",
|
|
1193
|
+
queryParams.excludeOrigin ? "origin" : "no-origin"
|
|
1194
|
+
];
|
|
1195
|
+
const events = db.executePrepared(
|
|
1196
|
+
`get-events-batch-${filterKeys.join("-")}`,
|
|
1197
|
+
queryParams,
|
|
1198
|
+
(db2, params) => db2.selectFrom(crdtEventsTable).where("sync_id", ">", params("afterSyncId")).where("status", "=", params("status")).$if(!!queryParams.excludeNodeId, (qb) => qb.where("source_node_id", "!=", params("excludeNodeId"))).$if(!!queryParams.excludeOrigin, (qb) => qb.where("origin", "!=", params("excludeOrigin"))).selectAll().limit(params("limit")).orderBy("sync_id", "asc")
|
|
1199
|
+
);
|
|
1029
1200
|
const hasMore = events.length > limit;
|
|
1030
1201
|
if (hasMore) {
|
|
1031
1202
|
events.pop();
|
|
@@ -1036,18 +1207,94 @@ function createCrdtStorage(storage) {
|
|
|
1036
1207
|
nextSyncId: events[events.length - 1]?.sync_id ?? options.afterSyncId ?? 0
|
|
1037
1208
|
};
|
|
1038
1209
|
};
|
|
1039
|
-
const
|
|
1210
|
+
const checkIsQuiescent = (pushedSyncId) => {
|
|
1211
|
+
if (hasPendingEvents()) {
|
|
1212
|
+
return false;
|
|
1213
|
+
}
|
|
1214
|
+
const unpushed = getEventsBatch({
|
|
1215
|
+
status: "applied",
|
|
1216
|
+
afterSyncId: pushedSyncId,
|
|
1217
|
+
excludeOrigin: "remote",
|
|
1218
|
+
limit: 1
|
|
1219
|
+
});
|
|
1220
|
+
return unpushed.events.length === 0;
|
|
1221
|
+
};
|
|
1222
|
+
const applyCrdtEvent = createSQLiteCrdtApplyFunction({
|
|
1223
|
+
db,
|
|
1224
|
+
dbConfig: storage.dbConfig
|
|
1225
|
+
});
|
|
1226
|
+
const eventHlcAccumulator = storage.eventHlcAccumulator ? createEventHlcAccumulator(storage.eventHlcAccumulator.current) : null;
|
|
1227
|
+
const persistEventHlcAccumulator = () => {
|
|
1228
|
+
if (eventHlcAccumulator && storage.eventHlcAccumulator) {
|
|
1229
|
+
storage.eventHlcAccumulator.current = eventHlcAccumulator.current;
|
|
1230
|
+
}
|
|
1231
|
+
};
|
|
1232
|
+
const recomputeEventHlcAccumulatorIfNeeded = () => {
|
|
1233
|
+
if (!eventHlcAccumulator || !storage.eventHlcAccumulator) {
|
|
1234
|
+
return;
|
|
1235
|
+
}
|
|
1236
|
+
if (storage.eventHlcAccumulator.current !== "") {
|
|
1237
|
+
return;
|
|
1238
|
+
}
|
|
1239
|
+
const batchSize = 1e3;
|
|
1240
|
+
let afterSyncId = 0;
|
|
1241
|
+
for (; ; ) {
|
|
1242
|
+
const rows = db.executePrepared(
|
|
1243
|
+
"get-applied-event-timestamps",
|
|
1244
|
+
{ status: "applied", afterSyncId, limit: batchSize },
|
|
1245
|
+
(db2, params) => db2.selectFrom(crdtEventsTable).select(["sync_id", "timestamp"]).where("status", "=", params("status")).where("sync_id", ">", params("afterSyncId")).orderBy("sync_id", "asc").limit(params("limit")),
|
|
1246
|
+
{ loggerLevel: "system" }
|
|
1247
|
+
);
|
|
1248
|
+
if (rows.length === 0) {
|
|
1249
|
+
break;
|
|
1250
|
+
}
|
|
1251
|
+
for (const row of rows) {
|
|
1252
|
+
eventHlcAccumulator.add(row.timestamp);
|
|
1253
|
+
afterSyncId = row.sync_id;
|
|
1254
|
+
}
|
|
1255
|
+
if (rows.length < batchSize) {
|
|
1256
|
+
break;
|
|
1257
|
+
}
|
|
1258
|
+
}
|
|
1259
|
+
persistEventHlcAccumulator();
|
|
1260
|
+
};
|
|
1261
|
+
const hasAcceptedEventWithTimestamp = (tx, event) => {
|
|
1262
|
+
const [existingEvent] = tx.executePrepared(
|
|
1263
|
+
"get-accepted-crdt-event-by-timestamp",
|
|
1264
|
+
{
|
|
1265
|
+
timestamp: event.timestamp,
|
|
1266
|
+
sync_id: event.sync_id
|
|
1267
|
+
},
|
|
1268
|
+
(db2, params) => db2.selectFrom(crdtEventsTable).select("sync_id").where("timestamp", "=", params("timestamp")).where("sync_id", "<", params("sync_id")).where("status", "=", sql3.lit("applied")).limit(sql3.lit(1)),
|
|
1269
|
+
{ loggerLevel: "system" }
|
|
1270
|
+
);
|
|
1271
|
+
return existingEvent !== void 0;
|
|
1272
|
+
};
|
|
1273
|
+
const processPersistedEvent = (tx, event) => {
|
|
1040
1274
|
if (event.status !== "pending") {
|
|
1041
1275
|
throw new Error(`Event ${event.sync_id} is not pending`);
|
|
1042
1276
|
}
|
|
1043
1277
|
try {
|
|
1278
|
+
if (hasAcceptedEventWithTimestamp(tx, event)) {
|
|
1279
|
+
event.status = "deduped";
|
|
1280
|
+
return event;
|
|
1281
|
+
}
|
|
1044
1282
|
if (event.origin === "local" || event.origin === "remote") {
|
|
1045
1283
|
storage.hlc.mergeHLC(deserializeHLC(event.timestamp));
|
|
1046
1284
|
}
|
|
1285
|
+
if (isNoOpCrdtEventPayload(event.payload)) {
|
|
1286
|
+
applyCrdtEvent(event);
|
|
1287
|
+
event.status = "applied";
|
|
1288
|
+
eventHlcAccumulator?.add(event.timestamp);
|
|
1289
|
+
return event;
|
|
1290
|
+
}
|
|
1047
1291
|
const migratedEvent = storage.migrator.migrateEvent(event, storage.migrator.latestSchemaVersion);
|
|
1048
1292
|
if (migratedEvent === null) {
|
|
1049
|
-
event.status = "skipped";
|
|
1050
1293
|
event.schema_version = storage.migrator.latestSchemaVersion;
|
|
1294
|
+
event.payload = CRDT_EVENT_NO_OP_PAYLOAD;
|
|
1295
|
+
applyCrdtEvent(event);
|
|
1296
|
+
event.status = "applied";
|
|
1297
|
+
eventHlcAccumulator?.add(event.timestamp);
|
|
1051
1298
|
return event;
|
|
1052
1299
|
}
|
|
1053
1300
|
event.schema_version = migratedEvent.schema_version;
|
|
@@ -1055,40 +1302,72 @@ function createCrdtStorage(storage) {
|
|
|
1055
1302
|
event.dataset = migratedEvent.dataset;
|
|
1056
1303
|
event.item_id = migratedEvent.item_id;
|
|
1057
1304
|
event.payload = migratedEvent.payload;
|
|
1058
|
-
|
|
1305
|
+
applyCrdtEvent(event);
|
|
1059
1306
|
event.status = "applied";
|
|
1307
|
+
eventHlcAccumulator?.add(event.timestamp);
|
|
1060
1308
|
} catch (error) {
|
|
1061
1309
|
console.error("Error applying enqueued CRDT event", error);
|
|
1062
1310
|
event.status = "failed";
|
|
1063
1311
|
} finally {
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1312
|
+
tx.executePrepared(
|
|
1313
|
+
"update-crdt-event",
|
|
1314
|
+
event,
|
|
1315
|
+
(db2, params) => db2.updateTable(crdtEventsTable).set({
|
|
1316
|
+
status: params("status"),
|
|
1317
|
+
schema_version: params("schema_version"),
|
|
1318
|
+
type: params("type"),
|
|
1319
|
+
dataset: params("dataset"),
|
|
1320
|
+
item_id: params("item_id"),
|
|
1321
|
+
payload: params("payload")
|
|
1322
|
+
}).where("sync_id", "=", params("sync_id")),
|
|
1323
|
+
{ loggerLevel: "system" }
|
|
1324
|
+
);
|
|
1072
1325
|
}
|
|
1073
1326
|
};
|
|
1074
1327
|
const processEnqueuedEvents = ensureSingletonExecution(async () => {
|
|
1075
1328
|
let hasMore = true;
|
|
1076
1329
|
while (hasMore) {
|
|
1077
1330
|
await Promise.resolve();
|
|
1078
|
-
const
|
|
1079
|
-
const events =
|
|
1080
|
-
|
|
1331
|
+
const batchSize = 100;
|
|
1332
|
+
const events = db.executePrepared(
|
|
1333
|
+
"get-enqueued-pending-events",
|
|
1334
|
+
{
|
|
1335
|
+
status: "pending",
|
|
1336
|
+
limit: batchSize + 1
|
|
1337
|
+
},
|
|
1338
|
+
(db2, params) => db2.selectFrom(crdtEventsTable).selectAll().where("status", "=", params("status")).limit(params("limit")).orderBy("sync_id", "asc")
|
|
1339
|
+
);
|
|
1340
|
+
hasMore = events.length > batchSize;
|
|
1341
|
+
if (hasMore) {
|
|
1342
|
+
events.pop();
|
|
1343
|
+
}
|
|
1081
1344
|
if (events.length === 0) {
|
|
1082
1345
|
break;
|
|
1083
1346
|
}
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1347
|
+
let appliedSyncId = null;
|
|
1348
|
+
const failedRemoteSyncIds = [];
|
|
1349
|
+
db.executeTransaction((tx) => {
|
|
1350
|
+
for (const event of events) {
|
|
1351
|
+
processPersistedEvent(tx, event);
|
|
1352
|
+
notifyEventApplied(event);
|
|
1353
|
+
if (event.status === "applied") {
|
|
1354
|
+
appliedSyncId = event.sync_id;
|
|
1355
|
+
} else if (event.status === "failed" && event.origin === "remote") {
|
|
1356
|
+
failedRemoteSyncIds.push(event.sync_id);
|
|
1357
|
+
}
|
|
1358
|
+
}
|
|
1359
|
+
persistEventHlcAccumulator();
|
|
1360
|
+
});
|
|
1361
|
+
if (appliedSyncId !== null) {
|
|
1362
|
+
dispatchEventsApplied(appliedSyncId);
|
|
1363
|
+
}
|
|
1364
|
+
for (const syncId of failedRemoteSyncIds) {
|
|
1365
|
+
eventTarget.dispatchEvent("remote-event-apply-failed", { syncId });
|
|
1088
1366
|
}
|
|
1089
|
-
dispatchEventsApplied();
|
|
1090
1367
|
}
|
|
1091
1368
|
});
|
|
1369
|
+
recomputeEventHlcAccumulatorIfNeeded();
|
|
1370
|
+
void processEnqueuedEvents();
|
|
1092
1371
|
return {
|
|
1093
1372
|
getEventsBatch,
|
|
1094
1373
|
enqueueLocalEvents,
|
|
@@ -1096,6 +1375,8 @@ function createCrdtStorage(storage) {
|
|
|
1096
1375
|
enqueueRemoteEvents,
|
|
1097
1376
|
applyOwnEvent,
|
|
1098
1377
|
dispatchEventsApplied,
|
|
1378
|
+
checkIsQuiescent,
|
|
1379
|
+
getEventHlcAccumulator: () => eventHlcAccumulator?.current ?? null,
|
|
1099
1380
|
addEventListener: eventTarget.addEventListener,
|
|
1100
1381
|
removeEventListener: eventTarget.removeEventListener
|
|
1101
1382
|
};
|
|
@@ -1104,12 +1385,23 @@ function createCrdtStorage(storage) {
|
|
|
1104
1385
|
// src/sqlite-crdt/crdt-sync-producer.ts
|
|
1105
1386
|
var createCrdtSyncProducer = ({ storage, broadcastEvents }) => {
|
|
1106
1387
|
storage.addEventListener("events-applied", (event) => {
|
|
1107
|
-
broadcastEvents({
|
|
1388
|
+
broadcastEvents({
|
|
1389
|
+
newSyncId: event.payload.syncId,
|
|
1390
|
+
eventHlcSum: event.payload.eventHlcSum
|
|
1391
|
+
});
|
|
1108
1392
|
});
|
|
1109
1393
|
};
|
|
1110
1394
|
|
|
1111
1395
|
// src/sqlite-crdt/crdt-sync-remote-source.ts
|
|
1112
1396
|
import retryAsPromised from "retry-as-promised";
|
|
1397
|
+
var SchemaVersionMismatchError = class extends Error {
|
|
1398
|
+
constructor(remoteSchemaVersion, localSchemaVersion) {
|
|
1399
|
+
super(`Schema version mismatch: remote ${remoteSchemaVersion} != local ${localSchemaVersion}`);
|
|
1400
|
+
this.remoteSchemaVersion = remoteSchemaVersion;
|
|
1401
|
+
this.localSchemaVersion = localSchemaVersion;
|
|
1402
|
+
this.name = "SchemaVersionMismatchError";
|
|
1403
|
+
}
|
|
1404
|
+
};
|
|
1113
1405
|
var createCrdtSyncRemoteSource = ({
|
|
1114
1406
|
bufferSize,
|
|
1115
1407
|
storage,
|
|
@@ -1138,8 +1430,8 @@ var createCrdtSyncRemoteSource = ({
|
|
|
1138
1430
|
setRemoteState({ type: "pending" });
|
|
1139
1431
|
const factoryResult = await tryCatchAsync(async () => {
|
|
1140
1432
|
return await remoteFactory?.({
|
|
1141
|
-
onEventsAvailable: (newSyncId) => {
|
|
1142
|
-
pullEvents({ remoteSyncId: newSyncId, includeSelf: false });
|
|
1433
|
+
onEventsAvailable: ({ newSyncId, remoteEventHlcSum }) => {
|
|
1434
|
+
pullEvents({ remoteSyncId: newSyncId, remoteEventHlcSum, includeSelf: false });
|
|
1143
1435
|
}
|
|
1144
1436
|
});
|
|
1145
1437
|
});
|
|
@@ -1198,6 +1490,7 @@ var createCrdtSyncRemoteSource = ({
|
|
|
1198
1490
|
}
|
|
1199
1491
|
const remoteSyncId = request?.remoteSyncId;
|
|
1200
1492
|
if (remoteSyncId !== void 0 && remoteSyncId <= pullSyncId.current) {
|
|
1493
|
+
checkRemoteConsistency(remoteSyncId, request?.remoteEventHlcSum ?? null);
|
|
1201
1494
|
return Promise.resolve();
|
|
1202
1495
|
}
|
|
1203
1496
|
if (pullPromise) {
|
|
@@ -1249,9 +1542,11 @@ var createCrdtSyncRemoteSource = ({
|
|
|
1249
1542
|
storage.enqueueRemoteEvents(
|
|
1250
1543
|
response.events.map((x) => {
|
|
1251
1544
|
if (x.schema_version > migrator.currentSchemaVersion) {
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1545
|
+
eventTarget.dispatchEvent("remote-schema-version-mismatch", {
|
|
1546
|
+
remoteSchemaVersion: x.schema_version,
|
|
1547
|
+
localSchemaVersion: migrator.currentSchemaVersion
|
|
1548
|
+
});
|
|
1549
|
+
throw new SchemaVersionMismatchError(x.schema_version, migrator.currentSchemaVersion);
|
|
1255
1550
|
}
|
|
1256
1551
|
return x;
|
|
1257
1552
|
})
|
|
@@ -1265,6 +1560,27 @@ var createCrdtSyncRemoteSource = ({
|
|
|
1265
1560
|
}
|
|
1266
1561
|
}
|
|
1267
1562
|
};
|
|
1563
|
+
const checkRemoteConsistency = (remoteSyncId, remoteEventHlcSum) => {
|
|
1564
|
+
if (remoteEventHlcSum === null) {
|
|
1565
|
+
return;
|
|
1566
|
+
}
|
|
1567
|
+
if (remoteSyncId !== pullSyncId.current) {
|
|
1568
|
+
return;
|
|
1569
|
+
}
|
|
1570
|
+
if (!storage.checkIsQuiescent(pushSyncId.current)) {
|
|
1571
|
+
return;
|
|
1572
|
+
}
|
|
1573
|
+
const localEventHlcSum = storage.getEventHlcAccumulator();
|
|
1574
|
+
if (localEventHlcSum === null) {
|
|
1575
|
+
return;
|
|
1576
|
+
}
|
|
1577
|
+
if (localEventHlcSum !== remoteEventHlcSum) {
|
|
1578
|
+
eventTarget.dispatchEvent("de-sync-detected", { reason: "CHECKSUM_MISMATCH" });
|
|
1579
|
+
console.warn(
|
|
1580
|
+
`[sqlite-sync] De-sync detected at syncId ${remoteSyncId}: local HLC checksum ${localEventHlcSum} != remote ${remoteEventHlcSum}. Local and remote have diverged despite being caught up.`
|
|
1581
|
+
);
|
|
1582
|
+
}
|
|
1583
|
+
};
|
|
1268
1584
|
const startPushingEvents = ensureSingletonExecution(async () => {
|
|
1269
1585
|
while (true) {
|
|
1270
1586
|
const eventsBatch = storage.getEventsBatch({
|
|
@@ -1280,42 +1596,53 @@ var createCrdtSyncRemoteSource = ({
|
|
|
1280
1596
|
break;
|
|
1281
1597
|
}
|
|
1282
1598
|
const source = remoteState.source;
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1599
|
+
let response;
|
|
1600
|
+
try {
|
|
1601
|
+
response = await retryAsPromised(
|
|
1602
|
+
() => source.pushEvents({
|
|
1603
|
+
nodeId,
|
|
1604
|
+
events: eventsBatch.events.map((event) => ({
|
|
1605
|
+
schema_version: event.schema_version,
|
|
1606
|
+
timestamp: event.timestamp,
|
|
1607
|
+
type: event.type,
|
|
1608
|
+
dataset: event.dataset,
|
|
1609
|
+
item_id: event.item_id,
|
|
1610
|
+
payload: event.payload
|
|
1611
|
+
}))
|
|
1612
|
+
}),
|
|
1613
|
+
{
|
|
1614
|
+
max: 3,
|
|
1615
|
+
backoffBase: 100,
|
|
1616
|
+
backoffExponent: 1.5,
|
|
1617
|
+
backoffJitter: 150,
|
|
1618
|
+
timeout: 1e4
|
|
1619
|
+
}
|
|
1620
|
+
);
|
|
1621
|
+
} catch (error) {
|
|
1622
|
+
console.error("Error pushing events. Going offline.", error);
|
|
1623
|
+
goOffline("REMOTE_PUSH_ERROR");
|
|
1624
|
+
return;
|
|
1304
1625
|
}
|
|
1305
1626
|
pushSyncId.current = eventsBatch.nextSyncId;
|
|
1627
|
+
if (response.ok && response.beforeSyncId !== void 0 && response.afterSyncId !== void 0 && response.beforeSyncId <= pullSyncId.current && response.afterSyncId > pullSyncId.current) {
|
|
1628
|
+
pullSyncId.current = response.afterSyncId;
|
|
1629
|
+
}
|
|
1306
1630
|
if (!eventsBatch.hasMore) {
|
|
1307
1631
|
break;
|
|
1308
1632
|
}
|
|
1309
1633
|
}
|
|
1310
1634
|
});
|
|
1311
|
-
const
|
|
1635
|
+
const eventsAppliedSubscription = storage.addEventListener("events-applied", () => {
|
|
1312
1636
|
startPushingEvents();
|
|
1313
|
-
};
|
|
1314
|
-
storage.addEventListener("
|
|
1637
|
+
});
|
|
1638
|
+
const remoteEventApplyFailedSubscription = storage.addEventListener("remote-event-apply-failed", () => {
|
|
1639
|
+
eventTarget.dispatchEvent("de-sync-detected", { reason: "ERROR_APPLYING_REMOTE_EVENT" });
|
|
1640
|
+
});
|
|
1315
1641
|
const getState = () => remoteState.type;
|
|
1316
1642
|
const dispose = async () => {
|
|
1317
1643
|
await goOffline("DISCONNECTED");
|
|
1318
|
-
|
|
1644
|
+
eventsAppliedSubscription.unsubscribe();
|
|
1645
|
+
remoteEventApplyFailedSubscription.unsubscribe();
|
|
1319
1646
|
};
|
|
1320
1647
|
return {
|
|
1321
1648
|
goOnline,
|
|
@@ -1328,21 +1655,74 @@ var createCrdtSyncRemoteSource = ({
|
|
|
1328
1655
|
};
|
|
1329
1656
|
};
|
|
1330
1657
|
|
|
1331
|
-
// src/
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1658
|
+
// src/worker-db/reset-state.ts
|
|
1659
|
+
import { createStore, del, get, set } from "idb-keyval";
|
|
1660
|
+
function createIdbResetStore() {
|
|
1661
|
+
const store = createStore("sqlite-sync", "kv");
|
|
1662
|
+
return {
|
|
1663
|
+
get: (key) => get(key, store),
|
|
1664
|
+
set: (key, value) => set(key, value, store),
|
|
1665
|
+
delete: (key) => del(key, store)
|
|
1666
|
+
};
|
|
1667
|
+
}
|
|
1668
|
+
var RESET_REQUEST_TTL_MS = 10 * 60 * 1e3;
|
|
1669
|
+
var resetRequestKey = (dbId) => `sqlite-sync-reset-request-${dbId}`;
|
|
1670
|
+
var resetAppliedKey = (dbId) => `sqlite-sync-reset-applied-${dbId}`;
|
|
1671
|
+
function createResetStateStore({ store, dbId, now = () => Date.now() }) {
|
|
1672
|
+
const requestKey = resetRequestKey(dbId);
|
|
1673
|
+
const appliedKey = resetAppliedKey(dbId);
|
|
1674
|
+
return {
|
|
1675
|
+
async writeResetRequest(epoch) {
|
|
1676
|
+
const request = { epoch, requestedAt: now() };
|
|
1677
|
+
await store.set(requestKey, request);
|
|
1678
|
+
return request;
|
|
1679
|
+
},
|
|
1680
|
+
/**
|
|
1681
|
+
* Read the pending reset request after winning the worker election.
|
|
1682
|
+
* Returns the request only when it has not been applied yet and is within
|
|
1683
|
+
* the TTL. Stale requests are deleted so they cannot fire on a later cold start.
|
|
1684
|
+
*/
|
|
1685
|
+
async resolvePendingReset() {
|
|
1686
|
+
const request = await store.get(requestKey);
|
|
1687
|
+
if (!request) {
|
|
1688
|
+
return void 0;
|
|
1689
|
+
}
|
|
1690
|
+
if (now() - request.requestedAt > RESET_REQUEST_TTL_MS) {
|
|
1691
|
+
await store.delete(requestKey);
|
|
1692
|
+
return void 0;
|
|
1693
|
+
}
|
|
1694
|
+
const appliedEpoch = await store.get(appliedKey);
|
|
1695
|
+
if (request.epoch === appliedEpoch) {
|
|
1696
|
+
return void 0;
|
|
1697
|
+
}
|
|
1698
|
+
return request;
|
|
1699
|
+
},
|
|
1700
|
+
/**
|
|
1701
|
+
* Record the epoch as applied. Must be called only after the worker has
|
|
1702
|
+
* successfully initialized with `clearOnInit: true`, so a failed init can
|
|
1703
|
+
* be retried by a later elected worker.
|
|
1704
|
+
*/
|
|
1705
|
+
async markResetApplied(epoch) {
|
|
1706
|
+
await store.set(appliedKey, epoch);
|
|
1707
|
+
}
|
|
1708
|
+
};
|
|
1709
|
+
}
|
|
1710
|
+
function createReloadRequestHandler({
|
|
1711
|
+
resetState,
|
|
1712
|
+
broadcast,
|
|
1713
|
+
generateEpoch = generateId
|
|
1714
|
+
}) {
|
|
1715
|
+
return async (options) => {
|
|
1716
|
+
const reloadEpoch = generateEpoch();
|
|
1717
|
+
if (options.clean) {
|
|
1718
|
+
await resetState.writeResetRequest(reloadEpoch);
|
|
1719
|
+
}
|
|
1720
|
+
broadcast({
|
|
1721
|
+
notificationType: "reload-requested",
|
|
1722
|
+
reloadEpoch,
|
|
1723
|
+
clean: options.clean
|
|
1724
|
+
});
|
|
1725
|
+
};
|
|
1346
1726
|
}
|
|
1347
1727
|
|
|
1348
1728
|
// src/worker-db/worker-common.ts
|
|
@@ -1376,6 +1756,7 @@ function isWorkerNotificationMessage(message) {
|
|
|
1376
1756
|
|
|
1377
1757
|
export {
|
|
1378
1758
|
dummyKysely,
|
|
1759
|
+
xxhash,
|
|
1379
1760
|
HLCCounter,
|
|
1380
1761
|
serializeHLC,
|
|
1381
1762
|
deserializeHLC,
|
|
@@ -1383,11 +1764,16 @@ export {
|
|
|
1383
1764
|
introspectDb,
|
|
1384
1765
|
startPerformanceLogger,
|
|
1385
1766
|
SQLiteDbWrapper,
|
|
1767
|
+
CRDT_EVENT_NO_OP_PAYLOAD,
|
|
1768
|
+
isNoOpCrdtEventPayload,
|
|
1386
1769
|
createMigrations,
|
|
1387
1770
|
createMigrator,
|
|
1388
1771
|
createStoredValue,
|
|
1389
1772
|
createKvStoreTableQuery,
|
|
1390
1773
|
createSQLiteKvStore,
|
|
1774
|
+
createSystemDbConfig,
|
|
1775
|
+
workerDbConfig,
|
|
1776
|
+
memoryDbConfig,
|
|
1391
1777
|
baseSystemMigrations,
|
|
1392
1778
|
runSystemMigrations,
|
|
1393
1779
|
applyWorkerDbSchema,
|
|
@@ -1397,7 +1783,6 @@ export {
|
|
|
1397
1783
|
createCrdtStorage,
|
|
1398
1784
|
createCrdtSyncProducer,
|
|
1399
1785
|
createCrdtSyncRemoteSource,
|
|
1400
|
-
applyKyselyEventsBatchFilters,
|
|
1401
1786
|
syncDbWorkerLockName,
|
|
1402
1787
|
syncDbClientLockName,
|
|
1403
1788
|
createBroadcastChannels,
|
|
@@ -1405,6 +1790,10 @@ export {
|
|
|
1405
1790
|
isWorkerRequestMessage,
|
|
1406
1791
|
isWorkerResponseMessage,
|
|
1407
1792
|
isWorkerErrorResponseMessage,
|
|
1408
|
-
isWorkerNotificationMessage
|
|
1793
|
+
isWorkerNotificationMessage,
|
|
1794
|
+
createIdbResetStore,
|
|
1795
|
+
RESET_REQUEST_TTL_MS,
|
|
1796
|
+
createResetStateStore,
|
|
1797
|
+
createReloadRequestHandler
|
|
1409
1798
|
};
|
|
1410
|
-
//# sourceMappingURL=chunk-
|
|
1799
|
+
//# sourceMappingURL=chunk-RKBTBPNC.js.map
|