@sqlite-sync/core 0.8.2 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,2298 +0,0 @@
1
- import {
2
- TypedBroadcastChannel,
3
- createTypedEventTarget,
4
- ensureSingletonExecution,
5
- generateId,
6
- parseTableName,
7
- quoteId,
8
- tryCatchAsync
9
- } from "./chunk-NHT3ELMN.js";
10
-
11
- // src/dummy-kysely.ts
12
- import { DummyDriver, Kysely, SqliteAdapter, SqliteIntrospector, SqliteQueryCompiler } from "kysely";
13
- var dummyKysely = new Kysely({
14
- dialect: {
15
- createAdapter: () => new SqliteAdapter(),
16
- createDriver: () => new DummyDriver(),
17
- createQueryCompiler: () => new SqliteQueryCompiler(),
18
- createIntrospector: (db) => new SqliteIntrospector(db)
19
- }
20
- });
21
-
22
- // src/export-import.ts
23
- function createTableNameNormalizer(tablesConfig) {
24
- const baseNamesByAlias = /* @__PURE__ */ new Map();
25
- for (const { baseTableName, crdtTableName } of tablesConfig ?? []) {
26
- baseNamesByAlias.set(baseTableName, baseTableName);
27
- baseNamesByAlias.set(crdtTableName, baseTableName);
28
- }
29
- return (tableName) => baseNamesByAlias.get(tableName) ?? tableName;
30
- }
31
- function createImportData({ migrator, tablesConfig, applyEvents }) {
32
- const normalizeTableName = createTableNameNormalizer(tablesConfig);
33
- return (data, opts) => {
34
- const validate = opts?.validate ?? true;
35
- const currentVersion = migrator.currentSchemaVersion;
36
- if (validate && data.schemaVersion > currentVersion) {
37
- throw new Error(
38
- `Cannot import data from schema version ${data.schemaVersion} into a database at older schema version ${currentVersion}. Exports migrate forward to newer schema versions, not backward.`
39
- );
40
- }
41
- const sourceEvents = [];
42
- for (const [tableName, rows] of Object.entries(data.tables)) {
43
- for (const row of rows) {
44
- sourceEvents.push({
45
- schema_version: data.schemaVersion,
46
- type: "item-created",
47
- dataset: normalizeTableName(tableName),
48
- item_id: row.id,
49
- payload: JSON.stringify(row)
50
- });
51
- }
52
- }
53
- const events = migrator.migrateEvents(sourceEvents, currentVersion).map((event) => ({
54
- type: event.type,
55
- dataset: event.dataset,
56
- item_id: event.item_id,
57
- payload: event.payload
58
- }));
59
- const applied = applyEvents(events);
60
- if (applied instanceof Promise) {
61
- return applied.then(() => ({ imported: events.length }));
62
- }
63
- return { imported: events.length };
64
- };
65
- }
66
- function createExportData({ reactiveDb, tablesConfig, schemaVersion }) {
67
- const tableNames = tablesConfig.flatMap((config) => [config.crdtTableName, config.baseTableName]);
68
- const resolveTables = (requested) => {
69
- if (!requested) {
70
- return tablesConfig;
71
- }
72
- const seenBaseTableNames = /* @__PURE__ */ new Set();
73
- const resolved = [];
74
- for (const name of requested) {
75
- const config = tablesConfig.find(
76
- (tableConfig) => tableConfig.crdtTableName === name || tableConfig.baseTableName === name
77
- );
78
- if (!config) {
79
- throw new Error(`Unknown table "${name}". Known synced tables: ${tableNames.join(", ")}`);
80
- }
81
- if (!seenBaseTableNames.has(config.baseTableName)) {
82
- seenBaseTableNames.add(config.baseTableName);
83
- resolved.push(config);
84
- }
85
- }
86
- return resolved;
87
- };
88
- return (opts) => {
89
- const tables = {};
90
- for (const { baseTableName, crdtTableName } of resolveTables(opts?.tables)) {
91
- const { rows } = reactiveDb.db.execute(`select * from ${quoteId(crdtTableName)}`, {
92
- loggerLevel: "system"
93
- });
94
- tables[baseTableName] = rows.map(({ tombstone: _tombstone, ...rest }) => rest);
95
- }
96
- return {
97
- schemaVersion,
98
- exportedAt: (/* @__PURE__ */ new Date()).toISOString(),
99
- tables
100
- };
101
- };
102
- }
103
-
104
- // src/hash.ts
105
- import initXxhash from "xxhash-wasm";
106
- var loadPromise = null;
107
- var api = null;
108
- function ensureLoaded() {
109
- if (!loadPromise) {
110
- loadPromise = initXxhash().then((hasher) => {
111
- api = hasher;
112
- });
113
- }
114
- return loadPromise;
115
- }
116
- function h64(input, seed = 0n) {
117
- if (!api) {
118
- throw new Error("xxhash is not initialized; call xxhash.ensureLoaded() first");
119
- }
120
- return api.h64(input, seed);
121
- }
122
- var xxhash = {
123
- ensureLoaded,
124
- h64
125
- };
126
-
127
- // src/hlc.ts
128
- var MAX_COUNTER = 36 ** 5 - 1;
129
- var DEFAULT_MAX_DRIFT_MS = 6 * 60 * 60 * 1e3;
130
- var HLCCounter = class {
131
- timestamp;
132
- counter;
133
- nodeId;
134
- getTimestamp;
135
- maxDrift;
136
- constructor(nodeId, getTimestamp, maxDrift = DEFAULT_MAX_DRIFT_MS) {
137
- this.timestamp = getTimestamp();
138
- this.counter = 0;
139
- this.nodeId = nodeId;
140
- this.getTimestamp = getTimestamp;
141
- this.maxDrift = maxDrift;
142
- }
143
- getCurrentHLC() {
144
- return {
145
- timestamp: this.timestamp,
146
- counter: this.counter,
147
- nodeId: this.nodeId
148
- };
149
- }
150
- getNextHLC() {
151
- const now = this.getTimestamp();
152
- if (now > this.timestamp) {
153
- this.timestamp = now;
154
- this.counter = 0;
155
- return this.getCurrentHLC();
156
- }
157
- this.counter++;
158
- if (this.counter > MAX_COUNTER) {
159
- throw new Error(`HLC counter overflow: exceeded max value ${MAX_COUNTER}`);
160
- }
161
- return this.getCurrentHLC();
162
- }
163
- mergeHLC(hlc) {
164
- const now = this.getTimestamp();
165
- if (hlc.timestamp - now > this.maxDrift) {
166
- console.warn(
167
- `HLC: ignoring far-future timestamp (remote=${hlc.timestamp}, local=${now}, drift=${hlc.timestamp - now}ms)`
168
- );
169
- return;
170
- }
171
- if (this.timestamp === hlc.timestamp) {
172
- this.counter = Math.max(this.counter, hlc.counter) + 1;
173
- } else if (this.timestamp > hlc.timestamp) {
174
- this.counter++;
175
- } else {
176
- this.timestamp = hlc.timestamp;
177
- this.counter = hlc.counter + 1;
178
- }
179
- if (this.counter > MAX_COUNTER) {
180
- throw new Error(`HLC counter overflow: exceeded max value ${MAX_COUNTER}`);
181
- }
182
- }
183
- };
184
- function serializeHLC(hlc) {
185
- return `${hlc.timestamp.toString().padStart(15, "0")}:${hlc.counter.toString(36).padStart(5, "0")}:${hlc.nodeId}`;
186
- }
187
- function deserializeHLC(serialized) {
188
- const parts = serialized.split(":");
189
- if (parts.length < 3) {
190
- throw new Error(`Invalid HLC format: expected at least 3 colon-separated segments, got ${parts.length}`);
191
- }
192
- const timestamp = parseInt(parts[0], 10);
193
- const counter = parseInt(parts[1], 36);
194
- if (Number.isNaN(timestamp) || Number.isNaN(counter)) {
195
- throw new Error(`Invalid HLC values: timestamp=${parts[0]}, counter=${parts[1]}`);
196
- }
197
- return {
198
- timestamp,
199
- counter,
200
- nodeId: parts.slice(2).join(":")
201
- };
202
- }
203
- function compareHLC(one, two) {
204
- if (one.timestamp === two.timestamp) {
205
- if (one.counter === two.counter) {
206
- if (one.nodeId === two.nodeId) {
207
- return 0;
208
- }
209
- return one.nodeId < two.nodeId ? -1 : 1;
210
- }
211
- return one.counter - two.counter;
212
- }
213
- return one.timestamp - two.timestamp;
214
- }
215
-
216
- // src/introspection.ts
217
- import { sql } from "kysely";
218
- function tablesQuery(qb) {
219
- return qb.selectFrom("sqlite_master").where("type", "in", ["table", "view"]).where("name", "not like", "sqlite_%").select(["name", "sql", "type"]).orderBy("name");
220
- }
221
- function introspectDb(_db) {
222
- const db = _db;
223
- const tables = db.executeKysely((db2) => tablesQuery(db2), {
224
- loggerLevel: "system"
225
- }).rows;
226
- const tablesMetadata = db.executeKysely(
227
- (db2) => db2.with("table_list", (qb) => tablesQuery(qb)).selectFrom(["table_list as tl", sql`pragma_table_info(tl.name)`.as("p")]).select(["tl.name as table", "p.cid", "p.name", "p.type", "p.notnull", "p.dflt_value", "p.pk"]).orderBy("tl.name").orderBy("p.cid"),
228
- { loggerLevel: "system" }
229
- ).rows;
230
- const columnsByTable = {};
231
- for (const row of tablesMetadata) {
232
- columnsByTable[row.table] ??= [];
233
- columnsByTable[row.table].push(row);
234
- }
235
- return Object.fromEntries(
236
- tables.map(({ name, sql: sql4, type }) => {
237
- let autoIncrementCol = sql4?.split(/[(),]/)?.find((it) => it.toLowerCase().includes("autoincrement"))?.trimStart()?.split(/\s+/)?.[0]?.replace(/["`]/g, "");
238
- const columns = columnsByTable[name] ?? [];
239
- if (!autoIncrementCol) {
240
- const pkCols = columns.filter((r) => r.pk > 0);
241
- if (pkCols.length === 1 && pkCols[0].type.toLowerCase() === "integer") {
242
- autoIncrementCol = pkCols[0].name;
243
- }
244
- }
245
- return [
246
- name,
247
- {
248
- name,
249
- isView: type === "view",
250
- columns: columns.map((col) => ({
251
- name: col.name,
252
- dataType: col.type,
253
- isNullable: !col.notnull,
254
- isAutoIncrementing: col.name === autoIncrementCol,
255
- hasDefaultValue: col.dflt_value != null,
256
- defaultValueSql: col.dflt_value == null ? null : String(col.dflt_value),
257
- isPrimaryKey: col.pk > 0,
258
- comment: void 0
259
- }))
260
- }
261
- ];
262
- })
263
- );
264
- }
265
-
266
- // src/logger.ts
267
- var startPerformanceLogger = (logger) => {
268
- let startTime = performance.now();
269
- return {
270
- restart: () => {
271
- startTime = performance.now();
272
- },
273
- logEnd: (type, message, level = "info") => {
274
- const elapsed = performance.now() - startTime;
275
- logger(type, `${elapsed.toFixed(2)}ms - ${message}`, level);
276
- }
277
- };
278
- };
279
-
280
- // src/sqlite-db-wrapper.ts
281
- var SQLiteDbWrapper = class {
282
- db = null;
283
- sqlite3;
284
- logger;
285
- loggerPrefix;
286
- loadedDbSchema = null;
287
- dataPointers = [];
288
- preparedStatements = [];
289
- preparedStatementsMap = /* @__PURE__ */ new Map();
290
- preparedRawStatementsMap = /* @__PURE__ */ new Map();
291
- constructor(opts) {
292
- this.db = opts.db();
293
- this.sqlite3 = opts.sqlite3;
294
- this.logger = opts.logger;
295
- this.loggerPrefix = opts.loggerPrefix;
296
- }
297
- get ensureDb() {
298
- if (!this.db) {
299
- throw new Error("Database is already closed");
300
- }
301
- return this.db;
302
- }
303
- get dbSchema() {
304
- if (!this.loadedDbSchema) {
305
- this.loadedDbSchema = introspectDb(this);
306
- }
307
- return this.loadedDbSchema;
308
- }
309
- execute(opts, meta) {
310
- const sql4 = typeof opts === "string" ? opts : opts.sql;
311
- const bind = typeof opts === "string" ? void 0 : opts.parameters;
312
- const perf = this.logger ? startPerformanceLogger(this.logger) : void 0;
313
- const rows = this.ensureDb.exec({
314
- sql: sql4,
315
- bind,
316
- returnValue: "resultRows",
317
- rowMode: "object"
318
- });
319
- perf?.logEnd(`${this.loggerPrefix ?? ""}:query`, sql4, meta?.loggerLevel);
320
- return { rows };
321
- }
322
- executeTransaction(callback) {
323
- const transaction = this.beginTransaction();
324
- try {
325
- const result = callback(this);
326
- transaction.commit();
327
- return result;
328
- } catch (error) {
329
- transaction.rollback();
330
- throw error;
331
- }
332
- }
333
- isInTransaction() {
334
- return this.sqlite3.capi.sqlite3_get_autocommit(this.ensureDb) === 0;
335
- }
336
- beginTransaction() {
337
- this.executePreparedRaw({
338
- key: "$begin-transaction",
339
- sql: "begin",
340
- meta: {
341
- loggerLevel: "system"
342
- }
343
- });
344
- return {
345
- commit: () => {
346
- this.executePreparedRaw({
347
- key: "$commit-transaction",
348
- sql: "commit",
349
- meta: {
350
- loggerLevel: "system"
351
- }
352
- });
353
- },
354
- rollback: () => {
355
- this.executePreparedRaw({
356
- key: "$rollback-transaction",
357
- sql: "rollback",
358
- meta: {
359
- loggerLevel: "system"
360
- }
361
- });
362
- }
363
- };
364
- }
365
- prepare(sql4, opts) {
366
- const perf = this.logger ? startPerformanceLogger(this.logger) : void 0;
367
- const stmt = this.ensureDb.prepare(sql4);
368
- perf?.logEnd(`${this.loggerPrefix ?? ""}:prepare`, sql4, opts?.loggerLevel);
369
- let isFinalized = false;
370
- const execute = (params) => {
371
- if (isFinalized) {
372
- throw new Error("Statement is finalized");
373
- }
374
- const perf2 = this.logger ? startPerformanceLogger(this.logger) : void 0;
375
- try {
376
- if (params.length > 0) {
377
- stmt.bind(params);
378
- }
379
- const results = [];
380
- while (stmt.step()) {
381
- results.push(stmt.get({}));
382
- }
383
- return results;
384
- } finally {
385
- stmt.reset(true);
386
- perf2?.logEnd(`${this.loggerPrefix ?? ""}:prepare-execute`, sql4, opts?.loggerLevel);
387
- }
388
- };
389
- const finalize = () => {
390
- isFinalized = true;
391
- stmt.finalize();
392
- };
393
- const preparedStatement = {
394
- execute,
395
- finalize,
396
- get isFinalized() {
397
- return isFinalized;
398
- }
399
- };
400
- this.preparedStatements.push(preparedStatement);
401
- return preparedStatement;
402
- }
403
- prepareKysely(opts) {
404
- return (factory) => {
405
- const query = factory(dummyKysely, (key) => key).compile();
406
- const statement = this.prepare(query.sql, opts);
407
- return {
408
- execute: (parameters) => {
409
- const params = query.parameters.map((param) => parameters[param]);
410
- const result = statement.execute(params);
411
- return result;
412
- }
413
- };
414
- };
415
- }
416
- executeKysely(factory, meta) {
417
- const query = factory(dummyKysely).compile();
418
- return this.execute(query, meta);
419
- }
420
- executePrepared(key, params, factory, meta) {
421
- let statement = this.preparedStatementsMap.get(key);
422
- if (!statement) {
423
- statement = this.prepareKysely(meta)(factory);
424
- this.preparedStatementsMap.set(key, statement);
425
- }
426
- return statement.execute(params);
427
- }
428
- executePreparedRaw({
429
- key,
430
- sql: sql4,
431
- params,
432
- meta
433
- }) {
434
- let statement = this.preparedRawStatementsMap.get(key);
435
- if (!statement) {
436
- statement = this.prepare(sql4, meta);
437
- this.preparedRawStatementsMap.set(key, statement);
438
- }
439
- return statement.execute(params ?? []);
440
- }
441
- sql(templateOrString, ...parameters) {
442
- if (typeof templateOrString === "string") {
443
- return this.execute({
444
- sql: templateOrString,
445
- parameters
446
- });
447
- }
448
- return this.execute({
449
- sql: templateOrString.join("?"),
450
- parameters
451
- });
452
- }
453
- createScalarFunction({
454
- name,
455
- callback,
456
- deterministic,
457
- directOnly,
458
- innocuous
459
- }) {
460
- return this.ensureDb.createFunction({
461
- name,
462
- xFunc: (_, ...args) => {
463
- const result = callback(...args);
464
- return result;
465
- },
466
- arity: callback.length,
467
- deterministic,
468
- directOnly,
469
- innocuous
470
- });
471
- }
472
- useSnapshot(snapshot) {
473
- const perf = this.logger ? startPerformanceLogger(this.logger) : void 0;
474
- const dataPointer = this.sqlite3.wasm.allocFromTypedArray(snapshot);
475
- this.dataPointers.push(dataPointer);
476
- const resultCode = this.sqlite3.capi.sqlite3_deserialize(
477
- this.ensureDb,
478
- "main",
479
- dataPointer,
480
- snapshot.byteLength,
481
- snapshot.byteLength,
482
- this.sqlite3.capi.SQLITE_DESERIALIZE_FREEONCLOSE | this.sqlite3.capi.SQLITE_DESERIALIZE_RESIZEABLE
483
- );
484
- this.ensureDb.checkRc(resultCode);
485
- this.invalidateDbSchema();
486
- perf?.logEnd("useSnapshot", "success", "system");
487
- }
488
- createSnapshot() {
489
- return this.sqlite3.capi.sqlite3_js_db_export(this.ensureDb);
490
- }
491
- invalidateDbSchema() {
492
- this.loadedDbSchema = null;
493
- }
494
- cleanup() {
495
- this.preparedStatements.forEach((stmt) => {
496
- stmt.finalize();
497
- });
498
- this.preparedStatements.splice(0);
499
- this.preparedStatementsMap.clear();
500
- this.preparedRawStatementsMap.clear();
501
- }
502
- close() {
503
- this.cleanup();
504
- this.db?.close();
505
- this.db = null;
506
- }
507
- };
508
-
509
- // src/sqlite-crdt/crdt-table-schema.ts
510
- var CRDT_EVENT_NO_OP_PAYLOAD = "no-op";
511
- function isNoOpCrdtEventPayload(payload) {
512
- return payload === CRDT_EVENT_NO_OP_PAYLOAD;
513
- }
514
-
515
- // src/migrations/migrator.ts
516
- var protectedColumns = ["id", "tombstone"];
517
- function assertColumnNotProtected(column, operation) {
518
- if (protectedColumns.includes(column)) {
519
- throw new Error(`Cannot ${operation} protected column "${column}"`);
520
- }
521
- }
522
- var migrationSteps = {
523
- createTable: (table, build) => ({
524
- sql: (db) => build(db.schema.createTable(table))
525
- }),
526
- dropTable: (table) => ({
527
- sql: (db) => db.schema.dropTable(table),
528
- eventTransformer: {
529
- [table]: () => null
530
- },
531
- tableDrops: [table]
532
- }),
533
- createIndex: (indexName, build) => ({
534
- sql: (db) => build(db.schema.createIndex(indexName))
535
- }),
536
- dropIndex: (indexName) => ({
537
- sql: (db) => db.schema.dropIndex(indexName)
538
- }),
539
- renameTable: ({ oldTable, newTable }) => ({
540
- sql: (db) => db.schema.alterTable(oldTable).renameTo(newTable),
541
- eventTransformer: {
542
- [oldTable]: (event) => {
543
- event.dataset = newTable;
544
- return event;
545
- }
546
- },
547
- tableRenames: [{ oldTable, newTable }]
548
- }),
549
- renameColumn: ({
550
- table,
551
- oldColumn,
552
- newColumn
553
- }) => {
554
- assertColumnNotProtected(oldColumn, "rename");
555
- assertColumnNotProtected(newColumn, "rename to");
556
- return {
557
- sql: (db) => db.schema.alterTable(table).renameColumn(oldColumn, newColumn),
558
- eventTransformer: {
559
- [table]: (event) => {
560
- if (event.type !== "item-updated" && event.type !== "item-created" || !(oldColumn in event.payload)) {
561
- return event;
562
- }
563
- const oldVal = event.payload[oldColumn];
564
- delete event.payload[oldColumn];
565
- event.payload[newColumn] = oldVal;
566
- return event;
567
- }
568
- }
569
- };
570
- },
571
- addColumn: ({
572
- table,
573
- column,
574
- type,
575
- defaultValue,
576
- build = (e) => e
577
- }) => ({
578
- sql: (db) => db.schema.alterTable(table).addColumn(column, type, (x) => build(x).defaultTo(defaultValue)),
579
- eventTransformer: {
580
- [table]: (event) => {
581
- if (event.type !== "item-created") {
582
- return event;
583
- }
584
- event.payload[column] = defaultValue;
585
- return event;
586
- }
587
- }
588
- }),
589
- dropColumn: ({ table, column }) => {
590
- assertColumnNotProtected(column, "drop");
591
- return {
592
- sql: (db) => db.schema.alterTable(table).dropColumn(column),
593
- eventTransformer: {
594
- [table]: (event) => {
595
- if (!(column in event.payload)) {
596
- return event;
597
- }
598
- delete event.payload[column];
599
- if (event.type === "item-updated" && Object.keys(event.payload).length === 0) {
600
- return null;
601
- }
602
- return event;
603
- }
604
- }
605
- };
606
- }
607
- };
608
- function buildMigrationSql(steps) {
609
- return steps.flatMap((step) => Array.isArray(step.sql) ? step.sql : [step.sql]).map((sql4) => {
610
- if (typeof sql4 === "string") {
611
- return { sql: sql4, parameters: [] };
612
- }
613
- if (typeof sql4 === "function") {
614
- const query = sql4(dummyKysely).compile();
615
- return { sql: query.sql, parameters: query.parameters };
616
- }
617
- if ("compile" in sql4) {
618
- const query = sql4.compile();
619
- return { sql: query.sql, parameters: query.parameters };
620
- }
621
- return {
622
- sql: sql4.sql,
623
- parameters: sql4.parameters ?? []
624
- };
625
- });
626
- }
627
- function buildMigrationEventTransformer(steps) {
628
- const transformers = [];
629
- for (const step of steps) {
630
- if (step.eventTransformer) {
631
- transformers.push(...Object.entries(step.eventTransformer));
632
- }
633
- }
634
- if (transformers.length === 0) {
635
- return void 0;
636
- }
637
- return (event) => {
638
- let transformedEvent = event;
639
- for (const [table, transformer] of transformers) {
640
- if (transformedEvent === null) {
641
- return null;
642
- }
643
- if (transformedEvent.dataset !== table) {
644
- continue;
645
- }
646
- transformedEvent = transformer(transformedEvent);
647
- if (transformedEvent === null) {
648
- return null;
649
- }
650
- }
651
- return transformedEvent;
652
- };
653
- }
654
- function createMigrations(buildMigrations) {
655
- const migrations = Object.fromEntries(
656
- Object.entries(buildMigrations(migrationSteps)).map(([version, steps]) => {
657
- const versionNumber = Number(version);
658
- if (Number.isNaN(versionNumber)) {
659
- throw new Error(`Invalid migration version: ${version}`);
660
- }
661
- if (versionNumber < 0) {
662
- throw new Error(`Migration version cannot be negative: ${version}`);
663
- }
664
- const tableRenames = steps.flatMap((s) => s.tableRenames ?? []);
665
- const tableDrops = steps.flatMap((s) => s.tableDrops ?? []);
666
- return [
667
- version,
668
- {
669
- sql: buildMigrationSql(steps),
670
- eventTransformer: buildMigrationEventTransformer(steps),
671
- ...tableRenames.length > 0 && { tableRenames },
672
- ...tableDrops.length > 0 && { tableDrops }
673
- }
674
- ];
675
- })
676
- );
677
- return migrations;
678
- }
679
- function createMigrator({
680
- migrations,
681
- schemaVersion,
682
- updateLogTableName
683
- }) {
684
- const latestSchemaVersion = Math.max(...Object.keys(migrations).map(Number));
685
- const sortedMigrations = Object.entries(migrations).map(([v, m]) => [Number(v), m]).sort((a, b) => a[0] - b[0]);
686
- const applyMigration = (db, version, migration) => {
687
- if (version <= schemaVersion.current) {
688
- throw new Error(`Cannot apply migration ${version} to schema version ${schemaVersion.current}`);
689
- }
690
- db.startTransaction((tx) => {
691
- for (const statement of migration.sql) {
692
- tx.execute(statement.sql, statement.parameters);
693
- }
694
- if (updateLogTableName) {
695
- if (migration.tableRenames) {
696
- for (const { oldTable, newTable } of migration.tableRenames) {
697
- tx.execute(`UPDATE ${updateLogTableName} SET "dataset" = ? WHERE "dataset" = ?`, [newTable, oldTable], {
698
- loggerLevel: "system"
699
- });
700
- }
701
- }
702
- if (migration.tableDrops) {
703
- for (const table of migration.tableDrops) {
704
- tx.execute(`DELETE FROM ${updateLogTableName} WHERE "dataset" = ?`, [table], { loggerLevel: "system" });
705
- }
706
- }
707
- }
708
- schemaVersion.current = version;
709
- });
710
- };
711
- const migrateEvent = (event, targetVersion) => {
712
- targetVersion ??= latestSchemaVersion;
713
- if (targetVersion > schemaVersion.current) {
714
- throw new Error(
715
- `Target schema version ${targetVersion} is greater than current schema version ${schemaVersion.current}`
716
- );
717
- }
718
- if (isNoOpCrdtEventPayload(event.payload)) {
719
- if (event.schema_version < targetVersion) {
720
- event.schema_version = targetVersion;
721
- }
722
- return event;
723
- }
724
- if (event.schema_version >= targetVersion) {
725
- return event;
726
- }
727
- const fromVersion = event.schema_version;
728
- let crdtEvent = {
729
- dataset: event.dataset,
730
- item_id: event.item_id,
731
- type: event.type,
732
- payload: JSON.parse(event.payload)
733
- };
734
- for (let i = 0; i < sortedMigrations.length; i++) {
735
- const [version, migration] = sortedMigrations[i];
736
- if (version <= fromVersion) continue;
737
- if (version > targetVersion) break;
738
- const transformer = migration.eventTransformer;
739
- if (transformer) {
740
- crdtEvent = transformer(crdtEvent);
741
- if (crdtEvent === null) return null;
742
- }
743
- }
744
- if (crdtEvent === null) {
745
- return null;
746
- }
747
- event.schema_version = targetVersion;
748
- event.dataset = crdtEvent.dataset;
749
- event.item_id = crdtEvent.item_id;
750
- event.type = crdtEvent.type;
751
- event.payload = JSON.stringify(crdtEvent.payload);
752
- return event;
753
- };
754
- const migrateEvents = (events, targetVersion) => {
755
- return events.map((event) => migrateEvent(event, targetVersion ?? latestSchemaVersion)).filter((event) => event !== null);
756
- };
757
- return {
758
- latestSchemaVersion,
759
- get currentSchemaVersion() {
760
- return schemaVersion.current;
761
- },
762
- migrateDbToLatest: (db) => {
763
- const currentSchemaVersion = schemaVersion.current;
764
- if (currentSchemaVersion >= latestSchemaVersion) {
765
- return;
766
- }
767
- for (let i = 0; i < sortedMigrations.length; i++) {
768
- const [version, migration] = sortedMigrations[i];
769
- if (version <= currentSchemaVersion) continue;
770
- applyMigration(db, version, migration);
771
- }
772
- },
773
- migrateEvent,
774
- migrateEvents
775
- };
776
- }
777
-
778
- // src/sqlite-crdt/stored-value.ts
779
- function createStoredValue({
780
- initialValue,
781
- saveToStorage
782
- }) {
783
- let currentValue = initialValue;
784
- return {
785
- get current() {
786
- return currentValue;
787
- },
788
- set current(newValue) {
789
- saveToStorage?.(newValue);
790
- currentValue = newValue;
791
- }
792
- };
793
- }
794
-
795
- // src/sqlite-kv-store.ts
796
- import { sql as sql2 } from "kysely";
797
- function createKvStoreTableQuery(schema, tableName) {
798
- return schema.createTable(tableName).ifNotExists().addColumn("key", "text", (col) => col.notNull().primaryKey()).addColumn("value", "text", (col) => col.notNull()).modifyEnd(sql2`without rowid`);
799
- }
800
- function createSQLiteKvStore({ db, metaTableName }) {
801
- const metaDb = db;
802
- const get2 = (key) => {
803
- const [result] = metaDb.executePrepared(
804
- "get-meta-value",
805
- { key },
806
- (db2, params) => db2.selectFrom(metaTableName).where("key", "=", params("key")).select("value").limit((eb) => eb.lit(1)),
807
- { loggerLevel: "system" }
808
- );
809
- return result?.value ?? null;
810
- };
811
- const set2 = (key, value) => {
812
- metaDb.executePrepared(
813
- "set-meta-value",
814
- { key, value },
815
- (db2, params) => db2.insertInto(metaTableName).values({ key: params("key"), value: params("value") }).onConflict((oc) => oc.doUpdateSet({ value: params("value") })),
816
- { loggerLevel: "system" }
817
- );
818
- };
819
- const remove = (key) => {
820
- metaDb.executePrepared(
821
- "remove-meta-value",
822
- { key },
823
- (db2, params) => db2.deleteFrom(metaTableName).where("key", "=", params("key")),
824
- { loggerLevel: "system" }
825
- );
826
- };
827
- const getNumberOrDefault = (key, defaultValue) => {
828
- const value = get2(key);
829
- if (!value) return defaultValue;
830
- const parsedValue = Number.parseInt(value, 10);
831
- return Number.isNaN(parsedValue) ? defaultValue : parsedValue;
832
- };
833
- return {
834
- get: get2,
835
- set: set2,
836
- remove,
837
- createStringStoredValue: (key, defaultValue) => createStoredValue({
838
- initialValue: get2(key) ?? defaultValue,
839
- saveToStorage: (val) => set2(key, val)
840
- }),
841
- createNumberStoredValue: (key, defaultValue) => createStoredValue({
842
- initialValue: getNumberOrDefault(key, defaultValue),
843
- saveToStorage: (val) => set2(key, val.toString())
844
- })
845
- };
846
- }
847
-
848
- // src/migrations/system-schema.ts
849
- function createSystemDbConfig({
850
- eventsTableName,
851
- updateLogTableName
852
- }) {
853
- return {
854
- eventsTable: parseTableName(eventsTableName),
855
- updateLogTable: parseTableName(updateLogTableName)
856
- };
857
- }
858
- var workerDbConfig = createSystemDbConfig({
859
- eventsTableName: "worker.crdt_events",
860
- updateLogTableName: "crdt_update_log"
861
- });
862
- var memoryDbConfig = createSystemDbConfig({
863
- eventsTableName: "persisted_crdt_events",
864
- updateLogTableName: "crdt_update_log"
865
- });
866
- var baseSystemMigrations = [
867
- {
868
- version: 0,
869
- up: (ctx) => {
870
- ctx.execute(`CREATE TABLE IF NOT EXISTS ${ctx.eventsTable.fullIdentifier} (
871
- "sync_id" integer NOT NULL PRIMARY KEY,
872
- "schema_version" integer NOT NULL,
873
- "status" text NOT NULL,
874
- "type" text NOT NULL,
875
- "timestamp" text NOT NULL,
876
- "origin" text NOT NULL,
877
- "dataset" text NOT NULL,
878
- "item_id" text NOT NULL,
879
- "payload" text NOT NULL
880
- )`);
881
- ctx.execute(`CREATE TABLE IF NOT EXISTS ${ctx.updateLogTable.fullIdentifier} (
882
- "dataset" text NOT NULL,
883
- "item_id" text NOT NULL,
884
- "payload" text NOT NULL,
885
- PRIMARY KEY ("item_id", "dataset")
886
- )`);
887
- }
888
- },
889
- {
890
- version: 1,
891
- up: (ctx) => {
892
- ctx.execute(`ALTER TABLE ${ctx.eventsTable.fullIdentifier} ADD COLUMN "source_node_id" TEXT NOT NULL DEFAULT ''`);
893
- }
894
- },
895
- {
896
- version: 2,
897
- up: (ctx) => {
898
- const indexName = `${ctx.eventsTable.table}_status_sync_id_idx`;
899
- ctx.execute(
900
- `CREATE INDEX IF NOT EXISTS ${ctx.eventsTable.schema}.${indexName} ON ${ctx.eventsTable.table} ("status", "sync_id")`
901
- );
902
- }
903
- },
904
- {
905
- version: 3,
906
- up: (ctx) => {
907
- const indexName = `${ctx.eventsTable.table}_timestamp_status_sync_id_idx`;
908
- ctx.execute(
909
- `CREATE INDEX IF NOT EXISTS ${ctx.eventsTable.schema}.${indexName} ON ${ctx.eventsTable.table} ("timestamp", "status", "sync_id")`
910
- );
911
- }
912
- }
913
- ];
914
- function runSystemMigrations(opts) {
915
- const ctx = {
916
- ...opts.dbConfig,
917
- execute: opts.execute
918
- };
919
- for (const migration of opts.migrations) {
920
- if (migration.version > opts.version.current) {
921
- opts.transaction(() => {
922
- migration.up(ctx);
923
- opts.version.current = migration.version;
924
- });
925
- }
926
- }
927
- }
928
- function applyWorkerDbSchema(db) {
929
- db.executeKysely((kysely) => createKvStoreTableQuery(kysely.schema, "worker.kv"), { loggerLevel: "system" });
930
- const kvStore = createSQLiteKvStore({ db, metaTableName: "worker.kv" });
931
- runSystemMigrations({
932
- migrations: baseSystemMigrations,
933
- version: kvStore.createNumberStoredValue("internal-schema-version", -1),
934
- dbConfig: workerDbConfig,
935
- execute: (sql4) => db.execute(sql4, { loggerLevel: "system" }),
936
- transaction: (callback) => db.executeTransaction(callback)
937
- });
938
- return { kvStore };
939
- }
940
- function applyMemoryDbSchema(db) {
941
- db.execute(
942
- `CREATE TABLE IF NOT EXISTS ${memoryDbConfig.eventsTable.fullIdentifier} (
943
- "sync_id" integer NOT NULL PRIMARY KEY,
944
- "schema_version" integer NOT NULL,
945
- "status" text NOT NULL,
946
- "type" text NOT NULL,
947
- "timestamp" text NOT NULL,
948
- "origin" text NOT NULL,
949
- "source_node_id" text NOT NULL DEFAULT '',
950
- "dataset" text NOT NULL,
951
- "item_id" text NOT NULL,
952
- "payload" text NOT NULL
953
- )`,
954
- { loggerLevel: "system" }
955
- );
956
- db.execute(
957
- `CREATE INDEX IF NOT EXISTS ${memoryDbConfig.eventsTable.table}_status_sync_id_idx ON ${memoryDbConfig.eventsTable.table} ("status", "sync_id")`,
958
- {
959
- loggerLevel: "system"
960
- }
961
- );
962
- db.execute(
963
- `CREATE INDEX IF NOT EXISTS ${memoryDbConfig.eventsTable.table}_timestamp_status_sync_id_idx ON ${memoryDbConfig.eventsTable.table} ("timestamp", "status", "sync_id")`,
964
- {
965
- loggerLevel: "system"
966
- }
967
- );
968
- db.execute(
969
- `CREATE TABLE IF NOT EXISTS ${memoryDbConfig.updateLogTable.fullIdentifier} (
970
- "dataset" text NOT NULL,
971
- "item_id" text NOT NULL,
972
- "payload" text NOT NULL,
973
- PRIMARY KEY ("item_id", "dataset")
974
- )`,
975
- { loggerLevel: "system" }
976
- );
977
- }
978
-
979
- // src/schema/validate-crdt-event.ts
980
- var CrdtEventValidationError = class extends Error {
981
- constructor(errors) {
982
- super(`Invalid CRDT events: ${errors.join("; ")}`);
983
- this.errors = errors;
984
- this.name = "CrdtEventValidationError";
985
- }
986
- };
987
- var crdtEventTypes = ["item-created", "item-updated", "item-deleted"];
988
- function validateNewCrdtEvent(schema, input) {
989
- const errors = [];
990
- if (!isRecord(input)) {
991
- return { success: false, errors: ["Event must be an object"] };
992
- }
993
- const type = input.type;
994
- const dataset = input.dataset;
995
- const itemId = input.item_id;
996
- let payload = input.payload;
997
- let payloadParseFailed = false;
998
- if (typeof payload === "string") {
999
- try {
1000
- payload = JSON.parse(payload);
1001
- } catch {
1002
- payloadParseFailed = true;
1003
- errors.push(`Event payload is not valid JSON`);
1004
- }
1005
- }
1006
- if (!isCrdtEventType(type)) {
1007
- errors.push(`Invalid event type "${String(type)}"`);
1008
- }
1009
- if (typeof dataset !== "string") {
1010
- errors.push(`Event dataset must be a string`);
1011
- }
1012
- if (typeof itemId !== "string") {
1013
- errors.push(`Event item_id must be a string`);
1014
- }
1015
- if (!payloadParseFailed && !isRecord(payload)) {
1016
- errors.push(`Event payload must be an object`);
1017
- }
1018
- const datasetMatch = typeof dataset === "string" ? findTableForDataset(schema, dataset) : null;
1019
- if (typeof dataset === "string" && !datasetMatch) {
1020
- errors.push(`Unknown dataset "${dataset}"`);
1021
- }
1022
- if (isCrdtEventType(type) && datasetMatch && isRecord(payload)) {
1023
- switch (type) {
1024
- case "item-created": {
1025
- const result = datasetMatch.table.validatePayload(payload, { event: "item-created" });
1026
- if (!result.success) {
1027
- errors.push(...result.errors.map((error) => `payload: ${error}`));
1028
- }
1029
- if (typeof itemId === "string" && "id" in payload && payload.id !== itemId) {
1030
- errors.push(`payload: Column "id" must match item_id "${itemId}"`);
1031
- }
1032
- break;
1033
- }
1034
- case "item-updated": {
1035
- const result = datasetMatch.table.validatePayload(payload, { event: "item-updated" });
1036
- if (!result.success) {
1037
- errors.push(...result.errors.map((error) => `payload: ${error}`));
1038
- }
1039
- break;
1040
- }
1041
- case "item-deleted":
1042
- if (Object.keys(payload).length > 0) {
1043
- errors.push(`payload: item-deleted events must have an empty payload`);
1044
- }
1045
- break;
1046
- }
1047
- }
1048
- if (errors.length > 0) {
1049
- return { success: false, errors };
1050
- }
1051
- return {
1052
- success: true,
1053
- event: {
1054
- type,
1055
- dataset: datasetMatch?.baseTableName ?? dataset,
1056
- item_id: itemId,
1057
- payload
1058
- }
1059
- };
1060
- }
1061
- function findTableForDataset(schema, dataset) {
1062
- const config = schema.tablesConfig.find(
1063
- (tableConfig) => tableConfig.baseTableName === dataset || tableConfig.crdtTableName === dataset
1064
- );
1065
- if (config) {
1066
- const table2 = schema.tables[config.crdtTableName];
1067
- return table2 ? { table: table2, baseTableName: config.baseTableName } : null;
1068
- }
1069
- const table = schema.tables[dataset];
1070
- return table ? { table, baseTableName: dataset } : null;
1071
- }
1072
- function isRecord(value) {
1073
- return typeof value === "object" && value !== null && !Array.isArray(value);
1074
- }
1075
- function isCrdtEventType(value) {
1076
- return typeof value === "string" && crdtEventTypes.includes(value);
1077
- }
1078
-
1079
- // src/schema/verify-sync-schema.ts
1080
- import sqlite3InitModule from "@sqlite.org/sqlite-wasm";
1081
- var acceptedDataTypes = {
1082
- text: ["TEXT"],
1083
- integer: ["INTEGER", "INT"],
1084
- real: ["REAL", "DOUBLE", "FLOAT"],
1085
- boolean: ["BOOLEAN", "INTEGER", "INT"],
1086
- enum: ["TEXT"]
1087
- };
1088
- var sqliteModule = null;
1089
- async function verifySyncSchema(schema) {
1090
- if (!sqliteModule) {
1091
- sqliteModule = await sqlite3InitModule();
1092
- }
1093
- const sqlite3 = sqliteModule;
1094
- const db = new SQLiteDbWrapper({
1095
- db: () => new sqlite3.oo1.DB({ filename: ":memory:" }),
1096
- logger: () => {
1097
- },
1098
- loggerPrefix: "schema-verify",
1099
- sqlite3
1100
- });
1101
- try {
1102
- const migrator = createMigrator({
1103
- migrations: schema.migrations,
1104
- schemaVersion: createStoredValue({ initialValue: -1 })
1105
- });
1106
- try {
1107
- migrator.migrateDbToLatest({
1108
- startTransaction: (callback) => {
1109
- db.executeTransaction(
1110
- (tx) => callback({ execute: (sql4, parameters, meta) => tx.execute({ sql: sql4, parameters }, meta) })
1111
- );
1112
- }
1113
- });
1114
- } catch (error) {
1115
- return [
1116
- {
1117
- table: "(migrations)",
1118
- message: `migration replay failed on an empty database: ${error instanceof Error ? error.message : String(error)}`
1119
- }
1120
- ];
1121
- }
1122
- const introspection = introspectDb(db);
1123
- const issues = [];
1124
- for (const { crdtTableName, baseTableName } of schema.tablesConfig) {
1125
- const declared = schema.tables[crdtTableName];
1126
- if (!declared) continue;
1127
- verifyTable(db, baseTableName, declared.columns, introspection[baseTableName], issues);
1128
- }
1129
- return issues;
1130
- } finally {
1131
- db.close();
1132
- }
1133
- }
1134
- function formatSchemaVerificationIssues(issues) {
1135
- const lines = issues.map(
1136
- (issue) => ` - table "${issue.table}"${issue.column ? ` column "${issue.column}"` : ""}: ${issue.message}`
1137
- );
1138
- return `sync schema verification failed (${issues.length} issue${issues.length === 1 ? "" : "s"}):
1139
- ${lines.join("\n")}`;
1140
- }
1141
- function verifyTable(db, table, declaredColumns, actual, issues) {
1142
- if (!actual) {
1143
- issues.push({
1144
- table,
1145
- message: "declared in the schema but missing after migrations \u2014 add a createTable step in a new version"
1146
- });
1147
- return;
1148
- }
1149
- if (actual.isView) {
1150
- issues.push({ table, message: "expected a base table, but migrations created a view with this name" });
1151
- return;
1152
- }
1153
- const actualColumns = new Map(actual.columns.map((column) => [column.name, column]));
1154
- for (const [name, meta] of Object.entries(declaredColumns)) {
1155
- const column = actualColumns.get(name);
1156
- if (!column) {
1157
- issues.push({
1158
- table,
1159
- column: name,
1160
- message: "declared in the schema but missing after migrations \u2014 add an addColumn step in a new version"
1161
- });
1162
- continue;
1163
- }
1164
- const accepted = acceptedDataTypes[meta.kind];
1165
- if (!accepted.includes(column.dataType.toUpperCase())) {
1166
- issues.push({
1167
- table,
1168
- column: name,
1169
- message: `type mismatch \u2014 schema declares ${meta.kind} (${accepted.join("/")}), migrated column is ${column.dataType.toUpperCase()}`
1170
- });
1171
- }
1172
- if (column.isNullable !== meta.nullable) {
1173
- issues.push({
1174
- table,
1175
- column: name,
1176
- message: meta.nullable ? "schema declares the column nullable, but the migrated column is NOT NULL" : "schema declares the column NOT NULL, but the migrated column is nullable"
1177
- });
1178
- }
1179
- if (name === "id" && !column.isPrimaryKey) {
1180
- issues.push({ table, column: name, message: "the id column must be the PRIMARY KEY" });
1181
- }
1182
- verifyDefault(db, table, name, meta, column.defaultValueSql, issues);
1183
- }
1184
- for (const column of actual.columns) {
1185
- if (!(column.name in declaredColumns)) {
1186
- issues.push({
1187
- table,
1188
- column: column.name,
1189
- message: "created by migrations but not declared in the schema \u2014 sync event payloads include every column, so declare it (or drop it in a new version)"
1190
- });
1191
- }
1192
- }
1193
- }
1194
- function verifyDefault(db, table, column, meta, defaultValueSql, issues) {
1195
- if (meta.hasDefault && meta.defaultValue !== null && defaultValueSql === null) {
1196
- issues.push({
1197
- table,
1198
- column,
1199
- message: `schema declares default ${formatExpected(meta)}, but the migrated column has no default`
1200
- });
1201
- return;
1202
- }
1203
- if (!meta.hasDefault && defaultValueSql !== null) {
1204
- issues.push({
1205
- table,
1206
- column,
1207
- message: `migrated column has default ${defaultValueSql}, but the schema declares none`
1208
- });
1209
- return;
1210
- }
1211
- if (!meta.hasDefault || defaultValueSql === null) {
1212
- return;
1213
- }
1214
- let actual;
1215
- try {
1216
- actual = db.execute({
1217
- sql: `SELECT (${defaultValueSql}) AS value`,
1218
- parameters: []
1219
- }).rows[0]?.value;
1220
- } catch {
1221
- actual = void 0;
1222
- }
1223
- if (actual !== expectedDefaultValue(meta)) {
1224
- issues.push({
1225
- table,
1226
- column,
1227
- message: `default mismatch \u2014 schema declares ${formatExpected(meta)}, migrated column default is ${defaultValueSql}`
1228
- });
1229
- }
1230
- }
1231
- function expectedDefaultValue(meta) {
1232
- if (meta.defaultValue === null) return null;
1233
- switch (meta.kind) {
1234
- case "boolean":
1235
- return meta.defaultValue ? 1 : 0;
1236
- case "integer":
1237
- case "real":
1238
- return Number(meta.defaultValue);
1239
- case "text":
1240
- case "enum":
1241
- return String(meta.defaultValue);
1242
- }
1243
- }
1244
- function formatExpected(meta) {
1245
- const expected = expectedDefaultValue(meta);
1246
- return typeof expected === "string" ? `'${expected}'` : String(expected);
1247
- }
1248
-
1249
- // src/sqlite-crdt/apply-crdt-event.ts
1250
- var createSQLiteCrdtApplyFunction = ({
1251
- db,
1252
- dbConfig
1253
- }) => {
1254
- const applyCrdtEvent = createCrdtApplyFunction({
1255
- getCrdtUpdateLog(opts) {
1256
- const [metaRow] = db.executePrepared(
1257
- "get-item-crdt-meta",
1258
- {
1259
- item_id: opts.itemId,
1260
- dataset: opts.dataset
1261
- },
1262
- (db2, params) => {
1263
- return db2.selectFrom(dbConfig.updateLogTable.fullIdentifier).select("payload").where("item_id", "=", params("item_id")).where("dataset", "=", params("dataset"));
1264
- },
1265
- { loggerLevel: "system" }
1266
- );
1267
- const meta = metaRow ? JSON.parse(metaRow.payload) : null;
1268
- return meta;
1269
- },
1270
- insertCrdtUpdateLog(opts) {
1271
- db.executePrepared(
1272
- "insert-crdt-update-log",
1273
- {
1274
- item_id: opts.itemId,
1275
- dataset: opts.dataset,
1276
- payload: opts.payload
1277
- },
1278
- (db2, params) => db2.insertInto(dbConfig.updateLogTable.fullIdentifier).values({
1279
- item_id: params("item_id"),
1280
- dataset: params("dataset"),
1281
- payload: params("payload")
1282
- }),
1283
- { loggerLevel: "system" }
1284
- );
1285
- },
1286
- updateCrdtUpdateLog(opts) {
1287
- db.executePrepared(
1288
- "update-crdt-update-log",
1289
- {
1290
- item_id: opts.itemId,
1291
- dataset: opts.dataset,
1292
- payload: opts.payload
1293
- },
1294
- (db2, params) => db2.updateTable(dbConfig.updateLogTable.fullIdentifier).set({
1295
- payload: params("payload")
1296
- }).where("item_id", "=", params("item_id")).where("dataset", "=", params("dataset")),
1297
- { loggerLevel: "system" }
1298
- );
1299
- },
1300
- insertItem(opts) {
1301
- const keys = Object.keys(opts.payload).sort();
1302
- const insertPayload = {};
1303
- for (const key of keys) {
1304
- insertPayload[key] = key;
1305
- }
1306
- db.executePrepared(
1307
- `crdt-insert-item-${opts.dataset}-${keys.join("-")}`,
1308
- opts.payload,
1309
- (db2) => db2.insertInto(opts.dataset).values(insertPayload),
1310
- { loggerLevel: "system" }
1311
- );
1312
- },
1313
- updateItem(opts) {
1314
- const keys = Array.from(Object.keys(opts.payload));
1315
- keys.sort();
1316
- db.executePreparedRaw({
1317
- key: `update-item-${opts.dataset}-${keys.join("-")}`,
1318
- sql: `update ${quoteId(opts.dataset)} set ${keys.map((key) => `${quoteId(key)} = ?`).join(",")} where id = ?`,
1319
- params: [...keys.map((key) => opts.payload[key]), opts.itemId],
1320
- meta: { loggerLevel: "system" }
1321
- });
1322
- }
1323
- });
1324
- return applyCrdtEvent;
1325
- };
1326
- function createCrdtApplyFunction({
1327
- getCrdtUpdateLog,
1328
- insertItem,
1329
- insertCrdtUpdateLog,
1330
- updateItem,
1331
- updateCrdtUpdateLog
1332
- }) {
1333
- const applyItemCreated = ({ event }) => {
1334
- const eventPayload = JSON.parse(event.payload);
1335
- eventPayload.tombstone = 0;
1336
- insertItem({ dataset: event.dataset, payload: eventPayload });
1337
- const newUpdateLog = {};
1338
- for (const key of Object.keys(eventPayload)) {
1339
- newUpdateLog[key] = event.timestamp;
1340
- }
1341
- insertCrdtUpdateLog({
1342
- dataset: event.dataset,
1343
- itemId: event.item_id,
1344
- payload: JSON.stringify(newUpdateLog)
1345
- });
1346
- };
1347
- const applyItemUpdated = ({ event, meta }) => {
1348
- if (!meta) {
1349
- throw new Error(`Item ${event.item_id} in dataset ${event.dataset} not found`);
1350
- }
1351
- const eventPayload = event.type === "item-deleted" ? { tombstone: 1 } : JSON.parse(event.payload);
1352
- const updatePayload = {};
1353
- let hasUpdates = false;
1354
- for (const [key, value] of Object.entries(eventPayload)) {
1355
- if (key === "id") {
1356
- continue;
1357
- }
1358
- const lastUpdateTimestamp = meta[key];
1359
- const currentUpdateTimestamp = event.timestamp;
1360
- if (!lastUpdateTimestamp || !currentUpdateTimestamp || currentUpdateTimestamp > lastUpdateTimestamp) {
1361
- updatePayload[key] = value;
1362
- meta[key] = currentUpdateTimestamp;
1363
- hasUpdates = true;
1364
- }
1365
- }
1366
- if (!hasUpdates) {
1367
- return;
1368
- }
1369
- updateItem({
1370
- dataset: event.dataset,
1371
- itemId: event.item_id,
1372
- payload: updatePayload
1373
- });
1374
- updateCrdtUpdateLog({
1375
- dataset: event.dataset,
1376
- itemId: event.item_id,
1377
- payload: JSON.stringify(meta)
1378
- });
1379
- };
1380
- return (event) => {
1381
- if (isNoOpCrdtEventPayload(event.payload)) {
1382
- return;
1383
- }
1384
- const meta = getCrdtUpdateLog({
1385
- itemId: event.item_id,
1386
- dataset: event.dataset
1387
- });
1388
- if (event.type !== "item-created" && event.type !== "item-updated" && event.type !== "item-deleted") {
1389
- throw new Error(`Unknown event type: ${event.type}`);
1390
- }
1391
- if (meta) {
1392
- applyItemUpdated({ event, meta });
1393
- return;
1394
- }
1395
- if (event.type === "item-created") {
1396
- applyItemCreated({ event });
1397
- return;
1398
- }
1399
- throw new Error(`Item ${event.item_id} in dataset ${event.dataset} not found`);
1400
- };
1401
- }
1402
-
1403
- // src/sqlite-crdt/crdt-storage.ts
1404
- import { sql as sql3 } from "kysely";
1405
-
1406
- // src/sqlite-crdt/event-consistency.ts
1407
- var MASK_128 = (1n << 128n) - 1n;
1408
- var HEX_128_PATTERN = /^[0-9a-f]{32}$/;
1409
- function createEventHlcAccumulator(initialValue) {
1410
- let current = parseHex128(initialValue);
1411
- return {
1412
- add(timestamp) {
1413
- current = current + hash128BigInt(timestamp) & MASK_128;
1414
- },
1415
- get current() {
1416
- return toHex128(current);
1417
- }
1418
- };
1419
- }
1420
- function parseHex128(value) {
1421
- if (value === "") {
1422
- return 0n;
1423
- }
1424
- const normalized = value.toLowerCase();
1425
- if (!HEX_128_PATTERN.test(normalized)) {
1426
- throw new Error(`Invalid event HLC accumulator value: ${value}`);
1427
- }
1428
- return BigInt(`0x${normalized}`);
1429
- }
1430
- function toHex128(value) {
1431
- return value.toString(16).padStart(32, "0");
1432
- }
1433
- function hash128BigInt(value) {
1434
- return xxhash.h64(value, 0n) | xxhash.h64(value, 1n) << 64n;
1435
- }
1436
-
1437
- // src/sqlite-crdt/crdt-storage.ts
1438
- function createCrdtStorage(storage) {
1439
- let localSyncId = storage.initialLocalSyncId;
1440
- const db = storage.db;
1441
- const crdtEventsTable = storage.dbConfig.eventsTable.fullIdentifier;
1442
- const getInitialSequentialSyncId = () => {
1443
- const [firstPendingEvent] = db.executePrepared(
1444
- "get-first-pending-event",
1445
- { status: "pending" },
1446
- (db2, params) => db2.selectFrom(crdtEventsTable).select("sync_id").where("status", "=", params("status")).orderBy("sync_id", "asc").limit(sql3.lit(1)),
1447
- { loggerLevel: "system" }
1448
- );
1449
- return firstPendingEvent ? firstPendingEvent.sync_id - 1 : localSyncId;
1450
- };
1451
- let sequentialSyncId = getInitialSequentialSyncId();
1452
- const eventTarget = createTypedEventTarget();
1453
- const persistEvent = (tx, event) => {
1454
- tx.executePrepared(
1455
- "persist-crdt-event",
1456
- event,
1457
- (db2, params) => db2.insertInto(crdtEventsTable).values({
1458
- type: params("type"),
1459
- dataset: params("dataset"),
1460
- item_id: params("item_id"),
1461
- payload: params("payload"),
1462
- schema_version: params("schema_version"),
1463
- sync_id: params("sync_id"),
1464
- status: params("status"),
1465
- timestamp: params("timestamp"),
1466
- origin: params("origin"),
1467
- source_node_id: params("source_node_id")
1468
- }),
1469
- { loggerLevel: "system" }
1470
- );
1471
- };
1472
- const enqueueEvents = (origin, sourceNodeId, events) => {
1473
- const beforeSyncId = localSyncId;
1474
- if (events.length === 0) {
1475
- return { beforeSyncId, afterSyncId: beforeSyncId, processed: Promise.resolve() };
1476
- }
1477
- db.executeTransaction((tx) => {
1478
- for (const event of events) {
1479
- persistEvent(tx, {
1480
- schema_version: event.schema_version ?? storage.migrator.currentSchemaVersion,
1481
- timestamp: event.timestamp ?? serializeHLC(storage.hlc.getNextHLC()),
1482
- type: event.type,
1483
- dataset: event.dataset,
1484
- item_id: event.item_id,
1485
- origin,
1486
- source_node_id: sourceNodeId,
1487
- payload: event.payload,
1488
- sync_id: ++localSyncId,
1489
- status: "pending"
1490
- });
1491
- }
1492
- });
1493
- return { beforeSyncId, afterSyncId: localSyncId, processed: processEnqueuedEvents() };
1494
- };
1495
- const enqueueLocalEvents = (events, sourceNodeId) => {
1496
- return enqueueEvents("local", sourceNodeId, events);
1497
- };
1498
- const enqueueOwnEvents = (events) => {
1499
- return enqueueEvents("own", storage.nodeId, events);
1500
- };
1501
- const enqueueRemoteEvents = (events) => {
1502
- return enqueueEvents("remote", "", events);
1503
- };
1504
- const notifyEventApplied = (event) => {
1505
- if (event.status === "applied") {
1506
- queueMicrotask(() => {
1507
- storage.onEventApplied?.(event);
1508
- });
1509
- }
1510
- };
1511
- const applyOwnEvents = (events) => {
1512
- const schema = storage.schema;
1513
- if (!schema) {
1514
- throw new Error("applyOwnEvents requires a sync schema; none was provided to createCrdtStorage");
1515
- }
1516
- const validatedEvents = [];
1517
- const errors = [];
1518
- for (const [index, event] of events.entries()) {
1519
- const result = validateNewCrdtEvent(schema, {
1520
- type: event.type,
1521
- dataset: event.dataset,
1522
- item_id: event.item_id,
1523
- payload: event.payload
1524
- });
1525
- if (!result.success) {
1526
- errors.push(...result.errors.map((error) => `[${index}] ${error}`));
1527
- continue;
1528
- }
1529
- validatedEvents.push(result.event);
1530
- }
1531
- if (errors.length > 0) {
1532
- throw new CrdtEventValidationError(errors);
1533
- }
1534
- db.executeTransaction((tx) => {
1535
- for (const event of validatedEvents) {
1536
- const persistedEvent = {
1537
- schema_version: storage.migrator.currentSchemaVersion,
1538
- timestamp: serializeHLC(storage.hlc.getNextHLC()),
1539
- type: event.type,
1540
- dataset: event.dataset,
1541
- item_id: event.item_id,
1542
- origin: "own-applied",
1543
- source_node_id: storage.nodeId,
1544
- payload: JSON.stringify(event.payload),
1545
- sync_id: ++localSyncId,
1546
- status: "pending"
1547
- };
1548
- persistEvent(tx, persistedEvent);
1549
- applyCrdtEvent(persistedEvent);
1550
- }
1551
- });
1552
- void processEnqueuedEvents();
1553
- };
1554
- const applyOwnEventFromTransaction = (tx, event) => {
1555
- const persistedEvent = {
1556
- schema_version: storage.migrator.currentSchemaVersion,
1557
- timestamp: serializeHLC(storage.hlc.getNextHLC()),
1558
- type: event.type,
1559
- dataset: event.dataset,
1560
- item_id: event.item_id,
1561
- origin: "own-applied",
1562
- source_node_id: storage.nodeId,
1563
- payload: event.payload,
1564
- sync_id: ++localSyncId,
1565
- status: "pending"
1566
- };
1567
- persistEvent(tx, persistedEvent);
1568
- applyCrdtEvent(persistedEvent);
1569
- };
1570
- const dispatchEventsApplied = (previousSequentialSyncId) => {
1571
- if (sequentialSyncId <= previousSequentialSyncId) {
1572
- return;
1573
- }
1574
- eventTarget.dispatchEvent("events-applied", {
1575
- syncId: sequentialSyncId,
1576
- eventHlcSum: sequentialSyncId === localSyncId ? eventHlcAccumulator?.current ?? null : null
1577
- });
1578
- };
1579
- const hasPendingEvents = () => {
1580
- const events = db.executePrepared(
1581
- "has-pending-events",
1582
- { status: "pending" },
1583
- (db2, params) => db2.selectFrom(crdtEventsTable).select("sync_id").where("status", "=", params("status")).limit(sql3.lit(1)),
1584
- { loggerLevel: "system" }
1585
- );
1586
- return events.length > 0;
1587
- };
1588
- const getEventsBatch = (options) => {
1589
- const limit = options.limit ?? 50;
1590
- const queryParams = {
1591
- limit: limit + 1,
1592
- status: options.status ?? null,
1593
- afterSyncId: options.afterSyncId ?? null,
1594
- excludeOrigin: options.excludeOrigin ?? null,
1595
- excludeNodeId: options.excludeNodeId ?? null,
1596
- maxSyncId: options.status === "applied" ? sequentialSyncId : localSyncId
1597
- };
1598
- const filterKeys = [
1599
- queryParams.excludeNodeId ? "nodeid" : "no-nodeid",
1600
- queryParams.excludeOrigin ? "origin" : "no-origin"
1601
- ];
1602
- const events = db.executePrepared(
1603
- `get-events-batch-${filterKeys.join("-")}`,
1604
- queryParams,
1605
- (db2, params) => db2.selectFrom(crdtEventsTable).where("sync_id", ">", params("afterSyncId")).where("sync_id", "<=", params("maxSyncId")).where("status", "=", params("status")).$if(!!queryParams.excludeNodeId, (qb) => qb.where("source_node_id", "!=", params("excludeNodeId"))).$if(!!queryParams.excludeOrigin, (qb) => qb.where("origin", "!=", params("excludeOrigin"))).selectAll().limit(params("limit")).orderBy("sync_id", "asc"),
1606
- { loggerLevel: "system" }
1607
- );
1608
- const hasMore = events.length > limit;
1609
- if (hasMore) {
1610
- events.pop();
1611
- }
1612
- const lastReturnedSyncId = events[events.length - 1]?.sync_id ?? options.afterSyncId ?? 0;
1613
- let nextSyncId = lastReturnedSyncId;
1614
- if (!hasMore && options.status === "applied") {
1615
- nextSyncId = Math.max(lastReturnedSyncId, queryParams.maxSyncId);
1616
- }
1617
- return {
1618
- events,
1619
- hasMore,
1620
- nextSyncId
1621
- };
1622
- };
1623
- const checkIsQuiescent = (pushedSyncId) => {
1624
- if (hasPendingEvents()) {
1625
- return false;
1626
- }
1627
- const unpushed = getEventsBatch({
1628
- status: "applied",
1629
- afterSyncId: pushedSyncId,
1630
- excludeOrigin: "remote",
1631
- limit: 1
1632
- });
1633
- return unpushed.events.length === 0;
1634
- };
1635
- const applyCrdtEvent = createSQLiteCrdtApplyFunction({
1636
- db,
1637
- dbConfig: storage.dbConfig
1638
- });
1639
- const eventHlcAccumulator = storage.eventHlcAccumulator ? createEventHlcAccumulator(storage.eventHlcAccumulator.current) : null;
1640
- const persistEventHlcAccumulator = () => {
1641
- if (eventHlcAccumulator && storage.eventHlcAccumulator) {
1642
- storage.eventHlcAccumulator.current = eventHlcAccumulator.current;
1643
- }
1644
- };
1645
- const recomputeEventHlcAccumulatorIfNeeded = () => {
1646
- if (!eventHlcAccumulator || !storage.eventHlcAccumulator) {
1647
- return;
1648
- }
1649
- if (storage.eventHlcAccumulator.current !== "") {
1650
- return;
1651
- }
1652
- const batchSize = 1e3;
1653
- let afterSyncId = 0;
1654
- for (; ; ) {
1655
- const rows = db.executePrepared(
1656
- "get-applied-event-timestamps",
1657
- { status: "applied", afterSyncId, limit: batchSize },
1658
- (db2, params) => db2.selectFrom(crdtEventsTable).select(["sync_id", "timestamp"]).where("status", "=", params("status")).where("sync_id", ">", params("afterSyncId")).orderBy("sync_id", "asc").limit(params("limit")),
1659
- { loggerLevel: "system" }
1660
- );
1661
- if (rows.length === 0) {
1662
- break;
1663
- }
1664
- for (const row of rows) {
1665
- eventHlcAccumulator.add(row.timestamp);
1666
- afterSyncId = row.sync_id;
1667
- }
1668
- if (rows.length < batchSize) {
1669
- break;
1670
- }
1671
- }
1672
- persistEventHlcAccumulator();
1673
- };
1674
- const hasAcceptedEventWithTimestamp = (tx, event) => {
1675
- const [existingEvent] = tx.executePrepared(
1676
- "get-accepted-crdt-event-by-timestamp",
1677
- {
1678
- timestamp: event.timestamp,
1679
- sync_id: event.sync_id
1680
- },
1681
- (db2, params) => db2.selectFrom(crdtEventsTable).select("sync_id").where("timestamp", "=", params("timestamp")).where("sync_id", "<", params("sync_id")).where("status", "=", sql3.lit("applied")).limit(sql3.lit(1)),
1682
- { loggerLevel: "system" }
1683
- );
1684
- return existingEvent !== void 0;
1685
- };
1686
- const processPersistedEvent = (tx, event) => {
1687
- if (event.status !== "pending") {
1688
- throw new Error(`Event ${event.sync_id} is not pending`);
1689
- }
1690
- try {
1691
- if (event.origin === "own-applied") {
1692
- event.status = "applied";
1693
- return event;
1694
- }
1695
- if (hasAcceptedEventWithTimestamp(tx, event)) {
1696
- event.status = "deduped";
1697
- return event;
1698
- }
1699
- if (event.origin === "local" || event.origin === "remote") {
1700
- storage.hlc.mergeHLC(deserializeHLC(event.timestamp));
1701
- }
1702
- if (isNoOpCrdtEventPayload(event.payload)) {
1703
- applyCrdtEvent(event);
1704
- event.status = "applied";
1705
- return event;
1706
- }
1707
- const migratedEvent = storage.migrator.migrateEvent(event, storage.migrator.latestSchemaVersion);
1708
- if (migratedEvent === null) {
1709
- event.schema_version = storage.migrator.latestSchemaVersion;
1710
- event.payload = CRDT_EVENT_NO_OP_PAYLOAD;
1711
- applyCrdtEvent(event);
1712
- event.status = "applied";
1713
- return event;
1714
- }
1715
- event.schema_version = migratedEvent.schema_version;
1716
- event.type = migratedEvent.type;
1717
- event.dataset = migratedEvent.dataset;
1718
- event.item_id = migratedEvent.item_id;
1719
- event.payload = migratedEvent.payload;
1720
- applyCrdtEvent(event);
1721
- event.status = "applied";
1722
- } catch (error) {
1723
- console.error("Error applying enqueued CRDT event", error);
1724
- event.status = "failed";
1725
- } finally {
1726
- tx.executePrepared(
1727
- "update-crdt-event",
1728
- event,
1729
- (db2, params) => db2.updateTable(crdtEventsTable).set({
1730
- status: params("status"),
1731
- schema_version: params("schema_version"),
1732
- type: params("type"),
1733
- dataset: params("dataset"),
1734
- item_id: params("item_id"),
1735
- payload: params("payload")
1736
- }).where("sync_id", "=", params("sync_id")),
1737
- { loggerLevel: "system" }
1738
- );
1739
- }
1740
- };
1741
- const processEnqueuedEvents = ensureSingletonExecution(async () => {
1742
- let hasMore = true;
1743
- while (hasMore) {
1744
- await Promise.resolve();
1745
- const batchSize = 100;
1746
- const events = db.executePrepared(
1747
- "get-enqueued-pending-events",
1748
- {
1749
- status: "pending",
1750
- limit: batchSize + 1
1751
- },
1752
- (db2, params) => db2.selectFrom(crdtEventsTable).selectAll().where("status", "=", params("status")).limit(params("limit")).orderBy("sync_id", "asc"),
1753
- { loggerLevel: "system" }
1754
- );
1755
- hasMore = events.length > batchSize;
1756
- if (hasMore) {
1757
- events.pop();
1758
- }
1759
- const previousSequentialSyncId = sequentialSyncId;
1760
- if (events.length === 0) {
1761
- sequentialSyncId = localSyncId;
1762
- dispatchEventsApplied(previousSequentialSyncId);
1763
- break;
1764
- }
1765
- const failedRemoteSyncIds = [];
1766
- db.executeTransaction((tx) => {
1767
- for (const event of events) {
1768
- processPersistedEvent(tx, event);
1769
- if (event.status === "applied") {
1770
- eventHlcAccumulator?.add(event.timestamp);
1771
- }
1772
- notifyEventApplied(event);
1773
- if (event.status === "failed" && event.origin === "remote") {
1774
- failedRemoteSyncIds.push(event.sync_id);
1775
- }
1776
- }
1777
- persistEventHlcAccumulator();
1778
- });
1779
- sequentialSyncId = hasMore ? events[events.length - 1]?.sync_id ?? sequentialSyncId : localSyncId;
1780
- dispatchEventsApplied(previousSequentialSyncId);
1781
- for (const syncId of failedRemoteSyncIds) {
1782
- eventTarget.dispatchEvent("remote-event-apply-failed", { syncId });
1783
- }
1784
- }
1785
- });
1786
- recomputeEventHlcAccumulatorIfNeeded();
1787
- void processEnqueuedEvents();
1788
- return {
1789
- getEventsBatch,
1790
- enqueueLocalEvents,
1791
- enqueueOwnEvents,
1792
- enqueueRemoteEvents,
1793
- applyOwnEvents,
1794
- checkIsQuiescent,
1795
- getEventHlcAccumulator: () => eventHlcAccumulator?.current ?? null,
1796
- addEventListener: eventTarget.addEventListener,
1797
- removeEventListener: eventTarget.removeEventListener,
1798
- internal: {
1799
- applyOwnEventFromTransaction,
1800
- processEnqueuedEvents
1801
- }
1802
- };
1803
- }
1804
-
1805
- // src/sqlite-crdt/crdt-sync-producer.ts
1806
- var createCrdtSyncProducer = ({ storage, broadcastEvents }) => {
1807
- storage.addEventListener("events-applied", (event) => {
1808
- broadcastEvents({
1809
- newSyncId: event.payload.syncId,
1810
- eventHlcSum: event.payload.eventHlcSum
1811
- });
1812
- });
1813
- };
1814
-
1815
- // src/sqlite-crdt/retry-remote-operation.ts
1816
- var REMOTE_RETRY_OPTIONS = {
1817
- maxAttempts: 3,
1818
- backoffBaseMs: 100,
1819
- backoffExponent: 1.5,
1820
- backoffJitterMs: 150,
1821
- timeoutMs: 1e4
1822
- };
1823
- var RetryTimeoutError = class extends Error {
1824
- constructor(message, previous) {
1825
- super(message);
1826
- this.previous = previous;
1827
- this.name = "TimeoutError";
1828
- }
1829
- };
1830
- var applyJitter = (delayMs, maxJitterMs) => {
1831
- const jitter = Math.random() * maxJitterMs * (Math.random() > 0.5 ? 1 : -1);
1832
- return Math.max(0, delayMs + jitter);
1833
- };
1834
- var delay = (delayMs) => new Promise((resolve) => setTimeout(resolve, delayMs));
1835
- var withTimeout = async (operation, timeoutMs, previousError) => {
1836
- let timeoutId;
1837
- try {
1838
- return await Promise.race([
1839
- operation(),
1840
- new Promise((_, reject) => {
1841
- timeoutId = setTimeout(
1842
- () => reject(new RetryTimeoutError("Remote operation timed out", previousError)),
1843
- timeoutMs
1844
- );
1845
- })
1846
- ]);
1847
- } finally {
1848
- if (timeoutId) {
1849
- clearTimeout(timeoutId);
1850
- }
1851
- }
1852
- };
1853
- var retryRemoteOperation = async (operation, options) => {
1854
- let lastError;
1855
- for (let attempt = 1; attempt <= options.maxAttempts; attempt++) {
1856
- try {
1857
- return await withTimeout(operation, options.timeoutMs, lastError);
1858
- } catch (error) {
1859
- lastError = error;
1860
- if (attempt >= options.maxAttempts) {
1861
- throw error;
1862
- }
1863
- const backoffDelay = applyJitter(
1864
- options.backoffBaseMs * options.backoffExponent ** (attempt - 1),
1865
- options.backoffJitterMs
1866
- );
1867
- if (backoffDelay > 0) {
1868
- await delay(backoffDelay);
1869
- }
1870
- }
1871
- }
1872
- throw lastError;
1873
- };
1874
-
1875
- // src/sqlite-crdt/crdt-sync-remote-source.ts
1876
- var SchemaVersionMismatchError = class extends Error {
1877
- constructor(remoteSchemaVersion, localSchemaVersion) {
1878
- super(`Schema version mismatch: remote ${remoteSchemaVersion} != local ${localSchemaVersion}`);
1879
- this.remoteSchemaVersion = remoteSchemaVersion;
1880
- this.localSchemaVersion = localSchemaVersion;
1881
- this.name = "SchemaVersionMismatchError";
1882
- }
1883
- };
1884
- var createCrdtSyncRemoteSource = ({
1885
- bufferSize,
1886
- storage,
1887
- migrator,
1888
- pullSyncId,
1889
- pushSyncId,
1890
- nodeId,
1891
- remoteFactory
1892
- }) => {
1893
- const eventTarget = createTypedEventTarget();
1894
- let remoteState = {
1895
- type: "offline",
1896
- reason: "NOT_INITIALIZED",
1897
- deSynced: false,
1898
- schemaVersionMismatched: false
1899
- };
1900
- const patchRemoteState = (state) => {
1901
- remoteState = { ...remoteState, ...state };
1902
- eventTarget.dispatchEvent("state-changed", remoteState.type);
1903
- };
1904
- const initRemote = ensureSingletonExecution(
1905
- async () => {
1906
- if (remoteState.type !== "offline") {
1907
- throw new Error("Remote source is not offline");
1908
- }
1909
- if (!remoteFactory) {
1910
- console.warn("Remote source factory not provided. Going offline.");
1911
- patchRemoteState({ type: "offline", reason: "NOT_INITIALIZED" });
1912
- return;
1913
- }
1914
- patchRemoteState({ type: "pending" });
1915
- const factoryResult = await tryCatchAsync(async () => {
1916
- return await remoteFactory?.({
1917
- onEventsAvailable: ({ newSyncId, remoteEventHlcSum }) => {
1918
- pullEvents({ remoteSyncId: newSyncId, remoteEventHlcSum, includeSelf: false });
1919
- }
1920
- });
1921
- });
1922
- if (!factoryResult.success) {
1923
- patchRemoteState({ type: "offline", reason: "INITIALIZATION_FAILED" });
1924
- console.warn("Failed to create remote source", factoryResult.error);
1925
- return;
1926
- }
1927
- patchRemoteState({
1928
- type: "online",
1929
- source: factoryResult.data,
1930
- deSynced: false,
1931
- schemaVersionMismatched: false
1932
- });
1933
- },
1934
- { queueReExecution: false }
1935
- );
1936
- const syncWithRemote = ensureSingletonExecution(
1937
- async () => {
1938
- if (remoteState.type !== "online") {
1939
- return;
1940
- }
1941
- await pullEvents();
1942
- await startPushingEvents();
1943
- },
1944
- { queueReExecution: false }
1945
- );
1946
- const goOffline = ensureSingletonExecution(
1947
- async (reason) => {
1948
- if (remoteState.type !== "online") {
1949
- return;
1950
- }
1951
- const source = remoteState.source;
1952
- patchRemoteState({ type: "pending" });
1953
- const disconnectResult = await tryCatchAsync(async () => {
1954
- return await source.disconnect?.();
1955
- });
1956
- if (!disconnectResult.success) {
1957
- console.warn("Error while disconnecting from remote source", disconnectResult.error);
1958
- }
1959
- patchRemoteState({ type: "offline", reason });
1960
- },
1961
- { queueReExecution: false }
1962
- );
1963
- const goOnline = async () => {
1964
- if (remoteState.type !== "online") {
1965
- await initRemote();
1966
- }
1967
- if (remoteState.type === "online") {
1968
- await syncWithRemote();
1969
- }
1970
- };
1971
- let requestedPullSyncId = null;
1972
- let pullPromise = null;
1973
- const pullEvents = (request) => {
1974
- if (remoteState.type !== "online") {
1975
- return Promise.resolve();
1976
- }
1977
- const remoteSyncId = request?.remoteSyncId;
1978
- if (remoteSyncId !== void 0 && remoteSyncId <= pullSyncId.current) {
1979
- checkRemoteConsistency(remoteSyncId, request?.remoteEventHlcSum ?? null);
1980
- return Promise.resolve();
1981
- }
1982
- if (pullPromise) {
1983
- if (remoteSyncId !== void 0 && (!requestedPullSyncId || requestedPullSyncId < remoteSyncId)) {
1984
- requestedPullSyncId = remoteSyncId;
1985
- }
1986
- return pullPromise;
1987
- }
1988
- pullPromise = pullAllEvents({
1989
- afterSyncId: pullSyncId.current,
1990
- excludeNodeId: request?.includeSelf ? void 0 : nodeId
1991
- }).catch((error) => {
1992
- console.error("Error pulling events. Going offline.", error);
1993
- goOffline("REMOTE_PULL_ERROR");
1994
- }).finally(() => {
1995
- pullPromise = null;
1996
- const nextTarget = requestedPullSyncId;
1997
- requestedPullSyncId = null;
1998
- if (nextTarget && nextTarget > pullSyncId.current) {
1999
- pullEvents({ remoteSyncId: nextTarget });
2000
- }
2001
- });
2002
- return pullPromise;
2003
- };
2004
- const pullAllEvents = async (opts) => {
2005
- let hasMore = true;
2006
- let afterSyncId = opts.afterSyncId;
2007
- while (hasMore) {
2008
- if (remoteState.type !== "online") {
2009
- return;
2010
- }
2011
- const source = remoteState.source;
2012
- const response = await retryRemoteOperation(
2013
- () => source.pullEvents({
2014
- ...opts,
2015
- afterSyncId
2016
- }),
2017
- REMOTE_RETRY_OPTIONS
2018
- );
2019
- hasMore = response.hasMore;
2020
- afterSyncId = response.nextSyncId;
2021
- if (response.events) {
2022
- storage.enqueueRemoteEvents(
2023
- response.events.map((x) => {
2024
- if (x.schema_version > migrator.currentSchemaVersion) {
2025
- eventTarget.dispatchEvent("remote-schema-version-mismatch", {
2026
- remoteSchemaVersion: x.schema_version,
2027
- localSchemaVersion: migrator.currentSchemaVersion
2028
- });
2029
- if (remoteState.type === "online" && !remoteState.schemaVersionMismatched) {
2030
- patchRemoteState({ schemaVersionMismatched: true });
2031
- }
2032
- throw new SchemaVersionMismatchError(x.schema_version, migrator.currentSchemaVersion);
2033
- }
2034
- return x;
2035
- })
2036
- );
2037
- }
2038
- if (response.nextSyncId <= pullSyncId.current) {
2039
- break;
2040
- }
2041
- if (response.nextSyncId > pullSyncId.current) {
2042
- pullSyncId.current = response.nextSyncId;
2043
- }
2044
- }
2045
- };
2046
- const checkRemoteConsistency = (remoteSyncId, remoteEventHlcSum) => {
2047
- if (remoteEventHlcSum === null) {
2048
- return;
2049
- }
2050
- if (remoteSyncId !== pullSyncId.current) {
2051
- return;
2052
- }
2053
- if (!storage.checkIsQuiescent(pushSyncId.current)) {
2054
- return;
2055
- }
2056
- const localEventHlcSum = storage.getEventHlcAccumulator();
2057
- if (localEventHlcSum === null) {
2058
- return;
2059
- }
2060
- if (localEventHlcSum === remoteEventHlcSum) {
2061
- return;
2062
- }
2063
- eventTarget.dispatchEvent("de-sync-detected", { reason: "CHECKSUM_MISMATCH" });
2064
- console.warn(
2065
- `[sqlite-sync] De-sync detected at syncId ${remoteSyncId}: local HLC checksum ${localEventHlcSum} != remote ${remoteEventHlcSum}. Local and remote have diverged despite being caught up.`
2066
- );
2067
- if (remoteState.type === "online" && !remoteState.deSynced) {
2068
- patchRemoteState({ deSynced: true });
2069
- }
2070
- };
2071
- const startPushingEvents = ensureSingletonExecution(async () => {
2072
- while (true) {
2073
- const eventsBatch = storage.getEventsBatch({
2074
- status: "applied",
2075
- afterSyncId: pushSyncId.current,
2076
- excludeOrigin: "remote",
2077
- limit: bufferSize
2078
- });
2079
- if (eventsBatch.events.length === 0) {
2080
- break;
2081
- }
2082
- if (remoteState.type !== "online") {
2083
- break;
2084
- }
2085
- const source = remoteState.source;
2086
- let response;
2087
- try {
2088
- response = await retryRemoteOperation(
2089
- () => source.pushEvents({
2090
- nodeId,
2091
- events: eventsBatch.events.map((event) => ({
2092
- schema_version: event.schema_version,
2093
- timestamp: event.timestamp,
2094
- type: event.type,
2095
- dataset: event.dataset,
2096
- item_id: event.item_id,
2097
- payload: event.payload
2098
- }))
2099
- }),
2100
- REMOTE_RETRY_OPTIONS
2101
- );
2102
- } catch (error) {
2103
- console.error("Error pushing events. Going offline.", error);
2104
- goOffline("REMOTE_PUSH_ERROR");
2105
- return;
2106
- }
2107
- if (!response.ok) {
2108
- console.error("Remote rejected pushed events. Going offline.");
2109
- goOffline("REMOTE_PUSH_ERROR");
2110
- return;
2111
- }
2112
- pushSyncId.current = eventsBatch.nextSyncId;
2113
- if (response.beforeSyncId !== void 0 && response.afterSyncId !== void 0 && response.beforeSyncId <= pullSyncId.current && response.afterSyncId > pullSyncId.current) {
2114
- pullSyncId.current = response.afterSyncId;
2115
- }
2116
- if (!eventsBatch.hasMore) {
2117
- break;
2118
- }
2119
- }
2120
- });
2121
- const eventsAppliedSubscription = storage.addEventListener("events-applied", () => {
2122
- startPushingEvents();
2123
- });
2124
- const remoteEventApplyFailedSubscription = storage.addEventListener("remote-event-apply-failed", () => {
2125
- eventTarget.dispatchEvent("de-sync-detected", { reason: "ERROR_APPLYING_REMOTE_EVENT" });
2126
- if (remoteState.type === "online" && !remoteState.deSynced) {
2127
- patchRemoteState({ deSynced: true });
2128
- }
2129
- });
2130
- const getState = () => ({
2131
- remoteState: remoteState.type,
2132
- deSynced: remoteState.deSynced,
2133
- schemaVersionMismatched: remoteState.schemaVersionMismatched
2134
- });
2135
- const dispose = async () => {
2136
- await goOffline("DISCONNECTED");
2137
- eventsAppliedSubscription.unsubscribe();
2138
- remoteEventApplyFailedSubscription.unsubscribe();
2139
- };
2140
- return {
2141
- goOnline,
2142
- goOffline,
2143
- syncWithRemote,
2144
- getState,
2145
- dispose,
2146
- addEventListener: eventTarget.addEventListener,
2147
- removeEventListener: eventTarget.removeEventListener
2148
- };
2149
- };
2150
-
2151
- // src/worker-db/reset-state.ts
2152
- import { createStore, del, get, set } from "idb-keyval";
2153
- function createIdbResetStore() {
2154
- const store = createStore("sqlite-sync", "kv");
2155
- return {
2156
- get: (key) => get(key, store),
2157
- set: (key, value) => set(key, value, store),
2158
- delete: (key) => del(key, store)
2159
- };
2160
- }
2161
- var RESET_REQUEST_TTL_MS = 10 * 60 * 1e3;
2162
- var resetRequestKey = (dbId) => `sqlite-sync-reset-request-${dbId}`;
2163
- var resetAppliedKey = (dbId) => `sqlite-sync-reset-applied-${dbId}`;
2164
- function createResetStateStore({ store, dbId, now = () => Date.now() }) {
2165
- const requestKey = resetRequestKey(dbId);
2166
- const appliedKey = resetAppliedKey(dbId);
2167
- return {
2168
- async writeResetRequest(epoch) {
2169
- const request = { epoch, requestedAt: now() };
2170
- await store.set(requestKey, request);
2171
- return request;
2172
- },
2173
- /**
2174
- * Read the pending reset request after winning the worker election.
2175
- * Returns the request only when it has not been applied yet and is within
2176
- * the TTL. Stale requests are deleted so they cannot fire on a later cold start.
2177
- */
2178
- async resolvePendingReset() {
2179
- const request = await store.get(requestKey);
2180
- if (!request) {
2181
- return void 0;
2182
- }
2183
- if (now() - request.requestedAt > RESET_REQUEST_TTL_MS) {
2184
- await store.delete(requestKey);
2185
- return void 0;
2186
- }
2187
- const appliedEpoch = await store.get(appliedKey);
2188
- if (request.epoch === appliedEpoch) {
2189
- return void 0;
2190
- }
2191
- return request;
2192
- },
2193
- /**
2194
- * Record the epoch as applied. Must be called only after the worker has
2195
- * successfully initialized with `clearOnInit: true`, so a failed init can
2196
- * be retried by a later elected worker.
2197
- */
2198
- async markResetApplied(epoch) {
2199
- await store.set(appliedKey, epoch);
2200
- }
2201
- };
2202
- }
2203
- function createReloadRequestHandler({
2204
- resetState,
2205
- broadcast,
2206
- generateEpoch = generateId
2207
- }) {
2208
- return async (options) => {
2209
- const reloadEpoch = generateEpoch();
2210
- if (options.clean) {
2211
- await resetState.writeResetRequest(reloadEpoch);
2212
- }
2213
- broadcast({
2214
- notificationType: "reload-requested",
2215
- reloadEpoch,
2216
- clean: options.clean
2217
- });
2218
- };
2219
- }
2220
-
2221
- // src/worker-db/worker-common.ts
2222
- var syncDbWorkerLockName = "sync-db-worker-lock";
2223
- var syncDbClientLockName = "sync-db-client-lock";
2224
- var broadcastChannelNames = {
2225
- requests: "sync-db-worker-requests",
2226
- responses: "sync-db-worker-responses"
2227
- };
2228
- var createBroadcastChannels = (prefix) => {
2229
- return {
2230
- requests: new TypedBroadcastChannel(`${prefix}-${broadcastChannelNames.requests}`),
2231
- responses: new TypedBroadcastChannel(`${prefix}-${broadcastChannelNames.responses}`)
2232
- };
2233
- };
2234
- function isWorkerInitMessage(message) {
2235
- return typeof message === "object" && message !== null && "type" in message && message.type === "init";
2236
- }
2237
- function isWorkerRequestMessage(message) {
2238
- return typeof message === "object" && message !== null && "type" in message && message.type === "request";
2239
- }
2240
- function isWorkerResponseMessage(message) {
2241
- return typeof message === "object" && message !== null && "type" in message && "requestId" in message && "data" in message;
2242
- }
2243
- function isWorkerErrorResponseMessage(message) {
2244
- return typeof message === "object" && message !== null && "type" in message && message.type === "error-response";
2245
- }
2246
- function isWorkerNotificationMessage(message) {
2247
- return typeof message === "object" && message !== null && "notificationType" in message && !!message.notificationType;
2248
- }
2249
-
2250
- export {
2251
- dummyKysely,
2252
- createImportData,
2253
- createExportData,
2254
- xxhash,
2255
- HLCCounter,
2256
- serializeHLC,
2257
- deserializeHLC,
2258
- compareHLC,
2259
- introspectDb,
2260
- startPerformanceLogger,
2261
- SQLiteDbWrapper,
2262
- CRDT_EVENT_NO_OP_PAYLOAD,
2263
- isNoOpCrdtEventPayload,
2264
- createMigrations,
2265
- createMigrator,
2266
- createStoredValue,
2267
- createKvStoreTableQuery,
2268
- createSQLiteKvStore,
2269
- createSystemDbConfig,
2270
- workerDbConfig,
2271
- memoryDbConfig,
2272
- baseSystemMigrations,
2273
- runSystemMigrations,
2274
- applyWorkerDbSchema,
2275
- applyMemoryDbSchema,
2276
- CrdtEventValidationError,
2277
- validateNewCrdtEvent,
2278
- verifySyncSchema,
2279
- formatSchemaVerificationIssues,
2280
- createSQLiteCrdtApplyFunction,
2281
- createCrdtApplyFunction,
2282
- createCrdtStorage,
2283
- createCrdtSyncProducer,
2284
- createCrdtSyncRemoteSource,
2285
- syncDbWorkerLockName,
2286
- syncDbClientLockName,
2287
- createBroadcastChannels,
2288
- isWorkerInitMessage,
2289
- isWorkerRequestMessage,
2290
- isWorkerResponseMessage,
2291
- isWorkerErrorResponseMessage,
2292
- isWorkerNotificationMessage,
2293
- createIdbResetStore,
2294
- RESET_REQUEST_TTL_MS,
2295
- createResetStateStore,
2296
- createReloadRequestHandler
2297
- };
2298
- //# sourceMappingURL=chunk-GMF45NFH.js.map