@syncular/server 0.22.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/admin.d.ts +3 -1
- package/dist/admin.js +7 -1
- package/dist/blob-handlers.js +1 -1
- package/dist/context.d.ts +8 -1
- package/dist/d1-storage.d.ts +11 -3
- package/dist/d1-storage.js +84 -2
- package/dist/errors.js +10 -0
- package/dist/handler.js +50 -4
- package/dist/operations.js +3 -3
- package/dist/postgres-storage.d.ts +12 -4
- package/dist/postgres-storage.js +266 -6
- package/dist/push.js +18 -0
- package/dist/readiness.d.ts +9 -2
- package/dist/readiness.js +27 -2
- package/dist/realtime.js +20 -1
- package/dist/segment-download.js +8 -0
- package/dist/sqlite-dialect.d.ts +1 -1
- package/dist/sqlite-dialect.js +15 -0
- package/dist/sqlite-storage.d.ts +11 -3
- package/dist/sqlite-storage.js +296 -33
- package/dist/storage-errors.d.ts +1 -1
- package/dist/storage-errors.js +4 -0
- package/dist/storage.d.ts +127 -2
- package/dist/storage.js +40 -1
- package/package.json +2 -2
- package/src/admin.ts +13 -1
- package/src/blob-handlers.ts +1 -1
- package/src/context.ts +8 -0
- package/src/d1-storage.ts +143 -1
- package/src/errors.ts +10 -0
- package/src/handler.ts +62 -4
- package/src/operations.ts +17 -13
- package/src/postgres-storage.ts +430 -3
- package/src/push.ts +18 -0
- package/src/readiness.ts +39 -4
- package/src/realtime.ts +22 -1
- package/src/segment-download.ts +16 -0
- package/src/sqlite-dialect.ts +15 -0
- package/src/sqlite-storage.ts +439 -36
- package/src/storage-errors.ts +13 -1
- package/src/storage.ts +218 -1
package/dist/sqlite-storage.js
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { validateCommitPruneQuery } from './prune.js';
|
|
2
2
|
import { StorageQueryError } from './storage-errors.js';
|
|
3
|
+
import { serveGateRefusal } from './storage.js';
|
|
4
|
+
import { serveNotReadyError } from './readiness.js';
|
|
3
5
|
/**
|
|
4
6
|
* SQLite server storage over the shared synchronous driver.
|
|
5
7
|
*
|
|
@@ -12,10 +14,22 @@ import { bindAuthoritativePartition, prepareAuthoritativeQuery, } from './author
|
|
|
12
14
|
import { syncError } from './errors.js';
|
|
13
15
|
import { assertPhysicalColumns, commitWindowPageSql, deleteRowSql, deleteSqliteRowScopesSql, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, retiredTableNames, rewritePlan, rewriteRowSql, rewriteValues, SCHEMA_META_DDL_SQLITE, scanRowPageSql, schemaDdl, selectRowScopesSql, selectRowSql, selectRowsForRewriteSql, upsertSql, upsertValues, } from './relational-rows.js';
|
|
14
16
|
import { matchesEffective } from './scopes.js';
|
|
15
|
-
import { collectCommitWindowPage, deserializePushResult, SQLITE_DDL, serializePushResult, toStoredRow, } from './sqlite-dialect.js';
|
|
17
|
+
import { collectCommitWindowPage, deserializePushResult, placeholders, SQLITE_DDL, serializePushResult, toStoredRow, } from './sqlite-dialect.js';
|
|
16
18
|
import { SqliteAdapterRequiredError, } from './sqlite-driver.js';
|
|
17
19
|
import { isSqliteConstraintError, StorageConstraintError, } from './storage-errors.js';
|
|
18
20
|
import { assertScopeIndexedScan, resolveIndexRowScan } from './storage-query.js';
|
|
21
|
+
function toStoredCheckpoint(record) {
|
|
22
|
+
return {
|
|
23
|
+
partition: record.partition,
|
|
24
|
+
name: record.name,
|
|
25
|
+
schemaVersion: record.schema_version,
|
|
26
|
+
state: record.state,
|
|
27
|
+
watermark: record.watermark,
|
|
28
|
+
ownerEpoch: record.owner_epoch,
|
|
29
|
+
observedRows: record.observed_rows,
|
|
30
|
+
updatedAtMs: record.updated_at_ms,
|
|
31
|
+
};
|
|
32
|
+
}
|
|
19
33
|
function toStoredReaction(record) {
|
|
20
34
|
return {
|
|
21
35
|
idempotencyKey: record.idempotency_key,
|
|
@@ -45,12 +59,15 @@ function toStoredReaction(record) {
|
|
|
45
59
|
class SqliteTransaction {
|
|
46
60
|
#storage;
|
|
47
61
|
#partition;
|
|
62
|
+
/** Schema version this transaction writes; rides every appended commit row. */
|
|
63
|
+
#writerVersion;
|
|
48
64
|
#open = true;
|
|
49
65
|
#pushApplySavepoint = false;
|
|
50
66
|
#release;
|
|
51
|
-
constructor(storage, partition, release) {
|
|
67
|
+
constructor(storage, partition, writerVersion, release) {
|
|
52
68
|
this.#storage = storage;
|
|
53
69
|
this.#partition = partition;
|
|
70
|
+
this.#writerVersion = writerVersion;
|
|
54
71
|
this.#release = release;
|
|
55
72
|
storage.db.exec('BEGIN IMMEDIATE');
|
|
56
73
|
}
|
|
@@ -95,6 +112,21 @@ class SqliteTransaction {
|
|
|
95
112
|
this.#storage.db.exec('SAVEPOINT syncular_push_candidate');
|
|
96
113
|
this.#pushApplySavepoint = true;
|
|
97
114
|
}
|
|
115
|
+
async advanceCheckpoint(name, ownerEpoch, watermark, observedRows, nowMs) {
|
|
116
|
+
this.#assertOpen();
|
|
117
|
+
const result = this.#storage.db
|
|
118
|
+
.query(`UPDATE sync_backfill_checkpoints
|
|
119
|
+
SET watermark=?, observed_rows=?, updated_at_ms=?
|
|
120
|
+
WHERE partition=? AND name=? AND owner_epoch=?`)
|
|
121
|
+
.run(watermark, observedRows, nowMs, this.#partition, name, ownerEpoch);
|
|
122
|
+
return Number(result.changes) === 1;
|
|
123
|
+
}
|
|
124
|
+
readServeGate(runningSchemaVersion) {
|
|
125
|
+
this.#assertOpen();
|
|
126
|
+
// Same shared connection: this read runs inside this transaction's
|
|
127
|
+
// BEGIN IMMEDIATE, so the gate and the write share one transaction.
|
|
128
|
+
return this.#storage.readServeGate(this.#partition, runningSchemaVersion);
|
|
129
|
+
}
|
|
98
130
|
async commitRejectedPushResult(clientId, clientCommitId, result) {
|
|
99
131
|
this.#assertOpen();
|
|
100
132
|
if (!this.#pushApplySavepoint) {
|
|
@@ -146,7 +178,7 @@ class SqliteTransaction {
|
|
|
146
178
|
.get(p);
|
|
147
179
|
const commitSeq = (row?.max_commit_seq ?? 0) + 1;
|
|
148
180
|
db.query('UPDATE sync_partitions SET max_commit_seq=? WHERE partition=?').run(commitSeq, p);
|
|
149
|
-
db.query('INSERT INTO sync_commits(partition, commit_seq, client_id, client_commit_id, actor_id, created_at_ms) VALUES (
|
|
181
|
+
db.query('INSERT INTO sync_commits(partition, commit_seq, client_id, client_commit_id, actor_id, created_at_ms, writer_version) VALUES (?,?,?,?,?,?,?)').run(p, commitSeq, commit.clientId, commit.clientCommitId, commit.actorId, commit.createdAtMs, this.#writerVersion);
|
|
150
182
|
commit.changes.forEach((change, idx) => {
|
|
151
183
|
db.query('INSERT INTO sync_changes(partition, commit_seq, idx, tbl, row_id, op, row_version, scopes, payload) VALUES (?,?,?,?,?,?,?,?,?)').run(p, commitSeq, idx, change.table, change.rowId, change.op === 'upsert' ? 1 : 2, change.rowVersion ?? null, JSON.stringify(change.scopes), change.payload ?? null);
|
|
152
184
|
if (change.op === 'delete') {
|
|
@@ -247,6 +279,30 @@ export class SqliteServerStorage {
|
|
|
247
279
|
if (!clientColumns.some((column) => column.name === 'wire_version')) {
|
|
248
280
|
this.db.exec('ALTER TABLE sync_clients ADD COLUMN wire_version INTEGER NOT NULL DEFAULT 1');
|
|
249
281
|
}
|
|
282
|
+
const commitColumns = this.db
|
|
283
|
+
.query('PRAGMA table_info("sync_commits")')
|
|
284
|
+
.all();
|
|
285
|
+
if (!commitColumns.some((column) => column.name === 'writer_version')) {
|
|
286
|
+
// Nullable by design: an old writer omits the column and the trigger's
|
|
287
|
+
// explicit NULL test rejects it. `ADD COLUMN NOT NULL` is unavailable in
|
|
288
|
+
// SQLite, and a default would hand old writers a passing value.
|
|
289
|
+
this.db.exec('ALTER TABLE sync_commits ADD COLUMN writer_version INTEGER');
|
|
290
|
+
}
|
|
291
|
+
// The fence is database-side: an old binary's INSERT never reaches JS. An
|
|
292
|
+
// absent `sync_writer_fence` row makes the WHEN clause false, so a
|
|
293
|
+
// partition with no barrier behaves exactly as before.
|
|
294
|
+
this.db.exec(`CREATE TRIGGER IF NOT EXISTS sync_commits_writer_fence
|
|
295
|
+
BEFORE INSERT ON sync_commits
|
|
296
|
+
FOR EACH ROW
|
|
297
|
+
WHEN EXISTS (
|
|
298
|
+
SELECT 1 FROM sync_writer_fence
|
|
299
|
+
WHERE partition = NEW.partition
|
|
300
|
+
AND (NEW.writer_version IS NULL
|
|
301
|
+
OR NEW.writer_version < required_writer_version)
|
|
302
|
+
)
|
|
303
|
+
BEGIN
|
|
304
|
+
SELECT RAISE(ABORT, 'sync.storage.writer_fence_rejected');
|
|
305
|
+
END`);
|
|
250
306
|
}
|
|
251
307
|
/** Resolve a table's compiled schema; row operations require `ensureSchema`. */
|
|
252
308
|
table(name) {
|
|
@@ -256,10 +312,24 @@ export class SqliteServerStorage {
|
|
|
256
312
|
}
|
|
257
313
|
return table;
|
|
258
314
|
}
|
|
259
|
-
async ensureSchema(schema) {
|
|
315
|
+
async ensureSchema(schema, checkpoints) {
|
|
260
316
|
// Memoized fast path: same instance, same schema version.
|
|
261
317
|
if (this.#schemaVersion === schema.version)
|
|
262
318
|
return;
|
|
319
|
+
// The barrier reads stored state: a checkpoint at this schema version that
|
|
320
|
+
// is not activated and that this caller did not declare means this process
|
|
321
|
+
// must not serve. An omitted or empty declaration set declares nothing and
|
|
322
|
+
// is refused here, never treated as a bypass.
|
|
323
|
+
const declarations = checkpoints ?? [];
|
|
324
|
+
const incomplete = this.db
|
|
325
|
+
.query(`SELECT partition, name FROM sync_backfill_checkpoints
|
|
326
|
+
WHERE schema_version=? AND state<>'activated'`)
|
|
327
|
+
.all(schema.version)
|
|
328
|
+
.filter((row) => !declarations.some((declaration) => declaration.partition === row.partition &&
|
|
329
|
+
declaration.name === row.name));
|
|
330
|
+
if (incomplete.length > 0) {
|
|
331
|
+
throw new StorageQueryError('sync.storage.checkpoint_incomplete');
|
|
332
|
+
}
|
|
263
333
|
this.db.exec(SCHEMA_META_DDL_SQLITE);
|
|
264
334
|
const marker = this.db
|
|
265
335
|
.query('SELECT schema_version, layouts FROM sync_schema_meta WHERE id=1')
|
|
@@ -340,36 +410,50 @@ export class SqliteServerStorage {
|
|
|
340
410
|
existingIndexes.set(table.name, new Set(indexes.map((index) => index.name)));
|
|
341
411
|
}
|
|
342
412
|
}
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
413
|
+
// The migration rewrites application rows (`#rewriteRows`) with no
|
|
414
|
+
// commit log entry. Serialize it through the same writer queue as
|
|
415
|
+
// pushes and prune (`#serializeWrite`) rather than trusting the
|
|
416
|
+
// migration as privileged; `BEGIN IMMEDIATE` then owns SQLite's writer
|
|
417
|
+
// lock for the whole rewrite.
|
|
418
|
+
await this.#serializeWrite(() => {
|
|
419
|
+
this.db.exec('BEGIN IMMEDIATE');
|
|
420
|
+
try {
|
|
421
|
+
for (const tableName of retiredTables) {
|
|
422
|
+
this.db
|
|
423
|
+
.query('DELETE FROM sync_row_scopes WHERE tbl=?')
|
|
424
|
+
.run(tableName);
|
|
425
|
+
this.db
|
|
426
|
+
.query('DELETE FROM sync_blob_refs WHERE tbl=?')
|
|
427
|
+
.run(tableName);
|
|
428
|
+
this.db.exec(dropTableDdl(tableName));
|
|
429
|
+
}
|
|
430
|
+
for (const statement of schemaDdl(schema, existing, 'sqlite', existingIndexes)) {
|
|
431
|
+
this.db.exec(statement);
|
|
432
|
+
}
|
|
433
|
+
for (const table of schema.tables.values()) {
|
|
434
|
+
const oldLayout = layouts[table.name];
|
|
435
|
+
const plan = rewritePlan(table, oldLayout, existing.get(table.name));
|
|
436
|
+
if (!plan.migrate && !plan.backfill)
|
|
437
|
+
continue;
|
|
438
|
+
this.#rewriteRows(table, plan.migrate ? oldLayout : undefined);
|
|
439
|
+
}
|
|
349
440
|
this.db
|
|
350
|
-
.query('
|
|
351
|
-
.run(
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
441
|
+
.query('INSERT INTO sync_schema_meta(id, schema_version, layouts) VALUES (1, ?, ?) ON CONFLICT(id) DO UPDATE SET schema_version=excluded.schema_version, layouts=excluded.layouts')
|
|
442
|
+
.run(schema.version, layoutsOf(schema));
|
|
443
|
+
// The host's declared checkpoints install in the schema-bump
|
|
444
|
+
// transaction, so no observer sees the bumped marker without the
|
|
445
|
+
// fence, or the fence without its declaration row.
|
|
446
|
+
const installedAtMs = Date.now();
|
|
447
|
+
for (const declaration of checkpoints ?? []) {
|
|
448
|
+
this.#installCheckpoint(declaration.partition, declaration.name, declaration.schemaVersion, installedAtMs);
|
|
449
|
+
}
|
|
450
|
+
this.db.exec('COMMIT');
|
|
356
451
|
}
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
if (!plan.migrate && !plan.backfill)
|
|
361
|
-
continue;
|
|
362
|
-
this.#rewriteRows(table, plan.migrate ? oldLayout : undefined);
|
|
452
|
+
catch (error) {
|
|
453
|
+
this.db.exec('ROLLBACK');
|
|
454
|
+
throw error;
|
|
363
455
|
}
|
|
364
|
-
|
|
365
|
-
.query('INSERT INTO sync_schema_meta(id, schema_version, layouts) VALUES (1, ?, ?) ON CONFLICT(id) DO UPDATE SET schema_version=excluded.schema_version, layouts=excluded.layouts')
|
|
366
|
-
.run(schema.version, layoutsOf(schema));
|
|
367
|
-
this.db.exec('COMMIT');
|
|
368
|
-
}
|
|
369
|
-
catch (error) {
|
|
370
|
-
this.db.exec('ROLLBACK');
|
|
371
|
-
throw error;
|
|
372
|
-
}
|
|
456
|
+
});
|
|
373
457
|
}
|
|
374
458
|
this.#tables = schema.tables;
|
|
375
459
|
this.#schemaVersion = schema.version;
|
|
@@ -485,7 +569,7 @@ export class SqliteServerStorage {
|
|
|
485
569
|
});
|
|
486
570
|
await previous;
|
|
487
571
|
try {
|
|
488
|
-
return new SqliteTransaction(this, partition, release);
|
|
572
|
+
return new SqliteTransaction(this, partition, this.#schemaVersion ?? 0, release);
|
|
489
573
|
}
|
|
490
574
|
catch (error) {
|
|
491
575
|
release();
|
|
@@ -513,7 +597,7 @@ export class SqliteServerStorage {
|
|
|
513
597
|
.get(partition);
|
|
514
598
|
return row?.max_commit_seq ?? 0;
|
|
515
599
|
}
|
|
516
|
-
async queryAuthoritative(partition, query) {
|
|
600
|
+
async queryAuthoritative(partition, query, checkpoints) {
|
|
517
601
|
if (this.#tables === undefined) {
|
|
518
602
|
throw new Error('ensureSchema(schema) must run before registered queries');
|
|
519
603
|
}
|
|
@@ -528,6 +612,14 @@ export class SqliteServerStorage {
|
|
|
528
612
|
try {
|
|
529
613
|
this.db.exec('BEGIN');
|
|
530
614
|
open = true;
|
|
615
|
+
// RFC 0007: evaluate the gate on this connection after BEGIN. SQLite is
|
|
616
|
+
// single-writer and the shared connection serializes writers, so the
|
|
617
|
+
// gate read and the query read cannot straddle a migration.
|
|
618
|
+
const runningVersion = this.#schemaVersion ?? 0;
|
|
619
|
+
const gate = await this.readServeGate(partition, runningVersion);
|
|
620
|
+
const refusal = serveGateRefusal(gate, runningVersion, checkpoints);
|
|
621
|
+
if (refusal !== undefined)
|
|
622
|
+
throw serveNotReadyError(refusal);
|
|
531
623
|
const rows = this.db
|
|
532
624
|
.query(prepared.sql)
|
|
533
625
|
.all(...prepared.params);
|
|
@@ -616,6 +708,177 @@ export class SqliteServerStorage {
|
|
|
616
708
|
.get(partition, createdBeforeMs);
|
|
617
709
|
return row?.seq ?? 0;
|
|
618
710
|
}
|
|
711
|
+
async readServeGate(partition, runningSchemaVersion) {
|
|
712
|
+
const marker = this.db
|
|
713
|
+
.query('SELECT schema_version FROM sync_schema_meta WHERE id=1')
|
|
714
|
+
.get();
|
|
715
|
+
const epoch = this.db
|
|
716
|
+
.query('SELECT log_epoch FROM sync_partition_registry WHERE partition=?')
|
|
717
|
+
.get(partition);
|
|
718
|
+
const pending = this.db
|
|
719
|
+
.query(`SELECT partition, name FROM sync_backfill_checkpoints
|
|
720
|
+
WHERE schema_version=? AND state<>'activated'`)
|
|
721
|
+
.all(runningSchemaVersion);
|
|
722
|
+
return {
|
|
723
|
+
storedSchemaVersion: marker?.schema_version ?? 0,
|
|
724
|
+
logEpoch: epoch?.log_epoch,
|
|
725
|
+
pending: pending.map((row) => ({
|
|
726
|
+
partition: row.partition,
|
|
727
|
+
name: row.name,
|
|
728
|
+
})),
|
|
729
|
+
};
|
|
730
|
+
}
|
|
731
|
+
async readCheckpoints(partition) {
|
|
732
|
+
return this.db
|
|
733
|
+
.query(`SELECT partition, name, schema_version, state, watermark,
|
|
734
|
+
owner_epoch, observed_rows, updated_at_ms
|
|
735
|
+
FROM sync_backfill_checkpoints WHERE partition=? ORDER BY name`)
|
|
736
|
+
.all(partition)
|
|
737
|
+
.map(toStoredCheckpoint);
|
|
738
|
+
}
|
|
739
|
+
/**
|
|
740
|
+
* Insert the `declared` row and raise the fence, inside the caller's open
|
|
741
|
+
* transaction. Shared by `declareCheckpoint` and the `ensureSchema`
|
|
742
|
+
* migration so the schema bump and the fence commit together. Never lowers a
|
|
743
|
+
* fence and never re-declares an existing row.
|
|
744
|
+
*/
|
|
745
|
+
#installCheckpoint(partition, name, schemaVersion, nowMs) {
|
|
746
|
+
this.db
|
|
747
|
+
.query(`INSERT INTO sync_backfill_checkpoints(
|
|
748
|
+
partition, name, schema_version, state, watermark,
|
|
749
|
+
owner_epoch, observed_rows, updated_at_ms)
|
|
750
|
+
VALUES (?,?,?,'declared',0,0,0,?)
|
|
751
|
+
ON CONFLICT(partition, name) DO NOTHING`)
|
|
752
|
+
.run(partition, name, schemaVersion, nowMs);
|
|
753
|
+
this.db
|
|
754
|
+
.query(`INSERT INTO sync_writer_fence(partition, required_writer_version)
|
|
755
|
+
VALUES (?,?)
|
|
756
|
+
ON CONFLICT(partition) DO UPDATE SET
|
|
757
|
+
required_writer_version=max(
|
|
758
|
+
sync_writer_fence.required_writer_version,
|
|
759
|
+
excluded.required_writer_version)`)
|
|
760
|
+
.run(partition, schemaVersion);
|
|
761
|
+
}
|
|
762
|
+
async declareCheckpoint(partition, name, schemaVersion, nowMs) {
|
|
763
|
+
// One transaction: the checkpoint becomes visible and the fence that
|
|
764
|
+
// protects it is raised together. The fence is raised at declaration, not
|
|
765
|
+
// at activation, so old writers are rejected for the whole backfill.
|
|
766
|
+
return this.#serializeWrite(() => {
|
|
767
|
+
this.db.exec('BEGIN IMMEDIATE');
|
|
768
|
+
try {
|
|
769
|
+
this.#installCheckpoint(partition, name, schemaVersion, nowMs);
|
|
770
|
+
const row = this.db
|
|
771
|
+
.query(`SELECT partition, name, schema_version, state, watermark,
|
|
772
|
+
owner_epoch, observed_rows, updated_at_ms
|
|
773
|
+
FROM sync_backfill_checkpoints WHERE partition=? AND name=?`)
|
|
774
|
+
.get(partition, name);
|
|
775
|
+
if (row === null)
|
|
776
|
+
throw new StorageQueryError('sync.storage.checkpoint_not_declared');
|
|
777
|
+
this.db.exec('COMMIT');
|
|
778
|
+
return toStoredCheckpoint(row);
|
|
779
|
+
}
|
|
780
|
+
catch (error) {
|
|
781
|
+
this.db.exec('ROLLBACK');
|
|
782
|
+
throw error;
|
|
783
|
+
}
|
|
784
|
+
});
|
|
785
|
+
}
|
|
786
|
+
async activateCheckpoint(partition, name, ownerEpoch, watermark, sources, nowMs) {
|
|
787
|
+
return this.#serializeWrite(() => {
|
|
788
|
+
// `BEGIN IMMEDIATE` is SQLite's partition write lock; the same writer
|
|
789
|
+
// lock a push takes. Nothing may observe activation without the fence.
|
|
790
|
+
this.db.exec('BEGIN IMMEDIATE');
|
|
791
|
+
try {
|
|
792
|
+
const fence = this.db
|
|
793
|
+
.query('SELECT required_writer_version FROM sync_writer_fence WHERE partition=?')
|
|
794
|
+
.get(partition);
|
|
795
|
+
const checkpoint = this.db
|
|
796
|
+
.query('SELECT schema_version FROM sync_backfill_checkpoints WHERE partition=? AND name=?')
|
|
797
|
+
.get(partition, name);
|
|
798
|
+
if (fence === null ||
|
|
799
|
+
checkpoint === null ||
|
|
800
|
+
fence.required_writer_version < checkpoint.schema_version) {
|
|
801
|
+
// The barrier is not installed. Activating would certify a state old
|
|
802
|
+
// writers can still decay; refuse instead of silently proceeding.
|
|
803
|
+
throw new StorageQueryError('sync.storage.checkpoint_fence_missing');
|
|
804
|
+
}
|
|
805
|
+
const coverage = this.#hasSourceChangesAbove(partition, sources, watermark);
|
|
806
|
+
if (coverage !== 'clean') {
|
|
807
|
+
this.db.exec('ROLLBACK');
|
|
808
|
+
return coverage === 'changed' ? 'stale' : 'unverifiable';
|
|
809
|
+
}
|
|
810
|
+
const updated = this.db
|
|
811
|
+
.query(`UPDATE sync_backfill_checkpoints
|
|
812
|
+
SET state='activated', watermark=?, updated_at_ms=?
|
|
813
|
+
WHERE partition=? AND name=? AND owner_epoch=?
|
|
814
|
+
AND state<>'activated'`)
|
|
815
|
+
.run(watermark, nowMs, partition, name, ownerEpoch);
|
|
816
|
+
this.db.exec('COMMIT');
|
|
817
|
+
return Number(updated.changes) === 1 ? 'activated' : 'stale';
|
|
818
|
+
}
|
|
819
|
+
catch (error) {
|
|
820
|
+
this.db.exec('ROLLBACK');
|
|
821
|
+
throw error;
|
|
822
|
+
}
|
|
823
|
+
});
|
|
824
|
+
}
|
|
825
|
+
async claimCheckpoint(partition, name, schemaVersion, nowMs) {
|
|
826
|
+
return this.#serializeWrite(() => {
|
|
827
|
+
const claimed = this.db
|
|
828
|
+
.query(`UPDATE sync_backfill_checkpoints
|
|
829
|
+
SET owner_epoch=owner_epoch+1,
|
|
830
|
+
state=CASE WHEN state='declared' THEN 'backfilling' ELSE state END,
|
|
831
|
+
schema_version=?, updated_at_ms=?
|
|
832
|
+
WHERE partition=? AND name=? AND state<>'activated'
|
|
833
|
+
RETURNING partition, name, schema_version, state, watermark,
|
|
834
|
+
owner_epoch, observed_rows, updated_at_ms`)
|
|
835
|
+
.get(schemaVersion, nowMs, partition, name);
|
|
836
|
+
if (claimed === null) {
|
|
837
|
+
throw new StorageQueryError('sync.storage.checkpoint_not_declared');
|
|
838
|
+
}
|
|
839
|
+
return toStoredCheckpoint(claimed);
|
|
840
|
+
});
|
|
841
|
+
}
|
|
842
|
+
async advanceCheckpoint(partition, name, ownerEpoch, watermark, observedRows, nowMs) {
|
|
843
|
+
return this.#serializeWrite(() => {
|
|
844
|
+
const result = this.db
|
|
845
|
+
.query(`UPDATE sync_backfill_checkpoints
|
|
846
|
+
SET watermark=?, observed_rows=?, updated_at_ms=?
|
|
847
|
+
WHERE partition=? AND name=? AND owner_epoch=?`)
|
|
848
|
+
.run(watermark, observedRows, nowMs, partition, name, ownerEpoch);
|
|
849
|
+
return Number(result.changes) === 1;
|
|
850
|
+
});
|
|
851
|
+
}
|
|
852
|
+
async sourceCoverageSeq(partition, tables) {
|
|
853
|
+
if (tables.length === 0)
|
|
854
|
+
return 0;
|
|
855
|
+
const row = this.db
|
|
856
|
+
.query(`SELECT max(commit_seq) AS seq FROM sync_changes
|
|
857
|
+
WHERE partition=? AND tbl IN (${placeholders(tables.length)})`)
|
|
858
|
+
.get(partition, ...tables);
|
|
859
|
+
return row?.seq ?? 0;
|
|
860
|
+
}
|
|
861
|
+
#hasSourceChangesAbove(partition, tables, seq) {
|
|
862
|
+
// Read the horizon before the window scan: a horizon past `seq` means the
|
|
863
|
+
// history that would answer the question is gone, so an empty scan proves
|
|
864
|
+
// nothing and must not be reported as `clean`.
|
|
865
|
+
const horizon = this.db
|
|
866
|
+
.query('SELECT horizon_seq FROM sync_partitions WHERE partition=?')
|
|
867
|
+
.get(partition)?.horizon_seq ?? 0;
|
|
868
|
+
if (tables.length === 0)
|
|
869
|
+
return 'clean';
|
|
870
|
+
const hit = this.db
|
|
871
|
+
.query(`SELECT 1 AS hit FROM sync_changes
|
|
872
|
+
WHERE partition=? AND tbl IN (${placeholders(tables.length)})
|
|
873
|
+
AND commit_seq>? LIMIT 1`)
|
|
874
|
+
.get(partition, ...tables, seq);
|
|
875
|
+
if (hit !== null && hit !== undefined)
|
|
876
|
+
return 'changed';
|
|
877
|
+
return horizon > seq ? 'unverifiable' : 'clean';
|
|
878
|
+
}
|
|
879
|
+
async hasSourceChangesAbove(partition, tables, seq) {
|
|
880
|
+
return this.#hasSourceChangesAbove(partition, tables, seq);
|
|
881
|
+
}
|
|
619
882
|
async getRow(partition, table, rowId) {
|
|
620
883
|
const record = this.db
|
|
621
884
|
.query(selectRowSql(this.table(table), 'sqlite'))
|
package/dist/storage-errors.d.ts
CHANGED
|
@@ -9,7 +9,7 @@ export declare class StorageConstraintError extends Error {
|
|
|
9
9
|
constructor(cause: unknown, opIndex?: number);
|
|
10
10
|
}
|
|
11
11
|
/** Stable, privacy-safe failures for trusted server storage queries. */
|
|
12
|
-
export type StorageQueryErrorCode = 'sync.storage.schema_migration_pending' | 'sync.storage.schema_migration_conflict' | 'sync.storage.schema_changed' | 'sync.storage.invalid_migration_budget' | 'sync.storage.scan_requires_scope' | 'sync.storage.index_not_found' | 'sync.storage.index_not_materialized' | 'sync.storage.index_value_count_mismatch' | 'sync.storage.invalid_limit' | 'sync.storage.prune_epoch_mismatch' | 'sync.storage.partition_unregistered' | 'sync.storage.invalid_prune_cursor';
|
|
12
|
+
export type StorageQueryErrorCode = 'sync.storage.schema_migration_pending' | 'sync.storage.schema_migration_conflict' | 'sync.storage.schema_changed' | 'sync.storage.invalid_migration_budget' | 'sync.storage.scan_requires_scope' | 'sync.storage.index_not_found' | 'sync.storage.index_not_materialized' | 'sync.storage.index_value_count_mismatch' | 'sync.storage.invalid_limit' | 'sync.storage.prune_epoch_mismatch' | 'sync.storage.partition_unregistered' | 'sync.storage.invalid_prune_cursor' | 'sync.storage.checkpoint_not_declared' | 'sync.storage.checkpoint_unsupported' | 'sync.storage.checkpoint_fence_missing' | 'sync.storage.checkpoint_incomplete';
|
|
13
13
|
/**
|
|
14
14
|
* Host-only query error. Messages never include identifiers, values, SQL,
|
|
15
15
|
* paths, or row data; callers branch on `code`, never message text.
|
package/dist/storage-errors.js
CHANGED
|
@@ -19,6 +19,10 @@ const STORAGE_QUERY_MESSAGES = {
|
|
|
19
19
|
'sync.storage.prune_epoch_mismatch': 'partition log epoch changed; recompute retention inputs',
|
|
20
20
|
'sync.storage.partition_unregistered': 'pruning requires a registered partition',
|
|
21
21
|
'sync.storage.invalid_prune_cursor': 'pruning requires a non-negative safe integer cursor and a non-empty log epoch',
|
|
22
|
+
'sync.storage.checkpoint_not_declared': 'checkpoint claim requires a declared, not yet activated row',
|
|
23
|
+
'sync.storage.checkpoint_unsupported': 'this storage backend cannot declare backfill checkpoints',
|
|
24
|
+
'sync.storage.checkpoint_fence_missing': 'checkpoint activation requires the writer fence raised at declaration',
|
|
25
|
+
'sync.storage.checkpoint_incomplete': 'a declared backfill checkpoint is not activated and this caller did not declare it',
|
|
22
26
|
'sync.storage.scan_requires_scope': 'scope-indexed row scans require at least one scope variable',
|
|
23
27
|
'sync.storage.index_not_found': 'trusted row lookup requires a declared relational index',
|
|
24
28
|
'sync.storage.index_not_materialized': 'trusted row lookup requires a materialized relational table',
|
package/dist/storage.d.ts
CHANGED
|
@@ -28,6 +28,72 @@ export interface CommitPruneResult {
|
|
|
28
28
|
readonly horizonSeq: number;
|
|
29
29
|
readonly removedCommits: number;
|
|
30
30
|
}
|
|
31
|
+
/** RFC 0007 backfill lifecycle. `activated` is terminal within a schema version. */
|
|
32
|
+
export type CheckpointState = 'declared' | 'backfilling' | 'activated';
|
|
33
|
+
/**
|
|
34
|
+
* Host-owned checkpoint installation carried on `ensureSchema`: the schema
|
|
35
|
+
* bump, the `declared` row, and the raised writer fence commit together.
|
|
36
|
+
*/
|
|
37
|
+
export interface CheckpointDeclaration {
|
|
38
|
+
readonly partition: string;
|
|
39
|
+
readonly name: string;
|
|
40
|
+
readonly schemaVersion: number;
|
|
41
|
+
}
|
|
42
|
+
/** One host-storage backfill checkpoint row. Never a synced application row. */
|
|
43
|
+
export interface StoredCheckpoint {
|
|
44
|
+
readonly partition: string;
|
|
45
|
+
readonly name: string;
|
|
46
|
+
readonly schemaVersion: number;
|
|
47
|
+
readonly state: CheckpointState;
|
|
48
|
+
/** Highest source `commit_seq` the projection was derived from. */
|
|
49
|
+
readonly watermark: number;
|
|
50
|
+
/** Monotonic claim counter; a batch is stale when it does not match. */
|
|
51
|
+
readonly ownerEpoch: number;
|
|
52
|
+
readonly observedRows: number;
|
|
53
|
+
readonly updatedAtMs: number;
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* RFC 0007 serve-gate token, read inside the transaction that produces the
|
|
57
|
+
* result. `storedSchemaVersion` is the published marker; `logEpoch` is the
|
|
58
|
+
* partition's log generation, so a restore that presents V -> other -> V is
|
|
59
|
+
* still distinguishable. `pending` names every incomplete declared checkpoint
|
|
60
|
+
* at the running schema version, across partitions.
|
|
61
|
+
*/
|
|
62
|
+
export interface ServeGate {
|
|
63
|
+
readonly storedSchemaVersion: number;
|
|
64
|
+
readonly logEpoch: string | undefined;
|
|
65
|
+
readonly pending: readonly {
|
|
66
|
+
readonly partition: string;
|
|
67
|
+
readonly name: string;
|
|
68
|
+
}[];
|
|
69
|
+
}
|
|
70
|
+
/** Why `serveGate` refuses, or `undefined` when this request may serve. */
|
|
71
|
+
export type ServeGateRefusal = {
|
|
72
|
+
readonly kind: 'schema';
|
|
73
|
+
readonly stored: number;
|
|
74
|
+
readonly running: number;
|
|
75
|
+
} | {
|
|
76
|
+
readonly kind: 'checkpoint';
|
|
77
|
+
readonly partition: string;
|
|
78
|
+
readonly name: string;
|
|
79
|
+
} | {
|
|
80
|
+
/** The token changed between two reads: a restore rotated the epoch. */
|
|
81
|
+
readonly kind: 'epoch';
|
|
82
|
+
};
|
|
83
|
+
/**
|
|
84
|
+
* Evaluate the RFC 0007 predicate against a gate token. A process that
|
|
85
|
+
* declares the pending checkpoint may serve while it backfills; any other
|
|
86
|
+
* process is refused, and a database newer than the running build is always
|
|
87
|
+
* refused.
|
|
88
|
+
*/
|
|
89
|
+
export declare function serveGateRefusal(gate: ServeGate, runningSchemaVersion: number, checkpoints: readonly CheckpointDeclaration[] | undefined): ServeGateRefusal | undefined;
|
|
90
|
+
/**
|
|
91
|
+
* RFC 0007: compare two gate tokens read around a data read. The token binds
|
|
92
|
+
* the published schema version and the partition log epoch, so a restore that
|
|
93
|
+
* presents V → other → V is distinguishable. Returns a refusal when either
|
|
94
|
+
* changed across the read.
|
|
95
|
+
*/
|
|
96
|
+
export declare function serveGateTokenChanged(before: ServeGate, after: ServeGate): ServeGateRefusal | undefined;
|
|
31
97
|
/** The current stored state of a synced row. */
|
|
32
98
|
export interface StoredRow {
|
|
33
99
|
readonly rowId: string;
|
|
@@ -352,6 +418,21 @@ export interface StorageTransaction {
|
|
|
352
418
|
* first writer wins; callers read the canonical value after commit.
|
|
353
419
|
*/
|
|
354
420
|
putPushResult(clientId: string, clientCommitId: string, result: StoredPushResult): Promise<void>;
|
|
421
|
+
/**
|
|
422
|
+
* Advance this partition's checkpoint only while `owner_epoch` still
|
|
423
|
+
* matches, so the owner-epoch CAS rides inside the caller's transaction.
|
|
424
|
+
* The host calls this from `processPushOperationsWithTrace`'s operation
|
|
425
|
+
* builder: the backfill is an ordinary server-authoritative write, and this
|
|
426
|
+
* CAS commits with that write's row versions and commit-log append. False
|
|
427
|
+
* means abort and write nothing, rather than a throw for a superseded owner.
|
|
428
|
+
*/
|
|
429
|
+
advanceCheckpoint(name: string, ownerEpoch: number, watermark: number, observedRows: number, nowMs: number): Promise<boolean>;
|
|
430
|
+
/**
|
|
431
|
+
* RFC 0007 serve gate on the caller's own connection, for write paths that
|
|
432
|
+
* already hold the partition lock: the gate and the write share one
|
|
433
|
+
* transaction, so a migration cannot interleave between them.
|
|
434
|
+
*/
|
|
435
|
+
readServeGate(runningSchemaVersion: number): Promise<ServeGate>;
|
|
355
436
|
/**
|
|
356
437
|
* Blob reference index (§5.9.4) — ADDITIVE, optional. Set the blobIds a
|
|
357
438
|
* row currently references (empty = clear), replacing any prior entries
|
|
@@ -379,7 +460,7 @@ export interface ServerStorage {
|
|
|
379
460
|
* before binding a public port; protocol handlers still call this method
|
|
380
461
|
* lazily as a defensive backstop.
|
|
381
462
|
*/
|
|
382
|
-
ensureSchema(schema: CompiledSchema): Promise<void>;
|
|
463
|
+
ensureSchema(schema: CompiledSchema, checkpoints?: readonly CheckpointDeclaration[]): Promise<void>;
|
|
383
464
|
/** Create or refresh the authenticated partition registry row (§2.1). */
|
|
384
465
|
touchPartition(partition: string, authenticatedAtMs: number, initialLogEpoch: string): Promise<PartitionRegistryEntry>;
|
|
385
466
|
/** Rotate log continuity after restore and discard stale cursor records. */
|
|
@@ -413,6 +494,50 @@ export interface ServerStorage {
|
|
|
413
494
|
pruneCommitsThrough(partition: string, query: CommitPruneQuery): Promise<CommitPruneResult>;
|
|
414
495
|
/** Newest commitSeq created strictly before the timestamp; 0 if none. */
|
|
415
496
|
getCommitSeqBefore(partition: string, createdBeforeMs: number): Promise<number>;
|
|
497
|
+
/** Every declared backfill checkpoint for the partition, ordered by name. */
|
|
498
|
+
readCheckpoints(partition: string): Promise<StoredCheckpoint[]>;
|
|
499
|
+
/**
|
|
500
|
+
* RFC 0007 serve gate: read the published schema version, the partition's
|
|
501
|
+
* log epoch, and every incomplete declared checkpoint at
|
|
502
|
+
* `runningSchemaVersion`. Never memoized; the serve paths call it per
|
|
503
|
+
* request so an already-serving process notices a migration.
|
|
504
|
+
*/
|
|
505
|
+
readServeGate(partition: string, runningSchemaVersion: number): Promise<ServeGate>;
|
|
506
|
+
/**
|
|
507
|
+
* Atomically take ownership of a declared or backfilling checkpoint. Increments
|
|
508
|
+
* `owner_epoch` and moves `declared` to `backfilling`. Throws
|
|
509
|
+
* `sync.storage.checkpoint_not_declared` when no claimable row exists (absent,
|
|
510
|
+
* or already `activated`).
|
|
511
|
+
*/
|
|
512
|
+
claimCheckpoint(partition: string, name: string, schemaVersion: number, nowMs: number): Promise<StoredCheckpoint>;
|
|
513
|
+
/**
|
|
514
|
+
* Advance the watermark only while `owner_epoch` still matches. Returns false
|
|
515
|
+
* for a superseded owner; the stale case never throws.
|
|
516
|
+
*/
|
|
517
|
+
advanceCheckpoint(partition: string, name: string, ownerEpoch: number, watermark: number, observedRows: number, nowMs: number): Promise<boolean>;
|
|
518
|
+
/** Highest commitSeq carrying a change to one of `tables`; 0 when none. */
|
|
519
|
+
sourceCoverageSeq(partition: string, tables: readonly string[]): Promise<number>;
|
|
520
|
+
/**
|
|
521
|
+
* Whether any source change sits above `seq`. `unverifiable` when the pruning
|
|
522
|
+
* horizon has passed `seq`, so the history that would answer the question is
|
|
523
|
+
* gone; a pruned window is never reported as `clean`.
|
|
524
|
+
*/
|
|
525
|
+
hasSourceChangesAbove(partition: string, tables: readonly string[], seq: number): Promise<'clean' | 'changed' | 'unverifiable'>;
|
|
526
|
+
/**
|
|
527
|
+
* Install the barrier: insert the checkpoint in `declared` and raise
|
|
528
|
+
* `required_writer_version` for the partition in one transaction. The fence
|
|
529
|
+
* is raised at declaration, before any backfill work, so old writers are
|
|
530
|
+
* rejected for the whole backfill window. Idempotent on an existing declared
|
|
531
|
+
* or backfilling row.
|
|
532
|
+
*/
|
|
533
|
+
declareCheckpoint(partition: string, name: string, schemaVersion: number, nowMs: number): Promise<StoredCheckpoint>;
|
|
534
|
+
/**
|
|
535
|
+
* Activate a declared checkpoint under the partition write lock. Refuses when
|
|
536
|
+
* the fence is not already raised at the checkpoint's schema version, when a
|
|
537
|
+
* source change sits above `watermark` (`stale`), or when the pruning horizon
|
|
538
|
+
* has passed `watermark` (`unverifiable`). `activated` is terminal.
|
|
539
|
+
*/
|
|
540
|
+
activateCheckpoint(partition: string, name: string, ownerEpoch: number, watermark: number, sources: readonly string[], nowMs: number): Promise<'activated' | 'stale' | 'unverifiable'>;
|
|
416
541
|
getRow(partition: string, table: string, rowId: string): Promise<StoredRow | undefined>;
|
|
417
542
|
/**
|
|
418
543
|
* Idempotency lookup (§2.3). Throws `SyncError sync.idempotency_cache_miss`
|
|
@@ -462,7 +587,7 @@ export interface ServerStorage {
|
|
|
462
587
|
* transaction to finish on a single-connection executor, and runs
|
|
463
588
|
* concurrently on a pool-backed one.
|
|
464
589
|
*/
|
|
465
|
-
queryAuthoritative?(partition: string, query: AuthoritativeQueryRequest): Promise<AuthoritativeQueryResult>;
|
|
590
|
+
queryAuthoritative?(partition: string, query: AuthoritativeQueryRequest, checkpoints?: readonly CheckpointDeclaration[]): Promise<AuthoritativeQueryResult>;
|
|
466
591
|
/**
|
|
467
592
|
* Optional trusted-host exact lookup through a declared relational index.
|
|
468
593
|
* This capability is outside client scope/subscription authorization and
|
package/dist/storage.js
CHANGED
|
@@ -1 +1,40 @@
|
|
|
1
|
-
|
|
1
|
+
/**
|
|
2
|
+
* Evaluate the RFC 0007 predicate against a gate token. A process that
|
|
3
|
+
* declares the pending checkpoint may serve while it backfills; any other
|
|
4
|
+
* process is refused, and a database newer than the running build is always
|
|
5
|
+
* refused.
|
|
6
|
+
*/
|
|
7
|
+
export function serveGateRefusal(gate, runningSchemaVersion, checkpoints) {
|
|
8
|
+
if (gate.storedSchemaVersion > runningSchemaVersion) {
|
|
9
|
+
return {
|
|
10
|
+
kind: 'schema',
|
|
11
|
+
stored: gate.storedSchemaVersion,
|
|
12
|
+
running: runningSchemaVersion,
|
|
13
|
+
};
|
|
14
|
+
}
|
|
15
|
+
for (const pending of gate.pending) {
|
|
16
|
+
const declared = (checkpoints ?? []).some((declaration) => declaration.partition === pending.partition &&
|
|
17
|
+
declaration.name === pending.name);
|
|
18
|
+
if (!declared) {
|
|
19
|
+
return {
|
|
20
|
+
kind: 'checkpoint',
|
|
21
|
+
partition: pending.partition,
|
|
22
|
+
name: pending.name,
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
return undefined;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* RFC 0007: compare two gate tokens read around a data read. The token binds
|
|
30
|
+
* the published schema version and the partition log epoch, so a restore that
|
|
31
|
+
* presents V → other → V is distinguishable. Returns a refusal when either
|
|
32
|
+
* changed across the read.
|
|
33
|
+
*/
|
|
34
|
+
export function serveGateTokenChanged(before, after) {
|
|
35
|
+
if (before.storedSchemaVersion !== after.storedSchemaVersion ||
|
|
36
|
+
before.logEpoch !== after.logEpoch) {
|
|
37
|
+
return { kind: 'epoch' };
|
|
38
|
+
}
|
|
39
|
+
return undefined;
|
|
40
|
+
}
|