@syncular/server 0.21.1 → 0.23.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/README.md +20 -6
  2. package/dist/admin.d.ts +3 -1
  3. package/dist/admin.js +7 -1
  4. package/dist/blob-handlers.js +1 -1
  5. package/dist/context.d.ts +8 -1
  6. package/dist/d1-storage.d.ts +11 -3
  7. package/dist/d1-storage.js +84 -2
  8. package/dist/errors.js +10 -0
  9. package/dist/handler.js +50 -4
  10. package/dist/operations.js +3 -3
  11. package/dist/pg-executor.d.ts +11 -1
  12. package/dist/postgres-storage.d.ts +20 -5
  13. package/dist/postgres-storage.js +345 -9
  14. package/dist/pull.js +9 -5
  15. package/dist/push.js +18 -0
  16. package/dist/readiness.d.ts +9 -2
  17. package/dist/readiness.js +27 -2
  18. package/dist/realtime.d.ts +40 -1
  19. package/dist/realtime.js +116 -19
  20. package/dist/relational-rows.d.ts +20 -0
  21. package/dist/relational-rows.js +29 -0
  22. package/dist/s3-segment-store.d.ts +20 -8
  23. package/dist/s3-segment-store.js +216 -68
  24. package/dist/schema.js +11 -0
  25. package/dist/segment-download.d.ts +1 -1
  26. package/dist/segment-download.js +50 -17
  27. package/dist/segment-store.d.ts +86 -0
  28. package/dist/segment-store.js +156 -17
  29. package/dist/signed-url.d.ts +7 -1
  30. package/dist/signed-url.js +10 -1
  31. package/dist/sqlite-dialect.d.ts +7 -1
  32. package/dist/sqlite-dialect.js +23 -0
  33. package/dist/sqlite-segment-store.js +153 -42
  34. package/dist/sqlite-storage.d.ts +11 -3
  35. package/dist/sqlite-storage.js +346 -34
  36. package/dist/storage-errors.d.ts +1 -1
  37. package/dist/storage-errors.js +4 -0
  38. package/dist/storage.d.ts +152 -2
  39. package/dist/storage.js +40 -1
  40. package/dist/validate.d.ts +20 -1
  41. package/package.json +2 -2
  42. package/src/admin.ts +13 -1
  43. package/src/blob-handlers.ts +1 -1
  44. package/src/context.ts +8 -0
  45. package/src/d1-storage.ts +143 -1
  46. package/src/errors.ts +10 -0
  47. package/src/handler.ts +62 -4
  48. package/src/operations.ts +17 -13
  49. package/src/pg-executor.ts +11 -1
  50. package/src/postgres-storage.ts +520 -5
  51. package/src/pull.ts +9 -1
  52. package/src/push.ts +18 -0
  53. package/src/readiness.ts +39 -4
  54. package/src/realtime.ts +117 -18
  55. package/src/relational-rows.ts +36 -0
  56. package/src/s3-segment-store.ts +275 -84
  57. package/src/schema.ts +15 -0
  58. package/src/segment-download.ts +79 -28
  59. package/src/segment-store.ts +250 -18
  60. package/src/signed-url.ts +12 -2
  61. package/src/sqlite-dialect.ts +23 -0
  62. package/src/sqlite-segment-store.ts +245 -65
  63. package/src/sqlite-storage.ts +494 -36
  64. package/src/storage-errors.ts +13 -1
  65. package/src/storage.ts +243 -1
  66. package/src/validate.ts +20 -1
@@ -1,6 +1,8 @@
1
1
  import { validateCommitPruneQuery } from './prune';
2
2
  import { StorageQueryError } from './storage-errors';
3
3
  import type { CommitPruneQuery, CommitPruneResult } from './storage';
4
+ import { serveGateRefusal } from './storage';
5
+ import { serveNotReadyError } from './readiness';
4
6
  /**
5
7
  * SQLite server storage over the shared synchronous driver.
6
8
  *
@@ -15,6 +17,7 @@ import {
15
17
  } from './authoritative-query';
16
18
  import { syncError } from './errors';
17
19
  import {
20
+ assertPhysicalColumns,
18
21
  commitWindowPageSql,
19
22
  deleteRowSql,
20
23
  deleteSqliteRowScopesSql,
@@ -42,6 +45,7 @@ import { matchesEffective } from './scopes';
42
45
  import {
43
46
  collectCommitWindowPage,
44
47
  deserializePushResult,
48
+ placeholders,
45
49
  SQLITE_DDL,
46
50
  type SqliteCommitWindowRecord,
47
51
  type SqliteRowRecord,
@@ -56,6 +60,7 @@ import type {
56
60
  AuthoritativeQueryRequest,
57
61
  AuthoritativeQueryResult,
58
62
  AuthoritativeQueryValue,
63
+ CheckpointDeclaration,
59
64
  ClientCursorInfo,
60
65
  ClientRecord,
61
66
  ClientSubscription,
@@ -77,8 +82,10 @@ import type {
77
82
  ScopeActivityQuery,
78
83
  ScopeCommitActivity,
79
84
  ServerStorage,
85
+ ServeGate,
80
86
  StorageTransaction,
81
87
  StoredCommit,
88
+ StoredCheckpoint,
82
89
  StoredPushResult,
83
90
  StoredReaction,
84
91
  StoredRow,
@@ -89,6 +96,30 @@ import {
89
96
  } from './storage-errors';
90
97
  import { assertScopeIndexedScan, resolveIndexRowScan } from './storage-query';
91
98
 
99
+ interface SqliteCheckpointRecord {
100
+ partition: string;
101
+ name: string;
102
+ schema_version: number;
103
+ state: StoredCheckpoint['state'];
104
+ watermark: number;
105
+ owner_epoch: number;
106
+ observed_rows: number;
107
+ updated_at_ms: number;
108
+ }
109
+
110
+ function toStoredCheckpoint(record: SqliteCheckpointRecord): StoredCheckpoint {
111
+ return {
112
+ partition: record.partition,
113
+ name: record.name,
114
+ schemaVersion: record.schema_version,
115
+ state: record.state,
116
+ watermark: record.watermark,
117
+ ownerEpoch: record.owner_epoch,
118
+ observedRows: record.observed_rows,
119
+ updatedAtMs: record.updated_at_ms,
120
+ };
121
+ }
122
+
92
123
  interface SqliteReactionRecord {
93
124
  partition: string;
94
125
  idempotency_key: string;
@@ -139,6 +170,8 @@ function toStoredReaction(record: SqliteReactionRecord): StoredReaction {
139
170
  class SqliteTransaction implements StorageTransaction {
140
171
  #storage: SqliteServerStorage;
141
172
  #partition: string;
173
+ /** Schema version this transaction writes; rides every appended commit row. */
174
+ #writerVersion: number;
142
175
  #open = true;
143
176
  #pushApplySavepoint = false;
144
177
  readonly #release: () => void;
@@ -146,10 +179,12 @@ class SqliteTransaction implements StorageTransaction {
146
179
  constructor(
147
180
  storage: SqliteServerStorage,
148
181
  partition: string,
182
+ writerVersion: number,
149
183
  release: () => void,
150
184
  ) {
151
185
  this.#storage = storage;
152
186
  this.#partition = partition;
187
+ this.#writerVersion = writerVersion;
153
188
  this.#release = release;
154
189
  storage.db.exec('BEGIN IMMEDIATE');
155
190
  }
@@ -216,6 +251,31 @@ class SqliteTransaction implements StorageTransaction {
216
251
  this.#pushApplySavepoint = true;
217
252
  }
218
253
 
254
+ async advanceCheckpoint(
255
+ name: string,
256
+ ownerEpoch: number,
257
+ watermark: number,
258
+ observedRows: number,
259
+ nowMs: number,
260
+ ): Promise<boolean> {
261
+ this.#assertOpen();
262
+ const result = this.#storage.db
263
+ .query(
264
+ `UPDATE sync_backfill_checkpoints
265
+ SET watermark=?, observed_rows=?, updated_at_ms=?
266
+ WHERE partition=? AND name=? AND owner_epoch=?`,
267
+ )
268
+ .run(watermark, observedRows, nowMs, this.#partition, name, ownerEpoch);
269
+ return Number(result.changes) === 1;
270
+ }
271
+
272
+ readServeGate(runningSchemaVersion: number): Promise<ServeGate> {
273
+ this.#assertOpen();
274
+ // Same shared connection: this read runs inside this transaction's
275
+ // BEGIN IMMEDIATE, so the gate and the write share one transaction.
276
+ return this.#storage.readServeGate(this.#partition, runningSchemaVersion);
277
+ }
278
+
219
279
  async commitRejectedPushResult(
220
280
  clientId: string,
221
281
  clientCommitId: string,
@@ -301,7 +361,7 @@ class SqliteTransaction implements StorageTransaction {
301
361
  'UPDATE sync_partitions SET max_commit_seq=? WHERE partition=?',
302
362
  ).run(commitSeq, p);
303
363
  db.query(
304
- 'INSERT INTO sync_commits(partition, commit_seq, client_id, client_commit_id, actor_id, created_at_ms) VALUES (?,?,?,?,?,?)',
364
+ 'INSERT INTO sync_commits(partition, commit_seq, client_id, client_commit_id, actor_id, created_at_ms, writer_version) VALUES (?,?,?,?,?,?,?)',
305
365
  ).run(
306
366
  p,
307
367
  commitSeq,
@@ -309,6 +369,7 @@ class SqliteTransaction implements StorageTransaction {
309
369
  commit.clientCommitId,
310
370
  commit.actorId,
311
371
  commit.createdAtMs,
372
+ this.#writerVersion,
312
373
  );
313
374
  commit.changes.forEach((change, idx) => {
314
375
  db.query(
@@ -454,6 +515,32 @@ export class SqliteServerStorage implements ServerStorage {
454
515
  'ALTER TABLE sync_clients ADD COLUMN wire_version INTEGER NOT NULL DEFAULT 1',
455
516
  );
456
517
  }
518
+ const commitColumns = this.db
519
+ .query<{ name: string }, []>('PRAGMA table_info("sync_commits")')
520
+ .all();
521
+ if (!commitColumns.some((column) => column.name === 'writer_version')) {
522
+ // Nullable by design: an old writer omits the column and the trigger's
523
+ // explicit NULL test rejects it. `ADD COLUMN NOT NULL` is unavailable in
524
+ // SQLite, and a default would hand old writers a passing value.
525
+ this.db.exec(
526
+ 'ALTER TABLE sync_commits ADD COLUMN writer_version INTEGER',
527
+ );
528
+ }
529
+ // The fence is database-side: an old binary's INSERT never reaches JS. An
530
+ // absent `sync_writer_fence` row makes the WHEN clause false, so a
531
+ // partition with no barrier behaves exactly as before.
532
+ this.db.exec(`CREATE TRIGGER IF NOT EXISTS sync_commits_writer_fence
533
+ BEFORE INSERT ON sync_commits
534
+ FOR EACH ROW
535
+ WHEN EXISTS (
536
+ SELECT 1 FROM sync_writer_fence
537
+ WHERE partition = NEW.partition
538
+ AND (NEW.writer_version IS NULL
539
+ OR NEW.writer_version < required_writer_version)
540
+ )
541
+ BEGIN
542
+ SELECT RAISE(ABORT, 'sync.storage.writer_fence_rejected');
543
+ END`);
457
544
  }
458
545
 
459
546
  /** Resolve a table's compiled schema; row operations require `ensureSchema`. */
@@ -467,9 +554,34 @@ export class SqliteServerStorage implements ServerStorage {
467
554
  return table;
468
555
  }
469
556
 
470
- async ensureSchema(schema: CompiledSchema): Promise<void> {
557
+ async ensureSchema(
558
+ schema: CompiledSchema,
559
+ checkpoints?: readonly CheckpointDeclaration[],
560
+ ): Promise<void> {
471
561
  // Memoized fast path: same instance, same schema version.
472
562
  if (this.#schemaVersion === schema.version) return;
563
+ // The barrier reads stored state: a checkpoint at this schema version that
564
+ // is not activated and that this caller did not declare means this process
565
+ // must not serve. An omitted or empty declaration set declares nothing and
566
+ // is refused here, never treated as a bypass.
567
+ const declarations = checkpoints ?? [];
568
+ const incomplete = this.db
569
+ .query<{ partition: string; name: string }, [number]>(
570
+ `SELECT partition, name FROM sync_backfill_checkpoints
571
+ WHERE schema_version=? AND state<>'activated'`,
572
+ )
573
+ .all(schema.version)
574
+ .filter(
575
+ (row) =>
576
+ !declarations.some(
577
+ (declaration) =>
578
+ declaration.partition === row.partition &&
579
+ declaration.name === row.name,
580
+ ),
581
+ );
582
+ if (incomplete.length > 0) {
583
+ throw new StorageQueryError('sync.storage.checkpoint_incomplete');
584
+ }
473
585
  this.db.exec(SCHEMA_META_DDL_SQLITE);
474
586
  const marker = this.db
475
587
  .query<{ schema_version: number; layouts: string }, []>(
@@ -481,6 +593,60 @@ export class SqliteServerStorage implements ServerStorage {
481
593
  `stored schema version ${marker.schema_version} is newer than the configured schema (${schema.version}) — refusing to run an older server against a migrated database`,
482
594
  );
483
595
  }
596
+ if (marker !== null && marker.schema_version === schema.version) {
597
+ // Version equality is not layout equality: a marker written by another
598
+ // build at the same version describes rows the running codec cannot
599
+ // decode. Compare the stored layouts instead of trusting the number.
600
+ const storedLayouts = parseLayouts(marker.layouts);
601
+ const configuredLayouts = parseLayouts(layoutsOf(schema));
602
+ let mismatch: string | undefined;
603
+ for (const [tableName, columns] of Object.entries(configuredLayouts)) {
604
+ const stored = storedLayouts[tableName];
605
+ const columnCount = Math.max(columns.length, stored?.length ?? 0);
606
+ for (let index = 0; index < columnCount; index++) {
607
+ const expected = columns[index];
608
+ const actual = stored?.[index];
609
+ if (
610
+ expected !== undefined &&
611
+ actual !== undefined &&
612
+ actual.name === expected.name &&
613
+ actual.type === expected.type &&
614
+ actual.nullable === expected.nullable
615
+ ) {
616
+ continue;
617
+ }
618
+ mismatch = `table ${JSON.stringify(tableName)} column ${JSON.stringify(expected?.name ?? actual?.name ?? '')}`;
619
+ break;
620
+ }
621
+ if (mismatch !== undefined) break;
622
+ }
623
+ if (mismatch === undefined) {
624
+ for (const tableName of Object.keys(storedLayouts)) {
625
+ if (!(tableName in configuredLayouts)) {
626
+ mismatch = `table ${JSON.stringify(tableName)}`;
627
+ break;
628
+ }
629
+ }
630
+ }
631
+ if (mismatch !== undefined) {
632
+ throw new Error(
633
+ `stored schema layouts disagree with the configured schema at version ${schema.version} (${mismatch}) — refusing to serve a database whose stored rows the running code cannot decode`,
634
+ );
635
+ }
636
+ // The persisted layouts describe the codec's app columns only: read the
637
+ // physical tables so a same-version database missing a
638
+ // storage-internal column is refused at startup instead of failing at
639
+ // the first write.
640
+ for (const table of schema.tables.values()) {
641
+ const escapedTableName = table.name.replaceAll('"', '""');
642
+ const columns = this.db
643
+ .query<{ name: string }, []>(
644
+ `PRAGMA table_info("${escapedTableName}")`,
645
+ )
646
+ .all();
647
+ assertPhysicalColumns(table, new Set(columns.map((c) => c.name)));
648
+ }
649
+ }
484
650
  if (marker === null || marker.schema_version < schema.version) {
485
651
  // Introspect existing app tables, then apply the migration subset
486
652
  // (CREATE TABLE / ADD COLUMN / rebuild indexes) to reach `schema`, then
@@ -512,41 +678,64 @@ export class SqliteServerStorage implements ServerStorage {
512
678
  );
513
679
  }
514
680
  }
515
- this.db.exec('BEGIN IMMEDIATE');
516
- try {
517
- for (const tableName of retiredTables) {
518
- this.db
519
- .query('DELETE FROM sync_row_scopes WHERE tbl=?')
520
- .run(tableName);
681
+ // The migration rewrites application rows (`#rewriteRows`) with no
682
+ // commit log entry. Serialize it through the same writer queue as
683
+ // pushes and prune (`#serializeWrite`) rather than trusting the
684
+ // migration as privileged; `BEGIN IMMEDIATE` then owns SQLite's writer
685
+ // lock for the whole rewrite.
686
+ await this.#serializeWrite(() => {
687
+ this.db.exec('BEGIN IMMEDIATE');
688
+ try {
689
+ for (const tableName of retiredTables) {
690
+ this.db
691
+ .query('DELETE FROM sync_row_scopes WHERE tbl=?')
692
+ .run(tableName);
693
+ this.db
694
+ .query('DELETE FROM sync_blob_refs WHERE tbl=?')
695
+ .run(tableName);
696
+ this.db.exec(dropTableDdl(tableName));
697
+ }
698
+ for (const statement of schemaDdl(
699
+ schema,
700
+ existing,
701
+ 'sqlite',
702
+ existingIndexes,
703
+ )) {
704
+ this.db.exec(statement);
705
+ }
706
+ for (const table of schema.tables.values()) {
707
+ const oldLayout = layouts[table.name];
708
+ const plan = rewritePlan(
709
+ table,
710
+ oldLayout,
711
+ existing.get(table.name),
712
+ );
713
+ if (!plan.migrate && !plan.backfill) continue;
714
+ this.#rewriteRows(table, plan.migrate ? oldLayout : undefined);
715
+ }
521
716
  this.db
522
- .query('DELETE FROM sync_blob_refs WHERE tbl=?')
523
- .run(tableName);
524
- this.db.exec(dropTableDdl(tableName));
525
- }
526
- for (const statement of schemaDdl(
527
- schema,
528
- existing,
529
- 'sqlite',
530
- existingIndexes,
531
- )) {
532
- this.db.exec(statement);
533
- }
534
- for (const table of schema.tables.values()) {
535
- const oldLayout = layouts[table.name];
536
- const plan = rewritePlan(table, oldLayout, existing.get(table.name));
537
- if (!plan.migrate && !plan.backfill) continue;
538
- this.#rewriteRows(table, plan.migrate ? oldLayout : undefined);
717
+ .query(
718
+ 'INSERT INTO sync_schema_meta(id, schema_version, layouts) VALUES (1, ?, ?) ON CONFLICT(id) DO UPDATE SET schema_version=excluded.schema_version, layouts=excluded.layouts',
719
+ )
720
+ .run(schema.version, layoutsOf(schema));
721
+ // The host's declared checkpoints install in the schema-bump
722
+ // transaction, so no observer sees the bumped marker without the
723
+ // fence, or the fence without its declaration row.
724
+ const installedAtMs = Date.now();
725
+ for (const declaration of checkpoints ?? []) {
726
+ this.#installCheckpoint(
727
+ declaration.partition,
728
+ declaration.name,
729
+ declaration.schemaVersion,
730
+ installedAtMs,
731
+ );
732
+ }
733
+ this.db.exec('COMMIT');
734
+ } catch (error) {
735
+ this.db.exec('ROLLBACK');
736
+ throw error;
539
737
  }
540
- this.db
541
- .query(
542
- 'INSERT INTO sync_schema_meta(id, schema_version, layouts) VALUES (1, ?, ?) ON CONFLICT(id) DO UPDATE SET schema_version=excluded.schema_version, layouts=excluded.layouts',
543
- )
544
- .run(schema.version, layoutsOf(schema));
545
- this.db.exec('COMMIT');
546
- } catch (error) {
547
- this.db.exec('ROLLBACK');
548
- throw error;
549
- }
738
+ });
550
739
  }
551
740
  this.#tables = schema.tables;
552
741
  this.#schemaVersion = schema.version;
@@ -709,7 +898,12 @@ export class SqliteServerStorage implements ServerStorage {
709
898
  });
710
899
  await previous;
711
900
  try {
712
- return new SqliteTransaction(this, partition, release);
901
+ return new SqliteTransaction(
902
+ this,
903
+ partition,
904
+ this.#schemaVersion ?? 0,
905
+ release,
906
+ );
713
907
  } catch (error) {
714
908
  release();
715
909
  throw error;
@@ -754,6 +948,7 @@ export class SqliteServerStorage implements ServerStorage {
754
948
  async queryAuthoritative(
755
949
  partition: string,
756
950
  query: AuthoritativeQueryRequest,
951
+ checkpoints?: readonly CheckpointDeclaration[],
757
952
  ): Promise<AuthoritativeQueryResult> {
758
953
  if (this.#tables === undefined) {
759
954
  throw new Error(
@@ -779,6 +974,13 @@ export class SqliteServerStorage implements ServerStorage {
779
974
  try {
780
975
  this.db.exec('BEGIN');
781
976
  open = true;
977
+ // RFC 0007: evaluate the gate on this connection after BEGIN. SQLite is
978
+ // single-writer and the shared connection serializes writers, so the
979
+ // gate read and the query read cannot straddle a migration.
980
+ const runningVersion = this.#schemaVersion ?? 0;
981
+ const gate = await this.readServeGate(partition, runningVersion);
982
+ const refusal = serveGateRefusal(gate, runningVersion, checkpoints);
983
+ if (refusal !== undefined) throw serveNotReadyError(refusal);
782
984
  const rows = this.db
783
985
  .query<Readonly<Record<string, unknown>>, AuthoritativeQueryValue[]>(
784
986
  prepared.sql,
@@ -894,6 +1096,262 @@ export class SqliteServerStorage implements ServerStorage {
894
1096
  return row?.seq ?? 0;
895
1097
  }
896
1098
 
1099
+ async readServeGate(
1100
+ partition: string,
1101
+ runningSchemaVersion: number,
1102
+ ): Promise<ServeGate> {
1103
+ const marker = this.db
1104
+ .query<{ schema_version: number }, []>(
1105
+ 'SELECT schema_version FROM sync_schema_meta WHERE id=1',
1106
+ )
1107
+ .get();
1108
+ const epoch = this.db
1109
+ .query<{ log_epoch: string }, [string]>(
1110
+ 'SELECT log_epoch FROM sync_partition_registry WHERE partition=?',
1111
+ )
1112
+ .get(partition);
1113
+ const pending = this.db
1114
+ .query<{ partition: string; name: string }, [number]>(
1115
+ `SELECT partition, name FROM sync_backfill_checkpoints
1116
+ WHERE schema_version=? AND state<>'activated'`,
1117
+ )
1118
+ .all(runningSchemaVersion);
1119
+ return {
1120
+ storedSchemaVersion: marker?.schema_version ?? 0,
1121
+ logEpoch: epoch?.log_epoch,
1122
+ pending: pending.map((row) => ({
1123
+ partition: row.partition,
1124
+ name: row.name,
1125
+ })),
1126
+ };
1127
+ }
1128
+
1129
+ async readCheckpoints(partition: string): Promise<StoredCheckpoint[]> {
1130
+ return this.db
1131
+ .query<SqliteCheckpointRecord, [string]>(
1132
+ `SELECT partition, name, schema_version, state, watermark,
1133
+ owner_epoch, observed_rows, updated_at_ms
1134
+ FROM sync_backfill_checkpoints WHERE partition=? ORDER BY name`,
1135
+ )
1136
+ .all(partition)
1137
+ .map(toStoredCheckpoint);
1138
+ }
1139
+
1140
+ /**
1141
+ * Insert the `declared` row and raise the fence, inside the caller's open
1142
+ * transaction. Shared by `declareCheckpoint` and the `ensureSchema`
1143
+ * migration so the schema bump and the fence commit together. Never lowers a
1144
+ * fence and never re-declares an existing row.
1145
+ */
1146
+ #installCheckpoint(
1147
+ partition: string,
1148
+ name: string,
1149
+ schemaVersion: number,
1150
+ nowMs: number,
1151
+ ): void {
1152
+ this.db
1153
+ .query(
1154
+ `INSERT INTO sync_backfill_checkpoints(
1155
+ partition, name, schema_version, state, watermark,
1156
+ owner_epoch, observed_rows, updated_at_ms)
1157
+ VALUES (?,?,?,'declared',0,0,0,?)
1158
+ ON CONFLICT(partition, name) DO NOTHING`,
1159
+ )
1160
+ .run(partition, name, schemaVersion, nowMs);
1161
+ this.db
1162
+ .query(
1163
+ `INSERT INTO sync_writer_fence(partition, required_writer_version)
1164
+ VALUES (?,?)
1165
+ ON CONFLICT(partition) DO UPDATE SET
1166
+ required_writer_version=max(
1167
+ sync_writer_fence.required_writer_version,
1168
+ excluded.required_writer_version)`,
1169
+ )
1170
+ .run(partition, schemaVersion);
1171
+ }
1172
+
1173
+ async declareCheckpoint(
1174
+ partition: string,
1175
+ name: string,
1176
+ schemaVersion: number,
1177
+ nowMs: number,
1178
+ ): Promise<StoredCheckpoint> {
1179
+ // One transaction: the checkpoint becomes visible and the fence that
1180
+ // protects it is raised together. The fence is raised at declaration, not
1181
+ // at activation, so old writers are rejected for the whole backfill.
1182
+ return this.#serializeWrite(() => {
1183
+ this.db.exec('BEGIN IMMEDIATE');
1184
+ try {
1185
+ this.#installCheckpoint(partition, name, schemaVersion, nowMs);
1186
+ const row = this.db
1187
+ .query<SqliteCheckpointRecord, [string, string]>(
1188
+ `SELECT partition, name, schema_version, state, watermark,
1189
+ owner_epoch, observed_rows, updated_at_ms
1190
+ FROM sync_backfill_checkpoints WHERE partition=? AND name=?`,
1191
+ )
1192
+ .get(partition, name);
1193
+ if (row === null)
1194
+ throw new StorageQueryError('sync.storage.checkpoint_not_declared');
1195
+ this.db.exec('COMMIT');
1196
+ return toStoredCheckpoint(row);
1197
+ } catch (error) {
1198
+ this.db.exec('ROLLBACK');
1199
+ throw error;
1200
+ }
1201
+ });
1202
+ }
1203
+
1204
+ async activateCheckpoint(
1205
+ partition: string,
1206
+ name: string,
1207
+ ownerEpoch: number,
1208
+ watermark: number,
1209
+ sources: readonly string[],
1210
+ nowMs: number,
1211
+ ): Promise<'activated' | 'stale' | 'unverifiable'> {
1212
+ return this.#serializeWrite(() => {
1213
+ // `BEGIN IMMEDIATE` is SQLite's partition write lock; the same writer
1214
+ // lock a push takes. Nothing may observe activation without the fence.
1215
+ this.db.exec('BEGIN IMMEDIATE');
1216
+ try {
1217
+ const fence = this.db
1218
+ .query<{ required_writer_version: number }, [string]>(
1219
+ 'SELECT required_writer_version FROM sync_writer_fence WHERE partition=?',
1220
+ )
1221
+ .get(partition);
1222
+ const checkpoint = this.db
1223
+ .query<{ schema_version: number }, [string, string]>(
1224
+ 'SELECT schema_version FROM sync_backfill_checkpoints WHERE partition=? AND name=?',
1225
+ )
1226
+ .get(partition, name);
1227
+ if (
1228
+ fence === null ||
1229
+ checkpoint === null ||
1230
+ fence.required_writer_version < checkpoint.schema_version
1231
+ ) {
1232
+ // The barrier is not installed. Activating would certify a state old
1233
+ // writers can still decay; refuse instead of silently proceeding.
1234
+ throw new StorageQueryError('sync.storage.checkpoint_fence_missing');
1235
+ }
1236
+ const coverage = this.#hasSourceChangesAbove(
1237
+ partition,
1238
+ sources,
1239
+ watermark,
1240
+ );
1241
+ if (coverage !== 'clean') {
1242
+ this.db.exec('ROLLBACK');
1243
+ return coverage === 'changed' ? 'stale' : 'unverifiable';
1244
+ }
1245
+ const updated = this.db
1246
+ .query(
1247
+ `UPDATE sync_backfill_checkpoints
1248
+ SET state='activated', watermark=?, updated_at_ms=?
1249
+ WHERE partition=? AND name=? AND owner_epoch=?
1250
+ AND state<>'activated'`,
1251
+ )
1252
+ .run(watermark, nowMs, partition, name, ownerEpoch);
1253
+ this.db.exec('COMMIT');
1254
+ return Number(updated.changes) === 1 ? 'activated' : 'stale';
1255
+ } catch (error) {
1256
+ this.db.exec('ROLLBACK');
1257
+ throw error;
1258
+ }
1259
+ });
1260
+ }
1261
+
1262
+ async claimCheckpoint(
1263
+ partition: string,
1264
+ name: string,
1265
+ schemaVersion: number,
1266
+ nowMs: number,
1267
+ ): Promise<StoredCheckpoint> {
1268
+ return this.#serializeWrite(() => {
1269
+ const claimed = this.db
1270
+ .query<SqliteCheckpointRecord, (string | number)[]>(
1271
+ `UPDATE sync_backfill_checkpoints
1272
+ SET owner_epoch=owner_epoch+1,
1273
+ state=CASE WHEN state='declared' THEN 'backfilling' ELSE state END,
1274
+ schema_version=?, updated_at_ms=?
1275
+ WHERE partition=? AND name=? AND state<>'activated'
1276
+ RETURNING partition, name, schema_version, state, watermark,
1277
+ owner_epoch, observed_rows, updated_at_ms`,
1278
+ )
1279
+ .get(schemaVersion, nowMs, partition, name);
1280
+ if (claimed === null) {
1281
+ throw new StorageQueryError('sync.storage.checkpoint_not_declared');
1282
+ }
1283
+ return toStoredCheckpoint(claimed);
1284
+ });
1285
+ }
1286
+
1287
+ async advanceCheckpoint(
1288
+ partition: string,
1289
+ name: string,
1290
+ ownerEpoch: number,
1291
+ watermark: number,
1292
+ observedRows: number,
1293
+ nowMs: number,
1294
+ ): Promise<boolean> {
1295
+ return this.#serializeWrite(() => {
1296
+ const result = this.db
1297
+ .query(
1298
+ `UPDATE sync_backfill_checkpoints
1299
+ SET watermark=?, observed_rows=?, updated_at_ms=?
1300
+ WHERE partition=? AND name=? AND owner_epoch=?`,
1301
+ )
1302
+ .run(watermark, observedRows, nowMs, partition, name, ownerEpoch);
1303
+ return Number(result.changes) === 1;
1304
+ });
1305
+ }
1306
+
1307
+ async sourceCoverageSeq(
1308
+ partition: string,
1309
+ tables: readonly string[],
1310
+ ): Promise<number> {
1311
+ if (tables.length === 0) return 0;
1312
+ const row = this.db
1313
+ .query<{ seq: number | null }, (string | number)[]>(
1314
+ `SELECT max(commit_seq) AS seq FROM sync_changes
1315
+ WHERE partition=? AND tbl IN (${placeholders(tables.length)})`,
1316
+ )
1317
+ .get(partition, ...tables);
1318
+ return row?.seq ?? 0;
1319
+ }
1320
+
1321
+ #hasSourceChangesAbove(
1322
+ partition: string,
1323
+ tables: readonly string[],
1324
+ seq: number,
1325
+ ): 'clean' | 'changed' | 'unverifiable' {
1326
+ // Read the horizon before the window scan: a horizon past `seq` means the
1327
+ // history that would answer the question is gone, so an empty scan proves
1328
+ // nothing and must not be reported as `clean`.
1329
+ const horizon =
1330
+ this.db
1331
+ .query<{ horizon_seq: number }, [string]>(
1332
+ 'SELECT horizon_seq FROM sync_partitions WHERE partition=?',
1333
+ )
1334
+ .get(partition)?.horizon_seq ?? 0;
1335
+ if (tables.length === 0) return 'clean';
1336
+ const hit = this.db
1337
+ .query<{ hit: number }, (string | number)[]>(
1338
+ `SELECT 1 AS hit FROM sync_changes
1339
+ WHERE partition=? AND tbl IN (${placeholders(tables.length)})
1340
+ AND commit_seq>? LIMIT 1`,
1341
+ )
1342
+ .get(partition, ...tables, seq);
1343
+ if (hit !== null && hit !== undefined) return 'changed';
1344
+ return horizon > seq ? 'unverifiable' : 'clean';
1345
+ }
1346
+
1347
+ async hasSourceChangesAbove(
1348
+ partition: string,
1349
+ tables: readonly string[],
1350
+ seq: number,
1351
+ ): Promise<'clean' | 'changed' | 'unverifiable'> {
1352
+ return this.#hasSourceChangesAbove(partition, tables, seq);
1353
+ }
1354
+
897
1355
  async getRow(
898
1356
  partition: string,
899
1357
  table: string,
@@ -26,7 +26,11 @@ export type StorageQueryErrorCode =
26
26
  | 'sync.storage.invalid_limit'
27
27
  | 'sync.storage.prune_epoch_mismatch'
28
28
  | 'sync.storage.partition_unregistered'
29
- | 'sync.storage.invalid_prune_cursor';
29
+ | 'sync.storage.invalid_prune_cursor'
30
+ | 'sync.storage.checkpoint_not_declared'
31
+ | 'sync.storage.checkpoint_unsupported'
32
+ | 'sync.storage.checkpoint_fence_missing'
33
+ | 'sync.storage.checkpoint_incomplete';
30
34
 
31
35
  const STORAGE_QUERY_MESSAGES: Readonly<Record<StorageQueryErrorCode, string>> =
32
36
  {
@@ -44,6 +48,14 @@ const STORAGE_QUERY_MESSAGES: Readonly<Record<StorageQueryErrorCode, string>> =
44
48
  'pruning requires a registered partition',
45
49
  'sync.storage.invalid_prune_cursor':
46
50
  'pruning requires a non-negative safe integer cursor and a non-empty log epoch',
51
+ 'sync.storage.checkpoint_not_declared':
52
+ 'checkpoint claim requires a declared, not yet activated row',
53
+ 'sync.storage.checkpoint_unsupported':
54
+ 'this storage backend cannot declare backfill checkpoints',
55
+ 'sync.storage.checkpoint_fence_missing':
56
+ 'checkpoint activation requires the writer fence raised at declaration',
57
+ 'sync.storage.checkpoint_incomplete':
58
+ 'a declared backfill checkpoint is not activated and this caller did not declare it',
47
59
  'sync.storage.scan_requires_scope':
48
60
  'scope-indexed row scans require at least one scope variable',
49
61
  'sync.storage.index_not_found':