@syncular/server 0.22.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/admin.d.ts +3 -1
- package/dist/admin.js +7 -1
- package/dist/blob-handlers.js +1 -1
- package/dist/context.d.ts +8 -1
- package/dist/d1-storage.d.ts +11 -3
- package/dist/d1-storage.js +84 -2
- package/dist/errors.js +10 -0
- package/dist/handler.js +50 -4
- package/dist/operations.js +3 -3
- package/dist/postgres-storage.d.ts +12 -4
- package/dist/postgres-storage.js +266 -6
- package/dist/push.js +18 -0
- package/dist/readiness.d.ts +9 -2
- package/dist/readiness.js +27 -2
- package/dist/realtime.js +20 -1
- package/dist/segment-download.js +8 -0
- package/dist/sqlite-dialect.d.ts +1 -1
- package/dist/sqlite-dialect.js +15 -0
- package/dist/sqlite-storage.d.ts +11 -3
- package/dist/sqlite-storage.js +296 -33
- package/dist/storage-errors.d.ts +1 -1
- package/dist/storage-errors.js +4 -0
- package/dist/storage.d.ts +127 -2
- package/dist/storage.js +40 -1
- package/package.json +2 -2
- package/src/admin.ts +13 -1
- package/src/blob-handlers.ts +1 -1
- package/src/context.ts +8 -0
- package/src/d1-storage.ts +143 -1
- package/src/errors.ts +10 -0
- package/src/handler.ts +62 -4
- package/src/operations.ts +17 -13
- package/src/postgres-storage.ts +430 -3
- package/src/push.ts +18 -0
- package/src/readiness.ts +39 -4
- package/src/realtime.ts +22 -1
- package/src/segment-download.ts +16 -0
- package/src/sqlite-dialect.ts +15 -0
- package/src/sqlite-storage.ts +439 -36
- package/src/storage-errors.ts +13 -1
- package/src/storage.ts +218 -1
package/src/realtime.ts
CHANGED
|
@@ -600,7 +600,28 @@ export class RealtimeSession {
|
|
|
600
600
|
// begin its next round as soon as END *arrives* (§8.7 rule 3), so
|
|
601
601
|
// in-flight must end with the send, not with generator cleanup.
|
|
602
602
|
const iterator = stream[Symbol.asyncIterator]();
|
|
603
|
-
let step
|
|
603
|
+
let step: IteratorResult<Uint8Array>;
|
|
604
|
+
try {
|
|
605
|
+
step = await iterator.next();
|
|
606
|
+
} catch (error) {
|
|
607
|
+
// The buffered read-verify refusal is thrown on the generator's first
|
|
608
|
+
// `next()`, before any chunk is yielded (the generator builds the whole
|
|
609
|
+
// response, then verifies). Nothing has escaped, so it is delivered as
|
|
610
|
+
// the same §8.4 ERROR frame HTTP returns rather than a silent close. A
|
|
611
|
+
// throw after the first chunk still takes the §8.7 violation path.
|
|
612
|
+
if (error instanceof DecodeError || error instanceof SyncError) {
|
|
613
|
+
const sync =
|
|
614
|
+
error instanceof SyncError
|
|
615
|
+
? error
|
|
616
|
+
: syncError(error.code, error.message);
|
|
617
|
+
finishRound();
|
|
618
|
+
await this.#sendRoundChunk(
|
|
619
|
+
errorResponseBytes(sync, this.wireVersion, this.logEpoch),
|
|
620
|
+
);
|
|
621
|
+
return;
|
|
622
|
+
}
|
|
623
|
+
throw error;
|
|
624
|
+
}
|
|
604
625
|
while (!step.done) {
|
|
605
626
|
const chunk = step.value;
|
|
606
627
|
step = await iterator.next();
|
package/src/segment-download.ts
CHANGED
|
@@ -196,6 +196,22 @@ async function downloadSegment(
|
|
|
196
196
|
}
|
|
197
197
|
const publication = candidates[candidates.length - 1]!;
|
|
198
198
|
|
|
199
|
+
// RFC 0007: a descriptor minted before this partition declared a backfill
|
|
200
|
+
// checkpoint is not a complete window once the projection becomes
|
|
201
|
+
// checkpointed. SPEC §9 mints segments per schema version, so refuse and
|
|
202
|
+
// let the client re-pull for a fresh descriptor.
|
|
203
|
+
const checkpoints = await ctx.storage.readCheckpoints(ctx.partition);
|
|
204
|
+
if (
|
|
205
|
+
checkpoints.some(
|
|
206
|
+
(checkpoint) => checkpoint.schemaVersion > publication.schemaVersion,
|
|
207
|
+
)
|
|
208
|
+
) {
|
|
209
|
+
throw syncError(
|
|
210
|
+
'sync.segment_expired',
|
|
211
|
+
'segment predates the partition backfill checkpoint — re-pull (§5.1)',
|
|
212
|
+
);
|
|
213
|
+
}
|
|
214
|
+
|
|
199
215
|
return {
|
|
200
216
|
// The returned record is projected onto the selected publication, so its
|
|
201
217
|
// `scopeDigests`/`publications` compatibility view cannot union a grant
|
package/src/sqlite-dialect.ts
CHANGED
|
@@ -66,6 +66,7 @@ CREATE TABLE IF NOT EXISTS sync_commits(
|
|
|
66
66
|
partition TEXT NOT NULL, commit_seq INTEGER NOT NULL,
|
|
67
67
|
client_id TEXT NOT NULL, client_commit_id TEXT NOT NULL,
|
|
68
68
|
actor_id TEXT NOT NULL, created_at_ms INTEGER NOT NULL,
|
|
69
|
+
writer_version INTEGER,
|
|
69
70
|
PRIMARY KEY(partition, commit_seq)
|
|
70
71
|
);
|
|
71
72
|
CREATE INDEX IF NOT EXISTS sync_commits_by_time
|
|
@@ -127,6 +128,20 @@ CREATE TABLE IF NOT EXISTS sync_tombstones(
|
|
|
127
128
|
commit_seq INTEGER NOT NULL,
|
|
128
129
|
PRIMARY KEY(partition, tbl, row_id)
|
|
129
130
|
);
|
|
131
|
+
CREATE TABLE IF NOT EXISTS sync_backfill_checkpoints(
|
|
132
|
+
partition TEXT NOT NULL, name TEXT NOT NULL,
|
|
133
|
+
schema_version INTEGER NOT NULL,
|
|
134
|
+
state TEXT NOT NULL CHECK(state IN ('declared','backfilling','activated')),
|
|
135
|
+
watermark INTEGER NOT NULL DEFAULT 0,
|
|
136
|
+
owner_epoch INTEGER NOT NULL DEFAULT 0,
|
|
137
|
+
observed_rows INTEGER NOT NULL DEFAULT 0,
|
|
138
|
+
updated_at_ms INTEGER NOT NULL,
|
|
139
|
+
PRIMARY KEY(partition, name)
|
|
140
|
+
);
|
|
141
|
+
CREATE TABLE IF NOT EXISTS sync_writer_fence(
|
|
142
|
+
partition TEXT PRIMARY KEY,
|
|
143
|
+
required_writer_version INTEGER NOT NULL
|
|
144
|
+
);
|
|
130
145
|
`;
|
|
131
146
|
|
|
132
147
|
/** Split the DDL into individual statements (D1 applies them one by one). */
|
package/src/sqlite-storage.ts
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { validateCommitPruneQuery } from './prune';
|
|
2
2
|
import { StorageQueryError } from './storage-errors';
|
|
3
3
|
import type { CommitPruneQuery, CommitPruneResult } from './storage';
|
|
4
|
+
import { serveGateRefusal } from './storage';
|
|
5
|
+
import { serveNotReadyError } from './readiness';
|
|
4
6
|
/**
|
|
5
7
|
* SQLite server storage over the shared synchronous driver.
|
|
6
8
|
*
|
|
@@ -43,6 +45,7 @@ import { matchesEffective } from './scopes';
|
|
|
43
45
|
import {
|
|
44
46
|
collectCommitWindowPage,
|
|
45
47
|
deserializePushResult,
|
|
48
|
+
placeholders,
|
|
46
49
|
SQLITE_DDL,
|
|
47
50
|
type SqliteCommitWindowRecord,
|
|
48
51
|
type SqliteRowRecord,
|
|
@@ -57,6 +60,7 @@ import type {
|
|
|
57
60
|
AuthoritativeQueryRequest,
|
|
58
61
|
AuthoritativeQueryResult,
|
|
59
62
|
AuthoritativeQueryValue,
|
|
63
|
+
CheckpointDeclaration,
|
|
60
64
|
ClientCursorInfo,
|
|
61
65
|
ClientRecord,
|
|
62
66
|
ClientSubscription,
|
|
@@ -78,8 +82,10 @@ import type {
|
|
|
78
82
|
ScopeActivityQuery,
|
|
79
83
|
ScopeCommitActivity,
|
|
80
84
|
ServerStorage,
|
|
85
|
+
ServeGate,
|
|
81
86
|
StorageTransaction,
|
|
82
87
|
StoredCommit,
|
|
88
|
+
StoredCheckpoint,
|
|
83
89
|
StoredPushResult,
|
|
84
90
|
StoredReaction,
|
|
85
91
|
StoredRow,
|
|
@@ -90,6 +96,30 @@ import {
|
|
|
90
96
|
} from './storage-errors';
|
|
91
97
|
import { assertScopeIndexedScan, resolveIndexRowScan } from './storage-query';
|
|
92
98
|
|
|
99
|
+
interface SqliteCheckpointRecord {
|
|
100
|
+
partition: string;
|
|
101
|
+
name: string;
|
|
102
|
+
schema_version: number;
|
|
103
|
+
state: StoredCheckpoint['state'];
|
|
104
|
+
watermark: number;
|
|
105
|
+
owner_epoch: number;
|
|
106
|
+
observed_rows: number;
|
|
107
|
+
updated_at_ms: number;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
function toStoredCheckpoint(record: SqliteCheckpointRecord): StoredCheckpoint {
|
|
111
|
+
return {
|
|
112
|
+
partition: record.partition,
|
|
113
|
+
name: record.name,
|
|
114
|
+
schemaVersion: record.schema_version,
|
|
115
|
+
state: record.state,
|
|
116
|
+
watermark: record.watermark,
|
|
117
|
+
ownerEpoch: record.owner_epoch,
|
|
118
|
+
observedRows: record.observed_rows,
|
|
119
|
+
updatedAtMs: record.updated_at_ms,
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
|
|
93
123
|
interface SqliteReactionRecord {
|
|
94
124
|
partition: string;
|
|
95
125
|
idempotency_key: string;
|
|
@@ -140,6 +170,8 @@ function toStoredReaction(record: SqliteReactionRecord): StoredReaction {
|
|
|
140
170
|
class SqliteTransaction implements StorageTransaction {
|
|
141
171
|
#storage: SqliteServerStorage;
|
|
142
172
|
#partition: string;
|
|
173
|
+
/** Schema version this transaction writes; rides every appended commit row. */
|
|
174
|
+
#writerVersion: number;
|
|
143
175
|
#open = true;
|
|
144
176
|
#pushApplySavepoint = false;
|
|
145
177
|
readonly #release: () => void;
|
|
@@ -147,10 +179,12 @@ class SqliteTransaction implements StorageTransaction {
|
|
|
147
179
|
constructor(
|
|
148
180
|
storage: SqliteServerStorage,
|
|
149
181
|
partition: string,
|
|
182
|
+
writerVersion: number,
|
|
150
183
|
release: () => void,
|
|
151
184
|
) {
|
|
152
185
|
this.#storage = storage;
|
|
153
186
|
this.#partition = partition;
|
|
187
|
+
this.#writerVersion = writerVersion;
|
|
154
188
|
this.#release = release;
|
|
155
189
|
storage.db.exec('BEGIN IMMEDIATE');
|
|
156
190
|
}
|
|
@@ -217,6 +251,31 @@ class SqliteTransaction implements StorageTransaction {
|
|
|
217
251
|
this.#pushApplySavepoint = true;
|
|
218
252
|
}
|
|
219
253
|
|
|
254
|
+
async advanceCheckpoint(
|
|
255
|
+
name: string,
|
|
256
|
+
ownerEpoch: number,
|
|
257
|
+
watermark: number,
|
|
258
|
+
observedRows: number,
|
|
259
|
+
nowMs: number,
|
|
260
|
+
): Promise<boolean> {
|
|
261
|
+
this.#assertOpen();
|
|
262
|
+
const result = this.#storage.db
|
|
263
|
+
.query(
|
|
264
|
+
`UPDATE sync_backfill_checkpoints
|
|
265
|
+
SET watermark=?, observed_rows=?, updated_at_ms=?
|
|
266
|
+
WHERE partition=? AND name=? AND owner_epoch=?`,
|
|
267
|
+
)
|
|
268
|
+
.run(watermark, observedRows, nowMs, this.#partition, name, ownerEpoch);
|
|
269
|
+
return Number(result.changes) === 1;
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
readServeGate(runningSchemaVersion: number): Promise<ServeGate> {
|
|
273
|
+
this.#assertOpen();
|
|
274
|
+
// Same shared connection: this read runs inside this transaction's
|
|
275
|
+
// BEGIN IMMEDIATE, so the gate and the write share one transaction.
|
|
276
|
+
return this.#storage.readServeGate(this.#partition, runningSchemaVersion);
|
|
277
|
+
}
|
|
278
|
+
|
|
220
279
|
async commitRejectedPushResult(
|
|
221
280
|
clientId: string,
|
|
222
281
|
clientCommitId: string,
|
|
@@ -302,7 +361,7 @@ class SqliteTransaction implements StorageTransaction {
|
|
|
302
361
|
'UPDATE sync_partitions SET max_commit_seq=? WHERE partition=?',
|
|
303
362
|
).run(commitSeq, p);
|
|
304
363
|
db.query(
|
|
305
|
-
'INSERT INTO sync_commits(partition, commit_seq, client_id, client_commit_id, actor_id, created_at_ms) VALUES (
|
|
364
|
+
'INSERT INTO sync_commits(partition, commit_seq, client_id, client_commit_id, actor_id, created_at_ms, writer_version) VALUES (?,?,?,?,?,?,?)',
|
|
306
365
|
).run(
|
|
307
366
|
p,
|
|
308
367
|
commitSeq,
|
|
@@ -310,6 +369,7 @@ class SqliteTransaction implements StorageTransaction {
|
|
|
310
369
|
commit.clientCommitId,
|
|
311
370
|
commit.actorId,
|
|
312
371
|
commit.createdAtMs,
|
|
372
|
+
this.#writerVersion,
|
|
313
373
|
);
|
|
314
374
|
commit.changes.forEach((change, idx) => {
|
|
315
375
|
db.query(
|
|
@@ -455,6 +515,32 @@ export class SqliteServerStorage implements ServerStorage {
|
|
|
455
515
|
'ALTER TABLE sync_clients ADD COLUMN wire_version INTEGER NOT NULL DEFAULT 1',
|
|
456
516
|
);
|
|
457
517
|
}
|
|
518
|
+
const commitColumns = this.db
|
|
519
|
+
.query<{ name: string }, []>('PRAGMA table_info("sync_commits")')
|
|
520
|
+
.all();
|
|
521
|
+
if (!commitColumns.some((column) => column.name === 'writer_version')) {
|
|
522
|
+
// Nullable by design: an old writer omits the column and the trigger's
|
|
523
|
+
// explicit NULL test rejects it. `ADD COLUMN NOT NULL` is unavailable in
|
|
524
|
+
// SQLite, and a default would hand old writers a passing value.
|
|
525
|
+
this.db.exec(
|
|
526
|
+
'ALTER TABLE sync_commits ADD COLUMN writer_version INTEGER',
|
|
527
|
+
);
|
|
528
|
+
}
|
|
529
|
+
// The fence is database-side: an old binary's INSERT never reaches JS. An
|
|
530
|
+
// absent `sync_writer_fence` row makes the WHEN clause false, so a
|
|
531
|
+
// partition with no barrier behaves exactly as before.
|
|
532
|
+
this.db.exec(`CREATE TRIGGER IF NOT EXISTS sync_commits_writer_fence
|
|
533
|
+
BEFORE INSERT ON sync_commits
|
|
534
|
+
FOR EACH ROW
|
|
535
|
+
WHEN EXISTS (
|
|
536
|
+
SELECT 1 FROM sync_writer_fence
|
|
537
|
+
WHERE partition = NEW.partition
|
|
538
|
+
AND (NEW.writer_version IS NULL
|
|
539
|
+
OR NEW.writer_version < required_writer_version)
|
|
540
|
+
)
|
|
541
|
+
BEGIN
|
|
542
|
+
SELECT RAISE(ABORT, 'sync.storage.writer_fence_rejected');
|
|
543
|
+
END`);
|
|
458
544
|
}
|
|
459
545
|
|
|
460
546
|
/** Resolve a table's compiled schema; row operations require `ensureSchema`. */
|
|
@@ -468,9 +554,34 @@ export class SqliteServerStorage implements ServerStorage {
|
|
|
468
554
|
return table;
|
|
469
555
|
}
|
|
470
556
|
|
|
471
|
-
async ensureSchema(
|
|
557
|
+
async ensureSchema(
|
|
558
|
+
schema: CompiledSchema,
|
|
559
|
+
checkpoints?: readonly CheckpointDeclaration[],
|
|
560
|
+
): Promise<void> {
|
|
472
561
|
// Memoized fast path: same instance, same schema version.
|
|
473
562
|
if (this.#schemaVersion === schema.version) return;
|
|
563
|
+
// The barrier reads stored state: a checkpoint at this schema version that
|
|
564
|
+
// is not activated and that this caller did not declare means this process
|
|
565
|
+
// must not serve. An omitted or empty declaration set declares nothing and
|
|
566
|
+
// is refused here, never treated as a bypass.
|
|
567
|
+
const declarations = checkpoints ?? [];
|
|
568
|
+
const incomplete = this.db
|
|
569
|
+
.query<{ partition: string; name: string }, [number]>(
|
|
570
|
+
`SELECT partition, name FROM sync_backfill_checkpoints
|
|
571
|
+
WHERE schema_version=? AND state<>'activated'`,
|
|
572
|
+
)
|
|
573
|
+
.all(schema.version)
|
|
574
|
+
.filter(
|
|
575
|
+
(row) =>
|
|
576
|
+
!declarations.some(
|
|
577
|
+
(declaration) =>
|
|
578
|
+
declaration.partition === row.partition &&
|
|
579
|
+
declaration.name === row.name,
|
|
580
|
+
),
|
|
581
|
+
);
|
|
582
|
+
if (incomplete.length > 0) {
|
|
583
|
+
throw new StorageQueryError('sync.storage.checkpoint_incomplete');
|
|
584
|
+
}
|
|
474
585
|
this.db.exec(SCHEMA_META_DDL_SQLITE);
|
|
475
586
|
const marker = this.db
|
|
476
587
|
.query<{ schema_version: number; layouts: string }, []>(
|
|
@@ -567,41 +678,64 @@ export class SqliteServerStorage implements ServerStorage {
|
|
|
567
678
|
);
|
|
568
679
|
}
|
|
569
680
|
}
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
681
|
+
// The migration rewrites application rows (`#rewriteRows`) with no
|
|
682
|
+
// commit log entry. Serialize it through the same writer queue as
|
|
683
|
+
// pushes and prune (`#serializeWrite`) rather than trusting the
|
|
684
|
+
// migration as privileged; `BEGIN IMMEDIATE` then owns SQLite's writer
|
|
685
|
+
// lock for the whole rewrite.
|
|
686
|
+
await this.#serializeWrite(() => {
|
|
687
|
+
this.db.exec('BEGIN IMMEDIATE');
|
|
688
|
+
try {
|
|
689
|
+
for (const tableName of retiredTables) {
|
|
690
|
+
this.db
|
|
691
|
+
.query('DELETE FROM sync_row_scopes WHERE tbl=?')
|
|
692
|
+
.run(tableName);
|
|
693
|
+
this.db
|
|
694
|
+
.query('DELETE FROM sync_blob_refs WHERE tbl=?')
|
|
695
|
+
.run(tableName);
|
|
696
|
+
this.db.exec(dropTableDdl(tableName));
|
|
697
|
+
}
|
|
698
|
+
for (const statement of schemaDdl(
|
|
699
|
+
schema,
|
|
700
|
+
existing,
|
|
701
|
+
'sqlite',
|
|
702
|
+
existingIndexes,
|
|
703
|
+
)) {
|
|
704
|
+
this.db.exec(statement);
|
|
705
|
+
}
|
|
706
|
+
for (const table of schema.tables.values()) {
|
|
707
|
+
const oldLayout = layouts[table.name];
|
|
708
|
+
const plan = rewritePlan(
|
|
709
|
+
table,
|
|
710
|
+
oldLayout,
|
|
711
|
+
existing.get(table.name),
|
|
712
|
+
);
|
|
713
|
+
if (!plan.migrate && !plan.backfill) continue;
|
|
714
|
+
this.#rewriteRows(table, plan.migrate ? oldLayout : undefined);
|
|
715
|
+
}
|
|
576
716
|
this.db
|
|
577
|
-
.query(
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
this
|
|
717
|
+
.query(
|
|
718
|
+
'INSERT INTO sync_schema_meta(id, schema_version, layouts) VALUES (1, ?, ?) ON CONFLICT(id) DO UPDATE SET schema_version=excluded.schema_version, layouts=excluded.layouts',
|
|
719
|
+
)
|
|
720
|
+
.run(schema.version, layoutsOf(schema));
|
|
721
|
+
// The host's declared checkpoints install in the schema-bump
|
|
722
|
+
// transaction, so no observer sees the bumped marker without the
|
|
723
|
+
// fence, or the fence without its declaration row.
|
|
724
|
+
const installedAtMs = Date.now();
|
|
725
|
+
for (const declaration of checkpoints ?? []) {
|
|
726
|
+
this.#installCheckpoint(
|
|
727
|
+
declaration.partition,
|
|
728
|
+
declaration.name,
|
|
729
|
+
declaration.schemaVersion,
|
|
730
|
+
installedAtMs,
|
|
731
|
+
);
|
|
732
|
+
}
|
|
733
|
+
this.db.exec('COMMIT');
|
|
734
|
+
} catch (error) {
|
|
735
|
+
this.db.exec('ROLLBACK');
|
|
736
|
+
throw error;
|
|
594
737
|
}
|
|
595
|
-
|
|
596
|
-
.query(
|
|
597
|
-
'INSERT INTO sync_schema_meta(id, schema_version, layouts) VALUES (1, ?, ?) ON CONFLICT(id) DO UPDATE SET schema_version=excluded.schema_version, layouts=excluded.layouts',
|
|
598
|
-
)
|
|
599
|
-
.run(schema.version, layoutsOf(schema));
|
|
600
|
-
this.db.exec('COMMIT');
|
|
601
|
-
} catch (error) {
|
|
602
|
-
this.db.exec('ROLLBACK');
|
|
603
|
-
throw error;
|
|
604
|
-
}
|
|
738
|
+
});
|
|
605
739
|
}
|
|
606
740
|
this.#tables = schema.tables;
|
|
607
741
|
this.#schemaVersion = schema.version;
|
|
@@ -764,7 +898,12 @@ export class SqliteServerStorage implements ServerStorage {
|
|
|
764
898
|
});
|
|
765
899
|
await previous;
|
|
766
900
|
try {
|
|
767
|
-
return new SqliteTransaction(
|
|
901
|
+
return new SqliteTransaction(
|
|
902
|
+
this,
|
|
903
|
+
partition,
|
|
904
|
+
this.#schemaVersion ?? 0,
|
|
905
|
+
release,
|
|
906
|
+
);
|
|
768
907
|
} catch (error) {
|
|
769
908
|
release();
|
|
770
909
|
throw error;
|
|
@@ -809,6 +948,7 @@ export class SqliteServerStorage implements ServerStorage {
|
|
|
809
948
|
async queryAuthoritative(
|
|
810
949
|
partition: string,
|
|
811
950
|
query: AuthoritativeQueryRequest,
|
|
951
|
+
checkpoints?: readonly CheckpointDeclaration[],
|
|
812
952
|
): Promise<AuthoritativeQueryResult> {
|
|
813
953
|
if (this.#tables === undefined) {
|
|
814
954
|
throw new Error(
|
|
@@ -834,6 +974,13 @@ export class SqliteServerStorage implements ServerStorage {
|
|
|
834
974
|
try {
|
|
835
975
|
this.db.exec('BEGIN');
|
|
836
976
|
open = true;
|
|
977
|
+
// RFC 0007: evaluate the gate on this connection after BEGIN. SQLite is
|
|
978
|
+
// single-writer and the shared connection serializes writers, so the
|
|
979
|
+
// gate read and the query read cannot straddle a migration.
|
|
980
|
+
const runningVersion = this.#schemaVersion ?? 0;
|
|
981
|
+
const gate = await this.readServeGate(partition, runningVersion);
|
|
982
|
+
const refusal = serveGateRefusal(gate, runningVersion, checkpoints);
|
|
983
|
+
if (refusal !== undefined) throw serveNotReadyError(refusal);
|
|
837
984
|
const rows = this.db
|
|
838
985
|
.query<Readonly<Record<string, unknown>>, AuthoritativeQueryValue[]>(
|
|
839
986
|
prepared.sql,
|
|
@@ -949,6 +1096,262 @@ export class SqliteServerStorage implements ServerStorage {
|
|
|
949
1096
|
return row?.seq ?? 0;
|
|
950
1097
|
}
|
|
951
1098
|
|
|
1099
|
+
async readServeGate(
|
|
1100
|
+
partition: string,
|
|
1101
|
+
runningSchemaVersion: number,
|
|
1102
|
+
): Promise<ServeGate> {
|
|
1103
|
+
const marker = this.db
|
|
1104
|
+
.query<{ schema_version: number }, []>(
|
|
1105
|
+
'SELECT schema_version FROM sync_schema_meta WHERE id=1',
|
|
1106
|
+
)
|
|
1107
|
+
.get();
|
|
1108
|
+
const epoch = this.db
|
|
1109
|
+
.query<{ log_epoch: string }, [string]>(
|
|
1110
|
+
'SELECT log_epoch FROM sync_partition_registry WHERE partition=?',
|
|
1111
|
+
)
|
|
1112
|
+
.get(partition);
|
|
1113
|
+
const pending = this.db
|
|
1114
|
+
.query<{ partition: string; name: string }, [number]>(
|
|
1115
|
+
`SELECT partition, name FROM sync_backfill_checkpoints
|
|
1116
|
+
WHERE schema_version=? AND state<>'activated'`,
|
|
1117
|
+
)
|
|
1118
|
+
.all(runningSchemaVersion);
|
|
1119
|
+
return {
|
|
1120
|
+
storedSchemaVersion: marker?.schema_version ?? 0,
|
|
1121
|
+
logEpoch: epoch?.log_epoch,
|
|
1122
|
+
pending: pending.map((row) => ({
|
|
1123
|
+
partition: row.partition,
|
|
1124
|
+
name: row.name,
|
|
1125
|
+
})),
|
|
1126
|
+
};
|
|
1127
|
+
}
|
|
1128
|
+
|
|
1129
|
+
async readCheckpoints(partition: string): Promise<StoredCheckpoint[]> {
|
|
1130
|
+
return this.db
|
|
1131
|
+
.query<SqliteCheckpointRecord, [string]>(
|
|
1132
|
+
`SELECT partition, name, schema_version, state, watermark,
|
|
1133
|
+
owner_epoch, observed_rows, updated_at_ms
|
|
1134
|
+
FROM sync_backfill_checkpoints WHERE partition=? ORDER BY name`,
|
|
1135
|
+
)
|
|
1136
|
+
.all(partition)
|
|
1137
|
+
.map(toStoredCheckpoint);
|
|
1138
|
+
}
|
|
1139
|
+
|
|
1140
|
+
/**
|
|
1141
|
+
* Insert the `declared` row and raise the fence, inside the caller's open
|
|
1142
|
+
* transaction. Shared by `declareCheckpoint` and the `ensureSchema`
|
|
1143
|
+
* migration so the schema bump and the fence commit together. Never lowers a
|
|
1144
|
+
* fence and never re-declares an existing row.
|
|
1145
|
+
*/
|
|
1146
|
+
#installCheckpoint(
|
|
1147
|
+
partition: string,
|
|
1148
|
+
name: string,
|
|
1149
|
+
schemaVersion: number,
|
|
1150
|
+
nowMs: number,
|
|
1151
|
+
): void {
|
|
1152
|
+
this.db
|
|
1153
|
+
.query(
|
|
1154
|
+
`INSERT INTO sync_backfill_checkpoints(
|
|
1155
|
+
partition, name, schema_version, state, watermark,
|
|
1156
|
+
owner_epoch, observed_rows, updated_at_ms)
|
|
1157
|
+
VALUES (?,?,?,'declared',0,0,0,?)
|
|
1158
|
+
ON CONFLICT(partition, name) DO NOTHING`,
|
|
1159
|
+
)
|
|
1160
|
+
.run(partition, name, schemaVersion, nowMs);
|
|
1161
|
+
this.db
|
|
1162
|
+
.query(
|
|
1163
|
+
`INSERT INTO sync_writer_fence(partition, required_writer_version)
|
|
1164
|
+
VALUES (?,?)
|
|
1165
|
+
ON CONFLICT(partition) DO UPDATE SET
|
|
1166
|
+
required_writer_version=max(
|
|
1167
|
+
sync_writer_fence.required_writer_version,
|
|
1168
|
+
excluded.required_writer_version)`,
|
|
1169
|
+
)
|
|
1170
|
+
.run(partition, schemaVersion);
|
|
1171
|
+
}
|
|
1172
|
+
|
|
1173
|
+
async declareCheckpoint(
|
|
1174
|
+
partition: string,
|
|
1175
|
+
name: string,
|
|
1176
|
+
schemaVersion: number,
|
|
1177
|
+
nowMs: number,
|
|
1178
|
+
): Promise<StoredCheckpoint> {
|
|
1179
|
+
// One transaction: the checkpoint becomes visible and the fence that
|
|
1180
|
+
// protects it is raised together. The fence is raised at declaration, not
|
|
1181
|
+
// at activation, so old writers are rejected for the whole backfill.
|
|
1182
|
+
return this.#serializeWrite(() => {
|
|
1183
|
+
this.db.exec('BEGIN IMMEDIATE');
|
|
1184
|
+
try {
|
|
1185
|
+
this.#installCheckpoint(partition, name, schemaVersion, nowMs);
|
|
1186
|
+
const row = this.db
|
|
1187
|
+
.query<SqliteCheckpointRecord, [string, string]>(
|
|
1188
|
+
`SELECT partition, name, schema_version, state, watermark,
|
|
1189
|
+
owner_epoch, observed_rows, updated_at_ms
|
|
1190
|
+
FROM sync_backfill_checkpoints WHERE partition=? AND name=?`,
|
|
1191
|
+
)
|
|
1192
|
+
.get(partition, name);
|
|
1193
|
+
if (row === null)
|
|
1194
|
+
throw new StorageQueryError('sync.storage.checkpoint_not_declared');
|
|
1195
|
+
this.db.exec('COMMIT');
|
|
1196
|
+
return toStoredCheckpoint(row);
|
|
1197
|
+
} catch (error) {
|
|
1198
|
+
this.db.exec('ROLLBACK');
|
|
1199
|
+
throw error;
|
|
1200
|
+
}
|
|
1201
|
+
});
|
|
1202
|
+
}
|
|
1203
|
+
|
|
1204
|
+
async activateCheckpoint(
|
|
1205
|
+
partition: string,
|
|
1206
|
+
name: string,
|
|
1207
|
+
ownerEpoch: number,
|
|
1208
|
+
watermark: number,
|
|
1209
|
+
sources: readonly string[],
|
|
1210
|
+
nowMs: number,
|
|
1211
|
+
): Promise<'activated' | 'stale' | 'unverifiable'> {
|
|
1212
|
+
return this.#serializeWrite(() => {
|
|
1213
|
+
// `BEGIN IMMEDIATE` is SQLite's partition write lock; the same writer
|
|
1214
|
+
// lock a push takes. Nothing may observe activation without the fence.
|
|
1215
|
+
this.db.exec('BEGIN IMMEDIATE');
|
|
1216
|
+
try {
|
|
1217
|
+
const fence = this.db
|
|
1218
|
+
.query<{ required_writer_version: number }, [string]>(
|
|
1219
|
+
'SELECT required_writer_version FROM sync_writer_fence WHERE partition=?',
|
|
1220
|
+
)
|
|
1221
|
+
.get(partition);
|
|
1222
|
+
const checkpoint = this.db
|
|
1223
|
+
.query<{ schema_version: number }, [string, string]>(
|
|
1224
|
+
'SELECT schema_version FROM sync_backfill_checkpoints WHERE partition=? AND name=?',
|
|
1225
|
+
)
|
|
1226
|
+
.get(partition, name);
|
|
1227
|
+
if (
|
|
1228
|
+
fence === null ||
|
|
1229
|
+
checkpoint === null ||
|
|
1230
|
+
fence.required_writer_version < checkpoint.schema_version
|
|
1231
|
+
) {
|
|
1232
|
+
// The barrier is not installed. Activating would certify a state old
|
|
1233
|
+
// writers can still decay; refuse instead of silently proceeding.
|
|
1234
|
+
throw new StorageQueryError('sync.storage.checkpoint_fence_missing');
|
|
1235
|
+
}
|
|
1236
|
+
const coverage = this.#hasSourceChangesAbove(
|
|
1237
|
+
partition,
|
|
1238
|
+
sources,
|
|
1239
|
+
watermark,
|
|
1240
|
+
);
|
|
1241
|
+
if (coverage !== 'clean') {
|
|
1242
|
+
this.db.exec('ROLLBACK');
|
|
1243
|
+
return coverage === 'changed' ? 'stale' : 'unverifiable';
|
|
1244
|
+
}
|
|
1245
|
+
const updated = this.db
|
|
1246
|
+
.query(
|
|
1247
|
+
`UPDATE sync_backfill_checkpoints
|
|
1248
|
+
SET state='activated', watermark=?, updated_at_ms=?
|
|
1249
|
+
WHERE partition=? AND name=? AND owner_epoch=?
|
|
1250
|
+
AND state<>'activated'`,
|
|
1251
|
+
)
|
|
1252
|
+
.run(watermark, nowMs, partition, name, ownerEpoch);
|
|
1253
|
+
this.db.exec('COMMIT');
|
|
1254
|
+
return Number(updated.changes) === 1 ? 'activated' : 'stale';
|
|
1255
|
+
} catch (error) {
|
|
1256
|
+
this.db.exec('ROLLBACK');
|
|
1257
|
+
throw error;
|
|
1258
|
+
}
|
|
1259
|
+
});
|
|
1260
|
+
}
|
|
1261
|
+
|
|
1262
|
+
async claimCheckpoint(
|
|
1263
|
+
partition: string,
|
|
1264
|
+
name: string,
|
|
1265
|
+
schemaVersion: number,
|
|
1266
|
+
nowMs: number,
|
|
1267
|
+
): Promise<StoredCheckpoint> {
|
|
1268
|
+
return this.#serializeWrite(() => {
|
|
1269
|
+
const claimed = this.db
|
|
1270
|
+
.query<SqliteCheckpointRecord, (string | number)[]>(
|
|
1271
|
+
`UPDATE sync_backfill_checkpoints
|
|
1272
|
+
SET owner_epoch=owner_epoch+1,
|
|
1273
|
+
state=CASE WHEN state='declared' THEN 'backfilling' ELSE state END,
|
|
1274
|
+
schema_version=?, updated_at_ms=?
|
|
1275
|
+
WHERE partition=? AND name=? AND state<>'activated'
|
|
1276
|
+
RETURNING partition, name, schema_version, state, watermark,
|
|
1277
|
+
owner_epoch, observed_rows, updated_at_ms`,
|
|
1278
|
+
)
|
|
1279
|
+
.get(schemaVersion, nowMs, partition, name);
|
|
1280
|
+
if (claimed === null) {
|
|
1281
|
+
throw new StorageQueryError('sync.storage.checkpoint_not_declared');
|
|
1282
|
+
}
|
|
1283
|
+
return toStoredCheckpoint(claimed);
|
|
1284
|
+
});
|
|
1285
|
+
}
|
|
1286
|
+
|
|
1287
|
+
async advanceCheckpoint(
|
|
1288
|
+
partition: string,
|
|
1289
|
+
name: string,
|
|
1290
|
+
ownerEpoch: number,
|
|
1291
|
+
watermark: number,
|
|
1292
|
+
observedRows: number,
|
|
1293
|
+
nowMs: number,
|
|
1294
|
+
): Promise<boolean> {
|
|
1295
|
+
return this.#serializeWrite(() => {
|
|
1296
|
+
const result = this.db
|
|
1297
|
+
.query(
|
|
1298
|
+
`UPDATE sync_backfill_checkpoints
|
|
1299
|
+
SET watermark=?, observed_rows=?, updated_at_ms=?
|
|
1300
|
+
WHERE partition=? AND name=? AND owner_epoch=?`,
|
|
1301
|
+
)
|
|
1302
|
+
.run(watermark, observedRows, nowMs, partition, name, ownerEpoch);
|
|
1303
|
+
return Number(result.changes) === 1;
|
|
1304
|
+
});
|
|
1305
|
+
}
|
|
1306
|
+
|
|
1307
|
+
async sourceCoverageSeq(
|
|
1308
|
+
partition: string,
|
|
1309
|
+
tables: readonly string[],
|
|
1310
|
+
): Promise<number> {
|
|
1311
|
+
if (tables.length === 0) return 0;
|
|
1312
|
+
const row = this.db
|
|
1313
|
+
.query<{ seq: number | null }, (string | number)[]>(
|
|
1314
|
+
`SELECT max(commit_seq) AS seq FROM sync_changes
|
|
1315
|
+
WHERE partition=? AND tbl IN (${placeholders(tables.length)})`,
|
|
1316
|
+
)
|
|
1317
|
+
.get(partition, ...tables);
|
|
1318
|
+
return row?.seq ?? 0;
|
|
1319
|
+
}
|
|
1320
|
+
|
|
1321
|
+
#hasSourceChangesAbove(
|
|
1322
|
+
partition: string,
|
|
1323
|
+
tables: readonly string[],
|
|
1324
|
+
seq: number,
|
|
1325
|
+
): 'clean' | 'changed' | 'unverifiable' {
|
|
1326
|
+
// Read the horizon before the window scan: a horizon past `seq` means the
|
|
1327
|
+
// history that would answer the question is gone, so an empty scan proves
|
|
1328
|
+
// nothing and must not be reported as `clean`.
|
|
1329
|
+
const horizon =
|
|
1330
|
+
this.db
|
|
1331
|
+
.query<{ horizon_seq: number }, [string]>(
|
|
1332
|
+
'SELECT horizon_seq FROM sync_partitions WHERE partition=?',
|
|
1333
|
+
)
|
|
1334
|
+
.get(partition)?.horizon_seq ?? 0;
|
|
1335
|
+
if (tables.length === 0) return 'clean';
|
|
1336
|
+
const hit = this.db
|
|
1337
|
+
.query<{ hit: number }, (string | number)[]>(
|
|
1338
|
+
`SELECT 1 AS hit FROM sync_changes
|
|
1339
|
+
WHERE partition=? AND tbl IN (${placeholders(tables.length)})
|
|
1340
|
+
AND commit_seq>? LIMIT 1`,
|
|
1341
|
+
)
|
|
1342
|
+
.get(partition, ...tables, seq);
|
|
1343
|
+
if (hit !== null && hit !== undefined) return 'changed';
|
|
1344
|
+
return horizon > seq ? 'unverifiable' : 'clean';
|
|
1345
|
+
}
|
|
1346
|
+
|
|
1347
|
+
async hasSourceChangesAbove(
|
|
1348
|
+
partition: string,
|
|
1349
|
+
tables: readonly string[],
|
|
1350
|
+
seq: number,
|
|
1351
|
+
): Promise<'clean' | 'changed' | 'unverifiable'> {
|
|
1352
|
+
return this.#hasSourceChangesAbove(partition, tables, seq);
|
|
1353
|
+
}
|
|
1354
|
+
|
|
952
1355
|
async getRow(
|
|
953
1356
|
partition: string,
|
|
954
1357
|
table: string,
|