@syncular/server 0.22.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/admin.d.ts +3 -1
- package/dist/admin.js +7 -1
- package/dist/blob-handlers.js +1 -1
- package/dist/context.d.ts +8 -1
- package/dist/d1-storage.d.ts +11 -3
- package/dist/d1-storage.js +84 -2
- package/dist/errors.js +10 -0
- package/dist/handler.js +50 -4
- package/dist/operations.js +3 -3
- package/dist/postgres-storage.d.ts +12 -4
- package/dist/postgres-storage.js +266 -6
- package/dist/push.js +18 -0
- package/dist/readiness.d.ts +9 -2
- package/dist/readiness.js +27 -2
- package/dist/realtime.js +20 -1
- package/dist/segment-download.js +8 -0
- package/dist/sqlite-dialect.d.ts +1 -1
- package/dist/sqlite-dialect.js +15 -0
- package/dist/sqlite-storage.d.ts +11 -3
- package/dist/sqlite-storage.js +296 -33
- package/dist/storage-errors.d.ts +1 -1
- package/dist/storage-errors.js +4 -0
- package/dist/storage.d.ts +127 -2
- package/dist/storage.js +40 -1
- package/package.json +2 -2
- package/src/admin.ts +13 -1
- package/src/blob-handlers.ts +1 -1
- package/src/context.ts +8 -0
- package/src/d1-storage.ts +143 -1
- package/src/errors.ts +10 -0
- package/src/handler.ts +62 -4
- package/src/operations.ts +17 -13
- package/src/postgres-storage.ts +430 -3
- package/src/push.ts +18 -0
- package/src/readiness.ts +39 -4
- package/src/realtime.ts +22 -1
- package/src/segment-download.ts +16 -0
- package/src/sqlite-dialect.ts +15 -0
- package/src/sqlite-storage.ts +439 -36
- package/src/storage-errors.ts +13 -1
- package/src/storage.ts +218 -1
package/src/postgres-storage.ts
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { validateCommitPruneQuery } from './prune';
|
|
2
2
|
import { StorageQueryError } from './storage-errors';
|
|
3
3
|
import type { CommitPruneQuery, CommitPruneResult } from './storage';
|
|
4
|
+
import { serveGateRefusal } from './storage';
|
|
5
|
+
import { serveNotReadyError } from './readiness';
|
|
4
6
|
/**
|
|
5
7
|
* Postgres server storage: the production database path.
|
|
6
8
|
*
|
|
@@ -79,6 +81,7 @@ import { matchesEffective } from './scopes';
|
|
|
79
81
|
import type {
|
|
80
82
|
AuthoritativeQueryRequest,
|
|
81
83
|
AuthoritativeQueryResult,
|
|
84
|
+
CheckpointDeclaration,
|
|
82
85
|
ClientCursorInfo,
|
|
83
86
|
ClientRecord,
|
|
84
87
|
ClientSubscription,
|
|
@@ -100,8 +103,10 @@ import type {
|
|
|
100
103
|
ScopeActivityQuery,
|
|
101
104
|
ScopeCommitActivity,
|
|
102
105
|
ServerStorage,
|
|
106
|
+
ServeGate,
|
|
103
107
|
StorageTransaction,
|
|
104
108
|
StoredChange,
|
|
109
|
+
StoredCheckpoint,
|
|
105
110
|
StoredCommit,
|
|
106
111
|
StoredPushResult,
|
|
107
112
|
StoredReaction,
|
|
@@ -165,6 +170,7 @@ CREATE TABLE IF NOT EXISTS sync_commits(
|
|
|
165
170
|
partition TEXT NOT NULL, commit_seq BIGINT NOT NULL,
|
|
166
171
|
client_id TEXT NOT NULL, client_commit_id TEXT NOT NULL,
|
|
167
172
|
actor_id TEXT NOT NULL, created_at_ms BIGINT NOT NULL,
|
|
173
|
+
writer_version BIGINT,
|
|
168
174
|
PRIMARY KEY(partition, commit_seq)
|
|
169
175
|
);
|
|
170
176
|
CREATE INDEX IF NOT EXISTS sync_commits_by_time
|
|
@@ -228,6 +234,20 @@ CREATE TABLE IF NOT EXISTS sync_tombstones(
|
|
|
228
234
|
commit_seq BIGINT NOT NULL,
|
|
229
235
|
PRIMARY KEY(partition, tbl, row_id)
|
|
230
236
|
);
|
|
237
|
+
CREATE TABLE IF NOT EXISTS sync_backfill_checkpoints(
|
|
238
|
+
partition TEXT NOT NULL, name TEXT NOT NULL,
|
|
239
|
+
schema_version INTEGER NOT NULL,
|
|
240
|
+
state TEXT NOT NULL CHECK(state IN ('declared','backfilling','activated')),
|
|
241
|
+
watermark BIGINT NOT NULL DEFAULT 0,
|
|
242
|
+
owner_epoch BIGINT NOT NULL DEFAULT 0,
|
|
243
|
+
observed_rows BIGINT NOT NULL DEFAULT 0,
|
|
244
|
+
updated_at_ms BIGINT NOT NULL,
|
|
245
|
+
PRIMARY KEY(partition, name)
|
|
246
|
+
);
|
|
247
|
+
CREATE TABLE IF NOT EXISTS sync_writer_fence(
|
|
248
|
+
partition TEXT PRIMARY KEY,
|
|
249
|
+
required_writer_version INTEGER NOT NULL
|
|
250
|
+
);
|
|
231
251
|
`;
|
|
232
252
|
|
|
233
253
|
interface SerializedResult {
|
|
@@ -264,6 +284,96 @@ async function lockPartitionOn(
|
|
|
264
284
|
);
|
|
265
285
|
}
|
|
266
286
|
|
|
287
|
+
/**
|
|
288
|
+
* Insert the `declared` row and raise the fence, on the caller's open
|
|
289
|
+
* transaction. Shared by `declareCheckpoint` and the `ensureSchema`
|
|
290
|
+
* migration so the schema bump and the fence commit together. Never lowers a
|
|
291
|
+
* fence and never re-declares an existing row.
|
|
292
|
+
*/
|
|
293
|
+
async function installCheckpointOn(
|
|
294
|
+
client: PgQueryable,
|
|
295
|
+
partition: string,
|
|
296
|
+
name: string,
|
|
297
|
+
schemaVersion: number,
|
|
298
|
+
nowMs: number,
|
|
299
|
+
): Promise<void> {
|
|
300
|
+
await client.query(
|
|
301
|
+
`INSERT INTO sync_backfill_checkpoints(
|
|
302
|
+
partition, name, schema_version, state, watermark,
|
|
303
|
+
owner_epoch, observed_rows, updated_at_ms)
|
|
304
|
+
VALUES ($1,$2,$3,'declared',0,0,0,$4)
|
|
305
|
+
ON CONFLICT (partition, name) DO NOTHING`,
|
|
306
|
+
[partition, name, schemaVersion, nowMs],
|
|
307
|
+
);
|
|
308
|
+
await client.query(
|
|
309
|
+
`INSERT INTO sync_writer_fence(partition, required_writer_version)
|
|
310
|
+
VALUES ($1,$2)
|
|
311
|
+
ON CONFLICT (partition) DO UPDATE SET
|
|
312
|
+
required_writer_version=GREATEST(
|
|
313
|
+
sync_writer_fence.required_writer_version,
|
|
314
|
+
EXCLUDED.required_writer_version)`,
|
|
315
|
+
[partition, schemaVersion],
|
|
316
|
+
);
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
/** RFC 0007 serve gate on either the pool or a transaction's client. */
|
|
320
|
+
async function readServeGateOn(
|
|
321
|
+
client: PgQueryable,
|
|
322
|
+
partition: string,
|
|
323
|
+
runningSchemaVersion: number,
|
|
324
|
+
): Promise<ServeGate> {
|
|
325
|
+
const marker = await client.query<{ schema_version: unknown }>(
|
|
326
|
+
'SELECT schema_version FROM sync_schema_meta WHERE id=1',
|
|
327
|
+
);
|
|
328
|
+
const epoch = await client.query<{ log_epoch: string }>(
|
|
329
|
+
'SELECT log_epoch FROM sync_partition_registry WHERE partition=$1',
|
|
330
|
+
[partition],
|
|
331
|
+
);
|
|
332
|
+
const pending = await client.query<{ partition: string; name: string }>(
|
|
333
|
+
`SELECT partition, name FROM sync_backfill_checkpoints
|
|
334
|
+
WHERE schema_version=$1 AND state<>'activated'`,
|
|
335
|
+
[runningSchemaVersion],
|
|
336
|
+
);
|
|
337
|
+
return {
|
|
338
|
+
storedSchemaVersion:
|
|
339
|
+
marker.rows[0] === undefined
|
|
340
|
+
? 0
|
|
341
|
+
: asNumber(marker.rows[0].schema_version),
|
|
342
|
+
logEpoch: epoch.rows[0]?.log_epoch,
|
|
343
|
+
pending: pending.rows.map((row) => ({
|
|
344
|
+
partition: row.partition,
|
|
345
|
+
name: row.name,
|
|
346
|
+
})),
|
|
347
|
+
};
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
async function hasSourceChangesAboveOn(
|
|
351
|
+
client: PgQueryable,
|
|
352
|
+
partition: string,
|
|
353
|
+
tables: readonly string[],
|
|
354
|
+
seq: number,
|
|
355
|
+
): Promise<'clean' | 'changed' | 'unverifiable'> {
|
|
356
|
+
// Read the horizon before the window scan: a horizon past `seq` means the
|
|
357
|
+
// history that would answer the question is gone, so an empty scan proves
|
|
358
|
+
// nothing and must not be reported as `clean`.
|
|
359
|
+
const horizon = await client.query<{ horizon_seq: unknown }>(
|
|
360
|
+
'SELECT horizon_seq FROM sync_partitions WHERE partition=$1',
|
|
361
|
+
[partition],
|
|
362
|
+
);
|
|
363
|
+
const horizonSeq =
|
|
364
|
+
horizon.rows[0] === undefined ? 0 : asNumber(horizon.rows[0].horizon_seq);
|
|
365
|
+
if (tables.length === 0) return 'clean';
|
|
366
|
+
const tableParams = tables.map((_, index) => `$${index + 2}`).join(',');
|
|
367
|
+
const hit = await client.query<{ hit: number }>(
|
|
368
|
+
`SELECT 1 AS hit FROM sync_changes
|
|
369
|
+
WHERE partition=$1 AND tbl IN (${tableParams}) AND commit_seq>$${tables.length + 2}
|
|
370
|
+
LIMIT 1`,
|
|
371
|
+
[partition, ...tables, seq],
|
|
372
|
+
);
|
|
373
|
+
if (hit.rows[0] !== undefined) return 'changed';
|
|
374
|
+
return horizonSeq > seq ? 'unverifiable' : 'clean';
|
|
375
|
+
}
|
|
376
|
+
|
|
267
377
|
function toBase64(bytes: Uint8Array): string {
|
|
268
378
|
return Buffer.from(bytes).toString('base64');
|
|
269
379
|
}
|
|
@@ -272,6 +382,32 @@ function fromBase64(text: string): Uint8Array {
|
|
|
272
382
|
return new Uint8Array(Buffer.from(text, 'base64'));
|
|
273
383
|
}
|
|
274
384
|
|
|
385
|
+
interface PostgresCheckpointRecord {
|
|
386
|
+
partition: string;
|
|
387
|
+
name: string;
|
|
388
|
+
schema_version: unknown;
|
|
389
|
+
state: StoredCheckpoint['state'];
|
|
390
|
+
watermark: unknown;
|
|
391
|
+
owner_epoch: unknown;
|
|
392
|
+
observed_rows: unknown;
|
|
393
|
+
updated_at_ms: unknown;
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
function toStoredCheckpoint(
|
|
397
|
+
record: PostgresCheckpointRecord,
|
|
398
|
+
): StoredCheckpoint {
|
|
399
|
+
return {
|
|
400
|
+
partition: record.partition,
|
|
401
|
+
name: record.name,
|
|
402
|
+
schemaVersion: asNumber(record.schema_version),
|
|
403
|
+
state: record.state,
|
|
404
|
+
watermark: asNumber(record.watermark),
|
|
405
|
+
ownerEpoch: asNumber(record.owner_epoch),
|
|
406
|
+
observedRows: asNumber(record.observed_rows),
|
|
407
|
+
updatedAtMs: asNumber(record.updated_at_ms),
|
|
408
|
+
};
|
|
409
|
+
}
|
|
410
|
+
|
|
275
411
|
interface PostgresReactionRecord {
|
|
276
412
|
idempotency_key: string;
|
|
277
413
|
type: string;
|
|
@@ -597,6 +733,8 @@ async function getPushResultOn(
|
|
|
597
733
|
class PostgresTransaction implements StorageTransaction {
|
|
598
734
|
#client: PgQueryable;
|
|
599
735
|
#partition: string;
|
|
736
|
+
/** Schema version this transaction writes; rides every appended commit row. */
|
|
737
|
+
#writerVersion: number;
|
|
600
738
|
#resolveTable: (name: string) => CompiledTable;
|
|
601
739
|
#open = true;
|
|
602
740
|
#pushApplySavepoint = false;
|
|
@@ -607,6 +745,7 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
607
745
|
constructor(
|
|
608
746
|
client: PgQueryable,
|
|
609
747
|
partition: string,
|
|
748
|
+
writerVersion: number,
|
|
610
749
|
resolveTable: (name: string) => CompiledTable,
|
|
611
750
|
resolve: () => void,
|
|
612
751
|
reject: (error: unknown) => void,
|
|
@@ -629,6 +768,7 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
629
768
|
},
|
|
630
769
|
};
|
|
631
770
|
this.#partition = partition;
|
|
771
|
+
this.#writerVersion = writerVersion;
|
|
632
772
|
this.#resolveTable = resolveTable;
|
|
633
773
|
this.#resolve = resolve;
|
|
634
774
|
this.#reject = reject;
|
|
@@ -738,6 +878,35 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
738
878
|
this.#pushApplySavepoint = true;
|
|
739
879
|
}
|
|
740
880
|
|
|
881
|
+
async advanceCheckpoint(
|
|
882
|
+
name: string,
|
|
883
|
+
ownerEpoch: number,
|
|
884
|
+
watermark: number,
|
|
885
|
+
observedRows: number,
|
|
886
|
+
nowMs: number,
|
|
887
|
+
): Promise<boolean> {
|
|
888
|
+
this.#assertOpen();
|
|
889
|
+
// `RETURNING`: the affected-row check must not depend on a driver's
|
|
890
|
+
// `rowCount` for a statement with no result rows. Bun.sql's canonical
|
|
891
|
+
// adapter derives `rowCount` from the returned rows, so a bare UPDATE
|
|
892
|
+
// reports 0 and the CAS would never advance on a real Bun.sql host.
|
|
893
|
+
const result = await this.#client.query<{ name: string }>(
|
|
894
|
+
`UPDATE sync_backfill_checkpoints
|
|
895
|
+
SET watermark=$1, observed_rows=$2, updated_at_ms=$3
|
|
896
|
+
WHERE partition=$4 AND name=$5 AND owner_epoch=$6
|
|
897
|
+
RETURNING name`,
|
|
898
|
+
[watermark, observedRows, nowMs, this.#partition, name, ownerEpoch],
|
|
899
|
+
);
|
|
900
|
+
return result.rows.length === 1;
|
|
901
|
+
}
|
|
902
|
+
|
|
903
|
+
readServeGate(runningSchemaVersion: number): Promise<ServeGate> {
|
|
904
|
+
this.#assertOpen();
|
|
905
|
+
// The transaction's own client: the gate and the write share one
|
|
906
|
+
// transaction, so a migration cannot interleave between them.
|
|
907
|
+
return readServeGateOn(this.#client, this.#partition, runningSchemaVersion);
|
|
908
|
+
}
|
|
909
|
+
|
|
741
910
|
async commitRejectedPushResult(
|
|
742
911
|
clientId: string,
|
|
743
912
|
clientCommitId: string,
|
|
@@ -826,8 +995,8 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
826
995
|
SET max_commit_seq = sync_partitions.max_commit_seq + 1
|
|
827
996
|
RETURNING max_commit_seq
|
|
828
997
|
)
|
|
829
|
-
INSERT INTO sync_commits(partition, commit_seq, client_id, client_commit_id, actor_id, created_at_ms)
|
|
830
|
-
SELECT $1, max_commit_seq, $2, $3, $4, $5 FROM allocated
|
|
998
|
+
INSERT INTO sync_commits(partition, commit_seq, client_id, client_commit_id, actor_id, created_at_ms, writer_version)
|
|
999
|
+
SELECT $1, max_commit_seq, $2, $3, $4, $5, $6 FROM allocated
|
|
831
1000
|
RETURNING commit_seq`,
|
|
832
1001
|
[
|
|
833
1002
|
p,
|
|
@@ -835,6 +1004,7 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
835
1004
|
commit.clientCommitId,
|
|
836
1005
|
commit.actorId,
|
|
837
1006
|
commit.createdAtMs,
|
|
1007
|
+
this.#writerVersion,
|
|
838
1008
|
],
|
|
839
1009
|
);
|
|
840
1010
|
const commitSeq = asNumber(rows[0]?.commit_seq);
|
|
@@ -972,6 +1142,33 @@ export class PostgresServerStorage implements ServerStorage {
|
|
|
972
1142
|
for (const statement of statements) {
|
|
973
1143
|
await this.#exec.query(statement);
|
|
974
1144
|
}
|
|
1145
|
+
await this.#exec.query(
|
|
1146
|
+
'ALTER TABLE sync_commits ADD COLUMN IF NOT EXISTS writer_version BIGINT',
|
|
1147
|
+
);
|
|
1148
|
+
// Database-side fence: an old binary's INSERT never reaches JS. An absent
|
|
1149
|
+
// `sync_writer_fence` row makes the EXISTS false, so a partition with no
|
|
1150
|
+
// barrier behaves exactly as before. Sent as separate statements because
|
|
1151
|
+
// the dollar-quoted body contains `;`.
|
|
1152
|
+
await this.#exec.query(`CREATE OR REPLACE FUNCTION syncular_writer_fence()
|
|
1153
|
+
RETURNS trigger AS $$
|
|
1154
|
+
BEGIN
|
|
1155
|
+
IF EXISTS (
|
|
1156
|
+
SELECT 1 FROM sync_writer_fence
|
|
1157
|
+
WHERE partition = NEW.partition
|
|
1158
|
+
AND (NEW.writer_version IS NULL
|
|
1159
|
+
OR NEW.writer_version < required_writer_version)
|
|
1160
|
+
) THEN
|
|
1161
|
+
RAISE EXCEPTION 'sync.storage.writer_fence_rejected';
|
|
1162
|
+
END IF;
|
|
1163
|
+
RETURN NEW;
|
|
1164
|
+
END;
|
|
1165
|
+
$$ LANGUAGE plpgsql`);
|
|
1166
|
+
await this.#exec.query(
|
|
1167
|
+
'DROP TRIGGER IF EXISTS sync_commits_writer_fence ON sync_commits',
|
|
1168
|
+
);
|
|
1169
|
+
await this.#exec.query(`CREATE TRIGGER sync_commits_writer_fence
|
|
1170
|
+
BEFORE INSERT ON sync_commits
|
|
1171
|
+
FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`);
|
|
975
1172
|
}
|
|
976
1173
|
|
|
977
1174
|
/** Resolve a table's compiled schema; row operations require `ensureSchema`. */
|
|
@@ -985,10 +1182,38 @@ export class PostgresServerStorage implements ServerStorage {
|
|
|
985
1182
|
return table;
|
|
986
1183
|
}
|
|
987
1184
|
|
|
988
|
-
async ensureSchema(
|
|
1185
|
+
async ensureSchema(
|
|
1186
|
+
schema: CompiledSchema,
|
|
1187
|
+
checkpoints?: readonly CheckpointDeclaration[],
|
|
1188
|
+
): Promise<void> {
|
|
989
1189
|
// Memoized fast path: same instance, same schema version.
|
|
990
1190
|
if (this.#schemaVersion === schema.version) return;
|
|
991
1191
|
await this.migrate();
|
|
1192
|
+
// The barrier reads stored state: a checkpoint at this schema version that
|
|
1193
|
+
// is not activated and that this caller did not declare means this process
|
|
1194
|
+
// must not serve. An omitted or empty declaration set declares nothing and
|
|
1195
|
+
// is refused here, never treated as a bypass.
|
|
1196
|
+
const declarations = checkpoints ?? [];
|
|
1197
|
+
const incomplete = await this.#exec.query<{
|
|
1198
|
+
partition: string;
|
|
1199
|
+
name: string;
|
|
1200
|
+
}>(
|
|
1201
|
+
`SELECT partition, name FROM sync_backfill_checkpoints
|
|
1202
|
+
WHERE schema_version=$1 AND state<>'activated'`,
|
|
1203
|
+
[schema.version],
|
|
1204
|
+
);
|
|
1205
|
+
if (
|
|
1206
|
+
incomplete.rows.some(
|
|
1207
|
+
(row) =>
|
|
1208
|
+
!declarations.some(
|
|
1209
|
+
(declaration) =>
|
|
1210
|
+
declaration.partition === row.partition &&
|
|
1211
|
+
declaration.name === row.name,
|
|
1212
|
+
),
|
|
1213
|
+
)
|
|
1214
|
+
) {
|
|
1215
|
+
throw new StorageQueryError('sync.storage.checkpoint_incomplete');
|
|
1216
|
+
}
|
|
992
1217
|
await this.#exec.query(SCHEMA_META_DDL_POSTGRES);
|
|
993
1218
|
const marker = await this.#exec.query<{
|
|
994
1219
|
schema_version: unknown;
|
|
@@ -1079,6 +1304,17 @@ export class PostgresServerStorage implements ServerStorage {
|
|
|
1079
1304
|
);
|
|
1080
1305
|
const retiredTables = retiredTableNames(schema, layouts);
|
|
1081
1306
|
await this.#exec.transaction(async (client) => {
|
|
1307
|
+
// RFC 0007 migration barrier: `LOCK TABLE ... EXCLUSIVE` is the FIRST
|
|
1308
|
+
// statement, before any read, DDL, rewrite, marker or checkpoint
|
|
1309
|
+
// install. EXCLUSIVE conflicts with the ROW SHARE that every writer's
|
|
1310
|
+
// `lockPartitionOn` takes (`SELECT … FOR UPDATE`) and with the ROW
|
|
1311
|
+
// EXCLUSIVE of its allocation (`UPDATE sync_partitions` / `INSERT … ON
|
|
1312
|
+
// CONFLICT DO UPDATE`), so an unaware old writer blocks at its first
|
|
1313
|
+
// statement with no cooperation; plain `SELECT` (ACCESS SHARE) still
|
|
1314
|
+
// works. The lock order is acyclic: the migration holds nothing else
|
|
1315
|
+
// when it takes this lock, so a writer already inside completes while
|
|
1316
|
+
// the migration waits.
|
|
1317
|
+
await client.query('LOCK TABLE sync_partitions IN EXCLUSIVE MODE');
|
|
1082
1318
|
const existing = new Map<string, ReadonlySet<string>>();
|
|
1083
1319
|
const existingIndexes = new Map<string, ReadonlySet<string>>();
|
|
1084
1320
|
for (const table of schema.tables.values()) {
|
|
@@ -1152,6 +1388,19 @@ export class PostgresServerStorage implements ServerStorage {
|
|
|
1152
1388
|
SET schema_version=EXCLUDED.schema_version, layouts=EXCLUDED.layouts`,
|
|
1153
1389
|
[schema.version, layoutsOf(schema)],
|
|
1154
1390
|
);
|
|
1391
|
+
// The host's declared checkpoints install in the schema-bump
|
|
1392
|
+
// transaction, so no observer sees the bumped marker without the
|
|
1393
|
+
// fence, or the fence without its declaration row.
|
|
1394
|
+
const installedAtMs = Date.now();
|
|
1395
|
+
for (const declaration of checkpoints ?? []) {
|
|
1396
|
+
await installCheckpointOn(
|
|
1397
|
+
client,
|
|
1398
|
+
declaration.partition,
|
|
1399
|
+
declaration.name,
|
|
1400
|
+
declaration.schemaVersion,
|
|
1401
|
+
installedAtMs,
|
|
1402
|
+
);
|
|
1403
|
+
}
|
|
1155
1404
|
});
|
|
1156
1405
|
}
|
|
1157
1406
|
this.#tables = schema.tables;
|
|
@@ -1263,6 +1512,7 @@ export class PostgresServerStorage implements ServerStorage {
|
|
|
1263
1512
|
const tx = new PostgresTransaction(
|
|
1264
1513
|
client,
|
|
1265
1514
|
partition,
|
|
1515
|
+
this.#schemaVersion ?? 0,
|
|
1266
1516
|
(name) => this.table(name),
|
|
1267
1517
|
resolveScope,
|
|
1268
1518
|
rejectScope,
|
|
@@ -1291,6 +1541,7 @@ export class PostgresServerStorage implements ServerStorage {
|
|
|
1291
1541
|
async queryAuthoritative(
|
|
1292
1542
|
partition: string,
|
|
1293
1543
|
query: AuthoritativeQueryRequest,
|
|
1544
|
+
checkpoints?: readonly CheckpointDeclaration[],
|
|
1294
1545
|
): Promise<AuthoritativeQueryResult> {
|
|
1295
1546
|
if (this.#tables === undefined) {
|
|
1296
1547
|
throw new Error(
|
|
@@ -1310,6 +1561,14 @@ export class PostgresServerStorage implements ServerStorage {
|
|
|
1310
1561
|
await client.query(
|
|
1311
1562
|
'SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY',
|
|
1312
1563
|
);
|
|
1564
|
+
// RFC 0007: evaluate the gate inside this pinned snapshot, so a
|
|
1565
|
+
// migration or an epoch rotation cannot land between the gate read and
|
|
1566
|
+
// the query's own read. The registered-query path uses the pinned
|
|
1567
|
+
// snapshot, not a request-entry check.
|
|
1568
|
+
const runningVersion = this.#schemaVersion ?? 0;
|
|
1569
|
+
const gate = await readServeGateOn(client, partition, runningVersion);
|
|
1570
|
+
const refusal = serveGateRefusal(gate, runningVersion, checkpoints);
|
|
1571
|
+
if (refusal !== undefined) throw serveNotReadyError(refusal);
|
|
1313
1572
|
const result = await client.query<Readonly<Record<string, unknown>>>(
|
|
1314
1573
|
postgresPlaceholders(prepared.sql),
|
|
1315
1574
|
prepared.params,
|
|
@@ -1412,6 +1671,174 @@ export class PostgresServerStorage implements ServerStorage {
|
|
|
1412
1671
|
return seq === null || seq === undefined ? 0 : asNumber(seq);
|
|
1413
1672
|
}
|
|
1414
1673
|
|
|
1674
|
+
async readCheckpoints(partition: string): Promise<StoredCheckpoint[]> {
|
|
1675
|
+
const { rows } = await this.#exec.query<PostgresCheckpointRecord>(
|
|
1676
|
+
`SELECT partition, name, schema_version, state, watermark,
|
|
1677
|
+
owner_epoch, observed_rows, updated_at_ms
|
|
1678
|
+
FROM sync_backfill_checkpoints WHERE partition=$1 ORDER BY name`,
|
|
1679
|
+
[partition],
|
|
1680
|
+
);
|
|
1681
|
+
return rows.map(toStoredCheckpoint);
|
|
1682
|
+
}
|
|
1683
|
+
|
|
1684
|
+
async readServeGate(
|
|
1685
|
+
partition: string,
|
|
1686
|
+
runningSchemaVersion: number,
|
|
1687
|
+
): Promise<ServeGate> {
|
|
1688
|
+
return readServeGateOn(this.#exec, partition, runningSchemaVersion);
|
|
1689
|
+
}
|
|
1690
|
+
|
|
1691
|
+
async declareCheckpoint(
|
|
1692
|
+
partition: string,
|
|
1693
|
+
name: string,
|
|
1694
|
+
schemaVersion: number,
|
|
1695
|
+
nowMs: number,
|
|
1696
|
+
): Promise<StoredCheckpoint> {
|
|
1697
|
+
// One transaction: the checkpoint becomes visible and the fence that
|
|
1698
|
+
// protects it is raised together. The fence is raised at declaration, not
|
|
1699
|
+
// at activation, so old writers are rejected for the whole backfill.
|
|
1700
|
+
//
|
|
1701
|
+
// RFC 0007 lock decision: this transaction writes only the checkpoint and
|
|
1702
|
+
// fence rows, no application rows and no schema marker, so it does not
|
|
1703
|
+
// take the migration table lock or a partition lock. The fence it raises is
|
|
1704
|
+
// enforced by the `sync_commits` trigger on every append from the commit
|
|
1705
|
+
// of this transaction onward; a writer that committed before the fence was
|
|
1706
|
+
// visible is pre-barrier by the design's own "from declaration onward"
|
|
1707
|
+
// rule, not a bypass of it.
|
|
1708
|
+
return this.#exec.transaction(async (client) => {
|
|
1709
|
+
await installCheckpointOn(client, partition, name, schemaVersion, nowMs);
|
|
1710
|
+
const { rows } = await client.query<PostgresCheckpointRecord>(
|
|
1711
|
+
`SELECT partition, name, schema_version, state, watermark,
|
|
1712
|
+
owner_epoch, observed_rows, updated_at_ms
|
|
1713
|
+
FROM sync_backfill_checkpoints WHERE partition=$1 AND name=$2`,
|
|
1714
|
+
[partition, name],
|
|
1715
|
+
);
|
|
1716
|
+
const row = rows[0];
|
|
1717
|
+
if (row === undefined) {
|
|
1718
|
+
throw new StorageQueryError('sync.storage.checkpoint_not_declared');
|
|
1719
|
+
}
|
|
1720
|
+
return toStoredCheckpoint(row);
|
|
1721
|
+
});
|
|
1722
|
+
}
|
|
1723
|
+
|
|
1724
|
+
async activateCheckpoint(
|
|
1725
|
+
partition: string,
|
|
1726
|
+
name: string,
|
|
1727
|
+
ownerEpoch: number,
|
|
1728
|
+
watermark: number,
|
|
1729
|
+
sources: readonly string[],
|
|
1730
|
+
nowMs: number,
|
|
1731
|
+
): Promise<'activated' | 'stale' | 'unverifiable'> {
|
|
1732
|
+
return this.#exec.transaction(async (client) => {
|
|
1733
|
+
// The same `SELECT … FOR UPDATE` partition row lock that
|
|
1734
|
+
// `lockPartitionForPush` takes; a bare `BEGIN` would race a push.
|
|
1735
|
+
await lockPartitionOn(client, partition);
|
|
1736
|
+
const fence = await client.query<{ required_writer_version: unknown }>(
|
|
1737
|
+
'SELECT required_writer_version FROM sync_writer_fence WHERE partition=$1',
|
|
1738
|
+
[partition],
|
|
1739
|
+
);
|
|
1740
|
+
const checkpoint = await client.query<{ schema_version: unknown }>(
|
|
1741
|
+
'SELECT schema_version FROM sync_backfill_checkpoints WHERE partition=$1 AND name=$2',
|
|
1742
|
+
[partition, name],
|
|
1743
|
+
);
|
|
1744
|
+
const fenceRow = fence.rows[0];
|
|
1745
|
+
const checkpointRow = checkpoint.rows[0];
|
|
1746
|
+
if (
|
|
1747
|
+
fenceRow === undefined ||
|
|
1748
|
+
checkpointRow === undefined ||
|
|
1749
|
+
asNumber(fenceRow.required_writer_version) <
|
|
1750
|
+
asNumber(checkpointRow.schema_version)
|
|
1751
|
+
) {
|
|
1752
|
+
// The barrier is not installed. Activating would certify a state old
|
|
1753
|
+
// writers can still decay; refuse instead of silently proceeding.
|
|
1754
|
+
throw new StorageQueryError('sync.storage.checkpoint_fence_missing');
|
|
1755
|
+
}
|
|
1756
|
+
const coverage = await hasSourceChangesAboveOn(
|
|
1757
|
+
client,
|
|
1758
|
+
partition,
|
|
1759
|
+
sources,
|
|
1760
|
+
watermark,
|
|
1761
|
+
);
|
|
1762
|
+
if (coverage !== 'clean') {
|
|
1763
|
+
return coverage === 'changed' ? 'stale' : 'unverifiable';
|
|
1764
|
+
}
|
|
1765
|
+
const updated = await client.query<{ partition: string }>(
|
|
1766
|
+
`UPDATE sync_backfill_checkpoints
|
|
1767
|
+
SET state='activated', watermark=$1, updated_at_ms=$2
|
|
1768
|
+
WHERE partition=$3 AND name=$4 AND owner_epoch=$5
|
|
1769
|
+
AND state<>'activated'
|
|
1770
|
+
RETURNING partition`,
|
|
1771
|
+
[watermark, nowMs, partition, name, ownerEpoch],
|
|
1772
|
+
);
|
|
1773
|
+
return updated.rows.length === 1 ? 'activated' : 'stale';
|
|
1774
|
+
});
|
|
1775
|
+
}
|
|
1776
|
+
|
|
1777
|
+
async claimCheckpoint(
|
|
1778
|
+
partition: string,
|
|
1779
|
+
name: string,
|
|
1780
|
+
schemaVersion: number,
|
|
1781
|
+
nowMs: number,
|
|
1782
|
+
): Promise<StoredCheckpoint> {
|
|
1783
|
+
// The UPDATE's own row lock serializes concurrent claims; owner_epoch+1
|
|
1784
|
+
// is evaluated under it, so two claimers never observe the same epoch.
|
|
1785
|
+
const { rows } = await this.#exec.query<PostgresCheckpointRecord>(
|
|
1786
|
+
`UPDATE sync_backfill_checkpoints
|
|
1787
|
+
SET owner_epoch=owner_epoch+1,
|
|
1788
|
+
state=CASE WHEN state='declared' THEN 'backfilling' ELSE state END,
|
|
1789
|
+
schema_version=$1, updated_at_ms=$2
|
|
1790
|
+
WHERE partition=$3 AND name=$4 AND state<>'activated'
|
|
1791
|
+
RETURNING partition, name, schema_version, state, watermark,
|
|
1792
|
+
owner_epoch, observed_rows, updated_at_ms`,
|
|
1793
|
+
[schemaVersion, nowMs, partition, name],
|
|
1794
|
+
);
|
|
1795
|
+
if (rows[0] === undefined) {
|
|
1796
|
+
throw new StorageQueryError('sync.storage.checkpoint_not_declared');
|
|
1797
|
+
}
|
|
1798
|
+
return toStoredCheckpoint(rows[0]);
|
|
1799
|
+
}
|
|
1800
|
+
|
|
1801
|
+
async advanceCheckpoint(
|
|
1802
|
+
partition: string,
|
|
1803
|
+
name: string,
|
|
1804
|
+
ownerEpoch: number,
|
|
1805
|
+
watermark: number,
|
|
1806
|
+
observedRows: number,
|
|
1807
|
+
nowMs: number,
|
|
1808
|
+
): Promise<boolean> {
|
|
1809
|
+
const result = await this.#exec.query<{ name: string }>(
|
|
1810
|
+
`UPDATE sync_backfill_checkpoints
|
|
1811
|
+
SET watermark=$1, observed_rows=$2, updated_at_ms=$3
|
|
1812
|
+
WHERE partition=$4 AND name=$5 AND owner_epoch=$6
|
|
1813
|
+
RETURNING name`,
|
|
1814
|
+
[watermark, observedRows, nowMs, partition, name, ownerEpoch],
|
|
1815
|
+
);
|
|
1816
|
+
return result.rows.length === 1;
|
|
1817
|
+
}
|
|
1818
|
+
|
|
1819
|
+
async sourceCoverageSeq(
|
|
1820
|
+
partition: string,
|
|
1821
|
+
tables: readonly string[],
|
|
1822
|
+
): Promise<number> {
|
|
1823
|
+
if (tables.length === 0) return 0;
|
|
1824
|
+
const tableParams = tables.map((_, index) => `$${index + 2}`).join(',');
|
|
1825
|
+
const { rows } = await this.#exec.query<{ seq: unknown }>(
|
|
1826
|
+
`SELECT max(commit_seq) AS seq FROM sync_changes
|
|
1827
|
+
WHERE partition=$1 AND tbl IN (${tableParams})`,
|
|
1828
|
+
[partition, ...tables],
|
|
1829
|
+
);
|
|
1830
|
+
const seq = rows[0]?.seq;
|
|
1831
|
+
return seq === null || seq === undefined ? 0 : asNumber(seq);
|
|
1832
|
+
}
|
|
1833
|
+
|
|
1834
|
+
hasSourceChangesAbove(
|
|
1835
|
+
partition: string,
|
|
1836
|
+
tables: readonly string[],
|
|
1837
|
+
seq: number,
|
|
1838
|
+
): Promise<'clean' | 'changed' | 'unverifiable'> {
|
|
1839
|
+
return hasSourceChangesAboveOn(this.#exec, partition, tables, seq);
|
|
1840
|
+
}
|
|
1841
|
+
|
|
1415
1842
|
getRow(
|
|
1416
1843
|
partition: string,
|
|
1417
1844
|
table: string,
|
package/src/push.ts
CHANGED
|
@@ -52,7 +52,9 @@ import type {
|
|
|
52
52
|
StoredPushResult,
|
|
53
53
|
StoredRow,
|
|
54
54
|
} from './storage';
|
|
55
|
+
import { serveGateRefusal } from './storage';
|
|
55
56
|
import { StorageConstraintError } from './storage-errors';
|
|
57
|
+
import { serveNotReadyError } from './readiness';
|
|
56
58
|
import type {
|
|
57
59
|
CommitValidationReader,
|
|
58
60
|
CommitValidator,
|
|
@@ -1356,6 +1358,15 @@ export async function processPushOperationsWithTrace(
|
|
|
1356
1358
|
);
|
|
1357
1359
|
}
|
|
1358
1360
|
await lockPartitionForPush();
|
|
1361
|
+
// RFC 0007: evaluate the gate on this transaction's own connection while
|
|
1362
|
+
// the partition lock is held, so the migration check and the write share
|
|
1363
|
+
// one transaction and no migration can interleave in between.
|
|
1364
|
+
const gate = await tx.readServeGate(schema.version);
|
|
1365
|
+
const refusal = serveGateRefusal(gate, schema.version, ctx.checkpoints);
|
|
1366
|
+
if (refusal !== undefined) {
|
|
1367
|
+
await tx.rollback();
|
|
1368
|
+
throw serveNotReadyError(refusal);
|
|
1369
|
+
}
|
|
1359
1370
|
// The optimistic lookup above may have raced another delivery. Re-check
|
|
1360
1371
|
// only after acquiring partition serialization and before any operation
|
|
1361
1372
|
// read, validation, merge, or staged write. The re-check runs on the
|
|
@@ -1551,6 +1562,13 @@ export async function processPushOperationsWithTrace(
|
|
|
1551
1562
|
}
|
|
1552
1563
|
}
|
|
1553
1564
|
if (ctx.realtime !== undefined && changes.length > 0) {
|
|
1565
|
+
// RFC 0007 fanout decision: the notification carries the durable commit
|
|
1566
|
+
// entry itself (this commit's `changes`), not a projection read or a pull
|
|
1567
|
+
// window, so fanning it out before the request's buffered read-verify can
|
|
1568
|
+
// never serve a mixed result. Each peer's next pull is gated at the
|
|
1569
|
+
// storage seam; a refused mixed push+pull is safe because the push is
|
|
1570
|
+
// durable and replayable under its commit id. This is commit-entry
|
|
1571
|
+
// durability, not gate inheritance.
|
|
1554
1572
|
await ctx.realtime.notifyCommit(partition, {
|
|
1555
1573
|
commitSeq,
|
|
1556
1574
|
createdAtMs,
|
package/src/readiness.ts
CHANGED
|
@@ -7,12 +7,17 @@
|
|
|
7
7
|
* a structured failure that cannot be confused with request authentication.
|
|
8
8
|
*/
|
|
9
9
|
import type { SyncServerConfig } from './context';
|
|
10
|
+
import { type SyncError, syncError } from './errors';
|
|
10
11
|
import { type CompiledSchema, compileSchema } from './schema';
|
|
12
|
+
import type { ServeGateRefusal } from './storage';
|
|
11
13
|
|
|
12
14
|
export const SYNC_SERVER_READINESS_ERROR_CODE =
|
|
13
15
|
'sync.schema_not_ready' as const;
|
|
14
16
|
|
|
15
|
-
export type SyncServerReadinessPhase =
|
|
17
|
+
export type SyncServerReadinessPhase =
|
|
18
|
+
| 'schema_compile'
|
|
19
|
+
| 'storage_migration'
|
|
20
|
+
| 'backfill_checkpoint';
|
|
16
21
|
|
|
17
22
|
export class SyncServerReadinessError extends Error {
|
|
18
23
|
override readonly name = 'SyncServerReadinessError';
|
|
@@ -42,7 +47,7 @@ export class SyncServerReadinessError extends Error {
|
|
|
42
47
|
* `cause` locally for the table/column or storage diagnostic.
|
|
43
48
|
*/
|
|
44
49
|
export async function ensureSyncServerReady(
|
|
45
|
-
config: Pick<SyncServerConfig, 'schema' | 'storage'>,
|
|
50
|
+
config: Pick<SyncServerConfig, 'schema' | 'storage' | 'checkpoints'>,
|
|
46
51
|
): Promise<void> {
|
|
47
52
|
let compiled: CompiledSchema;
|
|
48
53
|
try {
|
|
@@ -55,12 +60,42 @@ export async function ensureSyncServerReady(
|
|
|
55
60
|
});
|
|
56
61
|
}
|
|
57
62
|
try {
|
|
58
|
-
await config.storage.ensureSchema(compiled);
|
|
63
|
+
await config.storage.ensureSchema(compiled, config.checkpoints);
|
|
59
64
|
} catch (cause) {
|
|
65
|
+
// RFC 0007: a declared-but-incomplete checkpoint is a barrier refusal, not
|
|
66
|
+
// an ordinary migration failure — hosts distinguish it by `phase` while the
|
|
67
|
+
// client-visible identity stays `sync.schema_not_ready`.
|
|
68
|
+
const phase: SyncServerReadinessPhase =
|
|
69
|
+
(cause as { code?: unknown } | null | undefined)?.code ===
|
|
70
|
+
'sync.storage.checkpoint_incomplete'
|
|
71
|
+
? 'backfill_checkpoint'
|
|
72
|
+
: 'storage_migration';
|
|
60
73
|
throw new SyncServerReadinessError({
|
|
61
|
-
phase
|
|
74
|
+
phase,
|
|
62
75
|
schemaVersion: config.schema.version,
|
|
63
76
|
cause,
|
|
64
77
|
});
|
|
65
78
|
}
|
|
66
79
|
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* RFC 0007: refuse a serve path that observed a closed gate. The projection
|
|
83
|
+
* name travels in structured details, never in the message.
|
|
84
|
+
*/
|
|
85
|
+
export function serveNotReadyError(refusal: ServeGateRefusal): SyncError {
|
|
86
|
+
return syncError(
|
|
87
|
+
SYNC_SERVER_READINESS_ERROR_CODE,
|
|
88
|
+
'server schema is not ready for this request',
|
|
89
|
+
refusal.kind === 'checkpoint'
|
|
90
|
+
? JSON.stringify({
|
|
91
|
+
partition: refusal.partition,
|
|
92
|
+
projection: refusal.name,
|
|
93
|
+
})
|
|
94
|
+
: refusal.kind === 'epoch'
|
|
95
|
+
? JSON.stringify({ logEpochChanged: true })
|
|
96
|
+
: JSON.stringify({
|
|
97
|
+
storedSchemaVersion: refusal.stored,
|
|
98
|
+
runningSchemaVersion: refusal.running,
|
|
99
|
+
}),
|
|
100
|
+
);
|
|
101
|
+
}
|