@syncular/server 0.58.0 → 0.60.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -36
- package/dist/admin.d.ts +32 -20
- package/dist/admin.js +46 -40
- package/dist/blob-handlers.d.ts +25 -31
- package/dist/blob-handlers.js +63 -41
- package/dist/blob-store.d.ts +42 -24
- package/dist/blob-store.js +0 -0
- package/dist/bucket-blob-store.d.ts +12 -8
- package/dist/bucket-blob-store.js +54 -45
- package/dist/bucket-segment-store.js +18 -16
- package/dist/content-encoding.d.ts +5 -2
- package/dist/content-encoding.js +42 -13
- package/dist/context.d.ts +19 -5
- package/dist/context.js +19 -1
- package/dist/d1-storage.d.ts +14 -98
- package/dist/d1-storage.js +249 -1119
- package/dist/errors.js +16 -0
- package/dist/events.d.ts +36 -18
- package/dist/frame-bytes.js +24 -6
- package/dist/handler.js +503 -208
- package/dist/local-authority.d.ts +2 -0
- package/dist/local-authority.js +24 -12
- package/dist/object-bucket.d.ts +2 -0
- package/dist/operations-realtime.js +69 -50
- package/dist/operations.d.ts +5 -2
- package/dist/operations.js +20 -29
- package/dist/pg-executor-pglite.d.ts +5 -6
- package/dist/pg-executor-pglite.js +22 -14
- package/dist/pg-executor.d.ts +4 -1
- package/dist/postgres-storage.d.ts +24 -19
- package/dist/postgres-storage.js +420 -262
- package/dist/pull.d.ts +70 -47
- package/dist/pull.js +270 -301
- package/dist/push.d.ts +12 -3
- package/dist/push.js +361 -173
- package/dist/realtime.d.ts +85 -39
- package/dist/realtime.js +355 -219
- package/dist/relational-rows.d.ts +21 -66
- package/dist/relational-rows.js +28 -109
- package/dist/s3-blob-store.d.ts +11 -12
- package/dist/s3-blob-store.js +20 -13
- package/dist/s3-object-bucket.d.ts +8 -1
- package/dist/s3-object-bucket.js +16 -3
- package/dist/schema.d.ts +14 -0
- package/dist/schema.js +23 -0
- package/dist/scopes.d.ts +6 -2
- package/dist/scopes.js +22 -4
- package/dist/seed.js +10 -16
- package/dist/segment-download.d.ts +4 -1
- package/dist/segment-download.js +22 -6
- package/dist/signed-url.d.ts +2 -2
- package/dist/signed-url.js +9 -2
- package/dist/sigv4.d.ts +8 -2
- package/dist/sigv4.js +8 -4
- package/dist/sqlite-blob-store.d.ts +1 -1
- package/dist/sqlite-blob-store.js +23 -22
- package/dist/sqlite-dialect.d.ts +164 -84
- package/dist/sqlite-dialect.js +1145 -150
- package/dist/sqlite-lease-store.js +13 -18
- package/dist/sqlite-storage.d.ts +9 -62
- package/dist/sqlite-storage.js +303 -1123
- package/dist/storage-query.d.ts +30 -1
- package/dist/storage-query.js +68 -0
- package/dist/storage.d.ts +185 -63
- package/dist/storage.js +15 -0
- package/package.json +2 -2
- package/src/admin.ts +99 -57
- package/src/blob-handlers.ts +91 -72
- package/src/blob-store.ts +0 -0
- package/src/bucket-blob-store.ts +75 -48
- package/src/bucket-segment-store.ts +20 -15
- package/src/content-encoding.ts +48 -16
- package/src/context.ts +33 -6
- package/src/d1-storage.ts +350 -1700
- package/src/errors.ts +16 -0
- package/src/events.ts +37 -18
- package/src/frame-bytes.ts +26 -6
- package/src/handler.ts +654 -261
- package/src/local-authority.ts +27 -14
- package/src/object-bucket.ts +2 -0
- package/src/operations-realtime.ts +85 -61
- package/src/operations.ts +44 -42
- package/src/pg-executor-pglite.ts +27 -23
- package/src/pg-executor.ts +5 -1
- package/src/postgres-storage.ts +557 -354
- package/src/pull.ts +406 -440
- package/src/push.ts +466 -220
- package/src/realtime.ts +447 -290
- package/src/relational-rows.ts +43 -121
- package/src/s3-blob-store.ts +23 -13
- package/src/s3-object-bucket.ts +20 -2
- package/src/schema.ts +31 -0
- package/src/scopes.ts +23 -3
- package/src/seed.ts +13 -20
- package/src/segment-download.ts +33 -7
- package/src/signed-url.ts +10 -4
- package/src/sigv4.ts +17 -4
- package/src/sqlite-blob-store.ts +27 -26
- package/src/sqlite-dialect.ts +1851 -199
- package/src/sqlite-lease-store.ts +12 -23
- package/src/sqlite-storage.ts +390 -1821
- package/src/storage-query.ts +102 -0
- package/src/storage.ts +215 -67
package/dist/postgres-storage.js
CHANGED
|
@@ -1,25 +1,30 @@
|
|
|
1
1
|
import { validateCommitPruneQuery } from './prune.js';
|
|
2
2
|
import { StorageQueryError } from './storage-errors.js';
|
|
3
|
-
import { clientRoundState, PARTITION_TOUCH_INTERVAL_MS, rowsLiveAfterDelete, serveGateRefusal, } from './storage.js';
|
|
3
|
+
import { clientRoundState, PARTITION_TOUCH_INTERVAL_MS, commitDeleteFloor, rowsLiveAfterDelete, serveGateRefusal, } from './storage.js';
|
|
4
4
|
import { serveNotReadyError } from './readiness.js';
|
|
5
5
|
import { bindAuthoritativePartition, postgresPlaceholders, prepareAuthoritativeQuery, } from './authoritative-query.js';
|
|
6
6
|
import { syncError } from './errors.js';
|
|
7
7
|
import { asBytes, asNumber, } from './pg-executor.js';
|
|
8
|
-
import { assertPhysicalColumns, assertStoredLayouts,
|
|
8
|
+
import { assertPhysicalColumns, assertStoredLayouts, deleteRowSql, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, quoteIdent, tableColumnNames, retiredTableNames, rewritePlan, columnSqlType, rewriteValues, SCHEMA_META_DDL_POSTGRES, SYNC_COLUMN_VERSIONS_COLUMN, SYNC_PARTITION_COLUMN, SYNC_PAYLOAD_COLUMN, SYNC_ROW_ID_COLUMN, SYNC_SCOPES_COLUMN, SYNC_VERSION_COLUMN, scanRowBatchSql, scanRowPageSql, indexSignature, schemaDdl, serverIndexDdl, rowsReferencingBlobSql, selectRowSql, selectRowsSql, prefetchRowsBatchSql, selectRowsForRewriteSql, upsertSql, upsertValues, } from './relational-rows.js';
|
|
9
9
|
import { matchesEffective, sha256Hex } from './scopes.js';
|
|
10
10
|
import { isPostgresConstraintError, StorageConstraintError, } from './storage-errors.js';
|
|
11
|
-
import { assertScopeIndexedScan, resolveIndexRowScan } from './storage-query.js';
|
|
11
|
+
import { assertScopeIndexedScan, readLiveWindowPages, resolveIndexRowScan, } from './storage-query.js';
|
|
12
12
|
/**
|
|
13
13
|
* Schema DDL. Applied by `PostgresServerStorage.migrate()` (idempotent).
|
|
14
14
|
*
|
|
15
15
|
* Covering index design (the reason this file exists):
|
|
16
|
-
* - `
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
* - `
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
16
|
+
* - `sync_change_scopes_by_key (partition, var, value, commit_seq)`: the
|
|
17
|
+
* §4.5 live scan ranges once per unit key on the (partition, var, value)
|
|
18
|
+
* prefix across tables and returns `commit_seq` already ascending;
|
|
19
|
+
* - `sync_change_scopes_pk (partition, tbl, var, value, commit_seq)`:
|
|
20
|
+
* the §4.7 scope pin, erasure and pruning name the table;
|
|
21
|
+
* - `sync_row_scopes_pk (partition, tbl, var, value, row_id)`: the same
|
|
22
|
+
* shape for `scanRows`, ordered by `row_id`.
|
|
23
|
+
*
|
|
24
|
+
* The §6.3.2 watermark delete and the §9 settlement of results stored
|
|
25
|
+
* before wire version 4 (NULL `outbox_id`) range over the `(partition,
|
|
26
|
+
* client_id)` prefix of `sync_push_results`' PRIMARY KEY, which the
|
|
27
|
+
* watermark keeps to one request's results per outbox.
|
|
23
28
|
*
|
|
24
29
|
* `sync_row_scopes_by_row (partition, tbl, row_id)` is the one secondary
|
|
25
30
|
* scope index: `writeRowsOn` and `deleteRow` replace a row's scope entries
|
|
@@ -42,7 +47,8 @@ export const POSTGRES_DDL = `
|
|
|
42
47
|
CREATE TABLE IF NOT EXISTS sync_partitions(
|
|
43
48
|
partition TEXT PRIMARY KEY,
|
|
44
49
|
max_commit_seq BIGINT NOT NULL DEFAULT 0,
|
|
45
|
-
horizon_seq BIGINT NOT NULL DEFAULT 0
|
|
50
|
+
horizon_seq BIGINT NOT NULL DEFAULT 0,
|
|
51
|
+
delete_floor BIGINT NOT NULL DEFAULT 0
|
|
46
52
|
);
|
|
47
53
|
CREATE TABLE IF NOT EXISTS sync_partition_registry(
|
|
48
54
|
partition TEXT PRIMARY KEY,
|
|
@@ -79,11 +85,21 @@ CREATE TABLE IF NOT EXISTS sync_change_scopes(
|
|
|
79
85
|
var TEXT NOT NULL, value TEXT NOT NULL, commit_seq BIGINT NOT NULL,
|
|
80
86
|
PRIMARY KEY(partition, tbl, var, value, commit_seq)
|
|
81
87
|
);
|
|
88
|
+
CREATE INDEX IF NOT EXISTS sync_change_scopes_by_key
|
|
89
|
+
ON sync_change_scopes(partition, var, value, commit_seq);
|
|
82
90
|
CREATE TABLE IF NOT EXISTS sync_push_results(
|
|
83
91
|
partition TEXT NOT NULL, client_id TEXT NOT NULL,
|
|
84
92
|
client_commit_id TEXT NOT NULL, result JSONB NOT NULL,
|
|
93
|
+
outbox_id TEXT, client_seq BIGINT,
|
|
85
94
|
PRIMARY KEY(partition, client_id, client_commit_id)
|
|
86
95
|
);
|
|
96
|
+
ALTER TABLE sync_push_results ADD COLUMN IF NOT EXISTS outbox_id TEXT;
|
|
97
|
+
ALTER TABLE sync_push_results ADD COLUMN IF NOT EXISTS client_seq BIGINT;
|
|
98
|
+
CREATE TABLE IF NOT EXISTS sync_push_watermarks(
|
|
99
|
+
partition TEXT NOT NULL, client_id TEXT NOT NULL, outbox_id TEXT NOT NULL,
|
|
100
|
+
through_seq BIGINT NOT NULL,
|
|
101
|
+
PRIMARY KEY(partition, client_id, outbox_id)
|
|
102
|
+
);
|
|
87
103
|
CREATE TABLE IF NOT EXISTS sync_reactions(
|
|
88
104
|
partition TEXT NOT NULL, idempotency_key TEXT NOT NULL,
|
|
89
105
|
type TEXT NOT NULL, version INTEGER NOT NULL, payload JSONB NOT NULL,
|
|
@@ -107,10 +123,12 @@ CREATE INDEX IF NOT EXISTS sync_reactions_dead_letter
|
|
|
107
123
|
CREATE TABLE IF NOT EXISTS sync_clients(
|
|
108
124
|
partition TEXT NOT NULL, client_id TEXT NOT NULL, actor_id TEXT NOT NULL,
|
|
109
125
|
wire_version INTEGER NOT NULL DEFAULT 1,
|
|
110
|
-
cursor BIGINT NOT NULL,
|
|
126
|
+
cursor BIGINT NOT NULL,
|
|
111
127
|
updated_at_ms BIGINT NOT NULL,
|
|
112
128
|
last_round_at_ms BIGINT, last_error_code TEXT,
|
|
113
129
|
consecutive_failures INTEGER NOT NULL DEFAULT 0,
|
|
130
|
+
schema_version INTEGER, units JSONB, registration_digest BYTEA,
|
|
131
|
+
catching_up JSONB NOT NULL DEFAULT '[]',
|
|
114
132
|
PRIMARY KEY(partition, client_id)
|
|
115
133
|
);
|
|
116
134
|
ALTER TABLE sync_clients
|
|
@@ -119,6 +137,19 @@ ALTER TABLE sync_clients ADD COLUMN IF NOT EXISTS last_round_at_ms BIGINT;
|
|
|
119
137
|
ALTER TABLE sync_clients ADD COLUMN IF NOT EXISTS last_error_code TEXT;
|
|
120
138
|
ALTER TABLE sync_clients
|
|
121
139
|
ADD COLUMN IF NOT EXISTS consecutive_failures INTEGER NOT NULL DEFAULT 0;
|
|
140
|
+
ALTER TABLE sync_clients ADD COLUMN IF NOT EXISTS schema_version INTEGER;
|
|
141
|
+
ALTER TABLE sync_clients ADD COLUMN IF NOT EXISTS units JSONB;
|
|
142
|
+
ALTER TABLE sync_clients ADD COLUMN IF NOT EXISTS registration_digest BYTEA;
|
|
143
|
+
ALTER TABLE sync_clients
|
|
144
|
+
ADD COLUMN IF NOT EXISTS catching_up JSONB NOT NULL DEFAULT '[]';
|
|
145
|
+
-- §9: a database from before wire version 4 (its client records still hold
|
|
146
|
+
-- subscriptions) starts the horizon at its last commit.
|
|
147
|
+
UPDATE sync_partitions SET horizon_seq = max_commit_seq
|
|
148
|
+
WHERE EXISTS (SELECT 1 FROM information_schema.columns
|
|
149
|
+
WHERE table_schema = current_schema()
|
|
150
|
+
AND table_name = 'sync_clients'
|
|
151
|
+
AND column_name = 'subscriptions');
|
|
152
|
+
ALTER TABLE sync_clients DROP COLUMN IF EXISTS subscriptions;
|
|
122
153
|
CREATE TABLE IF NOT EXISTS sync_blob_refs(
|
|
123
154
|
partition TEXT NOT NULL, tbl TEXT NOT NULL, row_id TEXT NOT NULL,
|
|
124
155
|
blob_id TEXT NOT NULL,
|
|
@@ -128,9 +159,18 @@ CREATE INDEX IF NOT EXISTS sync_blob_refs_by_blob
|
|
|
128
159
|
ON sync_blob_refs(partition, blob_id);
|
|
129
160
|
CREATE TABLE IF NOT EXISTS sync_tombstones(
|
|
130
161
|
partition TEXT NOT NULL, tbl TEXT NOT NULL, row_id TEXT NOT NULL,
|
|
131
|
-
commit_seq BIGINT NOT NULL,
|
|
162
|
+
commit_seq BIGINT NOT NULL, server_version BIGINT, scopes JSONB,
|
|
132
163
|
PRIMARY KEY(partition, tbl, row_id)
|
|
133
164
|
);
|
|
165
|
+
ALTER TABLE sync_tombstones ADD COLUMN IF NOT EXISTS server_version BIGINT;
|
|
166
|
+
ALTER TABLE sync_tombstones ADD COLUMN IF NOT EXISTS scopes JSONB;
|
|
167
|
+
ALTER TABLE sync_partitions
|
|
168
|
+
ADD COLUMN IF NOT EXISTS delete_floor BIGINT NOT NULL DEFAULT 0;
|
|
169
|
+
UPDATE sync_partitions SET delete_floor = t.version
|
|
170
|
+
FROM (SELECT partition, max(server_version) AS version
|
|
171
|
+
FROM sync_tombstones GROUP BY partition) t
|
|
172
|
+
WHERE t.partition = sync_partitions.partition
|
|
173
|
+
AND t.version > sync_partitions.delete_floor;
|
|
134
174
|
CREATE TABLE IF NOT EXISTS sync_backfill_checkpoints(
|
|
135
175
|
partition TEXT NOT NULL, name TEXT NOT NULL,
|
|
136
176
|
schema_version INTEGER NOT NULL,
|
|
@@ -269,7 +309,8 @@ async function readAdmissionOn(client, partition, clientId, clientCommitId, runn
|
|
|
269
309
|
];
|
|
270
310
|
let statement = `SELECT ${SERVE_GATE_PROJECTION},
|
|
271
311
|
(SELECT result FROM sync_push_results
|
|
272
|
-
WHERE partition=$1 AND client_id=$3 AND client_commit_id=$4) AS result
|
|
312
|
+
WHERE partition=$1 AND client_id=$3 AND client_commit_id=$4) AS result,
|
|
313
|
+
(SELECT delete_floor FROM sync_partitions WHERE partition=$1) AS delete_floor`;
|
|
273
314
|
if (prefetch !== undefined && prefetch.specs.length > 0) {
|
|
274
315
|
params.push(JSON.stringify(prefetch.specs));
|
|
275
316
|
// The gate row pads the read-ahead columns with typed NULLs so both
|
|
@@ -278,11 +319,13 @@ async function readAdmissionOn(client, partition, clientId, clientCommitId, runn
|
|
|
278
319
|
NULL::text AS tbl, NULL::text AS lookup_id, NULL::text AS row_id,
|
|
279
320
|
NULL::bigint AS server_version, NULL::jsonb AS scopes,
|
|
280
321
|
NULL::bytea AS payload, NULL::bytea AS column_versions,
|
|
281
|
-
NULL::bigint AS tombstone_seq
|
|
322
|
+
NULL::bigint AS tombstone_seq, NULL::bigint AS tombstone_version,
|
|
323
|
+
NULL::jsonb AS tombstone_scopes
|
|
282
324
|
UNION ALL
|
|
283
|
-
SELECT NULL::bigint, NULL::text, NULL::json, NULL::jsonb,
|
|
325
|
+
SELECT NULL::bigint, NULL::text, NULL::json, NULL::jsonb, NULL::bigint,
|
|
284
326
|
p.tbl, p.lookup_id, p.row_id, p.server_version, p.scopes,
|
|
285
|
-
p.payload, p.column_versions, p.tombstone_seq
|
|
327
|
+
p.payload, p.column_versions, p.tombstone_seq,
|
|
328
|
+
p.tombstone_version, p.tombstone_scopes
|
|
286
329
|
FROM (${prefetchRowsBatchSql(prefetch.tables, {
|
|
287
330
|
partition: '$1',
|
|
288
331
|
specs: `$${params.length}`,
|
|
@@ -293,6 +336,8 @@ async function readAdmissionOn(client, partition, clientId, clientCommitId, runn
|
|
|
293
336
|
return {
|
|
294
337
|
gate: toServeGate(gateRow),
|
|
295
338
|
persisted: toPersistedPushResult(gateRow?.result),
|
|
339
|
+
// asNumber maps the NULL of a partition without a commit to 0.
|
|
340
|
+
deleteFloor: asNumber(gateRow?.delete_floor),
|
|
296
341
|
prefetched: rows.filter((row) => row.tbl !== null && row.tbl !== undefined),
|
|
297
342
|
};
|
|
298
343
|
}
|
|
@@ -466,6 +511,44 @@ function toStoredRow(record) {
|
|
|
466
511
|
: { columnVersions: asBytes(record.column_versions) }),
|
|
467
512
|
};
|
|
468
513
|
}
|
|
514
|
+
function toTombstone(seq, version, scopes) {
|
|
515
|
+
if (seq === null || seq === undefined)
|
|
516
|
+
return null;
|
|
517
|
+
return {
|
|
518
|
+
commitSeq: asNumber(seq),
|
|
519
|
+
...(version === null || version === undefined
|
|
520
|
+
? {}
|
|
521
|
+
: { serverVersion: asNumber(version) }),
|
|
522
|
+
...(scopes === null || scopes === undefined
|
|
523
|
+
? {}
|
|
524
|
+
: { scopes: asJson(scopes) }),
|
|
525
|
+
};
|
|
526
|
+
}
|
|
527
|
+
const CLIENT_RECORD_COLUMNS = 'client_id, actor_id, wire_version, cursor, updated_at_ms, schema_version, units, registration_digest, catching_up, last_round_at_ms, last_error_code, consecutive_failures';
|
|
528
|
+
function toClientRecord(record) {
|
|
529
|
+
return {
|
|
530
|
+
clientId: record.client_id,
|
|
531
|
+
actorId: record.actor_id,
|
|
532
|
+
wireVersion: asNumber(record.wire_version),
|
|
533
|
+
cursor: asNumber(record.cursor),
|
|
534
|
+
updatedAtMs: asNumber(record.updated_at_ms),
|
|
535
|
+
...(record.schema_version === null
|
|
536
|
+
? {}
|
|
537
|
+
: { schemaVersion: asNumber(record.schema_version) }),
|
|
538
|
+
...(record.units === null || record.registration_digest === null
|
|
539
|
+
? {}
|
|
540
|
+
: {
|
|
541
|
+
registration: {
|
|
542
|
+
units: asJson(record.units),
|
|
543
|
+
digest: asBytes(record.registration_digest),
|
|
544
|
+
},
|
|
545
|
+
}),
|
|
546
|
+
catchingUp: asJson(record.catching_up),
|
|
547
|
+
...clientRoundState(record.last_round_at_ms === null
|
|
548
|
+
? null
|
|
549
|
+
: asNumber(record.last_round_at_ms), record.last_error_code, asNumber(record.consecutive_failures)),
|
|
550
|
+
};
|
|
551
|
+
}
|
|
469
552
|
function toStoredChange(record) {
|
|
470
553
|
return {
|
|
471
554
|
table: record.tbl,
|
|
@@ -851,7 +934,7 @@ class PostgresTransaction {
|
|
|
851
934
|
// One statement for every id the admission read did not already answer.
|
|
852
935
|
// `#client` flushes this transaction's staged upserts first, so the rows
|
|
853
936
|
// are the final candidate state.
|
|
854
|
-
const { rows: records } = await this.#client.query(selectRowsSql(this.#resolveTable(table)), [this.#partition, missing]);
|
|
937
|
+
const { rows: records } = await this.#client.query(selectRowsSql(this.#resolveTable(table), 'postgres'), [this.#partition, missing]);
|
|
855
938
|
for (const record of records)
|
|
856
939
|
rows.set(record.row_id, toStoredRow(record));
|
|
857
940
|
return rows;
|
|
@@ -872,26 +955,25 @@ class PostgresTransaction {
|
|
|
872
955
|
const { rows } = await this.#client.query(prefetchRowsBatchSql(tables), [this.#partition, JSON.stringify(specs)]);
|
|
873
956
|
this.#recordPrefetched(rows);
|
|
874
957
|
}
|
|
875
|
-
/** Answer the following `getRow`/`
|
|
958
|
+
/** Answer the following `getRow`/`getTombstone` calls from a read snapshot. */
|
|
876
959
|
#recordPrefetched(records) {
|
|
877
960
|
for (const record of records) {
|
|
878
961
|
const key = `${record.tbl}\u0000${record.lookup_id}`;
|
|
879
962
|
this.#prefetchedRows.set(key, record.row_id === null || record.row_id === undefined
|
|
880
963
|
? null
|
|
881
964
|
: toStoredRow({ ...record, row_id: record.row_id }));
|
|
882
|
-
this.#prefetchedTombstones.set(key, record.tombstone_seq
|
|
883
|
-
? null
|
|
884
|
-
: asNumber(record.tombstone_seq));
|
|
965
|
+
this.#prefetchedTombstones.set(key, toTombstone(record.tombstone_seq, record.tombstone_version, record.tombstone_scopes));
|
|
885
966
|
}
|
|
886
967
|
}
|
|
887
|
-
async
|
|
968
|
+
async getTombstone(table, rowId) {
|
|
888
969
|
this.#assertOpen();
|
|
889
970
|
const prefetched = this.#prefetchedTombstones.get(`${table}\u0000${rowId}`);
|
|
890
971
|
if (prefetched !== undefined)
|
|
891
972
|
return prefetched ?? undefined;
|
|
892
|
-
const { rows } = await this.#client.query('SELECT commit_seq FROM sync_tombstones WHERE partition=$1 AND tbl=$2 AND row_id=$3', [this.#partition, table, rowId]);
|
|
893
|
-
const
|
|
894
|
-
return
|
|
973
|
+
const { rows } = await this.#client.query('SELECT commit_seq, server_version, scopes FROM sync_tombstones WHERE partition=$1 AND tbl=$2 AND row_id=$3', [this.#partition, table, rowId]);
|
|
974
|
+
const row = rows[0];
|
|
975
|
+
return (toTombstone(row?.commit_seq, row?.server_version, row?.scopes) ??
|
|
976
|
+
undefined);
|
|
895
977
|
}
|
|
896
978
|
async clearTombstone(table, rowId) {
|
|
897
979
|
this.#assertOpen();
|
|
@@ -957,7 +1039,7 @@ class PostgresTransaction {
|
|
|
957
1039
|
}
|
|
958
1040
|
return { rows: result.rows, maxCommitSeq: this.#cursorSeq };
|
|
959
1041
|
}
|
|
960
|
-
async preparePushApply(clientId, clientCommitId, runningSchemaVersion, targets = []) {
|
|
1042
|
+
async preparePushApply(clientId, clientCommitId, runningSchemaVersion, targets = [], outbox) {
|
|
961
1043
|
this.#assertOpen();
|
|
962
1044
|
// The lock is its own statement: a READ COMMITTED statement's snapshot is
|
|
963
1045
|
// fixed at its start, so the admission read must run *after* a duplicate's
|
|
@@ -965,6 +1047,9 @@ class PostgresTransaction {
|
|
|
965
1047
|
// and the read into one statement would miss a duplicate that committed
|
|
966
1048
|
// while this transaction waited.
|
|
967
1049
|
await this.lockPartition();
|
|
1050
|
+
const watermark = outbox === undefined
|
|
1051
|
+
? undefined
|
|
1052
|
+
: await this.#settleOutbox(clientId, clientCommitId, outbox);
|
|
968
1053
|
const tables = [];
|
|
969
1054
|
const specs = [];
|
|
970
1055
|
for (const target of targets) {
|
|
@@ -978,7 +1063,62 @@ class PostgresTransaction {
|
|
|
978
1063
|
// read-ahead (when the caller already knows its targets).
|
|
979
1064
|
const admission = await readAdmissionOn(this.#client, this.#partition, clientId, clientCommitId, runningSchemaVersion, specs.length === 0 ? undefined : { tables, specs });
|
|
980
1065
|
this.#recordPrefetched(admission.prefetched);
|
|
981
|
-
return {
|
|
1066
|
+
return {
|
|
1067
|
+
gate: admission.gate,
|
|
1068
|
+
persisted: admission.persisted,
|
|
1069
|
+
deleteFloor: admission.deleteFloor,
|
|
1070
|
+
...(watermark === undefined ? {} : { watermark }),
|
|
1071
|
+
};
|
|
1072
|
+
}
|
|
1073
|
+
/**
|
|
1074
|
+
* The §9 settlement and §6.3.2 steps 1 and 2 under the partition lock,
|
|
1075
|
+
* before the candidate savepoint, so a rejection keeps them. Two
|
|
1076
|
+
* statements: the settlement touches disjoint rows (outside and inside the
|
|
1077
|
+
* request), and the watermark delete must see its sequences.
|
|
1078
|
+
*/
|
|
1079
|
+
async #settleOutbox(clientId, clientCommitId, outbox) {
|
|
1080
|
+
const first = outbox.commits[0];
|
|
1081
|
+
const last = outbox.commits.at(-1);
|
|
1082
|
+
if (first === undefined ||
|
|
1083
|
+
last === undefined ||
|
|
1084
|
+
first.clientCommitId !== clientCommitId) {
|
|
1085
|
+
throw new Error("outbox settlement rides the request's first commit");
|
|
1086
|
+
}
|
|
1087
|
+
const commits = JSON.stringify(outbox.commits.map((commit) => ({
|
|
1088
|
+
id: commit.clientCommitId,
|
|
1089
|
+
seq: commit.clientSeq,
|
|
1090
|
+
})));
|
|
1091
|
+
// §9 (b): with no pre-upgrade result on the request's last commit, every
|
|
1092
|
+
// remaining one outside the request left the outbox; (a) assigns the
|
|
1093
|
+
// request's sequences to its pre-upgrade results.
|
|
1094
|
+
await this.#client.query(`WITH c AS (SELECT * FROM jsonb_to_recordset($4::text::jsonb) AS c(id text, seq bigint)),
|
|
1095
|
+
b AS (DELETE FROM sync_push_results
|
|
1096
|
+
WHERE partition=$1 AND client_id=$2 AND outbox_id IS NULL
|
|
1097
|
+
AND client_commit_id NOT IN (SELECT id FROM c)
|
|
1098
|
+
AND NOT EXISTS (SELECT 1 FROM sync_push_results
|
|
1099
|
+
WHERE partition=$1 AND client_id=$2 AND client_commit_id=$5
|
|
1100
|
+
AND outbox_id IS NULL)),
|
|
1101
|
+
a AS (UPDATE sync_push_results r SET outbox_id=$3, client_seq=c.seq FROM c
|
|
1102
|
+
WHERE r.partition=$1 AND r.client_id=$2 AND r.outbox_id IS NULL
|
|
1103
|
+
AND r.client_commit_id=c.id)
|
|
1104
|
+
SELECT 1`, [
|
|
1105
|
+
this.#partition,
|
|
1106
|
+
clientId,
|
|
1107
|
+
outbox.outboxId,
|
|
1108
|
+
commits,
|
|
1109
|
+
last.clientCommitId,
|
|
1110
|
+
]);
|
|
1111
|
+
const { rows } = await this.#client.query(`WITH w AS (
|
|
1112
|
+
INSERT INTO sync_push_watermarks(partition, client_id, outbox_id, through_seq)
|
|
1113
|
+
VALUES ($1,$2,$3,$4)
|
|
1114
|
+
ON CONFLICT (partition, client_id, outbox_id) DO UPDATE
|
|
1115
|
+
SET through_seq=GREATEST(sync_push_watermarks.through_seq, EXCLUDED.through_seq)
|
|
1116
|
+
RETURNING through_seq),
|
|
1117
|
+
d AS (DELETE FROM sync_push_results
|
|
1118
|
+
WHERE partition=$1 AND client_id=$2 AND outbox_id=$3
|
|
1119
|
+
AND client_seq<(SELECT through_seq FROM w))
|
|
1120
|
+
SELECT through_seq FROM w`, [this.#partition, clientId, outbox.outboxId, first.clientSeq]);
|
|
1121
|
+
return asNumber(rows[0]?.through_seq);
|
|
982
1122
|
}
|
|
983
1123
|
async lockPartition() {
|
|
984
1124
|
this.#assertOpen();
|
|
@@ -987,10 +1127,8 @@ class PostgresTransaction {
|
|
|
987
1127
|
async advanceCheckpoint(name, ownerEpoch, watermark, observedRows, nowMs) {
|
|
988
1128
|
this.#assertOpen();
|
|
989
1129
|
await this.#ensurePushSavepoint();
|
|
990
|
-
// `RETURNING`:
|
|
991
|
-
//
|
|
992
|
-
// adapter derives `rowCount` from the returned rows, so a bare UPDATE
|
|
993
|
-
// reports 0 and the CAS would never advance on a real Bun.sql host.
|
|
1130
|
+
// `RETURNING`: `PgQueryable` reports returned rows only (no affected-row
|
|
1131
|
+
// count), so every affected-row check counts returned rows.
|
|
994
1132
|
const result = await this.#client.query(`UPDATE sync_backfill_checkpoints
|
|
995
1133
|
SET watermark=$1, observed_rows=$2, updated_at_ms=$3
|
|
996
1134
|
WHERE partition=$4 AND name=$5 AND owner_epoch=$6
|
|
@@ -1003,7 +1141,7 @@ class PostgresTransaction {
|
|
|
1003
1141
|
// transaction, so a migration cannot interleave between them.
|
|
1004
1142
|
return readServeGateOn(this.#client, this.#partition, runningSchemaVersion);
|
|
1005
1143
|
}
|
|
1006
|
-
async commitRejectedPushResult(clientId, clientCommitId, result) {
|
|
1144
|
+
async commitRejectedPushResult(clientId, clientCommitId, result, sequence) {
|
|
1007
1145
|
this.#assertOpen();
|
|
1008
1146
|
// Candidate rows that never reached the database are discarded with the
|
|
1009
1147
|
// staging map; a savepoint exists only when one did.
|
|
@@ -1014,20 +1152,22 @@ class PostgresTransaction {
|
|
|
1014
1152
|
await this.#client.query('RELEASE SAVEPOINT syncular_push_candidate');
|
|
1015
1153
|
this.#pushApplySavepoint = false;
|
|
1016
1154
|
}
|
|
1017
|
-
const canonical = await this.#insertPushResult(clientId, clientCommitId, result);
|
|
1155
|
+
const canonical = await this.#insertPushResult(clientId, clientCommitId, result, sequence);
|
|
1018
1156
|
await this.commit();
|
|
1019
1157
|
return canonical;
|
|
1020
1158
|
}
|
|
1021
1159
|
/** Insert the §2.3 outcome and return the canonical persisted value. */
|
|
1022
|
-
async #insertPushResult(clientId, clientCommitId, result) {
|
|
1023
|
-
const { rows } = await this.#client.query(`INSERT INTO sync_push_results(partition, client_id, client_commit_id, result)
|
|
1024
|
-
VALUES ($1,$2,$3,$4)
|
|
1160
|
+
async #insertPushResult(clientId, clientCommitId, result, sequence) {
|
|
1161
|
+
const { rows } = await this.#client.query(`INSERT INTO sync_push_results(partition, client_id, client_commit_id, result, outbox_id, client_seq)
|
|
1162
|
+
VALUES ($1,$2,$3,$4,$5,$6)
|
|
1025
1163
|
ON CONFLICT (partition, client_id, client_commit_id) DO NOTHING
|
|
1026
1164
|
RETURNING result`, [
|
|
1027
1165
|
this.#partition,
|
|
1028
1166
|
clientId,
|
|
1029
1167
|
clientCommitId,
|
|
1030
1168
|
JSON.stringify(serializePushResult(result)),
|
|
1169
|
+
sequence?.outboxId ?? null,
|
|
1170
|
+
sequence?.clientSeq ?? null,
|
|
1031
1171
|
]);
|
|
1032
1172
|
if (rows[0] !== undefined) {
|
|
1033
1173
|
return toPersistedPushResult(rows[0].result) ?? result;
|
|
@@ -1119,9 +1259,11 @@ class PostgresTransaction {
|
|
|
1119
1259
|
// Allocate the next dense commitSeq under a per-partition row lock: the
|
|
1120
1260
|
// UPDATE … RETURNING serializes concurrent pushes to this partition and
|
|
1121
1261
|
// never leaves a gap on rollback (see the file header).
|
|
1122
|
-
|
|
1262
|
+
// §2.2: the allocation also raises the delete floor to the commit's.
|
|
1263
|
+
const allocate = `INSERT INTO sync_partitions(partition, max_commit_seq, delete_floor) VALUES ($1, 1, $7)
|
|
1123
1264
|
ON CONFLICT (partition) DO UPDATE
|
|
1124
|
-
SET max_commit_seq = sync_partitions.max_commit_seq + 1
|
|
1265
|
+
SET max_commit_seq = sync_partitions.max_commit_seq + 1,
|
|
1266
|
+
delete_floor = GREATEST(sync_partitions.delete_floor, EXCLUDED.delete_floor)
|
|
1125
1267
|
RETURNING max_commit_seq`;
|
|
1126
1268
|
const commitRow = `INSERT INTO sync_commits(partition, commit_seq, client_id, client_commit_id, actor_id, created_at_ms, writer_version)
|
|
1127
1269
|
SELECT $1, allocated.max_commit_seq, $2, $3, $4, $5, $6 FROM allocated
|
|
@@ -1133,6 +1275,7 @@ class PostgresTransaction {
|
|
|
1133
1275
|
commit.actorId,
|
|
1134
1276
|
commit.createdAtMs,
|
|
1135
1277
|
this.#writerVersion,
|
|
1278
|
+
commitDeleteFloor(commit.changes),
|
|
1136
1279
|
];
|
|
1137
1280
|
if (commit.changes.length === 0) {
|
|
1138
1281
|
const { rows } = await q.query(`WITH allocated AS (${allocate}) ${commitRow}`, head);
|
|
@@ -1152,11 +1295,12 @@ class PostgresTransaction {
|
|
|
1152
1295
|
// chunks run under the same commitSeq with their global idx. Serialized
|
|
1153
1296
|
// scopes bind as text before parsing JSONB; drivers that encode JSONB
|
|
1154
1297
|
// parameters would otherwise store the string as a scalar. A commit can
|
|
1155
|
-
// delete one row twice, so a chunk's tombstone insert keeps
|
|
1156
|
-
// key and a later chunk
|
|
1298
|
+
// delete one row twice, so a chunk's tombstone insert keeps the latest
|
|
1299
|
+
// delete per key and a later chunk overwrites it with its own. `version`
|
|
1300
|
+
// is the change's row version (§2.2), which a delete's tombstone keeps.
|
|
1157
1301
|
const valuesOf = (params, slice, firstIdx) => slice.map((change, offset) => {
|
|
1158
1302
|
const base = params.length;
|
|
1159
|
-
params.push(firstIdx + offset, change.table, change.rowId, change.op === 'upsert' ? 1 : 2, change.rowVersion
|
|
1303
|
+
params.push(firstIdx + offset, change.table, change.rowId, change.op === 'upsert' ? 1 : 2, change.rowVersion, JSON.stringify(change.scopes), change.payload ?? null);
|
|
1160
1304
|
return `($${base + 1}::int,$${base + 2}::text,$${base + 3}::text,$${base + 4}::smallint,$${base + 5}::bigint,$${base + 6}::text::jsonb,$${base + 7}::bytea)`;
|
|
1161
1305
|
});
|
|
1162
1306
|
const scopesCte = `scoped AS (
|
|
@@ -1165,16 +1309,21 @@ class PostgresTransaction {
|
|
|
1165
1309
|
FROM inserted CROSS JOIN LATERAL jsonb_each_text(inserted.scopes) AS scope
|
|
1166
1310
|
ON CONFLICT DO NOTHING
|
|
1167
1311
|
)`;
|
|
1168
|
-
const tombstones = (movedParam) => `INSERT INTO sync_tombstones(partition, tbl, row_id, commit_seq)
|
|
1169
|
-
SELECT DISTINCT ON (tbl, row_id) partition,
|
|
1170
|
-
|
|
1171
|
-
|
|
1312
|
+
const tombstones = (movedParam) => `INSERT INTO sync_tombstones(partition, tbl, row_id, commit_seq, server_version, scopes)
|
|
1313
|
+
SELECT DISTINCT ON (inserted.tbl, inserted.row_id) inserted.partition,
|
|
1314
|
+
inserted.tbl, inserted.row_id, inserted.commit_seq,
|
|
1315
|
+
v.version, inserted.scopes
|
|
1316
|
+
FROM inserted JOIN v ON v.idx = inserted.idx
|
|
1317
|
+
WHERE inserted.op = 2
|
|
1172
1318
|
AND NOT EXISTS (
|
|
1173
1319
|
SELECT 1
|
|
1174
1320
|
FROM jsonb_to_recordset(${movedParam}::text::jsonb) AS moved(tbl text, row_id text)
|
|
1175
1321
|
WHERE moved.tbl = inserted.tbl AND moved.row_id = inserted.row_id)
|
|
1322
|
+
ORDER BY inserted.tbl, inserted.row_id, inserted.idx DESC
|
|
1176
1323
|
ON CONFLICT (partition, tbl, row_id)
|
|
1177
|
-
DO UPDATE SET commit_seq = excluded.commit_seq
|
|
1324
|
+
DO UPDATE SET commit_seq = excluded.commit_seq,
|
|
1325
|
+
server_version = excluded.server_version,
|
|
1326
|
+
scopes = excluded.scopes`;
|
|
1178
1327
|
const firstSlice = commit.changes.slice(0, PG_CHANGES_PER_STATEMENT);
|
|
1179
1328
|
const params = [...head];
|
|
1180
1329
|
const values = valuesOf(params, firstSlice, 0);
|
|
@@ -1182,12 +1331,14 @@ class PostgresTransaction {
|
|
|
1182
1331
|
const movedParam = `$${params.length}`;
|
|
1183
1332
|
const { rows } = await q.query(`WITH allocated AS (${allocate}),
|
|
1184
1333
|
allocated_commit AS (${commitRow}),
|
|
1334
|
+
v(idx, tbl, row_id, op, version, scopes, payload) AS (
|
|
1335
|
+
VALUES ${values.join(',')}
|
|
1336
|
+
),
|
|
1185
1337
|
inserted AS (
|
|
1186
1338
|
INSERT INTO sync_changes(partition, commit_seq, idx, tbl, row_id, op, row_version, scopes, payload)
|
|
1187
|
-
SELECT $1, allocated.max_commit_seq, v.idx, v.tbl, v.row_id, v.op, v.
|
|
1188
|
-
FROM allocated
|
|
1189
|
-
|
|
1190
|
-
RETURNING partition, tbl, row_id, op, commit_seq, scopes
|
|
1339
|
+
SELECT $1, allocated.max_commit_seq, v.idx, v.tbl, v.row_id, v.op, v.version, v.scopes, v.payload
|
|
1340
|
+
FROM allocated CROSS JOIN v
|
|
1341
|
+
RETURNING partition, idx, tbl, row_id, op, commit_seq, scopes
|
|
1191
1342
|
),
|
|
1192
1343
|
${scopesCte},
|
|
1193
1344
|
tombstoned AS (${tombstones(movedParam)})
|
|
@@ -1198,11 +1349,14 @@ class PostgresTransaction {
|
|
|
1198
1349
|
const chunkValues = valuesOf(chunkParams, commit.changes.slice(first, first + PG_CHANGES_PER_STATEMENT), first);
|
|
1199
1350
|
chunkParams.push(moved);
|
|
1200
1351
|
const chunkMovedParam = `$${chunkParams.length}`;
|
|
1201
|
-
await q.query(`WITH
|
|
1352
|
+
await q.query(`WITH v(idx, tbl, row_id, op, version, scopes, payload) AS (
|
|
1353
|
+
VALUES ${chunkValues.join(',')}
|
|
1354
|
+
),
|
|
1355
|
+
inserted AS (
|
|
1202
1356
|
INSERT INTO sync_changes(partition, commit_seq, idx, tbl, row_id, op, row_version, scopes, payload)
|
|
1203
|
-
SELECT $1, $2, v.idx, v.tbl, v.row_id, v.op, v.
|
|
1204
|
-
FROM
|
|
1205
|
-
RETURNING partition, tbl, row_id, op, commit_seq, scopes
|
|
1357
|
+
SELECT $1, $2, v.idx, v.tbl, v.row_id, v.op, v.version, v.scopes, v.payload
|
|
1358
|
+
FROM v
|
|
1359
|
+
RETURNING partition, idx, tbl, row_id, op, commit_seq, scopes
|
|
1206
1360
|
),
|
|
1207
1361
|
${scopesCte},
|
|
1208
1362
|
tombstoned AS (${tombstones(chunkMovedParam)})
|
|
@@ -1210,15 +1364,17 @@ class PostgresTransaction {
|
|
|
1210
1364
|
}
|
|
1211
1365
|
return commitSeq;
|
|
1212
1366
|
}
|
|
1213
|
-
async putPushResult(clientId, clientCommitId, result) {
|
|
1367
|
+
async putPushResult(clientId, clientCommitId, result, sequence) {
|
|
1214
1368
|
this.#assertOpen();
|
|
1215
|
-
await this.#client.query(`INSERT INTO sync_push_results(partition, client_id, client_commit_id, result)
|
|
1216
|
-
VALUES ($1,$2,$3,$4)
|
|
1369
|
+
await this.#client.query(`INSERT INTO sync_push_results(partition, client_id, client_commit_id, result, outbox_id, client_seq)
|
|
1370
|
+
VALUES ($1,$2,$3,$4,$5,$6)
|
|
1217
1371
|
ON CONFLICT (partition, client_id, client_commit_id) DO NOTHING`, [
|
|
1218
1372
|
this.#partition,
|
|
1219
1373
|
clientId,
|
|
1220
1374
|
clientCommitId,
|
|
1221
1375
|
JSON.stringify(serializePushResult(result)),
|
|
1376
|
+
sequence?.outboxId ?? null,
|
|
1377
|
+
sequence?.clientSeq ?? null,
|
|
1222
1378
|
]);
|
|
1223
1379
|
}
|
|
1224
1380
|
async enqueueReactions(reactions) {
|
|
@@ -1654,9 +1810,12 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
1654
1810
|
*/
|
|
1655
1811
|
async begin(partition) {
|
|
1656
1812
|
let resolveReady;
|
|
1657
|
-
|
|
1658
|
-
|
|
1813
|
+
let rejectReady;
|
|
1814
|
+
const ready = new Promise((res, rej) => {
|
|
1815
|
+
resolveReady = res;
|
|
1816
|
+
rejectReady = rej;
|
|
1659
1817
|
});
|
|
1818
|
+
let opened = false;
|
|
1660
1819
|
let resolveScope;
|
|
1661
1820
|
let rejectScope;
|
|
1662
1821
|
const scope = new Promise((res, rej) => {
|
|
@@ -1666,6 +1825,7 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
1666
1825
|
const done = this.#exec
|
|
1667
1826
|
.transaction(async (client) => {
|
|
1668
1827
|
const tx = new PostgresTransaction(client, partition, this.#schemaVersion ?? 0, (name) => this.table(name), (query) => this.#prepareAuthoritativeQuery(partition, query), resolveScope, rejectScope);
|
|
1828
|
+
opened = true;
|
|
1669
1829
|
resolveReady(tx);
|
|
1670
1830
|
// Hold the transaction open until commit()/rollback() settles `scope`.
|
|
1671
1831
|
await scope;
|
|
@@ -1673,6 +1833,13 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
1673
1833
|
.catch((error) => {
|
|
1674
1834
|
if (error instanceof RollbackSignal)
|
|
1675
1835
|
return;
|
|
1836
|
+
// The executor failed before the callback ran (no connection, BEGIN
|
|
1837
|
+
// refused): `begin()` itself rejects, and no transaction exists to
|
|
1838
|
+
// observe `done`.
|
|
1839
|
+
if (!opened) {
|
|
1840
|
+
rejectReady(error);
|
|
1841
|
+
return;
|
|
1842
|
+
}
|
|
1676
1843
|
throw error;
|
|
1677
1844
|
});
|
|
1678
1845
|
const tx = await ready;
|
|
@@ -1728,15 +1895,23 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
1728
1895
|
const previousHorizonSeq = asNumber(previous.rows[0]?.horizon_seq);
|
|
1729
1896
|
const horizonSeq = Math.max(previousHorizonSeq, query.throughSeq);
|
|
1730
1897
|
await client.query('UPDATE sync_partitions SET horizon_seq=$2 WHERE partition=$1', [partition, horizonSeq]);
|
|
1731
|
-
const removed = await client.query('DELETE FROM sync_commits WHERE partition=$1 AND commit_seq<=$2', [partition, horizonSeq]);
|
|
1898
|
+
const removed = await client.query('DELETE FROM sync_commits WHERE partition=$1 AND commit_seq<=$2 RETURNING commit_seq', [partition, horizonSeq]);
|
|
1899
|
+
// Scope entries go by the keys of the changes being pruned, before
|
|
1900
|
+
// those changes are deleted: a `commit_seq` predicate alone would scan
|
|
1901
|
+
// the partition's whole scope index (`commit_seq` is its last key column).
|
|
1902
|
+
await client.query(`DELETE FROM sync_change_scopes s USING (
|
|
1903
|
+
SELECT DISTINCT c.tbl, e.key, e.value, c.commit_seq
|
|
1904
|
+
FROM sync_changes c CROSS JOIN LATERAL jsonb_each_text(c.scopes) e
|
|
1905
|
+
WHERE c.partition=$1 AND c.commit_seq<=$2) k
|
|
1906
|
+
WHERE s.partition=$1 AND s.tbl=k.tbl AND s.var=k.key
|
|
1907
|
+
AND s.value=k.value AND s.commit_seq=k.commit_seq`, [partition, horizonSeq]);
|
|
1732
1908
|
await client.query('DELETE FROM sync_changes WHERE partition=$1 AND commit_seq<=$2', [partition, horizonSeq]);
|
|
1733
|
-
await client.query('DELETE FROM sync_change_scopes WHERE partition=$1 AND commit_seq<=$2', [partition, horizonSeq]);
|
|
1734
1909
|
// §4.6: tombstones prune in the same pass as the commit log.
|
|
1735
1910
|
await client.query('DELETE FROM sync_tombstones WHERE partition=$1 AND commit_seq<=$2', [partition, horizonSeq]);
|
|
1736
1911
|
return {
|
|
1737
1912
|
previousHorizonSeq,
|
|
1738
1913
|
horizonSeq,
|
|
1739
|
-
removedCommits: removed.
|
|
1914
|
+
removedCommits: removed.rows.length,
|
|
1740
1915
|
};
|
|
1741
1916
|
});
|
|
1742
1917
|
}
|
|
@@ -1892,21 +2067,24 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
1892
2067
|
SET status='completed', completed_at_ms=$4,
|
|
1893
2068
|
lease_owner=NULL, lease_expires_at_ms=NULL
|
|
1894
2069
|
WHERE partition=$1 AND idempotency_key=$2
|
|
1895
|
-
AND status='leased' AND lease_owner=$3
|
|
1896
|
-
|
|
2070
|
+
AND status='leased' AND lease_owner=$3
|
|
2071
|
+
RETURNING idempotency_key`, [partition, idempotencyKey, leaseOwner, completedAtMs]);
|
|
2072
|
+
return result.rows.length === 1;
|
|
1897
2073
|
}
|
|
1898
2074
|
async extendReactionLease(partition, idempotencyKey, leaseOwner, leaseExpiresAtMs) {
|
|
1899
2075
|
const result = await this.#exec.query(`UPDATE sync_reactions SET lease_expires_at_ms=$4
|
|
1900
2076
|
WHERE partition=$1 AND idempotency_key=$2
|
|
1901
|
-
AND status='leased' AND lease_owner=$3
|
|
1902
|
-
|
|
2077
|
+
AND status='leased' AND lease_owner=$3
|
|
2078
|
+
RETURNING idempotency_key`, [partition, idempotencyKey, leaseOwner, leaseExpiresAtMs]);
|
|
2079
|
+
return result.rows.length === 1;
|
|
1903
2080
|
}
|
|
1904
2081
|
async failReaction(partition, idempotencyKey, update) {
|
|
1905
2082
|
const result = await this.#exec.query(`UPDATE sync_reactions
|
|
1906
2083
|
SET status=$4, available_at_ms=$5, last_failure=$6,
|
|
1907
2084
|
lease_owner=NULL, lease_expires_at_ms=NULL
|
|
1908
2085
|
WHERE partition=$1 AND idempotency_key=$2
|
|
1909
|
-
AND status='leased' AND lease_owner=$3
|
|
2086
|
+
AND status='leased' AND lease_owner=$3
|
|
2087
|
+
RETURNING idempotency_key`, [
|
|
1910
2088
|
partition,
|
|
1911
2089
|
idempotencyKey,
|
|
1912
2090
|
update.leaseOwner,
|
|
@@ -1914,15 +2092,16 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
1914
2092
|
update.retryAtMs ?? update.failure.atMs,
|
|
1915
2093
|
JSON.stringify(update.failure),
|
|
1916
2094
|
]);
|
|
1917
|
-
return result.
|
|
2095
|
+
return result.rows.length === 1;
|
|
1918
2096
|
}
|
|
1919
2097
|
async retryReaction(partition, idempotencyKey, nowMs) {
|
|
1920
2098
|
const result = await this.#exec.query(`UPDATE sync_reactions
|
|
1921
2099
|
SET status='pending', attempts=0, available_at_ms=$3,
|
|
1922
2100
|
last_failure=NULL, lease_owner=NULL, lease_expires_at_ms=NULL,
|
|
1923
2101
|
completed_at_ms=NULL
|
|
1924
|
-
WHERE partition=$1 AND idempotency_key=$2 AND status='dead-letter'
|
|
1925
|
-
|
|
2102
|
+
WHERE partition=$1 AND idempotency_key=$2 AND status='dead-letter'
|
|
2103
|
+
RETURNING idempotency_key`, [partition, idempotencyKey, nowMs]);
|
|
2104
|
+
return result.rows.length === 1;
|
|
1926
2105
|
}
|
|
1927
2106
|
async getReaction(partition, idempotencyKey) {
|
|
1928
2107
|
const { rows } = await this.#exec.query('SELECT * FROM sync_reactions WHERE partition=$1 AND idempotency_key=$2', [partition, idempotencyKey]);
|
|
@@ -1983,101 +2162,53 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
1983
2162
|
};
|
|
1984
2163
|
}
|
|
1985
2164
|
async latestScopedChangeSeq(partition, query) {
|
|
1986
|
-
const
|
|
1987
|
-
const
|
|
1988
|
-
|
|
1989
|
-
: (query.scopeFilter[firstVariable] ?? []);
|
|
1990
|
-
if (firstVariable === undefined || values.length === 0)
|
|
2165
|
+
const variable = Object.keys(query.scopes).sort()[0];
|
|
2166
|
+
const value = variable === undefined ? undefined : query.scopes[variable];
|
|
2167
|
+
if (variable === undefined || value === undefined)
|
|
1991
2168
|
return 0;
|
|
1992
|
-
// Concurrent reads of a pull share one statement across tables
|
|
1993
|
-
|
|
1994
|
-
|
|
1995
|
-
|
|
1996
|
-
|
|
1997
|
-
|
|
1998
|
-
|
|
1999
|
-
|
|
2000
|
-
|
|
2001
|
-
|
|
2002
|
-
|
|
2003
|
-
|
|
2004
|
-
//
|
|
2005
|
-
//
|
|
2006
|
-
//
|
|
2007
|
-
|
|
2008
|
-
|
|
2009
|
-
|
|
2010
|
-
|
|
2011
|
-
|
|
2012
|
-
|
|
2013
|
-
|
|
2014
|
-
|
|
2015
|
-
|
|
2016
|
-
|
|
2017
|
-
|
|
2018
|
-
|
|
2019
|
-
|
|
2020
|
-
|
|
2021
|
-
|
|
2022
|
-
|
|
2023
|
-
|
|
2024
|
-
|
|
2025
|
-
|
|
2026
|
-
|
|
2027
|
-
|
|
2028
|
-
|
|
2029
|
-
|
|
2030
|
-
|
|
2031
|
-
|
|
2032
|
-
|
|
2033
|
-
|
|
2034
|
-
|
|
2035
|
-
if (head === undefined)
|
|
2036
|
-
break;
|
|
2037
|
-
const commitSeq = asNumber(head.commit_seq);
|
|
2038
|
-
candidateCount += 1;
|
|
2039
|
-
afterSeq = commitSeq;
|
|
2040
|
-
const changes = [];
|
|
2041
|
-
for (; i < records.length; i++) {
|
|
2042
|
-
const record = records[i];
|
|
2043
|
-
if (record === undefined || asNumber(record.commit_seq) !== commitSeq)
|
|
2044
|
-
break;
|
|
2045
|
-
// NULL tbl: a candidate with no change rows for the table.
|
|
2046
|
-
if (record.tbl === null || record.row_id === null)
|
|
2047
|
-
continue;
|
|
2048
|
-
const change = toStoredChange({
|
|
2049
|
-
tbl: record.tbl,
|
|
2050
|
-
row_id: record.row_id,
|
|
2051
|
-
op: record.op,
|
|
2052
|
-
row_version: record.row_version,
|
|
2053
|
-
scopes: record.scopes,
|
|
2054
|
-
payload: record.payload,
|
|
2055
|
-
});
|
|
2056
|
-
if (matchesEffective(change.scopes, query.scopeFilter)) {
|
|
2057
|
-
changes.push(change);
|
|
2058
|
-
}
|
|
2059
|
-
}
|
|
2060
|
-
// NULL meta: the commit vanished — cursor advanced, nothing emitted.
|
|
2061
|
-
if (head.actor_id === null)
|
|
2062
|
-
continue;
|
|
2063
|
-
if (changes.length === 0)
|
|
2064
|
-
continue;
|
|
2065
|
-
commits.push({
|
|
2066
|
-
commitSeq,
|
|
2067
|
-
createdAtMs: asNumber(head.created_at_ms),
|
|
2068
|
-
actorId: head.actor_id,
|
|
2069
|
-
changes,
|
|
2070
|
-
});
|
|
2071
|
-
deliveredChanges += changes.length;
|
|
2072
|
-
if (deliveredChanges >= query.limitChanges)
|
|
2073
|
-
break;
|
|
2074
|
-
}
|
|
2075
|
-
if (deliveredChanges >= query.limitChanges)
|
|
2076
|
-
break;
|
|
2077
|
-
if (candidateCount < batchSize)
|
|
2078
|
-
break;
|
|
2079
|
-
}
|
|
2080
|
-
return commits;
|
|
2169
|
+
// Concurrent reads of a pull share one statement across tables: one
|
|
2170
|
+
// index-only max over the (partition, tbl, var, value) prefix per spec.
|
|
2171
|
+
const [row] = await this.#batchedPage('latestScopedChange', `SELECT s.i AS sub,
|
|
2172
|
+
(SELECT max(commit_seq) FROM sync_change_scopes
|
|
2173
|
+
WHERE partition=$1 AND tbl=s.tbl AND var=s.var AND value=s.value) AS seq
|
|
2174
|
+
FROM jsonb_to_recordset($2::text::jsonb) AS s(i int, tbl text, var text, value text)
|
|
2175
|
+
ORDER BY s.i`, partition, { tbl: query.table, var: variable, value });
|
|
2176
|
+
return row === undefined || row.seq === null ? 0 : asNumber(row.seq);
|
|
2177
|
+
}
|
|
2178
|
+
readLiveWindow(partition, query) {
|
|
2179
|
+
// One statement per page: one LATERAL range of
|
|
2180
|
+
// `sync_change_scopes_by_key` per unit key, each DISTINCT and LIMITed at
|
|
2181
|
+
// the index, merged in commit order and LIMITed again, then LEFT JOINed
|
|
2182
|
+
// to the commit meta and every change of the commit. A candidate whose
|
|
2183
|
+
// commit vanished comes back with NULL meta and still advances the scan.
|
|
2184
|
+
return readLiveWindowPages(query, async (keys, limit) => {
|
|
2185
|
+
const { rows } = await this.#exec.query(`SELECT c.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,
|
|
2186
|
+
ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op, ch.row_version AS row_version, ch.scopes AS scopes, ch.payload AS payload
|
|
2187
|
+
FROM (SELECT DISTINCT s.commit_seq
|
|
2188
|
+
FROM jsonb_to_recordset($2::text::jsonb) AS k(var text, value text, after bigint)
|
|
2189
|
+
CROSS JOIN LATERAL (SELECT DISTINCT commit_seq FROM sync_change_scopes
|
|
2190
|
+
WHERE partition=$1 AND var=k.var AND value=k.value
|
|
2191
|
+
AND commit_seq>k.after AND commit_seq<=$3
|
|
2192
|
+
ORDER BY commit_seq LIMIT $4) s
|
|
2193
|
+
ORDER BY s.commit_seq LIMIT $4) c
|
|
2194
|
+
LEFT JOIN sync_commits m ON m.partition=$1 AND m.commit_seq=c.commit_seq
|
|
2195
|
+
LEFT JOIN sync_changes ch ON ch.partition=$1 AND ch.commit_seq=c.commit_seq
|
|
2196
|
+
ORDER BY c.commit_seq, ch.idx`, [partition, JSON.stringify(keys), query.throughSeq, limit]);
|
|
2197
|
+
return rows.map((record) => ({
|
|
2198
|
+
commitSeq: asNumber(record.commit_seq),
|
|
2199
|
+
...(record.actor_id === null
|
|
2200
|
+
? {}
|
|
2201
|
+
: {
|
|
2202
|
+
meta: {
|
|
2203
|
+
actorId: record.actor_id,
|
|
2204
|
+
createdAtMs: asNumber(record.created_at_ms),
|
|
2205
|
+
},
|
|
2206
|
+
}),
|
|
2207
|
+
...(record.tbl === null
|
|
2208
|
+
? {}
|
|
2209
|
+
: { change: toStoredChange({ ...record, tbl: record.tbl }) }),
|
|
2210
|
+
}));
|
|
2211
|
+
});
|
|
2081
2212
|
}
|
|
2082
2213
|
async scanRows(partition, query) {
|
|
2083
2214
|
const firstVariable = assertScopeIndexedScan(query);
|
|
@@ -2124,31 +2255,24 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
2124
2255
|
return scanRowsByIndexOn(this.#exec, this.table(query.table), partition, query);
|
|
2125
2256
|
}
|
|
2126
2257
|
async getClientRecord(partition, clientId) {
|
|
2127
|
-
const { rows } = await this.#exec.query(
|
|
2128
|
-
|
|
2129
|
-
if (record === undefined)
|
|
2130
|
-
return undefined;
|
|
2131
|
-
return {
|
|
2132
|
-
clientId: record.client_id,
|
|
2133
|
-
actorId: record.actor_id,
|
|
2134
|
-
wireVersion: asNumber(record.wire_version),
|
|
2135
|
-
cursor: asNumber(record.cursor),
|
|
2136
|
-
updatedAtMs: asNumber(record.updated_at_ms),
|
|
2137
|
-
subscriptions: asJson(record.subscriptions),
|
|
2138
|
-
...clientRoundState(record.last_round_at_ms === null
|
|
2139
|
-
? null
|
|
2140
|
-
: asNumber(record.last_round_at_ms), record.last_error_code, asNumber(record.consecutive_failures)),
|
|
2141
|
-
};
|
|
2258
|
+
const { rows } = await this.#exec.query(`SELECT ${CLIENT_RECORD_COLUMNS} FROM sync_clients WHERE partition=$1 AND client_id=$2`, [partition, clientId]);
|
|
2259
|
+
return rows[0] === undefined ? undefined : toClientRecord(rows[0]);
|
|
2142
2260
|
}
|
|
2143
2261
|
async putClientRecord(partition, record, options) {
|
|
2144
|
-
|
|
2145
|
-
|
|
2262
|
+
// Every non-key field comes from the new record, except the registration
|
|
2263
|
+
// list and digest, which only a traveling list replaces (§4.5).
|
|
2264
|
+
const registration = record.registration;
|
|
2265
|
+
await this.#exec.query(`INSERT INTO sync_clients(partition, client_id, actor_id, wire_version, cursor, updated_at_ms, last_round_at_ms, schema_version, units, registration_digest, catching_up)
|
|
2266
|
+
VALUES ($1,$2,$3,$4,$5,$6,$11,$7,$8::text::jsonb,$9,$10::text::jsonb)
|
|
2146
2267
|
ON CONFLICT (partition, client_id) DO UPDATE
|
|
2147
2268
|
SET actor_id=EXCLUDED.actor_id, wire_version=EXCLUDED.wire_version,
|
|
2148
2269
|
cursor=EXCLUDED.cursor,
|
|
2149
|
-
subscriptions=EXCLUDED.subscriptions,
|
|
2150
2270
|
updated_at_ms=EXCLUDED.updated_at_ms,
|
|
2151
|
-
last_round_at_ms=EXCLUDED.last_round_at_ms
|
|
2271
|
+
last_round_at_ms=EXCLUDED.last_round_at_ms,
|
|
2272
|
+
schema_version=EXCLUDED.schema_version,${registration === undefined
|
|
2273
|
+
? ''
|
|
2274
|
+
: '\n units=EXCLUDED.units, registration_digest=EXCLUDED.registration_digest,'}
|
|
2275
|
+
catching_up=EXCLUDED.catching_up${options?.keepRoundOutcome === true
|
|
2152
2276
|
? ''
|
|
2153
2277
|
: ',\n last_error_code=NULL, consecutive_failures=0'}`, [
|
|
2154
2278
|
partition,
|
|
@@ -2156,8 +2280,12 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
2156
2280
|
record.actorId,
|
|
2157
2281
|
record.wireVersion,
|
|
2158
2282
|
record.cursor,
|
|
2159
|
-
JSON.stringify(record.subscriptions),
|
|
2160
2283
|
record.updatedAtMs,
|
|
2284
|
+
record.schemaVersion ?? null,
|
|
2285
|
+
registration === undefined ? null : JSON.stringify(registration.units),
|
|
2286
|
+
registration?.digest ?? null,
|
|
2287
|
+
JSON.stringify(record.catchingUp),
|
|
2288
|
+
record.lastRoundAtMs ?? record.updatedAtMs,
|
|
2161
2289
|
]);
|
|
2162
2290
|
}
|
|
2163
2291
|
async recordClientRoundFailure(partition, clientId, actorId, errorCode, atMs) {
|
|
@@ -2167,19 +2295,20 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
2167
2295
|
WHERE partition=$3 AND client_id=$4 AND actor_id=$5`, [atMs, errorCode, partition, clientId, actorId]);
|
|
2168
2296
|
}
|
|
2169
2297
|
async advanceClientCursor(partition, clientId, actorId, logEpoch, cursor, updatedAtMs) {
|
|
2298
|
+
// §8.2: the ack never raises the record cursor above a catching-up
|
|
2299
|
+
// unit's cursor, so pruning keeps that unit's history (§4.6).
|
|
2170
2300
|
await this.#exec.query(`UPDATE sync_clients
|
|
2171
|
-
SET cursor=GREATEST(cursor, $1
|
|
2301
|
+
SET cursor=GREATEST(cursor, LEAST($1::bigint, COALESCE((
|
|
2302
|
+
SELECT min((u->>'cursor')::bigint)
|
|
2303
|
+
FROM jsonb_array_elements(sync_clients.catching_up) AS u
|
|
2304
|
+
WHERE (u->>'cursor')::bigint>=0), $1::bigint))),
|
|
2305
|
+
updated_at_ms=GREATEST(updated_at_ms, $2)
|
|
2172
2306
|
WHERE partition=$3 AND client_id=$4 AND actor_id=$5
|
|
2173
2307
|
AND EXISTS (SELECT 1 FROM sync_partition_registry
|
|
2174
2308
|
WHERE partition=sync_clients.partition AND log_epoch=$6)`, [cursor, updatedAtMs, partition, clientId, actorId, logEpoch]);
|
|
2175
2309
|
}
|
|
2176
|
-
async updateClientCursor(partition, clientId, cursor, updatedAtMs) {
|
|
2177
|
-
await this.#exec.query(`UPDATE sync_clients
|
|
2178
|
-
SET cursor=GREATEST(cursor, $3), updated_at_ms=GREATEST(updated_at_ms, $4)
|
|
2179
|
-
WHERE partition=$1 AND client_id=$2`, [partition, clientId, cursor, updatedAtMs]);
|
|
2180
|
-
}
|
|
2181
2310
|
async getActiveClientCursorFloor(partition, cutoffMs) {
|
|
2182
|
-
const { rows } = await this.#exec.query('SELECT MIN(cursor) AS cursor FROM sync_clients WHERE partition=$1 AND updated_at_ms>=$2', [partition, cutoffMs]);
|
|
2311
|
+
const { rows } = await this.#exec.query('SELECT MIN(cursor) AS cursor FROM sync_clients WHERE partition=$1 AND updated_at_ms>=$2 AND cursor>=0', [partition, cutoffMs]);
|
|
2183
2312
|
return rows[0].cursor === null ? null : asNumber(rows[0].cursor);
|
|
2184
2313
|
}
|
|
2185
2314
|
async listClientCursors(partition) {
|
|
@@ -2191,25 +2320,24 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
2191
2320
|
}));
|
|
2192
2321
|
}
|
|
2193
2322
|
async listRowsReferencingBlob(partition, blobId) {
|
|
2194
|
-
//
|
|
2195
|
-
// stored scopes come from
|
|
2196
|
-
|
|
2197
|
-
|
|
2198
|
-
|
|
2199
|
-
|
|
2200
|
-
|
|
2201
|
-
|
|
2202
|
-
|
|
2203
|
-
|
|
2204
|
-
|
|
2205
|
-
|
|
2206
|
-
|
|
2207
|
-
|
|
2208
|
-
|
|
2209
|
-
|
|
2210
|
-
|
|
2211
|
-
|
|
2212
|
-
return out;
|
|
2323
|
+
// One statement: the by-blob index (§5.9.4/§5.9.5) supplies the candidate
|
|
2324
|
+
// rows and each row's stored scopes come from its row table for the §3.4
|
|
2325
|
+
// authorization test (see `rowsReferencingBlobSql`).
|
|
2326
|
+
if (this.#tables === undefined || this.#tables.size === 0)
|
|
2327
|
+
return [];
|
|
2328
|
+
const { rows } = await this.#exec.query(rowsReferencingBlobSql(this.#tables.values(), 'postgres'), [
|
|
2329
|
+
partition,
|
|
2330
|
+
blobId,
|
|
2331
|
+
]);
|
|
2332
|
+
return rows.flatMap((row) => row.scopes === null
|
|
2333
|
+
? []
|
|
2334
|
+
: [
|
|
2335
|
+
{
|
|
2336
|
+
table: row.tbl,
|
|
2337
|
+
rowId: row.row_id,
|
|
2338
|
+
scopes: asJson(row.scopes),
|
|
2339
|
+
},
|
|
2340
|
+
]);
|
|
2213
2341
|
}
|
|
2214
2342
|
async listReferencedBlobIds(partition) {
|
|
2215
2343
|
const { rows } = await this.#exec.query('SELECT DISTINCT blob_id FROM sync_blob_refs WHERE partition=$1', [partition]);
|
|
@@ -2241,7 +2369,8 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
2241
2369
|
WHERE s.partition=$1 AND s.tbl=$2 AND s.var=$3 AND s.value=$4
|
|
2242
2370
|
AND s.commit_seq<=$5 AND c2.op=1 AND c2.scopes->>$3=s.value
|
|
2243
2371
|
LIMIT $6) d
|
|
2244
|
-
WHERE c.partition=$1 AND c.commit_seq=d.commit_seq AND c.idx=d.idx
|
|
2372
|
+
WHERE c.partition=$1 AND c.commit_seq=d.commit_seq AND c.idx=d.idx
|
|
2373
|
+
RETURNING c.commit_seq`, params);
|
|
2245
2374
|
const scopes = await client.query(`DELETE FROM sync_change_scopes cs USING (
|
|
2246
2375
|
SELECT s.commit_seq FROM sync_change_scopes s
|
|
2247
2376
|
WHERE s.partition=$1 AND s.tbl=$2 AND s.var=$3 AND s.value=$4
|
|
@@ -2251,24 +2380,30 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
2251
2380
|
AND c.tbl=s.tbl AND c.scopes->>$3=s.value)
|
|
2252
2381
|
LIMIT $6) d
|
|
2253
2382
|
WHERE cs.partition=$1 AND cs.tbl=$2 AND cs.var=$3 AND cs.value=$4
|
|
2254
|
-
AND cs.commit_seq=d.commit_seq
|
|
2255
|
-
|
|
2383
|
+
AND cs.commit_seq=d.commit_seq
|
|
2384
|
+
RETURNING cs.commit_seq`, params);
|
|
2385
|
+
// §4.6.1: stored conflict results of the binding lose their row payload.
|
|
2386
|
+
const results = await client.query(`UPDATE sync_push_results p SET result=jsonb_set(p.result, '{results}', (
|
|
2387
|
+
SELECT jsonb_agg(e - 'serverRow' ORDER BY i)
|
|
2388
|
+
FROM jsonb_array_elements(p.result->'results')
|
|
2389
|
+
WITH ORDINALITY AS t(e, i)))
|
|
2390
|
+
FROM (
|
|
2391
|
+
SELECT client_id, client_commit_id FROM sync_push_results
|
|
2392
|
+
WHERE partition=$1
|
|
2393
|
+
AND result->'conflictAuthorization'->>'table'=$2
|
|
2394
|
+
AND result->'conflictAuthorization'->'scopes'->>$3=$4
|
|
2395
|
+
AND jsonb_path_exists(result, '$.results[*].serverRow')
|
|
2396
|
+
LIMIT $5) d
|
|
2397
|
+
WHERE p.partition=$1 AND p.client_id=d.client_id
|
|
2398
|
+
AND p.client_commit_id=d.client_commit_id
|
|
2399
|
+
RETURNING p.client_id`, [partition, query.table, query.variable, query.value, query.limit]);
|
|
2400
|
+
return changes.rows.length + scopes.rows.length + results.rows.length;
|
|
2256
2401
|
});
|
|
2257
2402
|
}
|
|
2258
2403
|
// -- admin/console read surface --------------------------------------------
|
|
2259
2404
|
async listClientRecords(partition) {
|
|
2260
|
-
const { rows } = await this.#exec.query(
|
|
2261
|
-
return rows.map(
|
|
2262
|
-
clientId: r.client_id,
|
|
2263
|
-
actorId: r.actor_id,
|
|
2264
|
-
wireVersion: asNumber(r.wire_version),
|
|
2265
|
-
cursor: asNumber(r.cursor),
|
|
2266
|
-
updatedAtMs: asNumber(r.updated_at_ms),
|
|
2267
|
-
subscriptions: (typeof r.subscriptions === 'string'
|
|
2268
|
-
? JSON.parse(r.subscriptions)
|
|
2269
|
-
: r.subscriptions),
|
|
2270
|
-
...clientRoundState(r.last_round_at_ms === null ? null : asNumber(r.last_round_at_ms), r.last_error_code, asNumber(r.consecutive_failures)),
|
|
2271
|
-
}));
|
|
2405
|
+
const { rows } = await this.#exec.query(`SELECT ${CLIENT_RECORD_COLUMNS} FROM sync_clients WHERE partition=$1 ORDER BY updated_at_ms DESC`, [partition]);
|
|
2406
|
+
return rows.map(toClientRecord);
|
|
2272
2407
|
}
|
|
2273
2408
|
async listCommitMetadata(partition, query) {
|
|
2274
2409
|
// Newest-first window, resumed by `afterSeq` (exclusive lower bound) and
|
|
@@ -2287,41 +2422,52 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
2287
2422
|
FROM sync_commits
|
|
2288
2423
|
WHERE partition=$1 AND commit_seq>$2 AND commit_seq<$3
|
|
2289
2424
|
ORDER BY commit_seq DESC LIMIT $4`, [partition, query.afterSeq, beforeSeq, query.limit]);
|
|
2290
|
-
//
|
|
2291
|
-
// `sample` changes (change-index order)
|
|
2425
|
+
// One grouped statement for the page's per-table change counts and, with
|
|
2426
|
+
// `changeSample`, one for the first `sample` changes (change-index order)
|
|
2427
|
+
// of every listed commit.
|
|
2292
2428
|
const sampleCount = query.changeSample !== undefined &&
|
|
2293
2429
|
Number.isInteger(query.changeSample) &&
|
|
2294
2430
|
query.changeSample > 0
|
|
2295
2431
|
? query.changeSample
|
|
2296
2432
|
: 0;
|
|
2433
|
+
const seqs = rows.map((row) => asNumber(row.commit_seq));
|
|
2434
|
+
const counts = new Map();
|
|
2435
|
+
for (const count of seqs.length === 0
|
|
2436
|
+
? []
|
|
2437
|
+
: (await this.#exec.query(`SELECT commit_seq, tbl, count(*) AS n FROM sync_changes
|
|
2438
|
+
WHERE partition=$1 AND commit_seq=ANY($2::bigint[])
|
|
2439
|
+
GROUP BY commit_seq, tbl ORDER BY commit_seq, tbl`, [partition, seqs])).rows) {
|
|
2440
|
+
const seq = asNumber(count.commit_seq);
|
|
2441
|
+
counts.set(seq, [...(counts.get(seq) ?? []), count]);
|
|
2442
|
+
}
|
|
2297
2443
|
const samples = new Map();
|
|
2298
|
-
|
|
2299
|
-
|
|
2300
|
-
|
|
2301
|
-
|
|
2302
|
-
|
|
2303
|
-
|
|
2304
|
-
|
|
2305
|
-
|
|
2444
|
+
for (const change of sampleCount === 0 || seqs.length === 0
|
|
2445
|
+
? []
|
|
2446
|
+
: (await this.#exec.query(`SELECT commit_seq, tbl, row_id, op FROM sync_changes
|
|
2447
|
+
WHERE partition=$1 AND commit_seq=ANY($2::bigint[]) AND idx<$3
|
|
2448
|
+
ORDER BY commit_seq, idx`, [partition, seqs, sampleCount])).rows) {
|
|
2449
|
+
const seq = asNumber(change.commit_seq);
|
|
2450
|
+
samples.set(seq, [
|
|
2451
|
+
...(samples.get(seq) ?? []),
|
|
2452
|
+
{
|
|
2306
2453
|
table: change.tbl,
|
|
2307
2454
|
rowId: change.row_id,
|
|
2308
2455
|
op: asNumber(change.op) === 1 ? 'upsert' : 'delete',
|
|
2309
|
-
}
|
|
2310
|
-
|
|
2311
|
-
}
|
|
2456
|
+
},
|
|
2457
|
+
]);
|
|
2312
2458
|
}
|
|
2313
2459
|
const out = [];
|
|
2314
2460
|
for (const row of rows) {
|
|
2315
2461
|
const commitSeq = asNumber(row.commit_seq);
|
|
2316
|
-
const changes =
|
|
2462
|
+
const changes = counts.get(commitSeq) ?? [];
|
|
2317
2463
|
out.push({
|
|
2318
2464
|
commitSeq,
|
|
2319
2465
|
clientId: row.client_id,
|
|
2320
2466
|
clientCommitId: row.client_commit_id,
|
|
2321
2467
|
actorId: row.actor_id,
|
|
2322
2468
|
createdAtMs: asNumber(row.created_at_ms),
|
|
2323
|
-
changeCount: changes.
|
|
2324
|
-
tables: changes.
|
|
2469
|
+
changeCount: changes.reduce((sum, c) => sum + asNumber(c.n), 0),
|
|
2470
|
+
tables: changes.map((c) => c.tbl),
|
|
2325
2471
|
...(sampleCount > 0 ? { sample: samples.get(commitSeq) ?? [] } : {}),
|
|
2326
2472
|
});
|
|
2327
2473
|
}
|
|
@@ -2394,27 +2540,39 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
2394
2540
|
}
|
|
2395
2541
|
});
|
|
2396
2542
|
}
|
|
2543
|
+
async listPushWatermarks(partition, clientCommitId) {
|
|
2544
|
+
// The clients come from the retained commit log, which has no index on
|
|
2545
|
+
// `client_commit_id`; the read is bounded by retention (§4.6).
|
|
2546
|
+
const { rows } = await this.#exec.query(`SELECT client_id, outbox_id, through_seq FROM sync_push_watermarks
|
|
2547
|
+
WHERE partition=$1 AND client_id IN (
|
|
2548
|
+
SELECT client_id FROM sync_commits WHERE partition=$1 AND client_commit_id=$2)
|
|
2549
|
+
ORDER BY client_id, outbox_id`, [partition, clientCommitId]);
|
|
2550
|
+
return rows.map((row) => ({
|
|
2551
|
+
clientId: row.client_id,
|
|
2552
|
+
outboxId: row.outbox_id,
|
|
2553
|
+
throughSeq: asNumber(row.through_seq),
|
|
2554
|
+
}));
|
|
2555
|
+
}
|
|
2397
2556
|
async scopeActivity(partition, query) {
|
|
2398
|
-
|
|
2399
|
-
|
|
2400
|
-
|
|
2401
|
-
|
|
2402
|
-
|
|
2403
|
-
|
|
2404
|
-
|
|
2405
|
-
|
|
2406
|
-
|
|
2407
|
-
|
|
2408
|
-
|
|
2409
|
-
|
|
2410
|
-
|
|
2411
|
-
|
|
2412
|
-
|
|
2413
|
-
|
|
2414
|
-
|
|
2415
|
-
|
|
2416
|
-
}
|
|
2417
|
-
return out;
|
|
2557
|
+
// One statement for the page: candidate (commit, table) pairs from the
|
|
2558
|
+
// `sync_change_scopes_by_key` range of the key, newest first, joined to
|
|
2559
|
+
// the commit metadata with the table's change count. A candidate whose
|
|
2560
|
+
// commit was pruned has no metadata and is skipped.
|
|
2561
|
+
const { rows } = await this.#exec.query(`SELECT s.commit_seq, s.tbl, m.actor_id, m.created_at_ms,
|
|
2562
|
+
(SELECT count(*) FROM sync_changes ch
|
|
2563
|
+
WHERE ch.partition=$1 AND ch.commit_seq=s.commit_seq AND ch.tbl=s.tbl) AS n
|
|
2564
|
+
FROM (SELECT DISTINCT commit_seq, tbl FROM sync_change_scopes
|
|
2565
|
+
WHERE partition=$1 AND var=$2 AND value=$3
|
|
2566
|
+
ORDER BY commit_seq DESC LIMIT $4) s
|
|
2567
|
+
JOIN sync_commits m ON m.partition=$1 AND m.commit_seq=s.commit_seq
|
|
2568
|
+
ORDER BY s.commit_seq DESC`, [partition, query.variable, query.value, query.limit]);
|
|
2569
|
+
return rows.map((row) => ({
|
|
2570
|
+
commitSeq: asNumber(row.commit_seq),
|
|
2571
|
+
table: row.tbl,
|
|
2572
|
+
createdAtMs: asNumber(row.created_at_ms),
|
|
2573
|
+
actorId: row.actor_id,
|
|
2574
|
+
changeCount: asNumber(row.n),
|
|
2575
|
+
}));
|
|
2418
2576
|
}
|
|
2419
2577
|
async listPartitions() {
|
|
2420
2578
|
return (await this.listPartitionRegistry()).map((entry) => entry.partition);
|