@syncular/server 0.59.0 → 0.60.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -16
- package/dist/admin.d.ts +32 -20
- package/dist/admin.js +42 -40
- package/dist/context.d.ts +6 -8
- package/dist/context.js +1 -3
- package/dist/d1-storage.js +42 -24
- package/dist/errors.js +8 -0
- package/dist/events.d.ts +35 -17
- package/dist/frame-bytes.js +24 -6
- package/dist/handler.js +486 -223
- package/dist/local-authority.js +7 -9
- package/dist/operations.d.ts +5 -2
- package/dist/operations.js +20 -29
- package/dist/postgres-storage.d.ts +24 -19
- package/dist/postgres-storage.js +249 -168
- package/dist/pull.d.ts +68 -53
- package/dist/pull.js +258 -287
- package/dist/push.d.ts +12 -3
- package/dist/push.js +95 -32
- package/dist/realtime.d.ts +17 -11
- package/dist/realtime.js +65 -90
- package/dist/relational-rows.d.ts +0 -57
- package/dist/relational-rows.js +0 -97
- package/dist/seed.js +10 -15
- package/dist/segment-download.d.ts +4 -1
- package/dist/segment-download.js +4 -2
- package/dist/sqlite-dialect.d.ts +36 -59
- package/dist/sqlite-dialect.js +255 -159
- package/dist/sqlite-storage.js +24 -20
- package/dist/storage-query.d.ts +30 -1
- package/dist/storage-query.js +68 -0
- package/dist/storage.d.ts +156 -54
- package/dist/storage.js +15 -0
- package/package.json +2 -2
- package/src/admin.ts +87 -59
- package/src/context.ts +7 -11
- package/src/d1-storage.ts +49 -33
- package/src/errors.ts +8 -0
- package/src/events.ts +36 -17
- package/src/frame-bytes.ts +26 -6
- package/src/handler.ts +628 -286
- package/src/local-authority.ts +10 -11
- package/src/operations.ts +44 -42
- package/src/postgres-storage.ts +336 -226
- package/src/pull.ts +394 -414
- package/src/push.ts +116 -21
- package/src/realtime.ts +85 -129
- package/src/relational-rows.ts +0 -105
- package/src/seed.ts +13 -19
- package/src/segment-download.ts +8 -3
- package/src/sqlite-dialect.ts +363 -204
- package/src/sqlite-storage.ts +45 -25
- package/src/storage-query.ts +102 -0
- package/src/storage.ts +185 -58
package/src/postgres-storage.ts
CHANGED
|
@@ -4,6 +4,7 @@ import type { CommitPruneQuery, CommitPruneResult } from './storage';
|
|
|
4
4
|
import {
|
|
5
5
|
clientRoundState,
|
|
6
6
|
PARTITION_TOUCH_INTERVAL_MS,
|
|
7
|
+
commitDeleteFloor,
|
|
7
8
|
rowsLiveAfterDelete,
|
|
8
9
|
serveGateRefusal,
|
|
9
10
|
} from './storage';
|
|
@@ -18,13 +19,13 @@ import { serveNotReadyError } from './readiness';
|
|
|
18
19
|
*
|
|
19
20
|
* - both the commit log and the current-row table carry a (table, var,
|
|
20
21
|
* value) inverted scope index;
|
|
21
|
-
* - `
|
|
22
|
-
* (`
|
|
23
|
-
* index, then verify the full multi-variable match against the
|
|
24
|
-
* scope map — never a log or table scan;
|
|
22
|
+
* - `readLiveWindow`/`scanRows` select candidates through an index
|
|
23
|
+
* (`sync_change_scopes_by_key` / `sync_row_scopes`), ordered + LIMITed
|
|
24
|
+
* at the index, then verify the full multi-variable match against the
|
|
25
|
+
* stored scope map — never a log or table scan;
|
|
25
26
|
* - the candidate indexes are *covering* for their selection: the ordered
|
|
26
|
-
* column (`commit_seq` / `row_id`) sits in the index key after
|
|
27
|
-
*
|
|
27
|
+
* column (`commit_seq` / `row_id`) sits in the index key after the scope
|
|
28
|
+
* key, so the planner does an index range scan and returns
|
|
28
29
|
* already-ordered candidates without a heap sort. `test/postgres-explain
|
|
29
30
|
* .test.ts` asserts an `Index` node (never `Seq Scan`) so the regression
|
|
30
31
|
* cannot silently return.
|
|
@@ -45,7 +46,7 @@ import { serveNotReadyError } from './readiness';
|
|
|
45
46
|
* would leave gaps on rollback, which the pull-window arithmetic in §4.5
|
|
46
47
|
* does not tolerate). Cross-partition pushes never contend.
|
|
47
48
|
*/
|
|
48
|
-
import type { PushOperationResult,
|
|
49
|
+
import type { PushOperationResult, UnitScopes } from '@syncular/core';
|
|
49
50
|
import {
|
|
50
51
|
bindAuthoritativePartition,
|
|
51
52
|
type BoundAuthoritativeQuery,
|
|
@@ -63,8 +64,6 @@ import {
|
|
|
63
64
|
import {
|
|
64
65
|
assertPhysicalColumns,
|
|
65
66
|
assertStoredLayouts,
|
|
66
|
-
COMMIT_WINDOW_BATCH_SQL,
|
|
67
|
-
LATEST_SCOPED_CHANGE_BATCH_SQL,
|
|
68
67
|
deleteRowSql,
|
|
69
68
|
dropTableDdl,
|
|
70
69
|
indexRowPageStatement,
|
|
@@ -108,20 +107,23 @@ import type {
|
|
|
108
107
|
ClientPushResult,
|
|
109
108
|
ClientRecord,
|
|
110
109
|
ClientRecordWriteOptions,
|
|
111
|
-
ClientSubscription,
|
|
112
110
|
CommitChangeSummary,
|
|
113
111
|
CommitDetail,
|
|
114
112
|
CommitMetadata,
|
|
115
113
|
CommitMetadataQuery,
|
|
116
|
-
CommitWindowQuery,
|
|
117
114
|
DurableJsonValue,
|
|
118
115
|
IndexRowScanQuery,
|
|
116
|
+
LiveWindow,
|
|
117
|
+
LiveWindowQuery,
|
|
119
118
|
NewChange,
|
|
120
119
|
NewCommit,
|
|
121
120
|
NewReaction,
|
|
121
|
+
OutboxSequence,
|
|
122
|
+
OutboxSettlement,
|
|
122
123
|
PartitionRegistryEntry,
|
|
123
124
|
PreparedPushApply,
|
|
124
125
|
PrunedReactionCounts,
|
|
126
|
+
PushWatermark,
|
|
125
127
|
ReactionClaimQuery,
|
|
126
128
|
ReactionFailure,
|
|
127
129
|
ReactionFailureUpdate,
|
|
@@ -140,7 +142,6 @@ import type {
|
|
|
140
142
|
StorageTransaction,
|
|
141
143
|
StoredChange,
|
|
142
144
|
StoredCheckpoint,
|
|
143
|
-
StoredCommit,
|
|
144
145
|
StoredPushResult,
|
|
145
146
|
StoredReaction,
|
|
146
147
|
StoredRow,
|
|
@@ -150,19 +151,28 @@ import {
|
|
|
150
151
|
isPostgresConstraintError,
|
|
151
152
|
StorageConstraintError,
|
|
152
153
|
} from './storage-errors';
|
|
153
|
-
import {
|
|
154
|
+
import {
|
|
155
|
+
assertScopeIndexedScan,
|
|
156
|
+
readLiveWindowPages,
|
|
157
|
+
resolveIndexRowScan,
|
|
158
|
+
} from './storage-query';
|
|
154
159
|
|
|
155
160
|
/**
|
|
156
161
|
* Schema DDL. Applied by `PostgresServerStorage.migrate()` (idempotent).
|
|
157
162
|
*
|
|
158
163
|
* Covering index design (the reason this file exists):
|
|
159
|
-
* - `
|
|
160
|
-
*
|
|
161
|
-
*
|
|
162
|
-
* - `
|
|
163
|
-
*
|
|
164
|
-
*
|
|
165
|
-
*
|
|
164
|
+
* - `sync_change_scopes_by_key (partition, var, value, commit_seq)`: the
|
|
165
|
+
* §4.5 live scan ranges once per unit key on the (partition, var, value)
|
|
166
|
+
* prefix across tables and returns `commit_seq` already ascending;
|
|
167
|
+
* - `sync_change_scopes_pk (partition, tbl, var, value, commit_seq)`:
|
|
168
|
+
* the §4.7 scope pin, erasure and pruning name the table;
|
|
169
|
+
* - `sync_row_scopes_pk (partition, tbl, var, value, row_id)`: the same
|
|
170
|
+
* shape for `scanRows`, ordered by `row_id`.
|
|
171
|
+
*
|
|
172
|
+
* The §6.3.2 watermark delete and the §9 settlement of results stored
|
|
173
|
+
* before wire version 4 (NULL `outbox_id`) range over the `(partition,
|
|
174
|
+
* client_id)` prefix of `sync_push_results`' PRIMARY KEY, which the
|
|
175
|
+
* watermark keeps to one request's results per outbox.
|
|
166
176
|
*
|
|
167
177
|
* `sync_row_scopes_by_row (partition, tbl, row_id)` is the one secondary
|
|
168
178
|
* scope index: `writeRowsOn` and `deleteRow` replace a row's scope entries
|
|
@@ -185,7 +195,8 @@ export const POSTGRES_DDL = `
|
|
|
185
195
|
CREATE TABLE IF NOT EXISTS sync_partitions(
|
|
186
196
|
partition TEXT PRIMARY KEY,
|
|
187
197
|
max_commit_seq BIGINT NOT NULL DEFAULT 0,
|
|
188
|
-
horizon_seq BIGINT NOT NULL DEFAULT 0
|
|
198
|
+
horizon_seq BIGINT NOT NULL DEFAULT 0,
|
|
199
|
+
delete_floor BIGINT NOT NULL DEFAULT 0
|
|
189
200
|
);
|
|
190
201
|
CREATE TABLE IF NOT EXISTS sync_partition_registry(
|
|
191
202
|
partition TEXT PRIMARY KEY,
|
|
@@ -222,11 +233,21 @@ CREATE TABLE IF NOT EXISTS sync_change_scopes(
|
|
|
222
233
|
var TEXT NOT NULL, value TEXT NOT NULL, commit_seq BIGINT NOT NULL,
|
|
223
234
|
PRIMARY KEY(partition, tbl, var, value, commit_seq)
|
|
224
235
|
);
|
|
236
|
+
CREATE INDEX IF NOT EXISTS sync_change_scopes_by_key
|
|
237
|
+
ON sync_change_scopes(partition, var, value, commit_seq);
|
|
225
238
|
CREATE TABLE IF NOT EXISTS sync_push_results(
|
|
226
239
|
partition TEXT NOT NULL, client_id TEXT NOT NULL,
|
|
227
240
|
client_commit_id TEXT NOT NULL, result JSONB NOT NULL,
|
|
241
|
+
outbox_id TEXT, client_seq BIGINT,
|
|
228
242
|
PRIMARY KEY(partition, client_id, client_commit_id)
|
|
229
243
|
);
|
|
244
|
+
ALTER TABLE sync_push_results ADD COLUMN IF NOT EXISTS outbox_id TEXT;
|
|
245
|
+
ALTER TABLE sync_push_results ADD COLUMN IF NOT EXISTS client_seq BIGINT;
|
|
246
|
+
CREATE TABLE IF NOT EXISTS sync_push_watermarks(
|
|
247
|
+
partition TEXT NOT NULL, client_id TEXT NOT NULL, outbox_id TEXT NOT NULL,
|
|
248
|
+
through_seq BIGINT NOT NULL,
|
|
249
|
+
PRIMARY KEY(partition, client_id, outbox_id)
|
|
250
|
+
);
|
|
230
251
|
CREATE TABLE IF NOT EXISTS sync_reactions(
|
|
231
252
|
partition TEXT NOT NULL, idempotency_key TEXT NOT NULL,
|
|
232
253
|
type TEXT NOT NULL, version INTEGER NOT NULL, payload JSONB NOT NULL,
|
|
@@ -250,10 +271,12 @@ CREATE INDEX IF NOT EXISTS sync_reactions_dead_letter
|
|
|
250
271
|
CREATE TABLE IF NOT EXISTS sync_clients(
|
|
251
272
|
partition TEXT NOT NULL, client_id TEXT NOT NULL, actor_id TEXT NOT NULL,
|
|
252
273
|
wire_version INTEGER NOT NULL DEFAULT 1,
|
|
253
|
-
cursor BIGINT NOT NULL,
|
|
274
|
+
cursor BIGINT NOT NULL,
|
|
254
275
|
updated_at_ms BIGINT NOT NULL,
|
|
255
276
|
last_round_at_ms BIGINT, last_error_code TEXT,
|
|
256
277
|
consecutive_failures INTEGER NOT NULL DEFAULT 0,
|
|
278
|
+
schema_version INTEGER, units JSONB, registration_digest BYTEA,
|
|
279
|
+
catching_up JSONB NOT NULL DEFAULT '[]',
|
|
257
280
|
PRIMARY KEY(partition, client_id)
|
|
258
281
|
);
|
|
259
282
|
ALTER TABLE sync_clients
|
|
@@ -262,6 +285,19 @@ ALTER TABLE sync_clients ADD COLUMN IF NOT EXISTS last_round_at_ms BIGINT;
|
|
|
262
285
|
ALTER TABLE sync_clients ADD COLUMN IF NOT EXISTS last_error_code TEXT;
|
|
263
286
|
ALTER TABLE sync_clients
|
|
264
287
|
ADD COLUMN IF NOT EXISTS consecutive_failures INTEGER NOT NULL DEFAULT 0;
|
|
288
|
+
ALTER TABLE sync_clients ADD COLUMN IF NOT EXISTS schema_version INTEGER;
|
|
289
|
+
ALTER TABLE sync_clients ADD COLUMN IF NOT EXISTS units JSONB;
|
|
290
|
+
ALTER TABLE sync_clients ADD COLUMN IF NOT EXISTS registration_digest BYTEA;
|
|
291
|
+
ALTER TABLE sync_clients
|
|
292
|
+
ADD COLUMN IF NOT EXISTS catching_up JSONB NOT NULL DEFAULT '[]';
|
|
293
|
+
-- §9: a database from before wire version 4 (its client records still hold
|
|
294
|
+
-- subscriptions) starts the horizon at its last commit.
|
|
295
|
+
UPDATE sync_partitions SET horizon_seq = max_commit_seq
|
|
296
|
+
WHERE EXISTS (SELECT 1 FROM information_schema.columns
|
|
297
|
+
WHERE table_schema = current_schema()
|
|
298
|
+
AND table_name = 'sync_clients'
|
|
299
|
+
AND column_name = 'subscriptions');
|
|
300
|
+
ALTER TABLE sync_clients DROP COLUMN IF EXISTS subscriptions;
|
|
265
301
|
CREATE TABLE IF NOT EXISTS sync_blob_refs(
|
|
266
302
|
partition TEXT NOT NULL, tbl TEXT NOT NULL, row_id TEXT NOT NULL,
|
|
267
303
|
blob_id TEXT NOT NULL,
|
|
@@ -276,6 +312,13 @@ CREATE TABLE IF NOT EXISTS sync_tombstones(
|
|
|
276
312
|
);
|
|
277
313
|
ALTER TABLE sync_tombstones ADD COLUMN IF NOT EXISTS server_version BIGINT;
|
|
278
314
|
ALTER TABLE sync_tombstones ADD COLUMN IF NOT EXISTS scopes JSONB;
|
|
315
|
+
ALTER TABLE sync_partitions
|
|
316
|
+
ADD COLUMN IF NOT EXISTS delete_floor BIGINT NOT NULL DEFAULT 0;
|
|
317
|
+
UPDATE sync_partitions SET delete_floor = t.version
|
|
318
|
+
FROM (SELECT partition, max(server_version) AS version
|
|
319
|
+
FROM sync_tombstones GROUP BY partition) t
|
|
320
|
+
WHERE t.partition = sync_partitions.partition
|
|
321
|
+
AND t.version > sync_partitions.delete_floor;
|
|
279
322
|
CREATE TABLE IF NOT EXISTS sync_backfill_checkpoints(
|
|
280
323
|
partition TEXT NOT NULL, name TEXT NOT NULL,
|
|
281
324
|
schema_version INTEGER NOT NULL,
|
|
@@ -433,6 +476,7 @@ interface ServeGateRow {
|
|
|
433
476
|
log_epoch: string | null;
|
|
434
477
|
pending: unknown;
|
|
435
478
|
result?: unknown;
|
|
479
|
+
delete_floor?: unknown;
|
|
436
480
|
}
|
|
437
481
|
|
|
438
482
|
/**
|
|
@@ -515,6 +559,7 @@ async function readAdmissionOn(
|
|
|
515
559
|
): Promise<{
|
|
516
560
|
gate: ServeGate;
|
|
517
561
|
persisted: StoredPushResult | undefined;
|
|
562
|
+
deleteFloor: number;
|
|
518
563
|
prefetched: readonly PrefetchRow[];
|
|
519
564
|
}> {
|
|
520
565
|
const params: unknown[] = [
|
|
@@ -525,7 +570,8 @@ async function readAdmissionOn(
|
|
|
525
570
|
];
|
|
526
571
|
let statement = `SELECT ${SERVE_GATE_PROJECTION},
|
|
527
572
|
(SELECT result FROM sync_push_results
|
|
528
|
-
WHERE partition=$1 AND client_id=$3 AND client_commit_id=$4) AS result
|
|
573
|
+
WHERE partition=$1 AND client_id=$3 AND client_commit_id=$4) AS result,
|
|
574
|
+
(SELECT delete_floor FROM sync_partitions WHERE partition=$1) AS delete_floor`;
|
|
529
575
|
if (prefetch !== undefined && prefetch.specs.length > 0) {
|
|
530
576
|
params.push(JSON.stringify(prefetch.specs));
|
|
531
577
|
// The gate row pads the read-ahead columns with typed NULLs so both
|
|
@@ -537,7 +583,7 @@ async function readAdmissionOn(
|
|
|
537
583
|
NULL::bigint AS tombstone_seq, NULL::bigint AS tombstone_version,
|
|
538
584
|
NULL::jsonb AS tombstone_scopes
|
|
539
585
|
UNION ALL
|
|
540
|
-
SELECT NULL::bigint, NULL::text, NULL::json, NULL::jsonb,
|
|
586
|
+
SELECT NULL::bigint, NULL::text, NULL::json, NULL::jsonb, NULL::bigint,
|
|
541
587
|
p.tbl, p.lookup_id, p.row_id, p.server_version, p.scopes,
|
|
542
588
|
p.payload, p.column_versions, p.tombstone_seq,
|
|
543
589
|
p.tombstone_version, p.tombstone_scopes
|
|
@@ -551,6 +597,8 @@ async function readAdmissionOn(
|
|
|
551
597
|
return {
|
|
552
598
|
gate: toServeGate(gateRow),
|
|
553
599
|
persisted: toPersistedPushResult(gateRow?.result),
|
|
600
|
+
// asNumber maps the NULL of a partition without a commit to 0.
|
|
601
|
+
deleteFloor: asNumber(gateRow?.delete_floor),
|
|
554
602
|
prefetched: rows.filter(
|
|
555
603
|
(row): row is AdmissionRow & PrefetchRow =>
|
|
556
604
|
row.tbl !== null && row.tbl !== undefined,
|
|
@@ -790,23 +838,6 @@ interface ChangeRecord {
|
|
|
790
838
|
payload: unknown;
|
|
791
839
|
}
|
|
792
840
|
|
|
793
|
-
/**
|
|
794
|
-
* One result row of `COMMIT_WINDOW_BATCH_SQL` (candidate LEFT JOIN commit meta
|
|
795
|
-
* LEFT JOIN changes): meta/change columns are NULL when the joined row
|
|
796
|
-
* vanished (LEFT JOIN contract).
|
|
797
|
-
*/
|
|
798
|
-
interface CommitWindowRecord {
|
|
799
|
-
commit_seq: unknown;
|
|
800
|
-
actor_id: string | null;
|
|
801
|
-
created_at_ms: unknown;
|
|
802
|
-
tbl: string | null;
|
|
803
|
-
row_id: string | null;
|
|
804
|
-
op: unknown;
|
|
805
|
-
row_version: unknown;
|
|
806
|
-
scopes: unknown;
|
|
807
|
-
payload: unknown;
|
|
808
|
-
}
|
|
809
|
-
|
|
810
841
|
function toStoredRow(record: RowRecord): StoredRow {
|
|
811
842
|
return {
|
|
812
843
|
rowId: record.row_id,
|
|
@@ -836,6 +867,51 @@ function toTombstone(
|
|
|
836
867
|
};
|
|
837
868
|
}
|
|
838
869
|
|
|
870
|
+
const CLIENT_RECORD_COLUMNS =
|
|
871
|
+
'client_id, actor_id, wire_version, cursor, updated_at_ms, schema_version, units, registration_digest, catching_up, last_round_at_ms, last_error_code, consecutive_failures';
|
|
872
|
+
|
|
873
|
+
function toClientRecord(record: {
|
|
874
|
+
client_id: string;
|
|
875
|
+
actor_id: string;
|
|
876
|
+
wire_version: unknown;
|
|
877
|
+
cursor: unknown;
|
|
878
|
+
updated_at_ms: unknown;
|
|
879
|
+
schema_version: unknown;
|
|
880
|
+
units: unknown;
|
|
881
|
+
registration_digest: unknown;
|
|
882
|
+
catching_up: unknown;
|
|
883
|
+
last_round_at_ms: unknown;
|
|
884
|
+
last_error_code: string | null;
|
|
885
|
+
consecutive_failures: unknown;
|
|
886
|
+
}): ClientRecord {
|
|
887
|
+
return {
|
|
888
|
+
clientId: record.client_id,
|
|
889
|
+
actorId: record.actor_id,
|
|
890
|
+
wireVersion: asNumber(record.wire_version),
|
|
891
|
+
cursor: asNumber(record.cursor),
|
|
892
|
+
updatedAtMs: asNumber(record.updated_at_ms),
|
|
893
|
+
...(record.schema_version === null
|
|
894
|
+
? {}
|
|
895
|
+
: { schemaVersion: asNumber(record.schema_version) }),
|
|
896
|
+
...(record.units === null || record.registration_digest === null
|
|
897
|
+
? {}
|
|
898
|
+
: {
|
|
899
|
+
registration: {
|
|
900
|
+
units: asJson<UnitScopes[]>(record.units),
|
|
901
|
+
digest: asBytes(record.registration_digest),
|
|
902
|
+
},
|
|
903
|
+
}),
|
|
904
|
+
catchingUp: asJson<ClientRecord['catchingUp']>(record.catching_up),
|
|
905
|
+
...clientRoundState(
|
|
906
|
+
record.last_round_at_ms === null
|
|
907
|
+
? null
|
|
908
|
+
: asNumber(record.last_round_at_ms),
|
|
909
|
+
record.last_error_code,
|
|
910
|
+
asNumber(record.consecutive_failures),
|
|
911
|
+
),
|
|
912
|
+
};
|
|
913
|
+
}
|
|
914
|
+
|
|
839
915
|
function toStoredChange(record: ChangeRecord): StoredChange {
|
|
840
916
|
return {
|
|
841
917
|
table: record.tbl,
|
|
@@ -1514,6 +1590,7 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1514
1590
|
readonly table: string;
|
|
1515
1591
|
readonly rowIds: readonly string[];
|
|
1516
1592
|
}[] = [],
|
|
1593
|
+
outbox?: OutboxSettlement,
|
|
1517
1594
|
): Promise<PreparedPushApply> {
|
|
1518
1595
|
this.#assertOpen();
|
|
1519
1596
|
// The lock is its own statement: a READ COMMITTED statement's snapshot is
|
|
@@ -1522,6 +1599,10 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1522
1599
|
// and the read into one statement would miss a duplicate that committed
|
|
1523
1600
|
// while this transaction waited.
|
|
1524
1601
|
await this.lockPartition();
|
|
1602
|
+
const watermark =
|
|
1603
|
+
outbox === undefined
|
|
1604
|
+
? undefined
|
|
1605
|
+
: await this.#settleOutbox(clientId, clientCommitId, outbox);
|
|
1525
1606
|
const tables: CompiledTable[] = [];
|
|
1526
1607
|
const specs: { tbl: string; ids: string[] }[] = [];
|
|
1527
1608
|
for (const target of targets) {
|
|
@@ -1543,7 +1624,77 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1543
1624
|
specs.length === 0 ? undefined : { tables, specs },
|
|
1544
1625
|
);
|
|
1545
1626
|
this.#recordPrefetched(admission.prefetched);
|
|
1546
|
-
return {
|
|
1627
|
+
return {
|
|
1628
|
+
gate: admission.gate,
|
|
1629
|
+
persisted: admission.persisted,
|
|
1630
|
+
deleteFloor: admission.deleteFloor,
|
|
1631
|
+
...(watermark === undefined ? {} : { watermark }),
|
|
1632
|
+
};
|
|
1633
|
+
}
|
|
1634
|
+
|
|
1635
|
+
/**
|
|
1636
|
+
* The §9 settlement and §6.3.2 steps 1 and 2 under the partition lock,
|
|
1637
|
+
* before the candidate savepoint, so a rejection keeps them. Two
|
|
1638
|
+
* statements: the settlement touches disjoint rows (outside and inside the
|
|
1639
|
+
* request), and the watermark delete must see its sequences.
|
|
1640
|
+
*/
|
|
1641
|
+
async #settleOutbox(
|
|
1642
|
+
clientId: string,
|
|
1643
|
+
clientCommitId: string,
|
|
1644
|
+
outbox: OutboxSettlement,
|
|
1645
|
+
): Promise<number> {
|
|
1646
|
+
const first = outbox.commits[0];
|
|
1647
|
+
const last = outbox.commits.at(-1);
|
|
1648
|
+
if (
|
|
1649
|
+
first === undefined ||
|
|
1650
|
+
last === undefined ||
|
|
1651
|
+
first.clientCommitId !== clientCommitId
|
|
1652
|
+
) {
|
|
1653
|
+
throw new Error("outbox settlement rides the request's first commit");
|
|
1654
|
+
}
|
|
1655
|
+
const commits = JSON.stringify(
|
|
1656
|
+
outbox.commits.map((commit) => ({
|
|
1657
|
+
id: commit.clientCommitId,
|
|
1658
|
+
seq: commit.clientSeq,
|
|
1659
|
+
})),
|
|
1660
|
+
);
|
|
1661
|
+
// §9 (b): with no pre-upgrade result on the request's last commit, every
|
|
1662
|
+
// remaining one outside the request left the outbox; (a) assigns the
|
|
1663
|
+
// request's sequences to its pre-upgrade results.
|
|
1664
|
+
await this.#client.query(
|
|
1665
|
+
`WITH c AS (SELECT * FROM jsonb_to_recordset($4::text::jsonb) AS c(id text, seq bigint)),
|
|
1666
|
+
b AS (DELETE FROM sync_push_results
|
|
1667
|
+
WHERE partition=$1 AND client_id=$2 AND outbox_id IS NULL
|
|
1668
|
+
AND client_commit_id NOT IN (SELECT id FROM c)
|
|
1669
|
+
AND NOT EXISTS (SELECT 1 FROM sync_push_results
|
|
1670
|
+
WHERE partition=$1 AND client_id=$2 AND client_commit_id=$5
|
|
1671
|
+
AND outbox_id IS NULL)),
|
|
1672
|
+
a AS (UPDATE sync_push_results r SET outbox_id=$3, client_seq=c.seq FROM c
|
|
1673
|
+
WHERE r.partition=$1 AND r.client_id=$2 AND r.outbox_id IS NULL
|
|
1674
|
+
AND r.client_commit_id=c.id)
|
|
1675
|
+
SELECT 1`,
|
|
1676
|
+
[
|
|
1677
|
+
this.#partition,
|
|
1678
|
+
clientId,
|
|
1679
|
+
outbox.outboxId,
|
|
1680
|
+
commits,
|
|
1681
|
+
last.clientCommitId,
|
|
1682
|
+
],
|
|
1683
|
+
);
|
|
1684
|
+
const { rows } = await this.#client.query<{ through_seq: unknown }>(
|
|
1685
|
+
`WITH w AS (
|
|
1686
|
+
INSERT INTO sync_push_watermarks(partition, client_id, outbox_id, through_seq)
|
|
1687
|
+
VALUES ($1,$2,$3,$4)
|
|
1688
|
+
ON CONFLICT (partition, client_id, outbox_id) DO UPDATE
|
|
1689
|
+
SET through_seq=GREATEST(sync_push_watermarks.through_seq, EXCLUDED.through_seq)
|
|
1690
|
+
RETURNING through_seq),
|
|
1691
|
+
d AS (DELETE FROM sync_push_results
|
|
1692
|
+
WHERE partition=$1 AND client_id=$2 AND outbox_id=$3
|
|
1693
|
+
AND client_seq<(SELECT through_seq FROM w))
|
|
1694
|
+
SELECT through_seq FROM w`,
|
|
1695
|
+
[this.#partition, clientId, outbox.outboxId, first.clientSeq],
|
|
1696
|
+
);
|
|
1697
|
+
return asNumber(rows[0]?.through_seq);
|
|
1547
1698
|
}
|
|
1548
1699
|
|
|
1549
1700
|
async lockPartition(): Promise<void> {
|
|
@@ -1583,6 +1734,7 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1583
1734
|
clientId: string,
|
|
1584
1735
|
clientCommitId: string,
|
|
1585
1736
|
result: StoredPushResult,
|
|
1737
|
+
sequence?: OutboxSequence,
|
|
1586
1738
|
): Promise<StoredPushResult> {
|
|
1587
1739
|
this.#assertOpen();
|
|
1588
1740
|
// Candidate rows that never reached the database are discarded with the
|
|
@@ -1598,6 +1750,7 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1598
1750
|
clientId,
|
|
1599
1751
|
clientCommitId,
|
|
1600
1752
|
result,
|
|
1753
|
+
sequence,
|
|
1601
1754
|
);
|
|
1602
1755
|
await this.commit();
|
|
1603
1756
|
return canonical;
|
|
@@ -1608,10 +1761,11 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1608
1761
|
clientId: string,
|
|
1609
1762
|
clientCommitId: string,
|
|
1610
1763
|
result: StoredPushResult,
|
|
1764
|
+
sequence: OutboxSequence | undefined,
|
|
1611
1765
|
): Promise<StoredPushResult> {
|
|
1612
1766
|
const { rows } = await this.#client.query<{ result: unknown }>(
|
|
1613
|
-
`INSERT INTO sync_push_results(partition, client_id, client_commit_id, result)
|
|
1614
|
-
VALUES ($1,$2,$3,$4)
|
|
1767
|
+
`INSERT INTO sync_push_results(partition, client_id, client_commit_id, result, outbox_id, client_seq)
|
|
1768
|
+
VALUES ($1,$2,$3,$4,$5,$6)
|
|
1615
1769
|
ON CONFLICT (partition, client_id, client_commit_id) DO NOTHING
|
|
1616
1770
|
RETURNING result`,
|
|
1617
1771
|
[
|
|
@@ -1619,6 +1773,8 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1619
1773
|
clientId,
|
|
1620
1774
|
clientCommitId,
|
|
1621
1775
|
JSON.stringify(serializePushResult(result)),
|
|
1776
|
+
sequence?.outboxId ?? null,
|
|
1777
|
+
sequence?.clientSeq ?? null,
|
|
1622
1778
|
],
|
|
1623
1779
|
);
|
|
1624
1780
|
if (rows[0] !== undefined) {
|
|
@@ -1734,9 +1890,11 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1734
1890
|
// Allocate the next dense commitSeq under a per-partition row lock: the
|
|
1735
1891
|
// UPDATE … RETURNING serializes concurrent pushes to this partition and
|
|
1736
1892
|
// never leaves a gap on rollback (see the file header).
|
|
1737
|
-
|
|
1893
|
+
// §2.2: the allocation also raises the delete floor to the commit's.
|
|
1894
|
+
const allocate = `INSERT INTO sync_partitions(partition, max_commit_seq, delete_floor) VALUES ($1, 1, $7)
|
|
1738
1895
|
ON CONFLICT (partition) DO UPDATE
|
|
1739
|
-
SET max_commit_seq = sync_partitions.max_commit_seq + 1
|
|
1896
|
+
SET max_commit_seq = sync_partitions.max_commit_seq + 1,
|
|
1897
|
+
delete_floor = GREATEST(sync_partitions.delete_floor, EXCLUDED.delete_floor)
|
|
1740
1898
|
RETURNING max_commit_seq`;
|
|
1741
1899
|
const commitRow = `INSERT INTO sync_commits(partition, commit_seq, client_id, client_commit_id, actor_id, created_at_ms, writer_version)
|
|
1742
1900
|
SELECT $1, allocated.max_commit_seq, $2, $3, $4, $5, $6 FROM allocated
|
|
@@ -1748,6 +1906,7 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1748
1906
|
commit.actorId,
|
|
1749
1907
|
commit.createdAtMs,
|
|
1750
1908
|
this.#writerVersion,
|
|
1909
|
+
commitDeleteFloor(commit.changes),
|
|
1751
1910
|
];
|
|
1752
1911
|
if (commit.changes.length === 0) {
|
|
1753
1912
|
const { rows } = await q.query<{ commit_seq: unknown }>(
|
|
@@ -1774,8 +1933,7 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1774
1933
|
// parameters would otherwise store the string as a scalar. A commit can
|
|
1775
1934
|
// delete one row twice, so a chunk's tombstone insert keeps the latest
|
|
1776
1935
|
// delete per key and a later chunk overwrites it with its own. `version`
|
|
1777
|
-
// is
|
|
1778
|
-
// the tombstone keeps the latter.
|
|
1936
|
+
// is the change's row version (§2.2), which a delete's tombstone keeps.
|
|
1779
1937
|
const valuesOf = (
|
|
1780
1938
|
params: unknown[],
|
|
1781
1939
|
slice: readonly NewChange[],
|
|
@@ -1788,7 +1946,7 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1788
1946
|
change.table,
|
|
1789
1947
|
change.rowId,
|
|
1790
1948
|
change.op === 'upsert' ? 1 : 2,
|
|
1791
|
-
change.rowVersion
|
|
1949
|
+
change.rowVersion,
|
|
1792
1950
|
JSON.stringify(change.scopes),
|
|
1793
1951
|
change.payload ?? null,
|
|
1794
1952
|
);
|
|
@@ -1830,7 +1988,7 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1830
1988
|
),
|
|
1831
1989
|
inserted AS (
|
|
1832
1990
|
INSERT INTO sync_changes(partition, commit_seq, idx, tbl, row_id, op, row_version, scopes, payload)
|
|
1833
|
-
SELECT $1, allocated.max_commit_seq, v.idx, v.tbl, v.row_id, v.op,
|
|
1991
|
+
SELECT $1, allocated.max_commit_seq, v.idx, v.tbl, v.row_id, v.op, v.version, v.scopes, v.payload
|
|
1834
1992
|
FROM allocated CROSS JOIN v
|
|
1835
1993
|
RETURNING partition, idx, tbl, row_id, op, commit_seq, scopes
|
|
1836
1994
|
),
|
|
@@ -1859,7 +2017,7 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1859
2017
|
),
|
|
1860
2018
|
inserted AS (
|
|
1861
2019
|
INSERT INTO sync_changes(partition, commit_seq, idx, tbl, row_id, op, row_version, scopes, payload)
|
|
1862
|
-
SELECT $1, $2, v.idx, v.tbl, v.row_id, v.op,
|
|
2020
|
+
SELECT $1, $2, v.idx, v.tbl, v.row_id, v.op, v.version, v.scopes, v.payload
|
|
1863
2021
|
FROM v
|
|
1864
2022
|
RETURNING partition, idx, tbl, row_id, op, commit_seq, scopes
|
|
1865
2023
|
),
|
|
@@ -1876,17 +2034,20 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1876
2034
|
clientId: string,
|
|
1877
2035
|
clientCommitId: string,
|
|
1878
2036
|
result: StoredPushResult,
|
|
2037
|
+
sequence?: OutboxSequence,
|
|
1879
2038
|
): Promise<void> {
|
|
1880
2039
|
this.#assertOpen();
|
|
1881
2040
|
await this.#client.query(
|
|
1882
|
-
`INSERT INTO sync_push_results(partition, client_id, client_commit_id, result)
|
|
1883
|
-
VALUES ($1,$2,$3,$4)
|
|
2041
|
+
`INSERT INTO sync_push_results(partition, client_id, client_commit_id, result, outbox_id, client_seq)
|
|
2042
|
+
VALUES ($1,$2,$3,$4,$5,$6)
|
|
1884
2043
|
ON CONFLICT (partition, client_id, client_commit_id) DO NOTHING`,
|
|
1885
2044
|
[
|
|
1886
2045
|
this.#partition,
|
|
1887
2046
|
clientId,
|
|
1888
2047
|
clientCommitId,
|
|
1889
2048
|
JSON.stringify(serializePushResult(result)),
|
|
2049
|
+
sequence?.outboxId ?? null,
|
|
2050
|
+
sequence?.clientSeq ?? null,
|
|
1890
2051
|
],
|
|
1891
2052
|
);
|
|
1892
2053
|
}
|
|
@@ -3068,107 +3229,76 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
3068
3229
|
|
|
3069
3230
|
async latestScopedChangeSeq(
|
|
3070
3231
|
partition: string,
|
|
3071
|
-
query: { readonly table: string; readonly
|
|
3232
|
+
query: { readonly table: string; readonly scopes: UnitScopes },
|
|
3072
3233
|
): Promise<number> {
|
|
3073
|
-
const
|
|
3074
|
-
const
|
|
3075
|
-
|
|
3076
|
-
|
|
3077
|
-
|
|
3078
|
-
if (firstVariable === undefined || values.length === 0) return 0;
|
|
3079
|
-
// Concurrent reads of a pull share one statement across tables.
|
|
3234
|
+
const variable = Object.keys(query.scopes).sort()[0];
|
|
3235
|
+
const value = variable === undefined ? undefined : query.scopes[variable];
|
|
3236
|
+
if (variable === undefined || value === undefined) return 0;
|
|
3237
|
+
// Concurrent reads of a pull share one statement across tables: one
|
|
3238
|
+
// index-only max over the (partition, tbl, var, value) prefix per spec.
|
|
3080
3239
|
const [row] = await this.#batchedPage<{ seq: unknown }>(
|
|
3081
3240
|
'latestScopedChange',
|
|
3082
|
-
|
|
3241
|
+
`SELECT s.i AS sub,
|
|
3242
|
+
(SELECT max(commit_seq) FROM sync_change_scopes
|
|
3243
|
+
WHERE partition=$1 AND tbl=s.tbl AND var=s.var AND value=s.value) AS seq
|
|
3244
|
+
FROM jsonb_to_recordset($2::text::jsonb) AS s(i int, tbl text, var text, value text)
|
|
3245
|
+
ORDER BY s.i`,
|
|
3083
3246
|
partition,
|
|
3084
|
-
{ tbl: query.table, var:
|
|
3247
|
+
{ tbl: query.table, var: variable, value },
|
|
3085
3248
|
);
|
|
3086
|
-
return row === undefined ? 0 : asNumber(row.seq);
|
|
3249
|
+
return row === undefined || row.seq === null ? 0 : asNumber(row.seq);
|
|
3087
3250
|
}
|
|
3088
3251
|
|
|
3089
|
-
|
|
3252
|
+
readLiveWindow(
|
|
3090
3253
|
partition: string,
|
|
3091
|
-
query:
|
|
3092
|
-
): Promise<
|
|
3093
|
-
|
|
3094
|
-
|
|
3095
|
-
|
|
3096
|
-
|
|
3097
|
-
|
|
3098
|
-
|
|
3099
|
-
|
|
3100
|
-
|
|
3101
|
-
|
|
3102
|
-
|
|
3103
|
-
|
|
3104
|
-
|
|
3105
|
-
|
|
3106
|
-
|
|
3107
|
-
|
|
3108
|
-
|
|
3109
|
-
|
|
3110
|
-
|
|
3111
|
-
|
|
3112
|
-
|
|
3113
|
-
|
|
3114
|
-
|
|
3115
|
-
|
|
3116
|
-
|
|
3117
|
-
|
|
3118
|
-
|
|
3119
|
-
|
|
3120
|
-
|
|
3121
|
-
|
|
3254
|
+
query: LiveWindowQuery,
|
|
3255
|
+
): Promise<LiveWindow> {
|
|
3256
|
+
// One statement per page: one LATERAL range of
|
|
3257
|
+
// `sync_change_scopes_by_key` per unit key, each DISTINCT and LIMITed at
|
|
3258
|
+
// the index, merged in commit order and LIMITed again, then LEFT JOINed
|
|
3259
|
+
// to the commit meta and every change of the commit. A candidate whose
|
|
3260
|
+
// commit vanished comes back with NULL meta and still advances the scan.
|
|
3261
|
+
return readLiveWindowPages(query, async (keys, limit) => {
|
|
3262
|
+
const { rows } = await this.#exec.query<{
|
|
3263
|
+
commit_seq: unknown;
|
|
3264
|
+
actor_id: string | null;
|
|
3265
|
+
created_at_ms: unknown;
|
|
3266
|
+
tbl: string | null;
|
|
3267
|
+
row_id: string;
|
|
3268
|
+
op: unknown;
|
|
3269
|
+
row_version: unknown;
|
|
3270
|
+
scopes: unknown;
|
|
3271
|
+
payload: unknown;
|
|
3272
|
+
}>(
|
|
3273
|
+
`SELECT c.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,
|
|
3274
|
+
ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op, ch.row_version AS row_version, ch.scopes AS scopes, ch.payload AS payload
|
|
3275
|
+
FROM (SELECT DISTINCT s.commit_seq
|
|
3276
|
+
FROM jsonb_to_recordset($2::text::jsonb) AS k(var text, value text, after bigint)
|
|
3277
|
+
CROSS JOIN LATERAL (SELECT DISTINCT commit_seq FROM sync_change_scopes
|
|
3278
|
+
WHERE partition=$1 AND var=k.var AND value=k.value
|
|
3279
|
+
AND commit_seq>k.after AND commit_seq<=$3
|
|
3280
|
+
ORDER BY commit_seq LIMIT $4) s
|
|
3281
|
+
ORDER BY s.commit_seq LIMIT $4) c
|
|
3282
|
+
LEFT JOIN sync_commits m ON m.partition=$1 AND m.commit_seq=c.commit_seq
|
|
3283
|
+
LEFT JOIN sync_changes ch ON ch.partition=$1 AND ch.commit_seq=c.commit_seq
|
|
3284
|
+
ORDER BY c.commit_seq, ch.idx`,
|
|
3285
|
+
[partition, JSON.stringify(keys), query.throughSeq, limit],
|
|
3122
3286
|
);
|
|
3123
|
-
|
|
3124
|
-
|
|
3125
|
-
|
|
3126
|
-
|
|
3127
|
-
|
|
3128
|
-
|
|
3129
|
-
|
|
3130
|
-
|
|
3131
|
-
|
|
3132
|
-
|
|
3133
|
-
|
|
3134
|
-
|
|
3135
|
-
|
|
3136
|
-
|
|
3137
|
-
|
|
3138
|
-
for (; i < records.length; i++) {
|
|
3139
|
-
const record = records[i];
|
|
3140
|
-
if (record === undefined || asNumber(record.commit_seq) !== commitSeq)
|
|
3141
|
-
break;
|
|
3142
|
-
// NULL tbl: a candidate with no change rows for the table.
|
|
3143
|
-
if (record.tbl === null || record.row_id === null) continue;
|
|
3144
|
-
const change = toStoredChange({
|
|
3145
|
-
tbl: record.tbl,
|
|
3146
|
-
row_id: record.row_id,
|
|
3147
|
-
op: record.op,
|
|
3148
|
-
row_version: record.row_version,
|
|
3149
|
-
scopes: record.scopes,
|
|
3150
|
-
payload: record.payload,
|
|
3151
|
-
});
|
|
3152
|
-
if (matchesEffective(change.scopes, query.scopeFilter)) {
|
|
3153
|
-
changes.push(change);
|
|
3154
|
-
}
|
|
3155
|
-
}
|
|
3156
|
-
// NULL meta: the commit vanished — cursor advanced, nothing emitted.
|
|
3157
|
-
if (head.actor_id === null) continue;
|
|
3158
|
-
if (changes.length === 0) continue;
|
|
3159
|
-
commits.push({
|
|
3160
|
-
commitSeq,
|
|
3161
|
-
createdAtMs: asNumber(head.created_at_ms),
|
|
3162
|
-
actorId: head.actor_id,
|
|
3163
|
-
changes,
|
|
3164
|
-
});
|
|
3165
|
-
deliveredChanges += changes.length;
|
|
3166
|
-
if (deliveredChanges >= query.limitChanges) break;
|
|
3167
|
-
}
|
|
3168
|
-
if (deliveredChanges >= query.limitChanges) break;
|
|
3169
|
-
if (candidateCount < batchSize) break;
|
|
3170
|
-
}
|
|
3171
|
-
return commits;
|
|
3287
|
+
return rows.map((record) => ({
|
|
3288
|
+
commitSeq: asNumber(record.commit_seq),
|
|
3289
|
+
...(record.actor_id === null
|
|
3290
|
+
? {}
|
|
3291
|
+
: {
|
|
3292
|
+
meta: {
|
|
3293
|
+
actorId: record.actor_id,
|
|
3294
|
+
createdAtMs: asNumber(record.created_at_ms),
|
|
3295
|
+
},
|
|
3296
|
+
}),
|
|
3297
|
+
...(record.tbl === null
|
|
3298
|
+
? {}
|
|
3299
|
+
: { change: toStoredChange({ ...record, tbl: record.tbl }) }),
|
|
3300
|
+
}));
|
|
3301
|
+
});
|
|
3172
3302
|
}
|
|
3173
3303
|
|
|
3174
3304
|
async scanRows(partition: string, query: RowScanQuery): Promise<StoredRow[]> {
|
|
@@ -3233,37 +3363,13 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
3233
3363
|
partition: string,
|
|
3234
3364
|
clientId: string,
|
|
3235
3365
|
): Promise<ClientRecord | undefined> {
|
|
3236
|
-
const { rows } = await this.#exec.query<
|
|
3237
|
-
|
|
3238
|
-
|
|
3239
|
-
|
|
3240
|
-
cursor: unknown;
|
|
3241
|
-
subscriptions: unknown;
|
|
3242
|
-
updated_at_ms: unknown;
|
|
3243
|
-
last_round_at_ms: unknown;
|
|
3244
|
-
last_error_code: string | null;
|
|
3245
|
-
consecutive_failures: unknown;
|
|
3246
|
-
}>(
|
|
3247
|
-
'SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms, last_round_at_ms, last_error_code, consecutive_failures FROM sync_clients WHERE partition=$1 AND client_id=$2',
|
|
3366
|
+
const { rows } = await this.#exec.query<
|
|
3367
|
+
Parameters<typeof toClientRecord>[0]
|
|
3368
|
+
>(
|
|
3369
|
+
`SELECT ${CLIENT_RECORD_COLUMNS} FROM sync_clients WHERE partition=$1 AND client_id=$2`,
|
|
3248
3370
|
[partition, clientId],
|
|
3249
3371
|
);
|
|
3250
|
-
|
|
3251
|
-
if (record === undefined) return undefined;
|
|
3252
|
-
return {
|
|
3253
|
-
clientId: record.client_id,
|
|
3254
|
-
actorId: record.actor_id,
|
|
3255
|
-
wireVersion: asNumber(record.wire_version),
|
|
3256
|
-
cursor: asNumber(record.cursor),
|
|
3257
|
-
updatedAtMs: asNumber(record.updated_at_ms),
|
|
3258
|
-
subscriptions: asJson<ClientSubscription[]>(record.subscriptions),
|
|
3259
|
-
...clientRoundState(
|
|
3260
|
-
record.last_round_at_ms === null
|
|
3261
|
-
? null
|
|
3262
|
-
: asNumber(record.last_round_at_ms),
|
|
3263
|
-
record.last_error_code,
|
|
3264
|
-
asNumber(record.consecutive_failures),
|
|
3265
|
-
),
|
|
3266
|
-
};
|
|
3372
|
+
return rows[0] === undefined ? undefined : toClientRecord(rows[0]);
|
|
3267
3373
|
}
|
|
3268
3374
|
|
|
3269
3375
|
async putClientRecord(
|
|
@@ -3271,15 +3377,23 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
3271
3377
|
record: ClientRecord,
|
|
3272
3378
|
options?: ClientRecordWriteOptions,
|
|
3273
3379
|
): Promise<void> {
|
|
3380
|
+
// Every non-key field comes from the new record, except the registration
|
|
3381
|
+
// list and digest, which only a traveling list replaces (§4.5).
|
|
3382
|
+
const registration = record.registration;
|
|
3274
3383
|
await this.#exec.query(
|
|
3275
|
-
`INSERT INTO sync_clients(partition, client_id, actor_id, wire_version, cursor,
|
|
3276
|
-
VALUES ($1,$2,$3,$4,$5,$6,$7,$
|
|
3384
|
+
`INSERT INTO sync_clients(partition, client_id, actor_id, wire_version, cursor, updated_at_ms, last_round_at_ms, schema_version, units, registration_digest, catching_up)
|
|
3385
|
+
VALUES ($1,$2,$3,$4,$5,$6,$11,$7,$8::text::jsonb,$9,$10::text::jsonb)
|
|
3277
3386
|
ON CONFLICT (partition, client_id) DO UPDATE
|
|
3278
3387
|
SET actor_id=EXCLUDED.actor_id, wire_version=EXCLUDED.wire_version,
|
|
3279
3388
|
cursor=EXCLUDED.cursor,
|
|
3280
|
-
subscriptions=EXCLUDED.subscriptions,
|
|
3281
3389
|
updated_at_ms=EXCLUDED.updated_at_ms,
|
|
3282
|
-
last_round_at_ms=EXCLUDED.last_round_at_ms
|
|
3390
|
+
last_round_at_ms=EXCLUDED.last_round_at_ms,
|
|
3391
|
+
schema_version=EXCLUDED.schema_version,${
|
|
3392
|
+
registration === undefined
|
|
3393
|
+
? ''
|
|
3394
|
+
: '\n units=EXCLUDED.units, registration_digest=EXCLUDED.registration_digest,'
|
|
3395
|
+
}
|
|
3396
|
+
catching_up=EXCLUDED.catching_up${
|
|
3283
3397
|
options?.keepRoundOutcome === true
|
|
3284
3398
|
? ''
|
|
3285
3399
|
: ',\n last_error_code=NULL, consecutive_failures=0'
|
|
@@ -3290,8 +3404,12 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
3290
3404
|
record.actorId,
|
|
3291
3405
|
record.wireVersion,
|
|
3292
3406
|
record.cursor,
|
|
3293
|
-
JSON.stringify(record.subscriptions),
|
|
3294
3407
|
record.updatedAtMs,
|
|
3408
|
+
record.schemaVersion ?? null,
|
|
3409
|
+
registration === undefined ? null : JSON.stringify(registration.units),
|
|
3410
|
+
registration?.digest ?? null,
|
|
3411
|
+
JSON.stringify(record.catchingUp),
|
|
3412
|
+
record.lastRoundAtMs ?? record.updatedAtMs,
|
|
3295
3413
|
],
|
|
3296
3414
|
);
|
|
3297
3415
|
}
|
|
@@ -3320,9 +3438,15 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
3320
3438
|
cursor: number,
|
|
3321
3439
|
updatedAtMs: number,
|
|
3322
3440
|
): Promise<void> {
|
|
3441
|
+
// §8.2: the ack never raises the record cursor above a catching-up
|
|
3442
|
+
// unit's cursor, so pruning keeps that unit's history (§4.6).
|
|
3323
3443
|
await this.#exec.query(
|
|
3324
3444
|
`UPDATE sync_clients
|
|
3325
|
-
SET cursor=GREATEST(cursor, $1
|
|
3445
|
+
SET cursor=GREATEST(cursor, LEAST($1::bigint, COALESCE((
|
|
3446
|
+
SELECT min((u->>'cursor')::bigint)
|
|
3447
|
+
FROM jsonb_array_elements(sync_clients.catching_up) AS u
|
|
3448
|
+
WHERE (u->>'cursor')::bigint>=0), $1::bigint))),
|
|
3449
|
+
updated_at_ms=GREATEST(updated_at_ms, $2)
|
|
3326
3450
|
WHERE partition=$3 AND client_id=$4 AND actor_id=$5
|
|
3327
3451
|
AND EXISTS (SELECT 1 FROM sync_partition_registry
|
|
3328
3452
|
WHERE partition=sync_clients.partition AND log_epoch=$6)`,
|
|
@@ -3330,20 +3454,6 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
3330
3454
|
);
|
|
3331
3455
|
}
|
|
3332
3456
|
|
|
3333
|
-
async updateClientCursor(
|
|
3334
|
-
partition: string,
|
|
3335
|
-
clientId: string,
|
|
3336
|
-
cursor: number,
|
|
3337
|
-
updatedAtMs: number,
|
|
3338
|
-
): Promise<void> {
|
|
3339
|
-
await this.#exec.query(
|
|
3340
|
-
`UPDATE sync_clients
|
|
3341
|
-
SET cursor=GREATEST(cursor, $3), updated_at_ms=GREATEST(updated_at_ms, $4)
|
|
3342
|
-
WHERE partition=$1 AND client_id=$2`,
|
|
3343
|
-
[partition, clientId, cursor, updatedAtMs],
|
|
3344
|
-
);
|
|
3345
|
-
}
|
|
3346
|
-
|
|
3347
3457
|
async getActiveClientCursorFloor(
|
|
3348
3458
|
partition: string,
|
|
3349
3459
|
cutoffMs: number,
|
|
@@ -3494,35 +3604,13 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
3494
3604
|
// -- admin/console read surface --------------------------------------------
|
|
3495
3605
|
|
|
3496
3606
|
async listClientRecords(partition: string): Promise<ClientRecord[]> {
|
|
3497
|
-
const { rows } = await this.#exec.query<
|
|
3498
|
-
|
|
3499
|
-
|
|
3500
|
-
|
|
3501
|
-
cursor: unknown;
|
|
3502
|
-
subscriptions: unknown;
|
|
3503
|
-
updated_at_ms: unknown;
|
|
3504
|
-
last_round_at_ms: unknown;
|
|
3505
|
-
last_error_code: string | null;
|
|
3506
|
-
consecutive_failures: unknown;
|
|
3507
|
-
}>(
|
|
3508
|
-
'SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms, last_round_at_ms, last_error_code, consecutive_failures FROM sync_clients WHERE partition=$1 ORDER BY updated_at_ms DESC',
|
|
3607
|
+
const { rows } = await this.#exec.query<
|
|
3608
|
+
Parameters<typeof toClientRecord>[0]
|
|
3609
|
+
>(
|
|
3610
|
+
`SELECT ${CLIENT_RECORD_COLUMNS} FROM sync_clients WHERE partition=$1 ORDER BY updated_at_ms DESC`,
|
|
3509
3611
|
[partition],
|
|
3510
3612
|
);
|
|
3511
|
-
return rows.map(
|
|
3512
|
-
clientId: r.client_id,
|
|
3513
|
-
actorId: r.actor_id,
|
|
3514
|
-
wireVersion: asNumber(r.wire_version),
|
|
3515
|
-
cursor: asNumber(r.cursor),
|
|
3516
|
-
updatedAtMs: asNumber(r.updated_at_ms),
|
|
3517
|
-
subscriptions: (typeof r.subscriptions === 'string'
|
|
3518
|
-
? JSON.parse(r.subscriptions)
|
|
3519
|
-
: r.subscriptions) as ClientSubscription[],
|
|
3520
|
-
...clientRoundState(
|
|
3521
|
-
r.last_round_at_ms === null ? null : asNumber(r.last_round_at_ms),
|
|
3522
|
-
r.last_error_code,
|
|
3523
|
-
asNumber(r.consecutive_failures),
|
|
3524
|
-
),
|
|
3525
|
-
}));
|
|
3613
|
+
return rows.map(toClientRecord);
|
|
3526
3614
|
}
|
|
3527
3615
|
|
|
3528
3616
|
async listCommitMetadata(
|
|
@@ -3750,16 +3838,38 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
3750
3838
|
});
|
|
3751
3839
|
}
|
|
3752
3840
|
|
|
3841
|
+
async listPushWatermarks(
|
|
3842
|
+
partition: string,
|
|
3843
|
+
clientCommitId: string,
|
|
3844
|
+
): Promise<PushWatermark[]> {
|
|
3845
|
+
// The clients come from the retained commit log, which has no index on
|
|
3846
|
+
// `client_commit_id`; the read is bounded by retention (§4.6).
|
|
3847
|
+
const { rows } = await this.#exec.query<{
|
|
3848
|
+
client_id: string;
|
|
3849
|
+
outbox_id: string;
|
|
3850
|
+
through_seq: unknown;
|
|
3851
|
+
}>(
|
|
3852
|
+
`SELECT client_id, outbox_id, through_seq FROM sync_push_watermarks
|
|
3853
|
+
WHERE partition=$1 AND client_id IN (
|
|
3854
|
+
SELECT client_id FROM sync_commits WHERE partition=$1 AND client_commit_id=$2)
|
|
3855
|
+
ORDER BY client_id, outbox_id`,
|
|
3856
|
+
[partition, clientCommitId],
|
|
3857
|
+
);
|
|
3858
|
+
return rows.map((row) => ({
|
|
3859
|
+
clientId: row.client_id,
|
|
3860
|
+
outboxId: row.outbox_id,
|
|
3861
|
+
throughSeq: asNumber(row.through_seq),
|
|
3862
|
+
}));
|
|
3863
|
+
}
|
|
3864
|
+
|
|
3753
3865
|
async scopeActivity(
|
|
3754
3866
|
partition: string,
|
|
3755
3867
|
query: ScopeActivityQuery,
|
|
3756
3868
|
): Promise<ScopeCommitActivity[]> {
|
|
3757
3869
|
// One statement for the page: candidate (commit, table) pairs from the
|
|
3758
|
-
//
|
|
3759
|
-
//
|
|
3760
|
-
//
|
|
3761
|
-
// entries; only the LIMIT bounds the output. A candidate whose commit was
|
|
3762
|
-
// pruned has no metadata and is skipped.
|
|
3870
|
+
// `sync_change_scopes_by_key` range of the key, newest first, joined to
|
|
3871
|
+
// the commit metadata with the table's change count. A candidate whose
|
|
3872
|
+
// commit was pruned has no metadata and is skipped.
|
|
3763
3873
|
const { rows } = await this.#exec.query<{
|
|
3764
3874
|
commit_seq: unknown;
|
|
3765
3875
|
tbl: string;
|