@syncular/server 0.58.0 → 0.59.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -22
- package/dist/admin.js +8 -4
- package/dist/blob-handlers.d.ts +25 -31
- package/dist/blob-handlers.js +63 -41
- package/dist/blob-store.d.ts +42 -24
- package/dist/blob-store.js +0 -0
- package/dist/bucket-blob-store.d.ts +12 -8
- package/dist/bucket-blob-store.js +54 -45
- package/dist/bucket-segment-store.js +18 -16
- package/dist/content-encoding.d.ts +5 -2
- package/dist/content-encoding.js +42 -13
- package/dist/context.d.ts +16 -0
- package/dist/context.js +21 -1
- package/dist/d1-storage.d.ts +14 -98
- package/dist/d1-storage.js +223 -1111
- package/dist/errors.js +8 -0
- package/dist/events.d.ts +1 -1
- package/dist/handler.js +135 -103
- package/dist/local-authority.d.ts +2 -0
- package/dist/local-authority.js +17 -3
- package/dist/object-bucket.d.ts +2 -0
- package/dist/operations-realtime.js +69 -50
- package/dist/pg-executor-pglite.d.ts +5 -6
- package/dist/pg-executor-pglite.js +22 -14
- package/dist/pg-executor.d.ts +4 -1
- package/dist/postgres-storage.d.ts +1 -1
- package/dist/postgres-storage.js +182 -105
- package/dist/pull.d.ts +28 -20
- package/dist/pull.js +85 -87
- package/dist/push.js +272 -147
- package/dist/realtime.d.ts +68 -28
- package/dist/realtime.js +297 -136
- package/dist/relational-rows.d.ts +27 -15
- package/dist/relational-rows.js +38 -22
- package/dist/s3-blob-store.d.ts +11 -12
- package/dist/s3-blob-store.js +20 -13
- package/dist/s3-object-bucket.d.ts +8 -1
- package/dist/s3-object-bucket.js +16 -3
- package/dist/schema.d.ts +14 -0
- package/dist/schema.js +23 -0
- package/dist/scopes.d.ts +6 -2
- package/dist/scopes.js +22 -4
- package/dist/seed.js +1 -2
- package/dist/segment-download.js +18 -4
- package/dist/signed-url.d.ts +2 -2
- package/dist/signed-url.js +9 -2
- package/dist/sigv4.d.ts +8 -2
- package/dist/sigv4.js +8 -4
- package/dist/sqlite-blob-store.d.ts +1 -1
- package/dist/sqlite-blob-store.js +23 -22
- package/dist/sqlite-dialect.d.ts +150 -47
- package/dist/sqlite-dialect.js +987 -88
- package/dist/sqlite-lease-store.js +13 -18
- package/dist/sqlite-storage.d.ts +9 -62
- package/dist/sqlite-storage.js +293 -1117
- package/dist/storage.d.ts +32 -12
- package/package.json +2 -2
- package/src/admin.ts +16 -2
- package/src/blob-handlers.ts +91 -72
- package/src/blob-store.ts +0 -0
- package/src/bucket-blob-store.ts +75 -48
- package/src/bucket-segment-store.ts +20 -15
- package/src/content-encoding.ts +48 -16
- package/src/context.ts +32 -1
- package/src/d1-storage.ts +326 -1692
- package/src/errors.ts +8 -0
- package/src/events.ts +1 -1
- package/src/handler.ts +178 -127
- package/src/local-authority.ts +17 -3
- package/src/object-bucket.ts +2 -0
- package/src/operations-realtime.ts +85 -61
- package/src/pg-executor-pglite.ts +27 -23
- package/src/pg-executor.ts +5 -1
- package/src/postgres-storage.ts +231 -138
- package/src/pull.ts +133 -147
- package/src/push.ts +356 -205
- package/src/realtime.ts +380 -179
- package/src/relational-rows.ts +54 -27
- package/src/s3-blob-store.ts +23 -13
- package/src/s3-object-bucket.ts +20 -2
- package/src/schema.ts +31 -0
- package/src/scopes.ts +23 -3
- package/src/seed.ts +1 -2
- package/src/segment-download.ts +25 -4
- package/src/signed-url.ts +10 -4
- package/src/sigv4.ts +17 -4
- package/src/sqlite-blob-store.ts +27 -26
- package/src/sqlite-dialect.ts +1607 -114
- package/src/sqlite-lease-store.ts +12 -23
- package/src/sqlite-storage.ts +360 -1811
- package/src/storage.ts +33 -12
package/src/postgres-storage.ts
CHANGED
|
@@ -90,7 +90,7 @@ import {
|
|
|
90
90
|
indexSignature,
|
|
91
91
|
schemaDdl,
|
|
92
92
|
serverIndexDdl,
|
|
93
|
-
|
|
93
|
+
rowsReferencingBlobSql,
|
|
94
94
|
selectRowSql,
|
|
95
95
|
selectRowsSql,
|
|
96
96
|
prefetchRowsBatchSql,
|
|
@@ -144,6 +144,7 @@ import type {
|
|
|
144
144
|
StoredPushResult,
|
|
145
145
|
StoredReaction,
|
|
146
146
|
StoredRow,
|
|
147
|
+
Tombstone,
|
|
147
148
|
} from './storage';
|
|
148
149
|
import {
|
|
149
150
|
isPostgresConstraintError,
|
|
@@ -270,9 +271,11 @@ CREATE INDEX IF NOT EXISTS sync_blob_refs_by_blob
|
|
|
270
271
|
ON sync_blob_refs(partition, blob_id);
|
|
271
272
|
CREATE TABLE IF NOT EXISTS sync_tombstones(
|
|
272
273
|
partition TEXT NOT NULL, tbl TEXT NOT NULL, row_id TEXT NOT NULL,
|
|
273
|
-
commit_seq BIGINT NOT NULL,
|
|
274
|
+
commit_seq BIGINT NOT NULL, server_version BIGINT, scopes JSONB,
|
|
274
275
|
PRIMARY KEY(partition, tbl, row_id)
|
|
275
276
|
);
|
|
277
|
+
ALTER TABLE sync_tombstones ADD COLUMN IF NOT EXISTS server_version BIGINT;
|
|
278
|
+
ALTER TABLE sync_tombstones ADD COLUMN IF NOT EXISTS scopes JSONB;
|
|
276
279
|
CREATE TABLE IF NOT EXISTS sync_backfill_checkpoints(
|
|
277
280
|
partition TEXT NOT NULL, name TEXT NOT NULL,
|
|
278
281
|
schema_version INTEGER NOT NULL,
|
|
@@ -480,6 +483,8 @@ interface PrefetchRow {
|
|
|
480
483
|
readonly payload: unknown;
|
|
481
484
|
readonly column_versions: unknown;
|
|
482
485
|
readonly tombstone_seq: unknown;
|
|
486
|
+
readonly tombstone_version: unknown;
|
|
487
|
+
readonly tombstone_scopes: unknown;
|
|
483
488
|
}
|
|
484
489
|
|
|
485
490
|
type AdmissionRow = ServeGateRow & Partial<PrefetchRow>;
|
|
@@ -529,11 +534,13 @@ async function readAdmissionOn(
|
|
|
529
534
|
NULL::text AS tbl, NULL::text AS lookup_id, NULL::text AS row_id,
|
|
530
535
|
NULL::bigint AS server_version, NULL::jsonb AS scopes,
|
|
531
536
|
NULL::bytea AS payload, NULL::bytea AS column_versions,
|
|
532
|
-
NULL::bigint AS tombstone_seq
|
|
537
|
+
NULL::bigint AS tombstone_seq, NULL::bigint AS tombstone_version,
|
|
538
|
+
NULL::jsonb AS tombstone_scopes
|
|
533
539
|
UNION ALL
|
|
534
540
|
SELECT NULL::bigint, NULL::text, NULL::json, NULL::jsonb,
|
|
535
541
|
p.tbl, p.lookup_id, p.row_id, p.server_version, p.scopes,
|
|
536
|
-
p.payload, p.column_versions, p.tombstone_seq
|
|
542
|
+
p.payload, p.column_versions, p.tombstone_seq,
|
|
543
|
+
p.tombstone_version, p.tombstone_scopes
|
|
537
544
|
FROM (${prefetchRowsBatchSql(prefetch.tables, {
|
|
538
545
|
partition: '$1',
|
|
539
546
|
specs: `$${params.length}`,
|
|
@@ -812,6 +819,23 @@ function toStoredRow(record: RowRecord): StoredRow {
|
|
|
812
819
|
};
|
|
813
820
|
}
|
|
814
821
|
|
|
822
|
+
function toTombstone(
|
|
823
|
+
seq: unknown,
|
|
824
|
+
version: unknown,
|
|
825
|
+
scopes: unknown,
|
|
826
|
+
): Tombstone | null {
|
|
827
|
+
if (seq === null || seq === undefined) return null;
|
|
828
|
+
return {
|
|
829
|
+
commitSeq: asNumber(seq),
|
|
830
|
+
...(version === null || version === undefined
|
|
831
|
+
? {}
|
|
832
|
+
: { serverVersion: asNumber(version) }),
|
|
833
|
+
...(scopes === null || scopes === undefined
|
|
834
|
+
? {}
|
|
835
|
+
: { scopes: asJson<Record<string, string>>(scopes) }),
|
|
836
|
+
};
|
|
837
|
+
}
|
|
838
|
+
|
|
815
839
|
function toStoredChange(record: ChangeRecord): StoredChange {
|
|
816
840
|
return {
|
|
817
841
|
table: record.tbl,
|
|
@@ -1144,7 +1168,7 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1144
1168
|
* key, so the next read goes to the database again.
|
|
1145
1169
|
*/
|
|
1146
1170
|
readonly #prefetchedRows = new Map<string, StoredRow | null>();
|
|
1147
|
-
readonly #prefetchedTombstones = new Map<string,
|
|
1171
|
+
readonly #prefetchedTombstones = new Map<string, Tombstone | null>();
|
|
1148
1172
|
/**
|
|
1149
1173
|
* Upserts staged per table (row id → row and its push operation index).
|
|
1150
1174
|
* Every statement of this transaction writes them first (see the
|
|
@@ -1325,7 +1349,7 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1325
1349
|
// `#client` flushes this transaction's staged upserts first, so the rows
|
|
1326
1350
|
// are the final candidate state.
|
|
1327
1351
|
const { rows: records } = await this.#client.query<RowRecord>(
|
|
1328
|
-
selectRowsSql(this.#resolveTable(table)),
|
|
1352
|
+
selectRowsSql(this.#resolveTable(table), 'postgres'),
|
|
1329
1353
|
[this.#partition, missing],
|
|
1330
1354
|
);
|
|
1331
1355
|
for (const record of records) rows.set(record.row_id, toStoredRow(record));
|
|
@@ -1357,7 +1381,7 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1357
1381
|
this.#recordPrefetched(rows);
|
|
1358
1382
|
}
|
|
1359
1383
|
|
|
1360
|
-
/** Answer the following `getRow`/`
|
|
1384
|
+
/** Answer the following `getRow`/`getTombstone` calls from a read snapshot. */
|
|
1361
1385
|
#recordPrefetched(records: readonly PrefetchRow[]): void {
|
|
1362
1386
|
for (const record of records) {
|
|
1363
1387
|
const key = `${record.tbl}\u0000${record.lookup_id}`;
|
|
@@ -1369,26 +1393,35 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1369
1393
|
);
|
|
1370
1394
|
this.#prefetchedTombstones.set(
|
|
1371
1395
|
key,
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1396
|
+
toTombstone(
|
|
1397
|
+
record.tombstone_seq,
|
|
1398
|
+
record.tombstone_version,
|
|
1399
|
+
record.tombstone_scopes,
|
|
1400
|
+
),
|
|
1375
1401
|
);
|
|
1376
1402
|
}
|
|
1377
1403
|
}
|
|
1378
1404
|
|
|
1379
|
-
async
|
|
1405
|
+
async getTombstone(
|
|
1380
1406
|
table: string,
|
|
1381
1407
|
rowId: string,
|
|
1382
|
-
): Promise<
|
|
1408
|
+
): Promise<Tombstone | undefined> {
|
|
1383
1409
|
this.#assertOpen();
|
|
1384
1410
|
const prefetched = this.#prefetchedTombstones.get(`${table}\u0000${rowId}`);
|
|
1385
1411
|
if (prefetched !== undefined) return prefetched ?? undefined;
|
|
1386
|
-
const { rows } = await this.#client.query<{
|
|
1387
|
-
|
|
1412
|
+
const { rows } = await this.#client.query<{
|
|
1413
|
+
commit_seq: unknown;
|
|
1414
|
+
server_version: unknown;
|
|
1415
|
+
scopes: unknown;
|
|
1416
|
+
}>(
|
|
1417
|
+
'SELECT commit_seq, server_version, scopes FROM sync_tombstones WHERE partition=$1 AND tbl=$2 AND row_id=$3',
|
|
1388
1418
|
[this.#partition, table, rowId],
|
|
1389
1419
|
);
|
|
1390
|
-
const
|
|
1391
|
-
return
|
|
1420
|
+
const row = rows[0];
|
|
1421
|
+
return (
|
|
1422
|
+
toTombstone(row?.commit_seq, row?.server_version, row?.scopes) ??
|
|
1423
|
+
undefined
|
|
1424
|
+
);
|
|
1392
1425
|
}
|
|
1393
1426
|
|
|
1394
1427
|
async clearTombstone(table: string, rowId: string): Promise<void> {
|
|
@@ -1527,10 +1560,8 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1527
1560
|
): Promise<boolean> {
|
|
1528
1561
|
this.#assertOpen();
|
|
1529
1562
|
await this.#ensurePushSavepoint();
|
|
1530
|
-
// `RETURNING`:
|
|
1531
|
-
//
|
|
1532
|
-
// adapter derives `rowCount` from the returned rows, so a bare UPDATE
|
|
1533
|
-
// reports 0 and the CAS would never advance on a real Bun.sql host.
|
|
1563
|
+
// `RETURNING`: `PgQueryable` reports returned rows only (no affected-row
|
|
1564
|
+
// count), so every affected-row check counts returned rows.
|
|
1534
1565
|
const result = await this.#client.query<{ name: string }>(
|
|
1535
1566
|
`UPDATE sync_backfill_checkpoints
|
|
1536
1567
|
SET watermark=$1, observed_rows=$2, updated_at_ms=$3
|
|
@@ -1741,8 +1772,10 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1741
1772
|
// chunks run under the same commitSeq with their global idx. Serialized
|
|
1742
1773
|
// scopes bind as text before parsing JSONB; drivers that encode JSONB
|
|
1743
1774
|
// parameters would otherwise store the string as a scalar. A commit can
|
|
1744
|
-
// delete one row twice, so a chunk's tombstone insert keeps
|
|
1745
|
-
// key and a later chunk
|
|
1775
|
+
// delete one row twice, so a chunk's tombstone insert keeps the latest
|
|
1776
|
+
// delete per key and a later chunk overwrites it with its own. `version`
|
|
1777
|
+
// is an upsert's row version or a delete's deleted version (§2.2); only
|
|
1778
|
+
// the tombstone keeps the latter.
|
|
1746
1779
|
const valuesOf = (
|
|
1747
1780
|
params: unknown[],
|
|
1748
1781
|
slice: readonly NewChange[],
|
|
@@ -1755,7 +1788,7 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1755
1788
|
change.table,
|
|
1756
1789
|
change.rowId,
|
|
1757
1790
|
change.op === 'upsert' ? 1 : 2,
|
|
1758
|
-
change.rowVersion ?? null,
|
|
1791
|
+
change.rowVersion ?? change.deletedVersion ?? null,
|
|
1759
1792
|
JSON.stringify(change.scopes),
|
|
1760
1793
|
change.payload ?? null,
|
|
1761
1794
|
);
|
|
@@ -1769,16 +1802,21 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1769
1802
|
)`;
|
|
1770
1803
|
const tombstones = (
|
|
1771
1804
|
movedParam: string,
|
|
1772
|
-
) => `INSERT INTO sync_tombstones(partition, tbl, row_id, commit_seq)
|
|
1773
|
-
SELECT DISTINCT ON (tbl, row_id) partition,
|
|
1774
|
-
|
|
1775
|
-
|
|
1805
|
+
) => `INSERT INTO sync_tombstones(partition, tbl, row_id, commit_seq, server_version, scopes)
|
|
1806
|
+
SELECT DISTINCT ON (inserted.tbl, inserted.row_id) inserted.partition,
|
|
1807
|
+
inserted.tbl, inserted.row_id, inserted.commit_seq,
|
|
1808
|
+
v.version, inserted.scopes
|
|
1809
|
+
FROM inserted JOIN v ON v.idx = inserted.idx
|
|
1810
|
+
WHERE inserted.op = 2
|
|
1776
1811
|
AND NOT EXISTS (
|
|
1777
1812
|
SELECT 1
|
|
1778
1813
|
FROM jsonb_to_recordset(${movedParam}::text::jsonb) AS moved(tbl text, row_id text)
|
|
1779
1814
|
WHERE moved.tbl = inserted.tbl AND moved.row_id = inserted.row_id)
|
|
1815
|
+
ORDER BY inserted.tbl, inserted.row_id, inserted.idx DESC
|
|
1780
1816
|
ON CONFLICT (partition, tbl, row_id)
|
|
1781
|
-
DO UPDATE SET commit_seq = excluded.commit_seq
|
|
1817
|
+
DO UPDATE SET commit_seq = excluded.commit_seq,
|
|
1818
|
+
server_version = excluded.server_version,
|
|
1819
|
+
scopes = excluded.scopes`;
|
|
1782
1820
|
const firstSlice = commit.changes.slice(0, PG_CHANGES_PER_STATEMENT);
|
|
1783
1821
|
const params = [...head];
|
|
1784
1822
|
const values = valuesOf(params, firstSlice, 0);
|
|
@@ -1787,12 +1825,14 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1787
1825
|
const { rows } = await q.query<{ max_commit_seq: unknown }>(
|
|
1788
1826
|
`WITH allocated AS (${allocate}),
|
|
1789
1827
|
allocated_commit AS (${commitRow}),
|
|
1828
|
+
v(idx, tbl, row_id, op, version, scopes, payload) AS (
|
|
1829
|
+
VALUES ${values.join(',')}
|
|
1830
|
+
),
|
|
1790
1831
|
inserted AS (
|
|
1791
1832
|
INSERT INTO sync_changes(partition, commit_seq, idx, tbl, row_id, op, row_version, scopes, payload)
|
|
1792
|
-
SELECT $1, allocated.max_commit_seq, v.idx, v.tbl, v.row_id, v.op, v.
|
|
1793
|
-
FROM allocated
|
|
1794
|
-
|
|
1795
|
-
RETURNING partition, tbl, row_id, op, commit_seq, scopes
|
|
1833
|
+
SELECT $1, allocated.max_commit_seq, v.idx, v.tbl, v.row_id, v.op, CASE WHEN v.op = 1 THEN v.version END, v.scopes, v.payload
|
|
1834
|
+
FROM allocated CROSS JOIN v
|
|
1835
|
+
RETURNING partition, idx, tbl, row_id, op, commit_seq, scopes
|
|
1796
1836
|
),
|
|
1797
1837
|
${scopesCte},
|
|
1798
1838
|
tombstoned AS (${tombstones(movedParam)})
|
|
@@ -1814,11 +1854,14 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1814
1854
|
chunkParams.push(moved);
|
|
1815
1855
|
const chunkMovedParam = `$${chunkParams.length}`;
|
|
1816
1856
|
await q.query(
|
|
1817
|
-
`WITH
|
|
1857
|
+
`WITH v(idx, tbl, row_id, op, version, scopes, payload) AS (
|
|
1858
|
+
VALUES ${chunkValues.join(',')}
|
|
1859
|
+
),
|
|
1860
|
+
inserted AS (
|
|
1818
1861
|
INSERT INTO sync_changes(partition, commit_seq, idx, tbl, row_id, op, row_version, scopes, payload)
|
|
1819
|
-
SELECT $1, $2, v.idx, v.tbl, v.row_id, v.op, v.
|
|
1820
|
-
FROM
|
|
1821
|
-
RETURNING partition, tbl, row_id, op, commit_seq, scopes
|
|
1862
|
+
SELECT $1, $2, v.idx, v.tbl, v.row_id, v.op, CASE WHEN v.op = 1 THEN v.version END, v.scopes, v.payload
|
|
1863
|
+
FROM v
|
|
1864
|
+
RETURNING partition, idx, tbl, row_id, op, commit_seq, scopes
|
|
1822
1865
|
),
|
|
1823
1866
|
${scopesCte},
|
|
1824
1867
|
tombstoned AS (${tombstones(chunkMovedParam)})
|
|
@@ -2445,9 +2488,12 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
2445
2488
|
*/
|
|
2446
2489
|
async begin(partition: string): Promise<StorageTransaction> {
|
|
2447
2490
|
let resolveReady!: (tx: PostgresTransaction) => void;
|
|
2448
|
-
|
|
2449
|
-
|
|
2491
|
+
let rejectReady!: (error: unknown) => void;
|
|
2492
|
+
const ready = new Promise<PostgresTransaction>((res, rej) => {
|
|
2493
|
+
resolveReady = res;
|
|
2494
|
+
rejectReady = rej;
|
|
2450
2495
|
});
|
|
2496
|
+
let opened = false;
|
|
2451
2497
|
let resolveScope!: () => void;
|
|
2452
2498
|
let rejectScope!: (error: unknown) => void;
|
|
2453
2499
|
const scope = new Promise<void>((res, rej) => {
|
|
@@ -2465,12 +2511,20 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
2465
2511
|
resolveScope,
|
|
2466
2512
|
rejectScope,
|
|
2467
2513
|
);
|
|
2514
|
+
opened = true;
|
|
2468
2515
|
resolveReady(tx);
|
|
2469
2516
|
// Hold the transaction open until commit()/rollback() settles `scope`.
|
|
2470
2517
|
await scope;
|
|
2471
2518
|
})
|
|
2472
2519
|
.catch((error: unknown) => {
|
|
2473
2520
|
if (error instanceof RollbackSignal) return;
|
|
2521
|
+
// The executor failed before the callback ran (no connection, BEGIN
|
|
2522
|
+
// refused): `begin()` itself rejects, and no transaction exists to
|
|
2523
|
+
// observe `done`.
|
|
2524
|
+
if (!opened) {
|
|
2525
|
+
rejectReady(error);
|
|
2526
|
+
return;
|
|
2527
|
+
}
|
|
2474
2528
|
throw error;
|
|
2475
2529
|
});
|
|
2476
2530
|
const tx = await ready;
|
|
@@ -2576,15 +2630,23 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
2576
2630
|
[partition, horizonSeq],
|
|
2577
2631
|
);
|
|
2578
2632
|
const removed = await client.query(
|
|
2579
|
-
'DELETE FROM sync_commits WHERE partition=$1 AND commit_seq<=$2',
|
|
2633
|
+
'DELETE FROM sync_commits WHERE partition=$1 AND commit_seq<=$2 RETURNING commit_seq',
|
|
2580
2634
|
[partition, horizonSeq],
|
|
2581
2635
|
);
|
|
2636
|
+
// Scope entries go by the keys of the changes being pruned, before
|
|
2637
|
+
// those changes are deleted: a `commit_seq` predicate alone would scan
|
|
2638
|
+
// the partition's whole scope index (`commit_seq` is its last key column).
|
|
2582
2639
|
await client.query(
|
|
2583
|
-
|
|
2640
|
+
`DELETE FROM sync_change_scopes s USING (
|
|
2641
|
+
SELECT DISTINCT c.tbl, e.key, e.value, c.commit_seq
|
|
2642
|
+
FROM sync_changes c CROSS JOIN LATERAL jsonb_each_text(c.scopes) e
|
|
2643
|
+
WHERE c.partition=$1 AND c.commit_seq<=$2) k
|
|
2644
|
+
WHERE s.partition=$1 AND s.tbl=k.tbl AND s.var=k.key
|
|
2645
|
+
AND s.value=k.value AND s.commit_seq=k.commit_seq`,
|
|
2584
2646
|
[partition, horizonSeq],
|
|
2585
2647
|
);
|
|
2586
2648
|
await client.query(
|
|
2587
|
-
'DELETE FROM
|
|
2649
|
+
'DELETE FROM sync_changes WHERE partition=$1 AND commit_seq<=$2',
|
|
2588
2650
|
[partition, horizonSeq],
|
|
2589
2651
|
);
|
|
2590
2652
|
// §4.6: tombstones prune in the same pass as the commit log.
|
|
@@ -2595,7 +2657,7 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
2595
2657
|
return {
|
|
2596
2658
|
previousHorizonSeq,
|
|
2597
2659
|
horizonSeq,
|
|
2598
|
-
removedCommits: removed.
|
|
2660
|
+
removedCommits: removed.rows.length,
|
|
2599
2661
|
};
|
|
2600
2662
|
});
|
|
2601
2663
|
}
|
|
@@ -2856,10 +2918,11 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
2856
2918
|
SET status='completed', completed_at_ms=$4,
|
|
2857
2919
|
lease_owner=NULL, lease_expires_at_ms=NULL
|
|
2858
2920
|
WHERE partition=$1 AND idempotency_key=$2
|
|
2859
|
-
AND status='leased' AND lease_owner=$3
|
|
2921
|
+
AND status='leased' AND lease_owner=$3
|
|
2922
|
+
RETURNING idempotency_key`,
|
|
2860
2923
|
[partition, idempotencyKey, leaseOwner, completedAtMs],
|
|
2861
2924
|
);
|
|
2862
|
-
return result.
|
|
2925
|
+
return result.rows.length === 1;
|
|
2863
2926
|
}
|
|
2864
2927
|
|
|
2865
2928
|
async extendReactionLease(
|
|
@@ -2871,10 +2934,11 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
2871
2934
|
const result = await this.#exec.query(
|
|
2872
2935
|
`UPDATE sync_reactions SET lease_expires_at_ms=$4
|
|
2873
2936
|
WHERE partition=$1 AND idempotency_key=$2
|
|
2874
|
-
AND status='leased' AND lease_owner=$3
|
|
2937
|
+
AND status='leased' AND lease_owner=$3
|
|
2938
|
+
RETURNING idempotency_key`,
|
|
2875
2939
|
[partition, idempotencyKey, leaseOwner, leaseExpiresAtMs],
|
|
2876
2940
|
);
|
|
2877
|
-
return result.
|
|
2941
|
+
return result.rows.length === 1;
|
|
2878
2942
|
}
|
|
2879
2943
|
|
|
2880
2944
|
async failReaction(
|
|
@@ -2887,7 +2951,8 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
2887
2951
|
SET status=$4, available_at_ms=$5, last_failure=$6,
|
|
2888
2952
|
lease_owner=NULL, lease_expires_at_ms=NULL
|
|
2889
2953
|
WHERE partition=$1 AND idempotency_key=$2
|
|
2890
|
-
AND status='leased' AND lease_owner=$3
|
|
2954
|
+
AND status='leased' AND lease_owner=$3
|
|
2955
|
+
RETURNING idempotency_key`,
|
|
2891
2956
|
[
|
|
2892
2957
|
partition,
|
|
2893
2958
|
idempotencyKey,
|
|
@@ -2897,7 +2962,7 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
2897
2962
|
JSON.stringify(update.failure),
|
|
2898
2963
|
],
|
|
2899
2964
|
);
|
|
2900
|
-
return result.
|
|
2965
|
+
return result.rows.length === 1;
|
|
2901
2966
|
}
|
|
2902
2967
|
|
|
2903
2968
|
async retryReaction(
|
|
@@ -2910,10 +2975,11 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
2910
2975
|
SET status='pending', attempts=0, available_at_ms=$3,
|
|
2911
2976
|
last_failure=NULL, lease_owner=NULL, lease_expires_at_ms=NULL,
|
|
2912
2977
|
completed_at_ms=NULL
|
|
2913
|
-
WHERE partition=$1 AND idempotency_key=$2 AND status='dead-letter'
|
|
2978
|
+
WHERE partition=$1 AND idempotency_key=$2 AND status='dead-letter'
|
|
2979
|
+
RETURNING idempotency_key`,
|
|
2914
2980
|
[partition, idempotencyKey, nowMs],
|
|
2915
2981
|
);
|
|
2916
|
-
return result.
|
|
2982
|
+
return result.rows.length === 1;
|
|
2917
2983
|
}
|
|
2918
2984
|
|
|
2919
2985
|
async getReaction(
|
|
@@ -3283,7 +3349,7 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
3283
3349
|
cutoffMs: number,
|
|
3284
3350
|
): Promise<number | null> {
|
|
3285
3351
|
const { rows } = await this.#exec.query<{ cursor: unknown }>(
|
|
3286
|
-
'SELECT MIN(cursor) AS cursor FROM sync_clients WHERE partition=$1 AND updated_at_ms>=$2',
|
|
3352
|
+
'SELECT MIN(cursor) AS cursor FROM sync_clients WHERE partition=$1 AND updated_at_ms>=$2 AND cursor>=0',
|
|
3287
3353
|
[partition, cutoffMs],
|
|
3288
3354
|
);
|
|
3289
3355
|
return rows[0]!.cursor === null ? null : asNumber(rows[0]!.cursor);
|
|
@@ -3315,36 +3381,29 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
3315
3381
|
readonly scopes: Record<string, string>;
|
|
3316
3382
|
}[]
|
|
3317
3383
|
> {
|
|
3318
|
-
//
|
|
3319
|
-
// stored scopes come from
|
|
3320
|
-
|
|
3384
|
+
// One statement: the by-blob index (§5.9.4/§5.9.5) supplies the candidate
|
|
3385
|
+
// rows and each row's stored scopes come from its row table for the §3.4
|
|
3386
|
+
// authorization test (see `rowsReferencingBlobSql`).
|
|
3387
|
+
if (this.#tables === undefined || this.#tables.size === 0) return [];
|
|
3388
|
+
const { rows } = await this.#exec.query<{
|
|
3321
3389
|
tbl: string;
|
|
3322
3390
|
row_id: string;
|
|
3323
|
-
|
|
3324
|
-
|
|
3325
|
-
|
|
3326
|
-
|
|
3327
|
-
|
|
3328
|
-
|
|
3329
|
-
|
|
3330
|
-
|
|
3331
|
-
|
|
3332
|
-
|
|
3333
|
-
|
|
3334
|
-
|
|
3335
|
-
|
|
3336
|
-
|
|
3337
|
-
|
|
3338
|
-
|
|
3339
|
-
const row = rows[0];
|
|
3340
|
-
if (row === undefined) continue;
|
|
3341
|
-
out.push({
|
|
3342
|
-
table: ref.tbl,
|
|
3343
|
-
rowId: ref.row_id,
|
|
3344
|
-
scopes: asJson<Record<string, string>>(row.scopes),
|
|
3345
|
-
});
|
|
3346
|
-
}
|
|
3347
|
-
return out;
|
|
3391
|
+
scopes: unknown;
|
|
3392
|
+
}>(rowsReferencingBlobSql(this.#tables.values(), 'postgres'), [
|
|
3393
|
+
partition,
|
|
3394
|
+
blobId,
|
|
3395
|
+
]);
|
|
3396
|
+
return rows.flatMap((row) =>
|
|
3397
|
+
row.scopes === null
|
|
3398
|
+
? []
|
|
3399
|
+
: [
|
|
3400
|
+
{
|
|
3401
|
+
table: row.tbl,
|
|
3402
|
+
rowId: row.row_id,
|
|
3403
|
+
scopes: asJson<Record<string, string>>(row.scopes),
|
|
3404
|
+
},
|
|
3405
|
+
],
|
|
3406
|
+
);
|
|
3348
3407
|
}
|
|
3349
3408
|
|
|
3350
3409
|
async listReferencedBlobIds(partition: string): Promise<string[]> {
|
|
@@ -3392,7 +3451,8 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
3392
3451
|
WHERE s.partition=$1 AND s.tbl=$2 AND s.var=$3 AND s.value=$4
|
|
3393
3452
|
AND s.commit_seq<=$5 AND c2.op=1 AND c2.scopes->>$3=s.value
|
|
3394
3453
|
LIMIT $6) d
|
|
3395
|
-
WHERE c.partition=$1 AND c.commit_seq=d.commit_seq AND c.idx=d.idx
|
|
3454
|
+
WHERE c.partition=$1 AND c.commit_seq=d.commit_seq AND c.idx=d.idx
|
|
3455
|
+
RETURNING c.commit_seq`,
|
|
3396
3456
|
params,
|
|
3397
3457
|
);
|
|
3398
3458
|
const scopes = await client.query(
|
|
@@ -3405,10 +3465,29 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
3405
3465
|
AND c.tbl=s.tbl AND c.scopes->>$3=s.value)
|
|
3406
3466
|
LIMIT $6) d
|
|
3407
3467
|
WHERE cs.partition=$1 AND cs.tbl=$2 AND cs.var=$3 AND cs.value=$4
|
|
3408
|
-
AND cs.commit_seq=d.commit_seq
|
|
3468
|
+
AND cs.commit_seq=d.commit_seq
|
|
3469
|
+
RETURNING cs.commit_seq`,
|
|
3409
3470
|
params,
|
|
3410
3471
|
);
|
|
3411
|
-
|
|
3472
|
+
// §4.6.1: stored conflict results of the binding lose their row payload.
|
|
3473
|
+
const results = await client.query(
|
|
3474
|
+
`UPDATE sync_push_results p SET result=jsonb_set(p.result, '{results}', (
|
|
3475
|
+
SELECT jsonb_agg(e - 'serverRow' ORDER BY i)
|
|
3476
|
+
FROM jsonb_array_elements(p.result->'results')
|
|
3477
|
+
WITH ORDINALITY AS t(e, i)))
|
|
3478
|
+
FROM (
|
|
3479
|
+
SELECT client_id, client_commit_id FROM sync_push_results
|
|
3480
|
+
WHERE partition=$1
|
|
3481
|
+
AND result->'conflictAuthorization'->>'table'=$2
|
|
3482
|
+
AND result->'conflictAuthorization'->'scopes'->>$3=$4
|
|
3483
|
+
AND jsonb_path_exists(result, '$.results[*].serverRow')
|
|
3484
|
+
LIMIT $5) d
|
|
3485
|
+
WHERE p.partition=$1 AND p.client_id=d.client_id
|
|
3486
|
+
AND p.client_commit_id=d.client_commit_id
|
|
3487
|
+
RETURNING p.client_id`,
|
|
3488
|
+
[partition, query.table, query.variable, query.value, query.limit],
|
|
3489
|
+
);
|
|
3490
|
+
return changes.rows.length + scopes.rows.length + results.rows.length;
|
|
3412
3491
|
});
|
|
3413
3492
|
}
|
|
3414
3493
|
|
|
@@ -3484,53 +3563,72 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
3484
3563
|
ORDER BY commit_seq DESC LIMIT $4`,
|
|
3485
3564
|
[partition, query.afterSeq, beforeSeq, query.limit],
|
|
3486
3565
|
);
|
|
3487
|
-
//
|
|
3488
|
-
// `sample` changes (change-index order)
|
|
3566
|
+
// One grouped statement for the page's per-table change counts and, with
|
|
3567
|
+
// `changeSample`, one for the first `sample` changes (change-index order)
|
|
3568
|
+
// of every listed commit.
|
|
3489
3569
|
const sampleCount =
|
|
3490
3570
|
query.changeSample !== undefined &&
|
|
3491
3571
|
Number.isInteger(query.changeSample) &&
|
|
3492
3572
|
query.changeSample > 0
|
|
3493
3573
|
? query.changeSample
|
|
3494
3574
|
: 0;
|
|
3575
|
+
const seqs = rows.map((row) => asNumber(row.commit_seq));
|
|
3576
|
+
const counts = new Map<number, { tbl: string; n: unknown }[]>();
|
|
3577
|
+
for (const count of seqs.length === 0
|
|
3578
|
+
? []
|
|
3579
|
+
: (
|
|
3580
|
+
await this.#exec.query<{
|
|
3581
|
+
commit_seq: unknown;
|
|
3582
|
+
tbl: string;
|
|
3583
|
+
n: unknown;
|
|
3584
|
+
}>(
|
|
3585
|
+
`SELECT commit_seq, tbl, count(*) AS n FROM sync_changes
|
|
3586
|
+
WHERE partition=$1 AND commit_seq=ANY($2::bigint[])
|
|
3587
|
+
GROUP BY commit_seq, tbl ORDER BY commit_seq, tbl`,
|
|
3588
|
+
[partition, seqs],
|
|
3589
|
+
)
|
|
3590
|
+
).rows) {
|
|
3591
|
+
const seq = asNumber(count.commit_seq);
|
|
3592
|
+
counts.set(seq, [...(counts.get(seq) ?? []), count]);
|
|
3593
|
+
}
|
|
3495
3594
|
const samples = new Map<number, CommitChangeSummary[]>();
|
|
3496
|
-
|
|
3497
|
-
|
|
3498
|
-
|
|
3499
|
-
|
|
3500
|
-
|
|
3501
|
-
|
|
3502
|
-
|
|
3503
|
-
|
|
3504
|
-
|
|
3505
|
-
|
|
3506
|
-
|
|
3507
|
-
|
|
3508
|
-
|
|
3509
|
-
|
|
3510
|
-
|
|
3511
|
-
|
|
3595
|
+
for (const change of sampleCount === 0 || seqs.length === 0
|
|
3596
|
+
? []
|
|
3597
|
+
: (
|
|
3598
|
+
await this.#exec.query<{
|
|
3599
|
+
commit_seq: unknown;
|
|
3600
|
+
tbl: string;
|
|
3601
|
+
row_id: string;
|
|
3602
|
+
op: unknown;
|
|
3603
|
+
}>(
|
|
3604
|
+
`SELECT commit_seq, tbl, row_id, op FROM sync_changes
|
|
3605
|
+
WHERE partition=$1 AND commit_seq=ANY($2::bigint[]) AND idx<$3
|
|
3606
|
+
ORDER BY commit_seq, idx`,
|
|
3607
|
+
[partition, seqs, sampleCount],
|
|
3608
|
+
)
|
|
3609
|
+
).rows) {
|
|
3610
|
+
const seq = asNumber(change.commit_seq);
|
|
3611
|
+
samples.set(seq, [
|
|
3612
|
+
...(samples.get(seq) ?? []),
|
|
3613
|
+
{
|
|
3512
3614
|
table: change.tbl,
|
|
3513
3615
|
rowId: change.row_id,
|
|
3514
3616
|
op: asNumber(change.op) === 1 ? 'upsert' : 'delete',
|
|
3515
|
-
}
|
|
3516
|
-
|
|
3517
|
-
}
|
|
3617
|
+
},
|
|
3618
|
+
]);
|
|
3518
3619
|
}
|
|
3519
3620
|
const out: CommitMetadata[] = [];
|
|
3520
3621
|
for (const row of rows) {
|
|
3521
3622
|
const commitSeq = asNumber(row.commit_seq);
|
|
3522
|
-
const changes =
|
|
3523
|
-
'SELECT tbl, count(*) AS n FROM sync_changes WHERE partition=$1 AND commit_seq=$2 GROUP BY tbl',
|
|
3524
|
-
[partition, commitSeq],
|
|
3525
|
-
);
|
|
3623
|
+
const changes = counts.get(commitSeq) ?? [];
|
|
3526
3624
|
out.push({
|
|
3527
3625
|
commitSeq,
|
|
3528
3626
|
clientId: row.client_id,
|
|
3529
3627
|
clientCommitId: row.client_commit_id,
|
|
3530
3628
|
actorId: row.actor_id,
|
|
3531
3629
|
createdAtMs: asNumber(row.created_at_ms),
|
|
3532
|
-
changeCount: changes.
|
|
3533
|
-
tables: changes.
|
|
3630
|
+
changeCount: changes.reduce((sum, c) => sum + asNumber(c.n), 0),
|
|
3631
|
+
tables: changes.map((c) => c.tbl),
|
|
3534
3632
|
...(sampleCount > 0 ? { sample: samples.get(commitSeq) ?? [] } : {}),
|
|
3535
3633
|
});
|
|
3536
3634
|
}
|
|
@@ -3656,41 +3754,36 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
3656
3754
|
partition: string,
|
|
3657
3755
|
query: ScopeActivityQuery,
|
|
3658
3756
|
): Promise<ScopeCommitActivity[]> {
|
|
3757
|
+
// One statement for the page: candidate (commit, table) pairs from the
|
|
3758
|
+
// change-scope index, joined to the commit metadata with the table's
|
|
3759
|
+
// change count. `tbl` sits between `partition` and `var` in the index key,
|
|
3760
|
+
// so the `var`/`value` filter ranges over the partition's change-scope
|
|
3761
|
+
// entries; only the LIMIT bounds the output. A candidate whose commit was
|
|
3762
|
+
// pruned has no metadata and is skipped.
|
|
3659
3763
|
const { rows } = await this.#exec.query<{
|
|
3660
3764
|
commit_seq: unknown;
|
|
3661
3765
|
tbl: string;
|
|
3766
|
+
actor_id: string;
|
|
3767
|
+
created_at_ms: unknown;
|
|
3768
|
+
n: unknown;
|
|
3662
3769
|
}>(
|
|
3663
|
-
`SELECT
|
|
3664
|
-
|
|
3665
|
-
|
|
3770
|
+
`SELECT s.commit_seq, s.tbl, m.actor_id, m.created_at_ms,
|
|
3771
|
+
(SELECT count(*) FROM sync_changes ch
|
|
3772
|
+
WHERE ch.partition=$1 AND ch.commit_seq=s.commit_seq AND ch.tbl=s.tbl) AS n
|
|
3773
|
+
FROM (SELECT DISTINCT commit_seq, tbl FROM sync_change_scopes
|
|
3774
|
+
WHERE partition=$1 AND var=$2 AND value=$3
|
|
3775
|
+
ORDER BY commit_seq DESC LIMIT $4) s
|
|
3776
|
+
JOIN sync_commits m ON m.partition=$1 AND m.commit_seq=s.commit_seq
|
|
3777
|
+
ORDER BY s.commit_seq DESC`,
|
|
3666
3778
|
[partition, query.variable, query.value, query.limit],
|
|
3667
3779
|
);
|
|
3668
|
-
|
|
3669
|
-
|
|
3670
|
-
|
|
3671
|
-
|
|
3672
|
-
|
|
3673
|
-
|
|
3674
|
-
|
|
3675
|
-
'SELECT actor_id, created_at_ms FROM sync_commits WHERE partition=$1 AND commit_seq=$2',
|
|
3676
|
-
[partition, commitSeq],
|
|
3677
|
-
);
|
|
3678
|
-
const metaRow = meta.rows[0];
|
|
3679
|
-
if (metaRow === undefined) continue;
|
|
3680
|
-
const count = await this.#exec.query<{ n: unknown }>(
|
|
3681
|
-
'SELECT count(*) AS n FROM sync_changes WHERE partition=$1 AND commit_seq=$2 AND tbl=$3',
|
|
3682
|
-
[partition, commitSeq, row.tbl],
|
|
3683
|
-
);
|
|
3684
|
-
out.push({
|
|
3685
|
-
commitSeq,
|
|
3686
|
-
table: row.tbl,
|
|
3687
|
-
createdAtMs: asNumber(metaRow.created_at_ms),
|
|
3688
|
-
actorId: metaRow.actor_id,
|
|
3689
|
-
changeCount:
|
|
3690
|
-
count.rows[0] === undefined ? 0 : asNumber(count.rows[0].n),
|
|
3691
|
-
});
|
|
3692
|
-
}
|
|
3693
|
-
return out;
|
|
3780
|
+
return rows.map((row) => ({
|
|
3781
|
+
commitSeq: asNumber(row.commit_seq),
|
|
3782
|
+
table: row.tbl,
|
|
3783
|
+
createdAtMs: asNumber(row.created_at_ms),
|
|
3784
|
+
actorId: row.actor_id,
|
|
3785
|
+
changeCount: asNumber(row.n),
|
|
3786
|
+
}));
|
|
3694
3787
|
}
|
|
3695
3788
|
|
|
3696
3789
|
async listPartitions(): Promise<string[]> {
|