@syncular/server 0.77.4 → 0.78.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/d1-storage.js +36 -27
- package/dist/handler.js +39 -20
- package/dist/postgres-storage.js +89 -77
- package/dist/push.d.ts +29 -31
- package/dist/push.js +353 -343
- package/dist/sqlite-dialect.d.ts +7 -7
- package/dist/sqlite-dialect.js +41 -25
- package/dist/sqlite-storage.js +17 -20
- package/dist/storage.d.ts +53 -31
- package/package.json +2 -2
- package/src/d1-storage.ts +48 -34
- package/src/handler.ts +53 -31
- package/src/postgres-storage.ts +106 -103
- package/src/push.ts +452 -475
- package/src/sqlite-dialect.ts +57 -34
- package/src/sqlite-storage.ts +29 -30
- package/src/storage.ts +54 -32
package/src/postgres-storage.ts
CHANGED
|
@@ -133,6 +133,7 @@ import type {
|
|
|
133
133
|
PartitionRegistryEntry,
|
|
134
134
|
PreparedPushApply,
|
|
135
135
|
PrunedReactionCounts,
|
|
136
|
+
PushApplyCommit,
|
|
136
137
|
PushWatermark,
|
|
137
138
|
ReactionClaimQuery,
|
|
138
139
|
ReactionFailure,
|
|
@@ -485,7 +486,7 @@ interface ServeGateRow {
|
|
|
485
486
|
schema_version: unknown;
|
|
486
487
|
log_epoch: string | null;
|
|
487
488
|
pending: unknown;
|
|
488
|
-
|
|
489
|
+
results?: unknown;
|
|
489
490
|
delete_floor?: unknown;
|
|
490
491
|
}
|
|
491
492
|
|
|
@@ -545,17 +546,18 @@ type AdmissionRow = ServeGateRow & Partial<PrefetchRow>;
|
|
|
545
546
|
|
|
546
547
|
/**
|
|
547
548
|
* The push admission read: the RFC 0007 serve gate, the §2.3 idempotency
|
|
548
|
-
*
|
|
549
|
-
* read-ahead rows, in one statement with one
|
|
550
|
-
* partition lock first: under READ COMMITTED a
|
|
551
|
-
* at its start, so the read must be the
|
|
552
|
-
* a duplicate that committed while this
|
|
549
|
+
* records of every commit of the request, and (when the caller already knows
|
|
550
|
+
* the commits' row targets) the read-ahead rows, in one statement with one
|
|
551
|
+
* snapshot. The caller takes the partition lock first: under READ COMMITTED a
|
|
552
|
+
* statement's snapshot is fixed at its start, so the read must be the
|
|
553
|
+
* statement *after* the lock to observe a duplicate that committed while this
|
|
554
|
+
* transaction waited for it.
|
|
553
555
|
*/
|
|
554
556
|
async function readAdmissionOn(
|
|
555
557
|
client: PgQueryable,
|
|
556
558
|
partition: string,
|
|
557
559
|
clientId: string,
|
|
558
|
-
|
|
560
|
+
clientCommitIds: readonly string[],
|
|
559
561
|
runningSchemaVersion: number,
|
|
560
562
|
prefetch:
|
|
561
563
|
| {
|
|
@@ -568,7 +570,7 @@ async function readAdmissionOn(
|
|
|
568
570
|
| undefined,
|
|
569
571
|
): Promise<{
|
|
570
572
|
gate: ServeGate;
|
|
571
|
-
persisted:
|
|
573
|
+
persisted: ReadonlyMap<string, unknown>;
|
|
572
574
|
deleteFloor: number;
|
|
573
575
|
prefetched: readonly PrefetchRow[];
|
|
574
576
|
}> {
|
|
@@ -576,11 +578,11 @@ async function readAdmissionOn(
|
|
|
576
578
|
partition,
|
|
577
579
|
runningSchemaVersion,
|
|
578
580
|
clientId,
|
|
579
|
-
|
|
581
|
+
[...clientCommitIds],
|
|
580
582
|
];
|
|
581
583
|
let statement = `SELECT ${SERVE_GATE_PROJECTION},
|
|
582
|
-
(SELECT result FROM sync_push_results
|
|
583
|
-
WHERE partition=$1 AND client_id=$3 AND client_commit_id
|
|
584
|
+
(SELECT jsonb_object_agg(client_commit_id, result) FROM sync_push_results
|
|
585
|
+
WHERE partition=$1 AND client_id=$3 AND client_commit_id = ANY($4::text[])) AS results,
|
|
584
586
|
(SELECT delete_floor FROM sync_partitions WHERE partition=$1) AS delete_floor`;
|
|
585
587
|
if (prefetch !== undefined && prefetch.specs.length > 0) {
|
|
586
588
|
params.push(JSON.stringify(prefetch.specs));
|
|
@@ -604,9 +606,17 @@ async function readAdmissionOn(
|
|
|
604
606
|
}
|
|
605
607
|
const { rows } = await client.query<AdmissionRow>(statement, params);
|
|
606
608
|
const gateRow = rows.find((row) => row.tbl === null) ?? rows[0];
|
|
609
|
+
const results =
|
|
610
|
+
gateRow?.results === null || gateRow?.results === undefined
|
|
611
|
+
? {}
|
|
612
|
+
: asJson(gateRow.results);
|
|
607
613
|
return {
|
|
608
614
|
gate: toServeGate(gateRow),
|
|
609
|
-
persisted:
|
|
615
|
+
persisted: new Map(
|
|
616
|
+
typeof results === 'object' && results !== null
|
|
617
|
+
? Object.entries(results)
|
|
618
|
+
: [],
|
|
619
|
+
),
|
|
610
620
|
// asNumber maps the NULL of a partition without a commit to 0.
|
|
611
621
|
deleteFloor: asNumber(gateRow?.delete_floor),
|
|
612
622
|
prefetched: rows.filter(
|
|
@@ -1274,7 +1284,20 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1274
1284
|
/** The storage's `chooseScanVariable` picks. */
|
|
1275
1285
|
readonly #scanVariables: Map<string, string>;
|
|
1276
1286
|
#open = true;
|
|
1287
|
+
/** True once the request's current commit has written behind its savepoint. */
|
|
1277
1288
|
#pushApplySavepoint = false;
|
|
1289
|
+
/**
|
|
1290
|
+
* The request's §2.3 results, written as one statement at `commit()`
|
|
1291
|
+
* (§6.4): each commit closes with its result, and none is read back before
|
|
1292
|
+
* the transaction ends.
|
|
1293
|
+
*/
|
|
1294
|
+
readonly #pushResults: {
|
|
1295
|
+
readonly client_id: string;
|
|
1296
|
+
readonly client_commit_id: string;
|
|
1297
|
+
readonly result: unknown;
|
|
1298
|
+
readonly outbox_id: string | null;
|
|
1299
|
+
readonly client_seq: number | null;
|
|
1300
|
+
}[] = [];
|
|
1278
1301
|
/**
|
|
1279
1302
|
* `prefetchRows` results keyed `table\u0000rowId`; `null` records a row
|
|
1280
1303
|
* (or tombstone) read as absent. A write in this transaction drops the
|
|
@@ -1423,9 +1446,12 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1423
1446
|
}
|
|
1424
1447
|
|
|
1425
1448
|
/**
|
|
1426
|
-
* The candidate savepoint, taken before
|
|
1427
|
-
* Every candidate write must be undoable by `
|
|
1428
|
-
*
|
|
1449
|
+
* The current commit's candidate savepoint, taken before its first write.
|
|
1450
|
+
* Every candidate write must be undoable by `rejectPushCommit`, and a
|
|
1451
|
+
* refused commit that wrote nothing must not pay to create one. Each commit
|
|
1452
|
+
* takes its own savepoint under the same name: `ROLLBACK TO` targets the
|
|
1453
|
+
* newest, and an earlier commit's savepoint stays open until the
|
|
1454
|
+
* transaction ends, so no `RELEASE` round trip is needed (§6.4).
|
|
1429
1455
|
*/
|
|
1430
1456
|
#ensurePushSavepoint(): Promise<unknown> | undefined {
|
|
1431
1457
|
if (this.#pushApplySavepoint) return undefined;
|
|
@@ -1634,12 +1660,8 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1634
1660
|
|
|
1635
1661
|
async preparePushApply(
|
|
1636
1662
|
clientId: string,
|
|
1637
|
-
|
|
1663
|
+
commits: readonly PushApplyCommit[],
|
|
1638
1664
|
runningSchemaVersion: number,
|
|
1639
|
-
targets: readonly {
|
|
1640
|
-
readonly table: string;
|
|
1641
|
-
readonly rowIds: readonly string[];
|
|
1642
|
-
}[] = [],
|
|
1643
1665
|
outbox?: OutboxSettlement,
|
|
1644
1666
|
): Promise<PreparedPushApply> {
|
|
1645
1667
|
this.#assertOpen();
|
|
@@ -1652,31 +1674,36 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1652
1674
|
const watermark =
|
|
1653
1675
|
outbox === undefined
|
|
1654
1676
|
? undefined
|
|
1655
|
-
: await this.#settleOutbox(clientId,
|
|
1656
|
-
const
|
|
1657
|
-
const
|
|
1658
|
-
|
|
1659
|
-
|
|
1660
|
-
(rowId
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
1677
|
+
: await this.#settleOutbox(clientId, outbox);
|
|
1678
|
+
const ids = new Map<string, Set<string>>();
|
|
1679
|
+
for (const commit of commits) {
|
|
1680
|
+
for (const target of commit.targets ?? []) {
|
|
1681
|
+
const table = ids.get(target.table) ?? new Set<string>();
|
|
1682
|
+
for (const rowId of target.rowIds) {
|
|
1683
|
+
if (!this.#prefetchedRows.has(`${target.table}\u0000${rowId}`)) {
|
|
1684
|
+
table.add(rowId);
|
|
1685
|
+
}
|
|
1686
|
+
}
|
|
1687
|
+
if (table.size > 0) ids.set(target.table, table);
|
|
1688
|
+
}
|
|
1665
1689
|
}
|
|
1666
|
-
|
|
1667
|
-
|
|
1690
|
+
const tables = [...ids.keys()].map((table) => this.#resolveTable(table));
|
|
1691
|
+
const specs = [...ids].map(([tbl, rowIds]) => ({ tbl, ids: [...rowIds] }));
|
|
1692
|
+
// One statement for the whole request: the serve gate, every commit's
|
|
1693
|
+
// idempotency record, and the read-ahead of every commit's known rows.
|
|
1668
1694
|
const admission = await readAdmissionOn(
|
|
1669
1695
|
this.#client,
|
|
1670
1696
|
this.#partition,
|
|
1671
1697
|
clientId,
|
|
1672
|
-
clientCommitId,
|
|
1698
|
+
commits.map((commit) => commit.clientCommitId),
|
|
1673
1699
|
runningSchemaVersion,
|
|
1674
1700
|
specs.length === 0 ? undefined : { tables, specs },
|
|
1675
1701
|
);
|
|
1676
1702
|
this.#recordPrefetched(admission.prefetched);
|
|
1677
1703
|
return {
|
|
1678
1704
|
gate: admission.gate,
|
|
1679
|
-
persisted:
|
|
1705
|
+
persisted: (clientCommitId) =>
|
|
1706
|
+
toPersistedPushResult(admission.persisted.get(clientCommitId)),
|
|
1680
1707
|
deleteFloor: admission.deleteFloor,
|
|
1681
1708
|
...(watermark === undefined ? {} : { watermark }),
|
|
1682
1709
|
};
|
|
@@ -1684,23 +1711,18 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1684
1711
|
|
|
1685
1712
|
/**
|
|
1686
1713
|
* The §9 settlement and §6.3.2 steps 1 and 2 under the partition lock,
|
|
1687
|
-
* before the
|
|
1714
|
+
* before the first commit's savepoint, so no rejection discards them. Two
|
|
1688
1715
|
* statements: the settlement touches disjoint rows (outside and inside the
|
|
1689
1716
|
* request), and the watermark delete must see its sequences.
|
|
1690
1717
|
*/
|
|
1691
1718
|
async #settleOutbox(
|
|
1692
1719
|
clientId: string,
|
|
1693
|
-
clientCommitId: string,
|
|
1694
1720
|
outbox: OutboxSettlement,
|
|
1695
1721
|
): Promise<number> {
|
|
1696
1722
|
const first = outbox.commits[0];
|
|
1697
1723
|
const last = outbox.commits.at(-1);
|
|
1698
|
-
if (
|
|
1699
|
-
|
|
1700
|
-
last === undefined ||
|
|
1701
|
-
first.clientCommitId !== clientCommitId
|
|
1702
|
-
) {
|
|
1703
|
-
throw new Error("outbox settlement rides the request's first commit");
|
|
1724
|
+
if (first === undefined || last === undefined) {
|
|
1725
|
+
throw new Error('outbox settlement needs the request commits');
|
|
1704
1726
|
}
|
|
1705
1727
|
const commits = JSON.stringify(
|
|
1706
1728
|
outbox.commits.map((commit) => ({
|
|
@@ -1780,68 +1802,38 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1780
1802
|
return readServeGateOn(this.#client, this.#partition, runningSchemaVersion);
|
|
1781
1803
|
}
|
|
1782
1804
|
|
|
1783
|
-
async
|
|
1805
|
+
async rejectPushCommit(
|
|
1784
1806
|
clientId: string,
|
|
1785
1807
|
clientCommitId: string,
|
|
1786
1808
|
result: StoredPushResult,
|
|
1787
1809
|
sequence?: OutboxSequence,
|
|
1788
|
-
): Promise<
|
|
1810
|
+
): Promise<void> {
|
|
1789
1811
|
this.#assertOpen();
|
|
1790
1812
|
// Candidate rows that never reached the database are discarded with the
|
|
1791
|
-
// staging map; a savepoint exists only when one did.
|
|
1813
|
+
// staging map; a savepoint exists only when one did. It stays open: the
|
|
1814
|
+
// rollback returns the database to where the next commit starts (the
|
|
1815
|
+
// results wait in memory), so the next commit writes behind it.
|
|
1792
1816
|
this.#staged.clear();
|
|
1793
1817
|
this.#stagedBlobRefs.clear();
|
|
1794
1818
|
if (this.#pushApplySavepoint) {
|
|
1795
|
-
await this.#
|
|
1796
|
-
await this.#client.query('RELEASE SAVEPOINT syncular_push_candidate');
|
|
1797
|
-
this.#pushApplySavepoint = false;
|
|
1819
|
+
await this.#serial.query('ROLLBACK TO SAVEPOINT syncular_push_candidate');
|
|
1798
1820
|
}
|
|
1799
|
-
|
|
1800
|
-
clientId,
|
|
1801
|
-
clientCommitId,
|
|
1802
|
-
result,
|
|
1803
|
-
sequence,
|
|
1804
|
-
);
|
|
1805
|
-
await this.commit();
|
|
1806
|
-
return canonical;
|
|
1821
|
+
this.#recordPushResult(clientId, clientCommitId, result, sequence);
|
|
1807
1822
|
}
|
|
1808
1823
|
|
|
1809
|
-
|
|
1810
|
-
async #insertPushResult(
|
|
1824
|
+
#recordPushResult(
|
|
1811
1825
|
clientId: string,
|
|
1812
1826
|
clientCommitId: string,
|
|
1813
1827
|
result: StoredPushResult,
|
|
1814
1828
|
sequence: OutboxSequence | undefined,
|
|
1815
|
-
):
|
|
1816
|
-
|
|
1817
|
-
|
|
1818
|
-
|
|
1819
|
-
|
|
1820
|
-
|
|
1821
|
-
|
|
1822
|
-
|
|
1823
|
-
clientId,
|
|
1824
|
-
clientCommitId,
|
|
1825
|
-
JSON.stringify(serializePushResult(result)),
|
|
1826
|
-
sequence?.outboxId ?? null,
|
|
1827
|
-
sequence?.clientSeq ?? null,
|
|
1828
|
-
],
|
|
1829
|
-
);
|
|
1830
|
-
if (rows[0] !== undefined) {
|
|
1831
|
-
return toPersistedPushResult(rows[0].result) ?? result;
|
|
1832
|
-
}
|
|
1833
|
-
// `ON CONFLICT DO NOTHING` means a previously committed outcome won; read
|
|
1834
|
-
// it rather than claiming this caller's candidate.
|
|
1835
|
-
const existing = await getPushResultOn(
|
|
1836
|
-
this.#client,
|
|
1837
|
-
this.#partition,
|
|
1838
|
-
clientId,
|
|
1839
|
-
clientCommitId,
|
|
1840
|
-
);
|
|
1841
|
-
if (existing === undefined) {
|
|
1842
|
-
throw new Error('push result insert did not persist an outcome');
|
|
1843
|
-
}
|
|
1844
|
-
return existing;
|
|
1829
|
+
): void {
|
|
1830
|
+
this.#pushResults.push({
|
|
1831
|
+
client_id: clientId,
|
|
1832
|
+
client_commit_id: clientCommitId,
|
|
1833
|
+
result: serializePushResult(result),
|
|
1834
|
+
outbox_id: sequence?.outboxId ?? null,
|
|
1835
|
+
client_seq: sequence?.clientSeq ?? null,
|
|
1836
|
+
});
|
|
1845
1837
|
}
|
|
1846
1838
|
|
|
1847
1839
|
async upsertRow(
|
|
@@ -1930,8 +1922,15 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1930
1922
|
|
|
1931
1923
|
async appendCommit(commit: NewCommit): Promise<number> {
|
|
1932
1924
|
this.#assertOpen();
|
|
1933
|
-
// The commit writes its delete tombstones below
|
|
1934
|
-
|
|
1925
|
+
// The commit writes its delete tombstones below; every other prefetched
|
|
1926
|
+
// tombstone still holds for the request's later commits.
|
|
1927
|
+
for (const change of commit.changes) {
|
|
1928
|
+
if (change.op === 'delete') {
|
|
1929
|
+
this.#prefetchedTombstones.delete(
|
|
1930
|
+
`${change.table}\u0000${change.rowId}`,
|
|
1931
|
+
);
|
|
1932
|
+
}
|
|
1933
|
+
}
|
|
1935
1934
|
// The append advances the partition cursor; a registered query that runs
|
|
1936
1935
|
// after it must read the new value.
|
|
1937
1936
|
this.#cursorSeq = undefined;
|
|
@@ -2087,19 +2086,10 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
2087
2086
|
sequence?: OutboxSequence,
|
|
2088
2087
|
): Promise<void> {
|
|
2089
2088
|
this.#assertOpen();
|
|
2090
|
-
|
|
2091
|
-
|
|
2092
|
-
|
|
2093
|
-
|
|
2094
|
-
[
|
|
2095
|
-
this.#partition,
|
|
2096
|
-
clientId,
|
|
2097
|
-
clientCommitId,
|
|
2098
|
-
JSON.stringify(serializePushResult(result)),
|
|
2099
|
-
sequence?.outboxId ?? null,
|
|
2100
|
-
sequence?.clientSeq ?? null,
|
|
2101
|
-
],
|
|
2102
|
-
);
|
|
2089
|
+
this.#recordPushResult(clientId, clientCommitId, result, sequence);
|
|
2090
|
+
// The commit is closed: the next commit's first write takes its own
|
|
2091
|
+
// savepoint.
|
|
2092
|
+
this.#pushApplySavepoint = false;
|
|
2103
2093
|
}
|
|
2104
2094
|
|
|
2105
2095
|
async enqueueReactions(reactions: readonly NewReaction[]): Promise<void> {
|
|
@@ -2129,6 +2119,19 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
2129
2119
|
|
|
2130
2120
|
async commit(): Promise<void> {
|
|
2131
2121
|
this.#assertOpen();
|
|
2122
|
+
if (this.#pushResults.length > 0) {
|
|
2123
|
+
// Every result of the request in one statement; `#client` flushes the
|
|
2124
|
+
// staged writes first.
|
|
2125
|
+
await this.#client.query(
|
|
2126
|
+
`INSERT INTO sync_push_results(partition, client_id, client_commit_id, result, outbox_id, client_seq)
|
|
2127
|
+
SELECT $1, r.client_id, r.client_commit_id, r.result, r.outbox_id, r.client_seq
|
|
2128
|
+
FROM jsonb_to_recordset($2::text::jsonb)
|
|
2129
|
+
AS r(client_id text, client_commit_id text, result jsonb, outbox_id text, client_seq bigint)
|
|
2130
|
+
ON CONFLICT (partition, client_id, client_commit_id) DO NOTHING`,
|
|
2131
|
+
[this.#partition, JSON.stringify(this.#pushResults)],
|
|
2132
|
+
);
|
|
2133
|
+
this.#pushResults.length = 0;
|
|
2134
|
+
}
|
|
2132
2135
|
await this.#writeStaged();
|
|
2133
2136
|
this.#open = false;
|
|
2134
2137
|
// Signal the transaction wrapper to COMMIT; wait for it to actually land.
|