@syncular/server 0.77.4 → 0.78.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/d1-storage.js +36 -27
- package/dist/handler.js +39 -20
- package/dist/postgres-storage.js +89 -77
- package/dist/push.d.ts +29 -31
- package/dist/push.js +353 -343
- package/dist/sqlite-dialect.d.ts +7 -7
- package/dist/sqlite-dialect.js +41 -25
- package/dist/sqlite-storage.js +17 -20
- package/dist/storage.d.ts +53 -31
- package/package.json +2 -2
- package/src/d1-storage.ts +48 -34
- package/src/handler.ts +53 -31
- package/src/postgres-storage.ts +106 -103
- package/src/push.ts +452 -475
- package/src/sqlite-dialect.ts +57 -34
- package/src/sqlite-storage.ts +29 -30
- package/src/storage.ts +54 -32
package/src/sqlite-dialect.ts
CHANGED
|
@@ -2062,32 +2062,50 @@ export abstract class SqliteFamilyTransaction {
|
|
|
2062
2062
|
}
|
|
2063
2063
|
|
|
2064
2064
|
/**
|
|
2065
|
-
* The §
|
|
2066
|
-
*
|
|
2067
|
-
*
|
|
2068
|
-
*
|
|
2065
|
+
* The §2.3 records of a request's commits and, with `outbox`, the §9
|
|
2066
|
+
* settlement and §6.3.2 steps 1 and 2. The read runs before the writes, so
|
|
2067
|
+
* D1, which buffers the writes, answers exactly as SQLite: a result the
|
|
2068
|
+
* writes delete counts as absent.
|
|
2069
2069
|
*/
|
|
2070
|
-
protected async
|
|
2070
|
+
protected async admitPushRequest(
|
|
2071
2071
|
clientId: string,
|
|
2072
|
-
|
|
2073
|
-
outbox: OutboxSettlement,
|
|
2072
|
+
clientCommitIds: readonly string[],
|
|
2073
|
+
outbox: OutboxSettlement | undefined,
|
|
2074
2074
|
): Promise<{
|
|
2075
|
-
readonly watermark
|
|
2076
|
-
readonly persisted:
|
|
2075
|
+
readonly watermark?: number;
|
|
2076
|
+
readonly persisted: (
|
|
2077
|
+
clientCommitId: string,
|
|
2078
|
+
) => StoredPushResult | undefined;
|
|
2077
2079
|
}> {
|
|
2080
|
+
const p = this.partition;
|
|
2081
|
+
const ids = JSON.stringify(clientCommitIds);
|
|
2082
|
+
const persistedOf =
|
|
2083
|
+
(records: ReadonlyMap<string, string>) =>
|
|
2084
|
+
(clientCommitId: string): StoredPushResult | undefined => {
|
|
2085
|
+
const record = records.get(clientCommitId);
|
|
2086
|
+
return record === undefined ? undefined : deserializePushResult(record);
|
|
2087
|
+
};
|
|
2088
|
+
if (outbox === undefined) {
|
|
2089
|
+
const rows = await this.all<{ client_commit_id: string; result: string }>(
|
|
2090
|
+
`SELECT client_commit_id, result FROM sync_push_results
|
|
2091
|
+
WHERE partition=? AND client_id=? AND client_commit_id IN (SELECT value FROM json_each(?))`,
|
|
2092
|
+
[p, clientId, ids],
|
|
2093
|
+
);
|
|
2094
|
+
return {
|
|
2095
|
+
persisted: persistedOf(
|
|
2096
|
+
new Map(rows.map((row) => [row.client_commit_id, row.result])),
|
|
2097
|
+
),
|
|
2098
|
+
};
|
|
2099
|
+
}
|
|
2078
2100
|
const first = outbox.commits[0];
|
|
2079
2101
|
const last = outbox.commits.at(-1);
|
|
2080
|
-
if (
|
|
2081
|
-
|
|
2082
|
-
last === undefined ||
|
|
2083
|
-
first.clientCommitId !== clientCommitId
|
|
2084
|
-
) {
|
|
2085
|
-
throw new Error("outbox settlement rides the request's first commit");
|
|
2102
|
+
if (first === undefined || last === undefined) {
|
|
2103
|
+
throw new Error('outbox settlement needs the request commits');
|
|
2086
2104
|
}
|
|
2087
|
-
const
|
|
2088
|
-
const [state] = await this.all<{
|
|
2105
|
+
const rows = await this.all<{
|
|
2089
2106
|
stored: number | null;
|
|
2090
2107
|
last_sequenceless: number;
|
|
2108
|
+
client_commit_id: string | null;
|
|
2091
2109
|
result: string | null;
|
|
2092
2110
|
outbox_id: string | null;
|
|
2093
2111
|
client_seq: number | null;
|
|
@@ -2096,11 +2114,14 @@ export abstract class SqliteFamilyTransaction {
|
|
|
2096
2114
|
WHERE partition=?1 AND client_id=?2 AND outbox_id=?3) AS stored,
|
|
2097
2115
|
EXISTS (SELECT 1 FROM sync_push_results WHERE partition=?1
|
|
2098
2116
|
AND client_id=?2 AND client_commit_id=?5 AND outbox_id IS NULL) AS last_sequenceless,
|
|
2099
|
-
r.
|
|
2117
|
+
r.client_commit_id AS client_commit_id, r.result AS result,
|
|
2118
|
+
r.outbox_id AS outbox_id, r.client_seq AS client_seq
|
|
2100
2119
|
FROM (SELECT 1) LEFT JOIN sync_push_results AS r
|
|
2101
|
-
ON r.partition=?1 AND r.client_id=?2
|
|
2102
|
-
|
|
2120
|
+
ON r.partition=?1 AND r.client_id=?2
|
|
2121
|
+
AND r.client_commit_id IN (SELECT value FROM json_each(?4))`,
|
|
2122
|
+
[p, clientId, outbox.outboxId, ids, last.clientCommitId],
|
|
2103
2123
|
);
|
|
2124
|
+
const state = rows[0];
|
|
2104
2125
|
const watermark = Math.max(state?.stored ?? 0, first.clientSeq);
|
|
2105
2126
|
const commits = JSON.stringify(
|
|
2106
2127
|
outbox.commits.map((commit) => [commit.clientCommitId, commit.clientSeq]),
|
|
@@ -2133,20 +2154,22 @@ export abstract class SqliteFamilyTransaction {
|
|
|
2133
2154
|
'DELETE FROM sync_push_results WHERE partition=? AND client_id=? AND outbox_id=? AND client_seq<?',
|
|
2134
2155
|
[p, clientId, outbox.outboxId, watermark],
|
|
2135
2156
|
);
|
|
2136
|
-
// The sequence step 2 sees: (a) gives a pre-upgrade result
|
|
2137
|
-
//
|
|
2138
|
-
const
|
|
2139
|
-
|
|
2140
|
-
|
|
2141
|
-
|
|
2142
|
-
|
|
2143
|
-
|
|
2144
|
-
|
|
2145
|
-
|
|
2146
|
-
|
|
2147
|
-
|
|
2148
|
-
|
|
2149
|
-
|
|
2157
|
+
// The sequence step 2 sees: (a) gives a pre-upgrade result its commit's
|
|
2158
|
+
// sequence in the request; a result of this outbox keeps its stored one.
|
|
2159
|
+
const sequences = new Map(
|
|
2160
|
+
outbox.commits.map((commit) => [commit.clientCommitId, commit.clientSeq]),
|
|
2161
|
+
);
|
|
2162
|
+
const records = new Map<string, string>();
|
|
2163
|
+
for (const row of rows) {
|
|
2164
|
+
if (row.client_commit_id === null || row.result === null) continue;
|
|
2165
|
+
const deleted =
|
|
2166
|
+
row.outbox_id === null
|
|
2167
|
+
? (sequences.get(row.client_commit_id) ?? 0) < watermark
|
|
2168
|
+
: row.outbox_id === outbox.outboxId &&
|
|
2169
|
+
(row.client_seq ?? 0) < watermark;
|
|
2170
|
+
if (!deleted) records.set(row.client_commit_id, row.result);
|
|
2171
|
+
}
|
|
2172
|
+
return { watermark, persisted: persistedOf(records) };
|
|
2150
2173
|
}
|
|
2151
2174
|
|
|
2152
2175
|
async enqueueReactions(reactions: readonly NewReaction[]): Promise<void> {
|
package/src/sqlite-storage.ts
CHANGED
|
@@ -68,6 +68,7 @@ import type {
|
|
|
68
68
|
OutboxSequence,
|
|
69
69
|
OutboxSettlement,
|
|
70
70
|
PreparedPushApply,
|
|
71
|
+
PushApplyCommit,
|
|
71
72
|
RowScanQuery,
|
|
72
73
|
ServerStorage,
|
|
73
74
|
ServeGate,
|
|
@@ -165,21 +166,19 @@ class SqliteTransaction
|
|
|
165
166
|
|
|
166
167
|
async preparePushApply(
|
|
167
168
|
clientId: string,
|
|
168
|
-
|
|
169
|
+
commits: readonly PushApplyCommit[],
|
|
169
170
|
runningSchemaVersion: number,
|
|
170
|
-
_targets: readonly {
|
|
171
|
-
readonly table: string;
|
|
172
|
-
readonly rowIds: readonly string[];
|
|
173
|
-
}[] = [],
|
|
174
171
|
outbox?: OutboxSettlement,
|
|
175
172
|
): Promise<PreparedPushApply> {
|
|
176
173
|
this.assertOpen();
|
|
177
174
|
// BEGIN IMMEDIATE in the constructor already owns SQLite's writer lock.
|
|
178
|
-
// The settlement precedes the savepoint, so
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
175
|
+
// The settlement precedes the first commit's savepoint, so no rejection
|
|
176
|
+
// discards it.
|
|
177
|
+
const admitted = await this.admitPushRequest(
|
|
178
|
+
clientId,
|
|
179
|
+
commits.map((commit) => commit.clientCommitId),
|
|
180
|
+
outbox,
|
|
181
|
+
);
|
|
183
182
|
this.#storage.db.exec('SAVEPOINT syncular_push_candidate');
|
|
184
183
|
this.#pushApplySavepoint = true;
|
|
185
184
|
const gate = await this.#storage.readServeGate(
|
|
@@ -190,18 +189,7 @@ class SqliteTransaction
|
|
|
190
189
|
'SELECT delete_floor FROM sync_partitions WHERE partition=?',
|
|
191
190
|
[this.partition],
|
|
192
191
|
);
|
|
193
|
-
|
|
194
|
-
return settled === undefined
|
|
195
|
-
? {
|
|
196
|
-
gate,
|
|
197
|
-
deleteFloor,
|
|
198
|
-
persisted: await this.#storage.getPushResult(
|
|
199
|
-
this.partition,
|
|
200
|
-
clientId,
|
|
201
|
-
clientCommitId,
|
|
202
|
-
),
|
|
203
|
-
}
|
|
204
|
-
: { gate, deleteFloor, ...settled };
|
|
192
|
+
return { gate, deleteFloor: floor?.delete_floor ?? 0, ...admitted };
|
|
205
193
|
}
|
|
206
194
|
|
|
207
195
|
async lockPartition(): Promise<void> {
|
|
@@ -234,24 +222,35 @@ class SqliteTransaction
|
|
|
234
222
|
return this.#storage.readServeGate(this.partition, runningSchemaVersion);
|
|
235
223
|
}
|
|
236
224
|
|
|
237
|
-
async
|
|
225
|
+
async rejectPushCommit(
|
|
238
226
|
clientId: string,
|
|
239
227
|
clientCommitId: string,
|
|
240
228
|
result: StoredPushResult,
|
|
241
229
|
sequence?: OutboxSequence,
|
|
242
|
-
): Promise<
|
|
230
|
+
): Promise<void> {
|
|
243
231
|
this.assertOpen();
|
|
244
232
|
if (!this.#pushApplySavepoint) {
|
|
245
233
|
throw new Error('push rejection requires its apply savepoint');
|
|
246
234
|
}
|
|
247
235
|
this.#storage.db.exec('ROLLBACK TO SAVEPOINT syncular_push_candidate');
|
|
248
|
-
this.#storage.db.exec('RELEASE SAVEPOINT syncular_push_candidate');
|
|
249
|
-
this.#pushApplySavepoint = false;
|
|
250
236
|
await this.putPushResult(clientId, clientCommitId, result, sequence);
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
/**
|
|
240
|
+
* Closes the request's current commit: its writes and result join the
|
|
241
|
+
* transaction, and the next commit writes behind a new savepoint (§6.4).
|
|
242
|
+
*/
|
|
243
|
+
override async putPushResult(
|
|
244
|
+
clientId: string,
|
|
245
|
+
clientCommitId: string,
|
|
246
|
+
result: StoredPushResult,
|
|
247
|
+
sequence?: OutboxSequence,
|
|
248
|
+
): Promise<void> {
|
|
249
|
+
await super.putPushResult(clientId, clientCommitId, result, sequence);
|
|
250
|
+
if (this.#pushApplySavepoint) {
|
|
251
|
+
this.#storage.db.exec('RELEASE SAVEPOINT syncular_push_candidate');
|
|
252
|
+
this.#storage.db.exec('SAVEPOINT syncular_push_candidate');
|
|
253
|
+
}
|
|
255
254
|
}
|
|
256
255
|
|
|
257
256
|
async upsertRow(
|
package/src/storage.ts
CHANGED
|
@@ -81,15 +81,19 @@ export interface ServeGate {
|
|
|
81
81
|
}
|
|
82
82
|
|
|
83
83
|
/**
|
|
84
|
-
*
|
|
85
|
-
* the idempotency
|
|
86
|
-
* network cost
|
|
84
|
+
* One push request's admission token (§6.4): the partition lock, the serve
|
|
85
|
+
* gate, and the idempotency records of every commit of the request. A storage
|
|
86
|
+
* with per-statement network cost reads them in one statement (see
|
|
87
87
|
* `PostgresServerStorage`); the lock is held from this call through commit.
|
|
88
88
|
*/
|
|
89
89
|
export interface PreparedPushApply {
|
|
90
90
|
readonly gate: ServeGate;
|
|
91
|
-
/**
|
|
92
|
-
|
|
91
|
+
/**
|
|
92
|
+
* The already-persisted outcome for one of the request's keys, or
|
|
93
|
+
* `undefined` on a miss. Throws `SyncError sync.idempotency_cache_miss`
|
|
94
|
+
* when the stored record is unreadable (§6.3).
|
|
95
|
+
*/
|
|
96
|
+
readonly persisted: (clientCommitId: string) => StoredPushResult | undefined;
|
|
93
97
|
/**
|
|
94
98
|
* The partition's delete floor (§2.2): the highest version any applied
|
|
95
99
|
* delete recorded, `0` before the first. An insert starts one above it.
|
|
@@ -104,6 +108,16 @@ export interface PreparedPushApply {
|
|
|
104
108
|
readonly watermark?: number;
|
|
105
109
|
}
|
|
106
110
|
|
|
111
|
+
/** One commit of a push request, as `preparePushApply` admits it. */
|
|
112
|
+
export interface PushApplyCommit {
|
|
113
|
+
readonly clientCommitId: string;
|
|
114
|
+
/** The rows the commit is known to touch before its operations are built. */
|
|
115
|
+
readonly targets?: readonly {
|
|
116
|
+
readonly table: string;
|
|
117
|
+
readonly rowIds: readonly string[];
|
|
118
|
+
}[];
|
|
119
|
+
}
|
|
120
|
+
|
|
107
121
|
/** The outbox position of one pushed commit (§6.3.2). */
|
|
108
122
|
export interface OutboxSequence {
|
|
109
123
|
readonly outboxId: string;
|
|
@@ -698,6 +712,13 @@ export interface AuthoritativeQueryResult {
|
|
|
698
712
|
* land together or not at all.
|
|
699
713
|
*/
|
|
700
714
|
export interface StorageTransaction {
|
|
715
|
+
/**
|
|
716
|
+
* True when `putPushResult` and `rejectPushCommit` persist the closed
|
|
717
|
+
* commit at once (D1, one atomic batch per commit), so a later failure of
|
|
718
|
+
* the request leaves it applied and the push layer still notifies it
|
|
719
|
+
* (§6.4). Absent means the request's commits persist only at `commit()`.
|
|
720
|
+
*/
|
|
721
|
+
readonly persistsCommitsOnClose?: boolean;
|
|
701
722
|
getRow(table: string, rowId: string): Promise<StoredRow | undefined>;
|
|
702
723
|
/**
|
|
703
724
|
* Optional candidate-state batch read by primary key. A storage with
|
|
@@ -775,46 +796,41 @@ export interface StorageTransaction {
|
|
|
775
796
|
lockPartition?(): Promise<void>;
|
|
776
797
|
/**
|
|
777
798
|
* Serialize every push apply for this partition before any operation read,
|
|
778
|
-
* validation, merge, or write, and read the admission token
|
|
779
|
-
*
|
|
780
|
-
* `
|
|
781
|
-
* so the idempotency read is exact and a racing
|
|
782
|
-
* `targets` are the rows
|
|
783
|
-
* are built; a storage with per-statement
|
|
784
|
-
* into the admission read and answers
|
|
785
|
-
* Missing support fails closed
|
|
799
|
+
* validation, merge, or write, and read the admission token once for the
|
|
800
|
+
* whole request (§6.4): the RFC 0007 serve gate and the §2.3 idempotency
|
|
801
|
+
* record of every commit in `commits`. The lock is retained until the
|
|
802
|
+
* transaction finishes, so the idempotency read is exact and a racing
|
|
803
|
+
* duplicate cannot rerun. A commit's `targets` are the rows it is known to
|
|
804
|
+
* touch before its operations are built; a storage with per-statement
|
|
805
|
+
* network cost folds that read-ahead into the admission read and answers
|
|
806
|
+
* the following `getRow` calls from it. Missing support fails closed
|
|
807
|
+
* before an app-row mutation.
|
|
786
808
|
*
|
|
787
|
-
* `outbox`, passed
|
|
788
|
-
*
|
|
789
|
-
*
|
|
790
|
-
*
|
|
791
|
-
* caller commits this transaction whatever the commit's outcome, so the
|
|
792
|
-
* settlement runs exactly once per request.
|
|
809
|
+
* `outbox`, passed when the request carries an `outboxId`, settles the
|
|
810
|
+
* client's results stored before wire version 4 (§9) and runs §6.3.2 steps
|
|
811
|
+
* 1 and 2 under the same lock, before the read of `persisted`. Those
|
|
812
|
+
* writes precede every commit's savepoint, so no rejection discards them.
|
|
793
813
|
*/
|
|
794
814
|
preparePushApply(
|
|
795
815
|
clientId: string,
|
|
796
|
-
|
|
816
|
+
commits: readonly PushApplyCommit[],
|
|
797
817
|
runningSchemaVersion: number,
|
|
798
|
-
targets?: readonly {
|
|
799
|
-
readonly table: string;
|
|
800
|
-
readonly rowIds: readonly string[];
|
|
801
|
-
}[],
|
|
802
818
|
outbox?: OutboxSettlement,
|
|
803
819
|
): Promise<PreparedPushApply>;
|
|
804
820
|
/**
|
|
805
|
-
*
|
|
806
|
-
* held: discard every candidate write
|
|
807
|
-
*
|
|
808
|
-
*
|
|
809
|
-
*
|
|
810
|
-
*
|
|
821
|
+
* Reject the request's current commit while the push-apply serialization
|
|
822
|
+
* lock is still held: discard every candidate write the commit staged since
|
|
823
|
+
* the previous commit closed (its savepoint), keep the request's earlier
|
|
824
|
+
* commits, and record the rejected idempotency result. Like
|
|
825
|
+
* `putPushResult`, it closes the commit; the transaction stays open for the
|
|
826
|
+
* request's next commit.
|
|
811
827
|
*/
|
|
812
|
-
|
|
828
|
+
rejectPushCommit(
|
|
813
829
|
clientId: string,
|
|
814
830
|
clientCommitId: string,
|
|
815
831
|
result: StoredPushResult,
|
|
816
832
|
sequence?: OutboxSequence,
|
|
817
|
-
): Promise<
|
|
833
|
+
): Promise<void>;
|
|
818
834
|
upsertRow(
|
|
819
835
|
table: string,
|
|
820
836
|
row: StoredRow,
|
|
@@ -834,6 +850,12 @@ export interface StorageTransaction {
|
|
|
834
850
|
* first writer wins; callers read the canonical value after commit. A
|
|
835
851
|
* result without `sequence` (no `outboxId`, §6.10, §12.3) is never deleted
|
|
836
852
|
* by a watermark.
|
|
853
|
+
*
|
|
854
|
+
* The call closes the request's current push commit: the next write opens
|
|
855
|
+
* the next commit's savepoint. PostgreSQL writes the request's results as
|
|
856
|
+
* one statement at `commit()`. D1, whose transaction cannot read its own
|
|
857
|
+
* staged writes across commits, flushes the commit's writes and this
|
|
858
|
+
* result as one atomic batch here (§6.4).
|
|
837
859
|
*/
|
|
838
860
|
putPushResult(
|
|
839
861
|
clientId: string,
|