@syncular/server 0.77.3 → 0.78.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -2062,32 +2062,50 @@ export abstract class SqliteFamilyTransaction {
2062
2062
  }
2063
2063
 
2064
2064
  /**
2065
- * The §9 settlement and §6.3.2 steps 1 and 2 for one request, then the
2066
- * idempotency read of its first commit. The reads run before the writes,
2067
- * so D1, which buffers the writes until commit, answers exactly as SQLite:
2068
- * the first commit's result counts as deleted when the writes delete it.
2065
+ * The §2.3 records of a request's commits and, with `outbox`, the §9
2066
+ * settlement and §6.3.2 steps 1 and 2. The read runs before the writes, so
2067
+ * D1, which buffers the writes, answers exactly as SQLite: a result the
2068
+ * writes delete counts as absent.
2069
2069
  */
2070
- protected async settleOutbox(
2070
+ protected async admitPushRequest(
2071
2071
  clientId: string,
2072
- clientCommitId: string,
2073
- outbox: OutboxSettlement,
2072
+ clientCommitIds: readonly string[],
2073
+ outbox: OutboxSettlement | undefined,
2074
2074
  ): Promise<{
2075
- readonly watermark: number;
2076
- readonly persisted: StoredPushResult | undefined;
2075
+ readonly watermark?: number;
2076
+ readonly persisted: (
2077
+ clientCommitId: string,
2078
+ ) => StoredPushResult | undefined;
2077
2079
  }> {
2080
+ const p = this.partition;
2081
+ const ids = JSON.stringify(clientCommitIds);
2082
+ const persistedOf =
2083
+ (records: ReadonlyMap<string, string>) =>
2084
+ (clientCommitId: string): StoredPushResult | undefined => {
2085
+ const record = records.get(clientCommitId);
2086
+ return record === undefined ? undefined : deserializePushResult(record);
2087
+ };
2088
+ if (outbox === undefined) {
2089
+ const rows = await this.all<{ client_commit_id: string; result: string }>(
2090
+ `SELECT client_commit_id, result FROM sync_push_results
2091
+ WHERE partition=? AND client_id=? AND client_commit_id IN (SELECT value FROM json_each(?))`,
2092
+ [p, clientId, ids],
2093
+ );
2094
+ return {
2095
+ persisted: persistedOf(
2096
+ new Map(rows.map((row) => [row.client_commit_id, row.result])),
2097
+ ),
2098
+ };
2099
+ }
2078
2100
  const first = outbox.commits[0];
2079
2101
  const last = outbox.commits.at(-1);
2080
- if (
2081
- first === undefined ||
2082
- last === undefined ||
2083
- first.clientCommitId !== clientCommitId
2084
- ) {
2085
- throw new Error("outbox settlement rides the request's first commit");
2102
+ if (first === undefined || last === undefined) {
2103
+ throw new Error('outbox settlement needs the request commits');
2086
2104
  }
2087
- const p = this.partition;
2088
- const [state] = await this.all<{
2105
+ const rows = await this.all<{
2089
2106
  stored: number | null;
2090
2107
  last_sequenceless: number;
2108
+ client_commit_id: string | null;
2091
2109
  result: string | null;
2092
2110
  outbox_id: string | null;
2093
2111
  client_seq: number | null;
@@ -2096,11 +2114,14 @@ export abstract class SqliteFamilyTransaction {
2096
2114
  WHERE partition=?1 AND client_id=?2 AND outbox_id=?3) AS stored,
2097
2115
  EXISTS (SELECT 1 FROM sync_push_results WHERE partition=?1
2098
2116
  AND client_id=?2 AND client_commit_id=?5 AND outbox_id IS NULL) AS last_sequenceless,
2099
- r.result AS result, r.outbox_id AS outbox_id, r.client_seq AS client_seq
2117
+ r.client_commit_id AS client_commit_id, r.result AS result,
2118
+ r.outbox_id AS outbox_id, r.client_seq AS client_seq
2100
2119
  FROM (SELECT 1) LEFT JOIN sync_push_results AS r
2101
- ON r.partition=?1 AND r.client_id=?2 AND r.client_commit_id=?4`,
2102
- [p, clientId, outbox.outboxId, first.clientCommitId, last.clientCommitId],
2120
+ ON r.partition=?1 AND r.client_id=?2
2121
+ AND r.client_commit_id IN (SELECT value FROM json_each(?4))`,
2122
+ [p, clientId, outbox.outboxId, ids, last.clientCommitId],
2103
2123
  );
2124
+ const state = rows[0];
2104
2125
  const watermark = Math.max(state?.stored ?? 0, first.clientSeq);
2105
2126
  const commits = JSON.stringify(
2106
2127
  outbox.commits.map((commit) => [commit.clientCommitId, commit.clientSeq]),
@@ -2133,20 +2154,22 @@ export abstract class SqliteFamilyTransaction {
2133
2154
  'DELETE FROM sync_push_results WHERE partition=? AND client_id=? AND outbox_id=? AND client_seq<?',
2134
2155
  [p, clientId, outbox.outboxId, watermark],
2135
2156
  );
2136
- // The sequence step 2 sees: (a) gives a pre-upgrade result the first
2137
- // commit's sequence; a result of this outbox keeps its stored one.
2138
- const deleted =
2139
- state?.outbox_id === null
2140
- ? first.clientSeq < watermark
2141
- : state?.outbox_id === outbox.outboxId &&
2142
- (state.client_seq ?? 0) < watermark;
2143
- return {
2144
- watermark,
2145
- persisted:
2146
- state?.result === null || state?.result === undefined || deleted
2147
- ? undefined
2148
- : deserializePushResult(state.result),
2149
- };
2157
+ // The sequence step 2 sees: (a) gives a pre-upgrade result its commit's
2158
+ // sequence in the request; a result of this outbox keeps its stored one.
2159
+ const sequences = new Map(
2160
+ outbox.commits.map((commit) => [commit.clientCommitId, commit.clientSeq]),
2161
+ );
2162
+ const records = new Map<string, string>();
2163
+ for (const row of rows) {
2164
+ if (row.client_commit_id === null || row.result === null) continue;
2165
+ const deleted =
2166
+ row.outbox_id === null
2167
+ ? (sequences.get(row.client_commit_id) ?? 0) < watermark
2168
+ : row.outbox_id === outbox.outboxId &&
2169
+ (row.client_seq ?? 0) < watermark;
2170
+ if (!deleted) records.set(row.client_commit_id, row.result);
2171
+ }
2172
+ return { watermark, persisted: persistedOf(records) };
2150
2173
  }
2151
2174
 
2152
2175
  async enqueueReactions(reactions: readonly NewReaction[]): Promise<void> {
@@ -68,6 +68,7 @@ import type {
68
68
  OutboxSequence,
69
69
  OutboxSettlement,
70
70
  PreparedPushApply,
71
+ PushApplyCommit,
71
72
  RowScanQuery,
72
73
  ServerStorage,
73
74
  ServeGate,
@@ -165,21 +166,19 @@ class SqliteTransaction
165
166
 
166
167
  async preparePushApply(
167
168
  clientId: string,
168
- clientCommitId: string,
169
+ commits: readonly PushApplyCommit[],
169
170
  runningSchemaVersion: number,
170
- _targets: readonly {
171
- readonly table: string;
172
- readonly rowIds: readonly string[];
173
- }[] = [],
174
171
  outbox?: OutboxSettlement,
175
172
  ): Promise<PreparedPushApply> {
176
173
  this.assertOpen();
177
174
  // BEGIN IMMEDIATE in the constructor already owns SQLite's writer lock.
178
- // The settlement precedes the savepoint, so a rejection keeps it.
179
- const settled =
180
- outbox === undefined
181
- ? undefined
182
- : await this.settleOutbox(clientId, clientCommitId, outbox);
175
+ // The settlement precedes the first commit's savepoint, so no rejection
176
+ // discards it.
177
+ const admitted = await this.admitPushRequest(
178
+ clientId,
179
+ commits.map((commit) => commit.clientCommitId),
180
+ outbox,
181
+ );
183
182
  this.#storage.db.exec('SAVEPOINT syncular_push_candidate');
184
183
  this.#pushApplySavepoint = true;
185
184
  const gate = await this.#storage.readServeGate(
@@ -190,18 +189,7 @@ class SqliteTransaction
190
189
  'SELECT delete_floor FROM sync_partitions WHERE partition=?',
191
190
  [this.partition],
192
191
  );
193
- const deleteFloor = floor?.delete_floor ?? 0;
194
- return settled === undefined
195
- ? {
196
- gate,
197
- deleteFloor,
198
- persisted: await this.#storage.getPushResult(
199
- this.partition,
200
- clientId,
201
- clientCommitId,
202
- ),
203
- }
204
- : { gate, deleteFloor, ...settled };
192
+ return { gate, deleteFloor: floor?.delete_floor ?? 0, ...admitted };
205
193
  }
206
194
 
207
195
  async lockPartition(): Promise<void> {
@@ -234,24 +222,35 @@ class SqliteTransaction
234
222
  return this.#storage.readServeGate(this.partition, runningSchemaVersion);
235
223
  }
236
224
 
237
- async commitRejectedPushResult(
225
+ async rejectPushCommit(
238
226
  clientId: string,
239
227
  clientCommitId: string,
240
228
  result: StoredPushResult,
241
229
  sequence?: OutboxSequence,
242
- ): Promise<StoredPushResult> {
230
+ ): Promise<void> {
243
231
  this.assertOpen();
244
232
  if (!this.#pushApplySavepoint) {
245
233
  throw new Error('push rejection requires its apply savepoint');
246
234
  }
247
235
  this.#storage.db.exec('ROLLBACK TO SAVEPOINT syncular_push_candidate');
248
- this.#storage.db.exec('RELEASE SAVEPOINT syncular_push_candidate');
249
- this.#pushApplySavepoint = false;
250
236
  await this.putPushResult(clientId, clientCommitId, result, sequence);
251
- await this.commit();
252
- // The idempotency re-check found no record under the writer lock, so the
253
- // insert above is the canonical outcome.
254
- return result;
237
+ }
238
+
239
+ /**
240
+ * Closes the request's current commit: its writes and result join the
241
+ * transaction, and the next commit writes behind a new savepoint (§6.4).
242
+ */
243
+ override async putPushResult(
244
+ clientId: string,
245
+ clientCommitId: string,
246
+ result: StoredPushResult,
247
+ sequence?: OutboxSequence,
248
+ ): Promise<void> {
249
+ await super.putPushResult(clientId, clientCommitId, result, sequence);
250
+ if (this.#pushApplySavepoint) {
251
+ this.#storage.db.exec('RELEASE SAVEPOINT syncular_push_candidate');
252
+ this.#storage.db.exec('SAVEPOINT syncular_push_candidate');
253
+ }
255
254
  }
256
255
 
257
256
  async upsertRow(
package/src/storage.ts CHANGED
@@ -81,15 +81,19 @@ export interface ServeGate {
81
81
  }
82
82
 
83
83
  /**
84
- * The push apply's admission token: the partition lock, the serve gate, and
85
- * the idempotency record read in one round trip. A storage with per-statement
86
- * network cost answers all three with one statement (see
84
+ * One push request's admission token (§6.4): the partition lock, the serve
85
+ * gate, and the idempotency records of every commit of the request. A storage
86
+ * with per-statement network cost reads them in one statement (see
87
87
  * `PostgresServerStorage`); the lock is held from this call through commit.
88
88
  */
89
89
  export interface PreparedPushApply {
90
90
  readonly gate: ServeGate;
91
- /** The already-persisted outcome for this key, or `undefined` on a miss. */
92
- readonly persisted: StoredPushResult | undefined;
91
+ /**
92
+ * The already-persisted outcome for one of the request's keys, or
93
+ * `undefined` on a miss. Throws `SyncError sync.idempotency_cache_miss`
94
+ * when the stored record is unreadable (§6.3).
95
+ */
96
+ readonly persisted: (clientCommitId: string) => StoredPushResult | undefined;
93
97
  /**
94
98
  * The partition's delete floor (§2.2): the highest version any applied
95
99
  * delete recorded, `0` before the first. An insert starts one above it.
@@ -104,6 +108,16 @@ export interface PreparedPushApply {
104
108
  readonly watermark?: number;
105
109
  }
106
110
 
111
+ /** One commit of a push request, as `preparePushApply` admits it. */
112
+ export interface PushApplyCommit {
113
+ readonly clientCommitId: string;
114
+ /** The rows the commit is known to touch before its operations are built. */
115
+ readonly targets?: readonly {
116
+ readonly table: string;
117
+ readonly rowIds: readonly string[];
118
+ }[];
119
+ }
120
+
107
121
  /** The outbox position of one pushed commit (§6.3.2). */
108
122
  export interface OutboxSequence {
109
123
  readonly outboxId: string;
@@ -698,6 +712,13 @@ export interface AuthoritativeQueryResult {
698
712
  * land together or not at all.
699
713
  */
700
714
  export interface StorageTransaction {
715
+ /**
716
+ * True when `putPushResult` and `rejectPushCommit` persist the closed
717
+ * commit at once (D1, one atomic batch per commit), so a later failure of
718
+ * the request leaves it applied and the push layer still notifies it
719
+ * (§6.4). Absent means the request's commits persist only at `commit()`.
720
+ */
721
+ readonly persistsCommitsOnClose?: boolean;
701
722
  getRow(table: string, rowId: string): Promise<StoredRow | undefined>;
702
723
  /**
703
724
  * Optional candidate-state batch read by primary key. A storage with
@@ -775,46 +796,41 @@ export interface StorageTransaction {
775
796
  lockPartition?(): Promise<void>;
776
797
  /**
777
798
  * Serialize every push apply for this partition before any operation read,
778
- * validation, merge, or write, and read the admission token in the same
779
- * round trip: the RFC 0007 serve gate and the §2.3 idempotency record for
780
- * `clientCommitId`. The lock is retained through terminal-result commit,
781
- * so the idempotency read is exact and a racing duplicate cannot rerun.
782
- * `targets` are the rows the commit is known to touch before its operations
783
- * are built; a storage with per-statement network cost folds that read-ahead
784
- * into the admission read and answers the following `getRow` calls from it.
785
- * Missing support fails closed before an app-row mutation.
799
+ * validation, merge, or write, and read the admission token once for the
800
+ * whole request (§6.4): the RFC 0007 serve gate and the §2.3 idempotency
801
+ * record of every commit in `commits`. The lock is retained until the
802
+ * transaction finishes, so the idempotency read is exact and a racing
803
+ * duplicate cannot rerun. A commit's `targets` are the rows it is known to
804
+ * touch before its operations are built; a storage with per-statement
805
+ * network cost folds that read-ahead into the admission read and answers
806
+ * the following `getRow` calls from it. Missing support fails closed
807
+ * before an app-row mutation.
786
808
  *
787
- * `outbox`, passed with the first commit of a request that carries an
788
- * `outboxId`, settles the client's results stored before wire version 4
789
- * (§9) and runs §6.3.2 steps 1 and 2 under the same lock, before the read
790
- * of `persisted`. Those writes survive `commitRejectedPushResult`; the
791
- * caller commits this transaction whatever the commit's outcome, so the
792
- * settlement runs exactly once per request.
809
+ * `outbox`, passed when the request carries an `outboxId`, settles the
810
+ * client's results stored before wire version 4 (§9) and runs §6.3.2 steps
811
+ * 1 and 2 under the same lock, before the read of `persisted`. Those
812
+ * writes precede every commit's savepoint, so no rejection discards them.
793
813
  */
794
814
  preparePushApply(
795
815
  clientId: string,
796
- clientCommitId: string,
816
+ commits: readonly PushApplyCommit[],
797
817
  runningSchemaVersion: number,
798
- targets?: readonly {
799
- readonly table: string;
800
- readonly rowIds: readonly string[];
801
- }[],
802
818
  outbox?: OutboxSettlement,
803
819
  ): Promise<PreparedPushApply>;
804
820
  /**
805
- * Rejection finalization while the push-apply serialization lock is still
806
- * held: discard every candidate write, persist the rejected idempotency
807
- * result, and finish the transaction atomically. Returns the persisted
808
- * canonical outcome, so the caller never re-reads it. Required for every
809
- * push so a concurrent duplicate cannot rerun operations, validators, or
810
- * merges.
821
+ * Reject the request's current commit while the push-apply serialization
822
+ * lock is still held: discard every candidate write the commit staged since
823
+ * the previous commit closed (its savepoint), keep the request's earlier
824
+ * commits, and record the rejected idempotency result. Like
825
+ * `putPushResult`, it closes the commit; the transaction stays open for the
826
+ * request's next commit.
811
827
  */
812
- commitRejectedPushResult(
828
+ rejectPushCommit(
813
829
  clientId: string,
814
830
  clientCommitId: string,
815
831
  result: StoredPushResult,
816
832
  sequence?: OutboxSequence,
817
- ): Promise<StoredPushResult>;
833
+ ): Promise<void>;
818
834
  upsertRow(
819
835
  table: string,
820
836
  row: StoredRow,
@@ -834,6 +850,12 @@ export interface StorageTransaction {
834
850
  * first writer wins; callers read the canonical value after commit. A
835
851
  * result without `sequence` (no `outboxId`, §6.10, §12.3) is never deleted
836
852
  * by a watermark.
853
+ *
854
+ * The call closes the request's current push commit: the next write opens
855
+ * the next commit's savepoint. PostgreSQL writes the request's results as
856
+ * one statement at `commit()`. D1, whose transaction cannot read its own
857
+ * staged writes across commits, flushes the commit's writes and this
858
+ * result as one atomic batch here (§6.4).
837
859
  */
838
860
  putPushResult(
839
861
  clientId: string,