@syncular/server 0.77.4 → 0.78.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/push.ts CHANGED
@@ -10,8 +10,9 @@ import { retainRollbackFailure } from './storage-errors';
10
10
  * - a lost `baseVersion = 0` insert race re-authorizes the winner's row
11
11
  * before disclosing it in a conflict record (§6.2).
12
12
  *
13
- * Per-commit atomicity (§6.4): one storage transaction per commit; the
14
- * idempotency record persists in the same transaction as the writes.
13
+ * Atomicity (§6.4): one storage transaction per push request, one savepoint
14
+ * per commit; every idempotency record persists in the same transaction as
15
+ * the writes.
15
16
  *
16
17
  * Optional per-table write validation (§6.7) runs after decode + the §3.4
17
18
  * scope check, on the row that will persist (post scope-strip, post CRDT
@@ -22,7 +23,6 @@ import {
22
23
  decodeRow,
23
24
  decodeSparseRow,
24
25
  encodeRow,
25
- type PushCommitFrame,
26
26
  type PushOperation,
27
27
  type PushOperationResult,
28
28
  type PushResultFrame,
@@ -48,15 +48,18 @@ import { authorizeWrite, renderScopeValue, storedScopesForRow } from './scopes';
48
48
  import { decodeColumnVersions, encodeColumnVersions } from './relational-rows';
49
49
  import type {
50
50
  NewChange,
51
- OutboxSequence,
51
+ NewReaction,
52
52
  OutboxSettlement,
53
- PreparedPushApply,
53
+ PushApplyCommit,
54
54
  StorageTransaction,
55
- StoredCommit,
56
55
  StoredPushResult,
57
56
  StoredRow,
58
57
  } from './storage';
59
- import { readCandidateRows, serveGateRefusal } from './storage';
58
+ import {
59
+ commitDeleteFloor,
60
+ readCandidateRows,
61
+ serveGateRefusal,
62
+ } from './storage';
60
63
  import { StorageConstraintError } from './storage-errors';
61
64
  import { transactionQuery } from './storage-query';
62
65
  import { serveNotReadyError } from './readiness';
@@ -1729,28 +1732,31 @@ export interface ProcessedPushCommit {
1729
1732
  readonly replayed: boolean;
1730
1733
  readonly recordedAtMs?: number;
1731
1734
  readonly cacheIdentity?: string;
1732
- /** The outbox watermark `W` the commit's settlement read (§6.3.2). */
1733
- readonly watermark?: number;
1734
1735
  }
1735
1736
 
1736
- /** A pushed commit's outbox position and the request's watermark (§6.3.2). */
1737
- export interface PushOutbox extends OutboxSequence {
1738
- /** `W` after the request's first commit. */
1739
- readonly watermark: number;
1740
- /** Present with the request's first commit: steps 1 and 2 run with it. */
1741
- readonly settlement?: OutboxSettlement;
1737
+ /** One commit of a push request (§6.4). */
1738
+ export interface PushRequestCommit {
1739
+ readonly clientCommitId: string;
1740
+ /**
1741
+ * Runs after the partition lock and the idempotency read, inside the
1742
+ * request's transaction, so its reads and the returned operations share
1743
+ * the transaction that commits.
1744
+ */
1745
+ readonly buildOperations: (
1746
+ tx: StorageTransaction,
1747
+ ) => Promise<readonly ApplyOperation[]>;
1748
+ /** The rows the commit names before its operations are built. */
1749
+ readonly targets?: PushApplyCommit['targets'];
1742
1750
  }
1743
1751
 
1744
1752
  function processedPushCommit(
1745
1753
  clientCommitId: string,
1746
1754
  stored: StoredPushResult,
1747
1755
  replayed: boolean,
1748
- watermark?: number,
1749
1756
  ): ProcessedPushCommit {
1750
1757
  return {
1751
1758
  frame: resultFrame(clientCommitId, stored, replayed),
1752
1759
  replayed,
1753
- ...(watermark !== undefined ? { watermark } : {}),
1754
1760
  ...(stored.recordedAtMs !== undefined
1755
1761
  ? { recordedAtMs: stored.recordedAtMs }
1756
1762
  : {}),
@@ -1809,90 +1815,51 @@ function commitExpiredFrame(clientCommitId: string): PushResultFrame {
1809
1815
  };
1810
1816
  }
1811
1817
 
1812
- export interface AppliedCommitEvent {
1813
- readonly commit: StoredCommit;
1814
- }
1815
-
1816
- /**
1817
- * Process one `PUSH_COMMIT` frame: idempotency replay (§2.3), sequential
1818
- * atomic apply (§6.4), realtime notification for applied commits.
1819
- */
1820
- export async function processPushCommit(
1821
- ctx: SyncRequestContext,
1822
- schema: CompiledSchema,
1823
- resolved: ResolvedScopes,
1824
- clientId: string,
1825
- frame: PushCommitFrame,
1826
- ): Promise<PushResultFrame> {
1827
- return (
1828
- await processPushCommitWithTrace(ctx, schema, resolved, clientId, frame)
1829
- ).frame;
1830
- }
1831
-
1832
1818
  /**
1833
- * Host-observable variant of `processPushCommit`. The SSP2 wire frame keeps
1834
- * rejected replays as `status: rejected`; this companion result preserves the
1835
- * cache provenance needed by structured events and server helpers.
1819
+ * The serialized apply of one commit outside an SSP2 request: an
1820
+ * authoritative command or an erasure.
1836
1821
  */
1837
- export async function processPushCommitWithTrace(
1822
+ export async function processPushOperationsWithTrace(
1838
1823
  ctx: SyncRequestContext,
1839
1824
  schema: CompiledSchema,
1840
1825
  resolved: ResolvedScopes,
1841
1826
  clientId: string,
1842
- frame: PushCommitFrame,
1827
+ clientCommitId: string,
1828
+ buildOperations: PushRequestCommit['buildOperations'],
1843
1829
  wireSchema: CompiledSchema = schema,
1844
- outbox?: PushOutbox,
1845
1830
  ): Promise<ProcessedPushCommit> {
1846
- // The frame names its rows before the transaction opens, so the admission
1847
- // read carries the commit's read-ahead and the apply issues no separate
1848
- // prefetch statement.
1849
- const targets = new Map<string, string[]>();
1850
- for (const op of frame.operations) {
1851
- // A rowId holding U+0000 is rejected per operation (§6.1) before any
1852
- // store reads it.
1853
- if (!schema.tables.has(op.table) || op.rowId.includes('\u0000')) continue;
1854
- const ids = targets.get(op.table) ?? [];
1855
- ids.push(op.rowId);
1856
- targets.set(op.table, ids);
1857
- }
1858
- return processPushOperationsWithTrace(
1831
+ const [processed] = await processPushCommitsWithTrace(
1859
1832
  ctx,
1860
1833
  schema,
1861
1834
  resolved,
1862
1835
  clientId,
1863
- frame.clientCommitId,
1864
- async () => frame.operations,
1836
+ [{ clientCommitId, buildOperations }],
1865
1837
  wireSchema,
1866
- targets.size === 0
1867
- ? undefined
1868
- : [...targets].map(([table, rowIds]) => ({ table, rowIds })),
1869
- outbox,
1870
1838
  );
1839
+ return processed!;
1871
1840
  }
1872
1841
 
1873
1842
  /**
1874
- * Shared serialized apply path for SSP2 commits and authoritative commands.
1875
- * The builder runs after the partition lock and idempotency re-check, so its
1876
- * reads and the returned operations share the transaction that is committed.
1843
+ * Apply one push request (§6.4): one storage transaction under one partition
1844
+ * lock, one serve-gate and idempotency read for every commit, each commit
1845
+ * inside its own savepoint in frame order. A commit's rejection rolls back
1846
+ * only that commit; any other failure rolls back the whole request and
1847
+ * throws. Reaction events and realtime notifications leave after the
1848
+ * transaction commits, once per applied commit in `commitSeq` order.
1849
+ *
1850
+ * `outbox` lists every commit of the request in the same order as
1851
+ * `commits`; it carries the §6.3.2 sequences and the §9 settlement.
1877
1852
  */
1878
- export async function processPushOperationsWithTrace(
1853
+ export async function processPushCommitsWithTrace(
1879
1854
  ctx: SyncRequestContext,
1880
1855
  schema: CompiledSchema,
1881
1856
  resolved: ResolvedScopes,
1882
1857
  clientId: string,
1883
- clientCommitId: string,
1884
- buildOperations: (
1885
- tx: StorageTransaction,
1886
- ) => Promise<readonly ApplyOperation[]>,
1858
+ commits: readonly PushRequestCommit[],
1887
1859
  wireSchema: CompiledSchema = schema,
1888
- prefetchTargets?: readonly {
1889
- readonly table: string;
1890
- readonly rowIds: readonly string[];
1891
- }[],
1892
- outbox?: PushOutbox,
1893
- ): Promise<ProcessedPushCommit> {
1860
+ outbox?: OutboxSettlement,
1861
+ ): Promise<ProcessedPushCommit[]> {
1894
1862
  const { partition } = ctx;
1895
- const createdAtMs = clockOf(ctx)();
1896
1863
  const blobCtx: BlobApplyContext = {
1897
1864
  store: ctx.blobs,
1898
1865
  partition,
@@ -1902,14 +1869,53 @@ export async function processPushOperationsWithTrace(
1902
1869
  const validators = ctx.validators;
1903
1870
  const commitValidator = ctx.commitValidator;
1904
1871
  const reactionPlanner = ctx.reactionPlanner;
1872
+ const processed: ProcessedPushCommit[] = [];
1873
+ const applied: {
1874
+ readonly clientCommitId: string;
1875
+ readonly commitSeq: number;
1876
+ readonly createdAtMs: number;
1877
+ readonly changes: readonly NewChange[];
1878
+ readonly reactions: readonly NewReaction[];
1879
+ }[] = [];
1880
+ // Reaction events and realtime notifications of the applied commits, in
1881
+ // commitSeq order, once they are durable.
1882
+ const announce = async (): Promise<void> => {
1883
+ for (const commit of applied) {
1884
+ if (ctx.events !== undefined) {
1885
+ const atMs = clockOf(ctx)();
1886
+ for (const reaction of commit.reactions) {
1887
+ emitEvent(ctx.events, {
1888
+ type: 'reaction.queued',
1889
+ atMs,
1890
+ partition,
1891
+ actorId: ctx.actorId,
1892
+ clientId,
1893
+ clientCommitId: commit.clientCommitId,
1894
+ commitSeq: commit.commitSeq,
1895
+ idempotencyKey: reaction.idempotencyKey,
1896
+ reactionType: reaction.type,
1897
+ version: reaction.version,
1898
+ });
1899
+ }
1900
+ }
1901
+ if (ctx.realtime !== undefined && commit.changes.length > 0) {
1902
+ // RFC 0007 fanout decision: the notification carries the durable commit
1903
+ // entry itself (this commit's `changes`), not a projection read or a pull
1904
+ // window, so fanning it out before the request's buffered read-verify can
1905
+ // never serve a mixed result. Each peer's next pull is gated at the
1906
+ // storage seam; a refused mixed push+pull is safe because the push is
1907
+ // durable and replayable under its commit id. This is commit-entry
1908
+ // durability, not gate inheritance.
1909
+ await ctx.realtime.notifyCommit(partition, {
1910
+ commitSeq: commit.commitSeq,
1911
+ createdAtMs: commit.createdAtMs,
1912
+ actorId: ctx.actorId,
1913
+ changes: commit.changes,
1914
+ });
1915
+ }
1916
+ }
1917
+ };
1905
1918
  const tx = await ctx.storage.begin(partition);
1906
- // §6.3.2: the settlement's writes commit whatever the commit's outcome.
1907
- const release = (): Promise<void> =>
1908
- outbox?.settlement !== undefined ? tx.commit() : tx.rollback();
1909
- const sequence =
1910
- outbox === undefined
1911
- ? undefined
1912
- : { outboxId: outbox.outboxId, clientSeq: outbox.clientSeq };
1913
1919
  try {
1914
1920
  if (reactionPlanner !== undefined && tx.enqueueReactions === undefined) {
1915
1921
  throw new Error(
@@ -1918,36 +1924,21 @@ export async function processPushOperationsWithTrace(
1918
1924
  }
1919
1925
  // The storage takes the partition lock and then reads the admission token
1920
1926
  // on the transaction's own connection: the RFC 0007 serve gate and the
1921
- // §2.3 idempotency record, plus any read-ahead the caller already knows.
1922
- // A racing duplicate's result commits before the lock releases, so the
1923
- // later read is exact and a duplicate never reruns an operation,
1924
- // validator, or merge.
1925
- let prepared: PreparedPushApply;
1926
- try {
1927
- prepared = await tx.preparePushApply(
1928
- clientId,
1929
- clientCommitId,
1930
- schema.version,
1931
- prefetchTargets,
1932
- outbox?.settlement,
1933
- );
1934
- } catch (error) {
1935
- if (
1936
- error instanceof SyncError &&
1937
- error.code === 'sync.idempotency_cache_miss'
1938
- ) {
1939
- // §6.3: answer the retryable cache-miss for this commit rather than
1940
- // re-applying. Not persisted — a retry may find a readable record.
1941
- await release();
1942
- return {
1943
- frame: idempotencyCacheMissFrame(clientCommitId, error),
1944
- replayed: false,
1945
- };
1946
- }
1947
- throw error;
1948
- }
1927
+ // §2.3 idempotency record of every commit, plus any read-ahead the
1928
+ // commits already name. A racing duplicate's result commits before the
1929
+ // lock releases, so the later read is exact and a duplicate never reruns
1930
+ // an operation, validator, or merge.
1931
+ const prepared = await tx.preparePushApply(
1932
+ clientId,
1933
+ commits.map((commit) => ({
1934
+ clientCommitId: commit.clientCommitId,
1935
+ ...(commit.targets === undefined ? {} : { targets: commit.targets }),
1936
+ })),
1937
+ schema.version,
1938
+ outbox,
1939
+ );
1949
1940
  // RFC 0007: the gate is evaluated while the partition lock is held, so
1950
- // the migration check and the write share one transaction and no
1941
+ // the migration check and the writes share one transaction and no
1951
1942
  // migration can interleave in between.
1952
1943
  const refusal = serveGateRefusal(
1953
1944
  prepared.gate,
@@ -1955,395 +1946,381 @@ export async function processPushOperationsWithTrace(
1955
1946
  ctx.checkpoints,
1956
1947
  );
1957
1948
  if (refusal !== undefined) {
1958
- await release();
1949
+ // §6.3.2: the settlement commits whatever the request's outcome.
1950
+ await tx.commit();
1959
1951
  throw serveNotReadyError(refusal);
1960
1952
  }
1961
- const watermark = prepared.watermark ?? outbox?.watermark;
1962
- if (prepared.persisted !== undefined) {
1963
- const conflict = prepared.persisted.results.find(
1964
- (result) => result.status === 'conflict',
1965
- );
1966
- const authorization = prepared.persisted.conflictAuthorization;
1967
- const table = authorization && schema.tables.get(authorization.table);
1968
- if (
1969
- conflict !== undefined &&
1970
- (authorization === undefined ||
1971
- authorization.schemaVersion !== schema.version ||
1972
- table === undefined ||
1973
- !authorizeWrite(table, authorization.scopes, resolved))
1974
- ) {
1975
- await release();
1976
- return processedPushCommit(
1977
- clientCommitId,
1978
- {
1979
- ...prepared.persisted,
1980
- results: [
1981
- {
1982
- opIndex: conflict.opIndex,
1983
- status: 'error',
1984
- code: 'sync.forbidden',
1985
- message: 'conflict replay denied by scope authorization (§3.4)',
1986
- retryable: false,
1987
- },
1988
- ],
1989
- },
1990
- true,
1991
- watermark,
1992
- );
1993
- }
1994
- await release();
1995
- return processedPushCommit(
1996
- clientCommitId,
1997
- prepared.persisted,
1998
- true,
1999
- watermark,
2000
- );
2001
- }
2002
- // §6.3.2 step 3: below the watermark without a retained result.
2003
- if (
2004
- outbox !== undefined &&
2005
- watermark !== undefined &&
2006
- outbox.clientSeq < watermark
2007
- ) {
2008
- await release();
2009
- return {
2010
- frame: commitExpiredFrame(clientCommitId),
2011
- replayed: false,
2012
- watermark,
2013
- };
2014
- }
2015
- const results: PushOperationResult[] = [];
2016
- const changes: NewChange[] = [];
2017
- const validatedOperations: ValidateCommitOperation[] = [];
2018
- const stagedWrites: StagedWrite[] = [];
2019
- let terminated: PushOperationResult | undefined;
2020
- const operations = await buildOperations(tx);
2021
- if (tx.prefetchRows !== undefined) {
2022
- const targets = new Map<string, string[]>();
2023
- for (const op of operations) {
2024
- if (!schema.tables.has(op.table) || op.rowId.includes('\u0000'))
1953
+ const watermark = prepared.watermark;
1954
+ // §2.2: inserts start above the floor, which applied deletes raise.
1955
+ let requestDeleteFloor = prepared.deleteFloor;
1956
+ // §6.3: a clientCommitId repeated within the request replays the outcome
1957
+ // its first occurrence recorded in this transaction.
1958
+ const recorded = new Map<string, StoredPushResult>();
1959
+ for (const [index, commit] of commits.entries()) {
1960
+ const { clientCommitId } = commit;
1961
+ const sequence =
1962
+ outbox === undefined
1963
+ ? undefined
1964
+ : {
1965
+ outboxId: outbox.outboxId,
1966
+ clientSeq: outbox.commits[index]!.clientSeq,
1967
+ };
1968
+ let persisted: StoredPushResult | undefined;
1969
+ try {
1970
+ persisted =
1971
+ recorded.get(clientCommitId) ?? prepared.persisted(clientCommitId);
1972
+ } catch (error) {
1973
+ if (
1974
+ error instanceof SyncError &&
1975
+ error.code === 'sync.idempotency_cache_miss'
1976
+ ) {
1977
+ // §6.3: answer the retryable cache-miss for this commit rather than
1978
+ // re-applying. Not persisted: a retry may find a readable record.
1979
+ processed.push({
1980
+ frame: idempotencyCacheMissFrame(clientCommitId, error),
1981
+ replayed: false,
1982
+ });
2025
1983
  continue;
2026
- const ids = targets.get(op.table) ?? [];
2027
- ids.push(op.rowId);
2028
- targets.set(op.table, ids);
1984
+ }
1985
+ throw error;
2029
1986
  }
2030
- if (targets.size > 0) {
2031
- await tx.prefetchRows(
2032
- [...targets].map(([table, rowIds]) => ({ table, rowIds })),
1987
+ if (persisted !== undefined) {
1988
+ const conflict = persisted.results.find(
1989
+ (result) => result.status === 'conflict',
1990
+ );
1991
+ const authorization = persisted.conflictAuthorization;
1992
+ const table = authorization && schema.tables.get(authorization.table);
1993
+ processed.push(
1994
+ processedPushCommit(
1995
+ clientCommitId,
1996
+ conflict !== undefined &&
1997
+ (authorization === undefined ||
1998
+ authorization.schemaVersion !== schema.version ||
1999
+ table === undefined ||
2000
+ !authorizeWrite(table, authorization.scopes, resolved))
2001
+ ? {
2002
+ ...persisted,
2003
+ results: [
2004
+ {
2005
+ opIndex: conflict.opIndex,
2006
+ status: 'error',
2007
+ code: 'sync.forbidden',
2008
+ message:
2009
+ 'conflict replay denied by scope authorization (§3.4)',
2010
+ retryable: false,
2011
+ },
2012
+ ],
2013
+ }
2014
+ : persisted,
2015
+ true,
2016
+ ),
2033
2017
  );
2018
+ continue;
2034
2019
  }
2035
- }
2036
- // §6.6: the blobIds the commit pushes are checked once, concurrently,
2037
- // before the operation loop. A payload that fails decode is skipped here
2038
- // and rejected by its operation.
2039
- const pushedBlobIds: string[] = [];
2040
- for (const op of operations) {
2041
- const table = schema.tables.get(op.table);
2042
- const wireTable = wireSchema.tables.get(op.table);
2020
+ // §6.3.2 step 3: below the watermark without a retained result.
2043
2021
  if (
2044
- op.payload === undefined ||
2045
- table === undefined ||
2046
- wireTable === undefined ||
2047
- table.blobRefColumnIndices.length === 0
2022
+ sequence !== undefined &&
2023
+ watermark !== undefined &&
2024
+ sequence.clientSeq < watermark
2048
2025
  ) {
2026
+ processed.push({
2027
+ frame: commitExpiredFrame(clientCommitId),
2028
+ replayed: false,
2029
+ });
2049
2030
  continue;
2050
2031
  }
2051
- let sparse: SparseRowValue[];
2032
+ const createdAtMs = clockOf(ctx)();
2033
+ let stored: StoredPushResult;
2052
2034
  try {
2053
- sparse = decodeSparseRow(
2054
- wireTable.columns,
2055
- wireTable.primaryKeyIndex,
2056
- op.payload,
2057
- );
2058
- } catch {
2059
- continue;
2060
- }
2061
- for (const index of table.blobRefColumnIndices) {
2062
- const value = sparse[index];
2063
- if (typeof value === 'string') {
2064
- pushedBlobIds.push(parseBlobRef(value).blobId);
2035
+ const results: PushOperationResult[] = [];
2036
+ const changes: NewChange[] = [];
2037
+ const validatedOperations: ValidateCommitOperation[] = [];
2038
+ const stagedWrites: StagedWrite[] = [];
2039
+ let terminated: PushOperationResult | undefined;
2040
+ const operations = await commit.buildOperations(tx);
2041
+ if (tx.prefetchRows !== undefined) {
2042
+ const targets = new Map<string, string[]>();
2043
+ for (const op of operations) {
2044
+ if (!schema.tables.has(op.table) || op.rowId.includes('\u0000'))
2045
+ continue;
2046
+ const ids = targets.get(op.table) ?? [];
2047
+ ids.push(op.rowId);
2048
+ targets.set(op.table, ids);
2049
+ }
2050
+ if (targets.size > 0) {
2051
+ await tx.prefetchRows(
2052
+ [...targets].map(([table, rowIds]) => ({ table, rowIds })),
2053
+ );
2054
+ }
2065
2055
  }
2066
- }
2067
- }
2068
- await resolveBlobPresence(blobCtx, pushedBlobIds);
2069
- // §6.4: the row each touched (table, rowId) held before this commit, so a
2070
- // conflict reports the row as stored once the commit rolls back.
2071
- const storedBeforeCommit = new Map<string, StoredRow | undefined>();
2072
- // §2.2: inserts start above the floor, which this commit's deletes raise
2073
- // as they stage.
2074
- let deleteFloor = prepared.deleteFloor;
2075
- for (let opIndex = 0; opIndex < operations.length; opIndex++) {
2076
- const op = operations[opIndex];
2077
- if (op === undefined) continue;
2078
- const outcome = await applyOperation(
2079
- tx,
2080
- schema,
2081
- wireSchema,
2082
- resolved,
2083
- op,
2084
- opIndex,
2085
- blobCtx,
2086
- crdtMergers,
2087
- validators,
2088
- partition,
2089
- ctx.actorId,
2090
- ctx.context,
2091
- deleteFloor,
2092
- );
2093
- if (outcome.kind === 'terminate') {
2094
- terminated = outcome.record;
2095
- break;
2096
- }
2097
- for (const change of outcome.changes) {
2098
- if (change.op === 'delete') {
2099
- deleteFloor = Math.max(deleteFloor, change.rowVersion);
2056
+ // §6.6: the blobIds the commit pushes are checked once, concurrently,
2057
+ // before the operation loop. A payload that fails decode is skipped
2058
+ // here and rejected by its operation.
2059
+ const pushedBlobIds: string[] = [];
2060
+ for (const op of operations) {
2061
+ const table = schema.tables.get(op.table);
2062
+ const wireTable = wireSchema.tables.get(op.table);
2063
+ if (
2064
+ op.payload === undefined ||
2065
+ table === undefined ||
2066
+ wireTable === undefined ||
2067
+ table.blobRefColumnIndices.length === 0
2068
+ ) {
2069
+ continue;
2070
+ }
2071
+ let sparse: SparseRowValue[];
2072
+ try {
2073
+ sparse = decodeSparseRow(
2074
+ wireTable.columns,
2075
+ wireTable.primaryKeyIndex,
2076
+ op.payload,
2077
+ );
2078
+ } catch {
2079
+ continue;
2080
+ }
2081
+ for (const index of table.blobRefColumnIndices) {
2082
+ const value = sparse[index];
2083
+ if (typeof value === 'string') {
2084
+ pushedBlobIds.push(parseBlobRef(value).blobId);
2085
+ }
2086
+ }
2087
+ }
2088
+ await resolveBlobPresence(blobCtx, pushedBlobIds);
2089
+ // §6.4: the row each touched (table, rowId) held before this commit,
2090
+ // so a conflict reports the row as stored once the commit rolls back.
2091
+ const storedBeforeCommit = new Map<string, StoredRow | undefined>();
2092
+ // §2.2: this commit's deletes raise the floor as they stage.
2093
+ let deleteFloor = requestDeleteFloor;
2094
+ for (let opIndex = 0; opIndex < operations.length; opIndex++) {
2095
+ const op = operations[opIndex];
2096
+ if (op === undefined) continue;
2097
+ const outcome = await applyOperation(
2098
+ tx,
2099
+ schema,
2100
+ wireSchema,
2101
+ resolved,
2102
+ op,
2103
+ opIndex,
2104
+ blobCtx,
2105
+ crdtMergers,
2106
+ validators,
2107
+ partition,
2108
+ ctx.actorId,
2109
+ ctx.context,
2110
+ deleteFloor,
2111
+ );
2112
+ if (outcome.kind === 'terminate') {
2113
+ terminated = outcome.record;
2114
+ break;
2115
+ }
2116
+ for (const change of outcome.changes) {
2117
+ if (change.op === 'delete') {
2118
+ deleteFloor = Math.max(deleteFloor, change.rowVersion);
2119
+ }
2120
+ }
2121
+ const rowKey = `${op.table}\u0000${op.rowId}`;
2122
+ if (!storedBeforeCommit.has(rowKey)) {
2123
+ storedBeforeCommit.set(rowKey, outcome.staged.stored);
2124
+ }
2125
+ results.push({ opIndex, status: 'applied' });
2126
+ validatedOperations.push(...outcome.operations);
2127
+ stagedWrites.push(outcome.staged);
2128
+ changes.push(...outcome.changes);
2100
2129
  }
2101
- }
2102
- const rowKey = `${op.table}\u0000${op.rowId}`;
2103
- if (!storedBeforeCommit.has(rowKey)) {
2104
- storedBeforeCommit.set(rowKey, outcome.staged.stored);
2105
- }
2106
- results.push({ opIndex, status: 'applied' });
2107
- validatedOperations.push(...outcome.operations);
2108
- stagedWrites.push(outcome.staged);
2109
- changes.push(...outcome.changes);
2110
- }
2111
2130
 
2112
- // §6.11: reference enforcement runs after every client operation is
2113
- // staged (so the candidate state is final for the client's own writes)
2114
- // and before whole-commit validation.
2115
- let appendedOrigin: readonly number[] = [];
2116
- if (terminated === undefined) {
2117
- const referencePass = await enforceReferences(
2118
- tx,
2119
- schema,
2120
- resolved,
2121
- stagedWrites,
2122
- operations.length,
2123
- limitsOf(ctx).maxCascadeOperationsPerCommit,
2124
- validators,
2125
- partition,
2126
- ctx.actorId,
2127
- ctx.context,
2128
- );
2129
- if (referencePass.kind === 'terminate') {
2130
- terminated = referencePass.record;
2131
- } else {
2132
- changes.push(...referencePass.changes);
2133
- validatedOperations.push(...referencePass.operations);
2134
- appendedOrigin = referencePass.originByIndex;
2135
- }
2136
- }
2131
+ // §6.11: reference enforcement runs after every client operation is
2132
+ // staged (so the candidate state is final for the client's own
2133
+ // writes) and before whole-commit validation.
2134
+ let appendedOrigin: readonly number[] = [];
2135
+ if (terminated === undefined) {
2136
+ const referencePass = await enforceReferences(
2137
+ tx,
2138
+ schema,
2139
+ resolved,
2140
+ stagedWrites,
2141
+ operations.length,
2142
+ limitsOf(ctx).maxCascadeOperationsPerCommit,
2143
+ validators,
2144
+ partition,
2145
+ ctx.actorId,
2146
+ ctx.context,
2147
+ );
2148
+ if (referencePass.kind === 'terminate') {
2149
+ terminated = referencePass.record;
2150
+ } else {
2151
+ changes.push(...referencePass.changes);
2152
+ validatedOperations.push(...referencePass.operations);
2153
+ appendedOrigin = referencePass.originByIndex;
2154
+ }
2155
+ }
2137
2156
 
2138
- if (terminated === undefined) {
2139
- const commitReject = await runCommitValidator(
2140
- commitValidator,
2141
- ctx.onError,
2142
- tx,
2143
- schema,
2144
- clientId,
2145
- clientCommitId,
2146
- ctx.actorId,
2147
- partition,
2148
- validatedOperations,
2149
- ctx.context,
2150
- );
2151
- if (commitReject?.kind === 'terminate') {
2152
- // An appended §6.11 cascade operation carries a synthetic opIndex;
2153
- // surface the originating client operation instead.
2154
- const rejected = commitReject.record;
2155
- terminated =
2156
- rejected.opIndex >= operations.length
2157
- ? {
2158
- ...rejected,
2159
- opIndex:
2160
- appendedOrigin[rejected.opIndex - operations.length] ?? 0,
2161
- }
2162
- : rejected;
2163
- }
2164
- }
2157
+ if (terminated === undefined) {
2158
+ const commitReject = await runCommitValidator(
2159
+ commitValidator,
2160
+ ctx.onError,
2161
+ tx,
2162
+ schema,
2163
+ clientId,
2164
+ clientCommitId,
2165
+ ctx.actorId,
2166
+ partition,
2167
+ validatedOperations,
2168
+ ctx.context,
2169
+ );
2170
+ if (commitReject?.kind === 'terminate') {
2171
+ // An appended §6.11 cascade operation carries a synthetic
2172
+ // opIndex; surface the originating client operation instead.
2173
+ const rejected = commitReject.record;
2174
+ terminated =
2175
+ rejected.opIndex >= operations.length
2176
+ ? {
2177
+ ...rejected,
2178
+ opIndex:
2179
+ appendedOrigin[rejected.opIndex - operations.length] ?? 0,
2180
+ }
2181
+ : rejected;
2182
+ }
2183
+ }
2165
2184
 
2166
- let preparedReactions: PreparedReaction[] = [];
2167
- if (terminated === undefined && reactionPlanner !== undefined) {
2168
- preparedReactions = await prepareReactions(reactionPlanner, {
2169
- clientId,
2170
- clientCommitId,
2171
- actorId: ctx.actorId,
2172
- partition,
2173
- operations: validatedOperations,
2174
- read: commitValidationReader(tx, schema),
2175
- });
2176
- }
2185
+ let preparedReactions: PreparedReaction[] = [];
2186
+ if (terminated === undefined && reactionPlanner !== undefined) {
2187
+ preparedReactions = await prepareReactions(reactionPlanner, {
2188
+ clientId,
2189
+ clientCommitId,
2190
+ actorId: ctx.actorId,
2191
+ partition,
2192
+ operations: validatedOperations,
2193
+ read: commitValidationReader(tx, schema),
2194
+ });
2195
+ }
2177
2196
 
2178
- if (terminated !== undefined) {
2179
- // §6.3 rejected: only the terminating operation's record; §6.4:
2180
- // every write of the commit rolls back.
2181
- const conflictOperation =
2182
- terminated.status === 'conflict'
2183
- ? operations[terminated.opIndex]
2184
- : undefined;
2185
- // §6.4: a conflict only arises in the operation loop, so a row no
2186
- // earlier operation touched still reads as stored before the commit.
2187
- const conflictKey =
2188
- conflictOperation === undefined
2189
- ? undefined
2190
- : `${conflictOperation.table}\u0000${conflictOperation.rowId}`;
2191
- const conflictRow =
2192
- conflictOperation === undefined || conflictKey === undefined
2193
- ? undefined
2194
- : storedBeforeCommit.has(conflictKey)
2195
- ? storedBeforeCommit.get(conflictKey)
2196
- : await tx.getRow(conflictOperation.table, conflictOperation.rowId);
2197
- if (terminated.status === 'conflict') {
2198
- terminated =
2199
- conflictRow === undefined
2200
- ? errorRecord(
2201
- terminated.opIndex,
2202
- 'sync.row_missing',
2203
- 'the conflicting row exists only through an earlier operation of the same commit (§6.4)',
2204
- ).record
2205
- : {
2206
- ...terminated,
2207
- serverVersion: conflictRow.serverVersion,
2208
- serverRow: conflictRow.payload,
2209
- };
2210
- }
2211
- const stored = newStoredPushResult(createdAtMs, {
2212
- status: 'rejected',
2213
- ...(terminated.status === 'conflict' &&
2214
- conflictOperation !== undefined &&
2215
- conflictRow !== undefined
2216
- ? {
2217
- conflictAuthorization: {
2218
- table: conflictOperation.table,
2219
- schemaVersion: schema.version,
2220
- scopes: conflictRow.scopes,
2221
- },
2197
+ if (terminated !== undefined) {
2198
+ // §6.3 rejected: only the terminating operation's record; §6.4:
2199
+ // every write of the commit rolls back.
2200
+ const conflictOperation =
2201
+ terminated.status === 'conflict'
2202
+ ? operations[terminated.opIndex]
2203
+ : undefined;
2204
+ // §6.4: a conflict only arises in the operation loop, so a row no
2205
+ // earlier operation touched still reads as stored before the commit.
2206
+ const conflictKey =
2207
+ conflictOperation === undefined
2208
+ ? undefined
2209
+ : `${conflictOperation.table}\u0000${conflictOperation.rowId}`;
2210
+ const conflictRow =
2211
+ conflictOperation === undefined || conflictKey === undefined
2212
+ ? undefined
2213
+ : storedBeforeCommit.has(conflictKey)
2214
+ ? storedBeforeCommit.get(conflictKey)
2215
+ : await tx.getRow(
2216
+ conflictOperation.table,
2217
+ conflictOperation.rowId,
2218
+ );
2219
+ if (terminated.status === 'conflict') {
2220
+ terminated =
2221
+ conflictRow === undefined
2222
+ ? errorRecord(
2223
+ terminated.opIndex,
2224
+ 'sync.row_missing',
2225
+ 'the conflicting row exists only through an earlier operation of the same commit (§6.4)',
2226
+ ).record
2227
+ : {
2228
+ ...terminated,
2229
+ serverVersion: conflictRow.serverVersion,
2230
+ serverRow: conflictRow.payload,
2231
+ };
2232
+ }
2233
+ stored = newStoredPushResult(createdAtMs, {
2234
+ status: 'rejected',
2235
+ ...(terminated.status === 'conflict' &&
2236
+ conflictOperation !== undefined &&
2237
+ conflictRow !== undefined
2238
+ ? {
2239
+ conflictAuthorization: {
2240
+ table: conflictOperation.table,
2241
+ schemaVersion: schema.version,
2242
+ scopes: conflictRow.scopes,
2243
+ },
2244
+ }
2245
+ : {}),
2246
+ results: [terminated],
2247
+ });
2248
+ } else {
2249
+ const commitSeq = await tx.appendCommit({
2250
+ clientId,
2251
+ clientCommitId,
2252
+ actorId: ctx.actorId,
2253
+ createdAtMs,
2254
+ changes,
2255
+ });
2256
+ const reactions = toNewReactions(preparedReactions, {
2257
+ clientId,
2258
+ clientCommitId,
2259
+ commitSeq,
2260
+ createdAtMs,
2261
+ });
2262
+ if (reactions.length > 0) {
2263
+ const enqueueReactions = tx.enqueueReactions;
2264
+ if (enqueueReactions === undefined) {
2265
+ throw new Error('storage lost durable reaction enqueue support');
2222
2266
  }
2223
- : {}),
2224
- results: [terminated],
2225
- });
2226
- // Discard candidates and persist the rejection while retaining the same
2227
- // partition lock. There is no unlock gap in which a duplicate can rerun,
2228
- // and the finalization returns the canonical outcome it persisted.
2229
- const canonical = await tx.commitRejectedPushResult(
2230
- clientId,
2231
- clientCommitId,
2232
- stored,
2233
- sequence,
2234
- );
2235
- return processedPushCommit(
2236
- clientCommitId,
2237
- canonical,
2238
- canonical.cacheIdentity !== stored.cacheIdentity,
2239
- watermark,
2240
- );
2241
- }
2242
-
2243
- const commitSeq = await tx.appendCommit({
2244
- clientId,
2245
- clientCommitId,
2246
- actorId: ctx.actorId,
2247
- createdAtMs,
2248
- changes,
2249
- });
2250
- const newReactions = toNewReactions(preparedReactions, {
2251
- clientId,
2252
- clientCommitId,
2253
- commitSeq,
2254
- createdAtMs,
2255
- });
2256
- if (newReactions.length > 0) {
2257
- const enqueueReactions = tx.enqueueReactions;
2258
- if (enqueueReactions === undefined) {
2259
- throw new Error('storage lost durable reaction enqueue support');
2260
- }
2261
- await enqueueReactions.call(tx, newReactions);
2262
- }
2263
- const stored = newStoredPushResult(createdAtMs, {
2264
- status: 'applied',
2265
- commitSeq,
2266
- results,
2267
- });
2268
- await tx.putPushResult(clientId, clientCommitId, stored, sequence);
2269
- await tx.commit();
2270
- if (ctx.events !== undefined) {
2271
- const atMs = clockOf(ctx)();
2272
- for (const reaction of newReactions) {
2273
- emitEvent(ctx.events, {
2274
- type: 'reaction.queued',
2275
- atMs,
2276
- partition,
2277
- actorId: ctx.actorId,
2278
- clientId,
2279
- clientCommitId,
2280
- commitSeq,
2281
- idempotencyKey: reaction.idempotencyKey,
2282
- reactionType: reaction.type,
2283
- version: reaction.version,
2267
+ await enqueueReactions.call(tx, reactions);
2268
+ }
2269
+ stored = newStoredPushResult(createdAtMs, {
2270
+ status: 'applied',
2271
+ commitSeq,
2272
+ results,
2273
+ });
2274
+ // Closes the commit; D1 writes the commit's batch here.
2275
+ await tx.putPushResult(clientId, clientCommitId, stored, sequence);
2276
+ applied.push({
2277
+ clientCommitId,
2278
+ commitSeq,
2279
+ createdAtMs,
2280
+ changes,
2281
+ reactions,
2282
+ });
2283
+ requestDeleteFloor = Math.max(
2284
+ requestDeleteFloor,
2285
+ commitDeleteFloor(changes),
2286
+ );
2287
+ }
2288
+ } catch (error) {
2289
+ if (!(error instanceof StorageConstraintError)) throw error;
2290
+ stored = newStoredPushResult(createdAtMs, {
2291
+ status: 'rejected',
2292
+ results: [
2293
+ {
2294
+ opIndex: error.opIndex ?? 0,
2295
+ status: 'error',
2296
+ code: 'sync.constraint_violation',
2297
+ message: 'write violates a relational constraint',
2298
+ retryable: false,
2299
+ },
2300
+ ],
2284
2301
  });
2285
2302
  }
2286
- }
2287
- if (ctx.realtime !== undefined && changes.length > 0) {
2288
- // RFC 0007 fanout decision: the notification carries the durable commit
2289
- // entry itself (this commit's `changes`), not a projection read or a pull
2290
- // window, so fanning it out before the request's buffered read-verify can
2291
- // never serve a mixed result. Each peer's next pull is gated at the
2292
- // storage seam; a refused mixed push+pull is safe because the push is
2293
- // durable and replayable under its commit id. This is commit-entry
2294
- // durability, not gate inheritance.
2295
- await ctx.realtime.notifyCommit(partition, {
2296
- commitSeq,
2297
- createdAtMs,
2298
- actorId: ctx.actorId,
2299
- changes,
2300
- });
2301
- }
2302
- return processedPushCommit(clientCommitId, stored, false, watermark);
2303
- } catch (error) {
2304
- if (error instanceof StorageConstraintError) {
2305
- const stored = newStoredPushResult(createdAtMs, {
2306
- status: 'rejected',
2307
- results: [
2308
- {
2309
- opIndex: error.opIndex ?? 0,
2310
- status: 'error',
2311
- code: 'sync.constraint_violation',
2312
- message: 'write violates a relational constraint',
2313
- retryable: false,
2314
- },
2315
- ],
2316
- });
2317
- try {
2318
- const canonical = await tx.commitRejectedPushResult(
2319
- clientId,
2320
- clientCommitId,
2321
- stored,
2322
- sequence,
2323
- );
2324
- return processedPushCommit(
2325
- clientCommitId,
2326
- canonical,
2327
- canonical.cacheIdentity !== stored.cacheIdentity,
2328
- );
2329
- } catch (finalizationError) {
2330
- // A failed finalization must still release the transaction: on
2331
- // SQLite an unreleased BEGIN wedges the storage's global begin()
2332
- // queue, on Postgres it leaks the pinned pool client. rollback() is
2333
- // a no-op when the finalization already committed.
2334
- try {
2335
- await tx.rollback();
2336
- } catch (rollbackError) {
2337
- retainRollbackFailure(finalizationError, rollbackError);
2338
- }
2339
- throw finalizationError;
2303
+ if (stored.status === 'rejected') {
2304
+ // Discard the commit's candidates and record the rejection while the
2305
+ // partition lock is retained: no unlock gap lets a duplicate rerun.
2306
+ await tx.rejectPushCommit(clientId, clientCommitId, stored, sequence);
2340
2307
  }
2308
+ recorded.set(clientCommitId, stored);
2309
+ processed.push(processedPushCommit(clientCommitId, stored, false));
2341
2310
  }
2311
+ await tx.commit();
2312
+ } catch (error) {
2342
2313
  try {
2343
2314
  await tx.rollback();
2344
2315
  } catch (rollbackError) {
2345
2316
  retainRollbackFailure(error, rollbackError);
2346
2317
  }
2318
+ // §6.4: a storage that persists each commit when it closes (D1) keeps the
2319
+ // commits that closed before the failure; they are durable, so peers
2320
+ // learn of them now rather than at their next pull.
2321
+ if (tx.persistsCommitsOnClose === true) await announce();
2347
2322
  throw error;
2348
2323
  }
2324
+ await announce();
2325
+ return processed;
2349
2326
  }