@syncular/server 0.59.0 → 0.60.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/README.md +20 -16
  2. package/dist/admin.d.ts +32 -20
  3. package/dist/admin.js +42 -40
  4. package/dist/context.d.ts +6 -8
  5. package/dist/context.js +1 -3
  6. package/dist/d1-storage.js +42 -24
  7. package/dist/errors.js +8 -0
  8. package/dist/events.d.ts +35 -17
  9. package/dist/frame-bytes.js +24 -6
  10. package/dist/handler.js +486 -223
  11. package/dist/local-authority.js +7 -9
  12. package/dist/operations.d.ts +5 -2
  13. package/dist/operations.js +20 -29
  14. package/dist/postgres-storage.d.ts +24 -19
  15. package/dist/postgres-storage.js +249 -168
  16. package/dist/pull.d.ts +68 -53
  17. package/dist/pull.js +258 -287
  18. package/dist/push.d.ts +12 -3
  19. package/dist/push.js +95 -32
  20. package/dist/realtime.d.ts +17 -11
  21. package/dist/realtime.js +65 -90
  22. package/dist/relational-rows.d.ts +0 -57
  23. package/dist/relational-rows.js +0 -97
  24. package/dist/seed.js +10 -15
  25. package/dist/segment-download.d.ts +4 -1
  26. package/dist/segment-download.js +4 -2
  27. package/dist/sqlite-dialect.d.ts +36 -59
  28. package/dist/sqlite-dialect.js +255 -159
  29. package/dist/sqlite-storage.js +24 -20
  30. package/dist/storage-query.d.ts +30 -1
  31. package/dist/storage-query.js +68 -0
  32. package/dist/storage.d.ts +156 -54
  33. package/dist/storage.js +15 -0
  34. package/package.json +2 -2
  35. package/src/admin.ts +87 -59
  36. package/src/context.ts +7 -11
  37. package/src/d1-storage.ts +49 -33
  38. package/src/errors.ts +8 -0
  39. package/src/events.ts +36 -17
  40. package/src/frame-bytes.ts +26 -6
  41. package/src/handler.ts +628 -286
  42. package/src/local-authority.ts +10 -11
  43. package/src/operations.ts +44 -42
  44. package/src/postgres-storage.ts +336 -226
  45. package/src/pull.ts +394 -414
  46. package/src/push.ts +116 -21
  47. package/src/realtime.ts +85 -129
  48. package/src/relational-rows.ts +0 -105
  49. package/src/seed.ts +13 -19
  50. package/src/segment-download.ts +8 -3
  51. package/src/sqlite-dialect.ts +363 -204
  52. package/src/sqlite-storage.ts +45 -25
  53. package/src/storage-query.ts +102 -0
  54. package/src/storage.ts +185 -58
package/src/push.ts CHANGED
@@ -48,6 +48,8 @@ import { authorizeWrite, renderScopeValue, storedScopesForRow } from './scopes';
48
48
  import { decodeColumnVersions, encodeColumnVersions } from './relational-rows';
49
49
  import type {
50
50
  NewChange,
51
+ OutboxSequence,
52
+ OutboxSettlement,
51
53
  PreparedPushApply,
52
54
  StorageTransaction,
53
55
  StoredCommit,
@@ -344,6 +346,8 @@ async function applyOperation(
344
346
  validators: ValidatorRegistry | undefined,
345
347
  partition: string,
346
348
  actorId: string,
349
+ /** §2.2: the partition's delete floor, raised by this commit's deletes. */
350
+ deleteFloor: number,
347
351
  ): Promise<OperationOutcome> {
348
352
  const table = schema.tables.get(op.table);
349
353
  const wireTable = wireSchema.tables.get(op.table);
@@ -369,8 +373,9 @@ async function applyOperation(
369
373
 
370
374
  if (op.op === 'rescope') {
371
375
  // §2.2 explicit scope move. One stored row changes its scope values;
372
- // apply rewrites it in place, keeps `server_version`, and emits the
373
- // delete under the old scopes plus the upsert under the new ones.
376
+ // apply rewrites it in place, advances `server_version` by 1, and emits
377
+ // the delete under the old scopes plus the upsert under the new ones,
378
+ // both at the new version.
374
379
  if (stored === undefined) {
375
380
  return errorRecord(
376
381
  opIndex,
@@ -502,6 +507,22 @@ async function applyOperation(
502
507
  );
503
508
  if (blobCheck !== undefined) return blobCheck;
504
509
  const payload = encodeRow(table.columns, values);
510
+ // §2.2: the moved scope columns take the new version as their column
511
+ // version; every other column keeps its own (a row stored before column
512
+ // versions holds every column at its current version).
513
+ const movedVersion = stored.serverVersion + 1;
514
+ const storedVersions = decodeColumnVersions(stored.columnVersions);
515
+ const movedVersions = new Map<number, number>();
516
+ for (let i = 0; i < table.columns.length; i++) {
517
+ const version =
518
+ scopeIndexes.has(i) &&
519
+ !sameValue(values[i] ?? null, storedValues[i] ?? null)
520
+ ? movedVersion
521
+ : (storedVersions.get(i) ??
522
+ (stored.columnVersions === undefined ? stored.serverVersion : 1));
523
+ if (version > 1) movedVersions.set(i, version);
524
+ }
525
+ const columnVersions = encodeColumnVersions(movedVersions);
505
526
  // The move clears any tombstone from an earlier incarnation: the row is
506
527
  // live at the end of this commit, so the next move must not hit §5.2.
507
528
  await tx.clearTombstone(op.table, op.rowId);
@@ -509,12 +530,10 @@ async function applyOperation(
509
530
  op.table,
510
531
  {
511
532
  rowId: op.rowId,
512
- serverVersion: stored.serverVersion,
533
+ serverVersion: movedVersion,
513
534
  scopes: extracted.scopes,
514
535
  payload,
515
- ...(stored.columnVersions !== undefined
516
- ? { columnVersions: stored.columnVersions }
517
- : {}),
536
+ ...(columnVersions !== undefined ? { columnVersions } : {}),
518
537
  },
519
538
  { opIndex },
520
539
  );
@@ -526,13 +545,14 @@ async function applyOperation(
526
545
  table: op.table,
527
546
  rowId: op.rowId,
528
547
  op: 'delete',
548
+ rowVersion: movedVersion,
529
549
  scopes: stored.scopes,
530
550
  },
531
551
  {
532
552
  table: op.table,
533
553
  rowId: op.rowId,
534
554
  op: 'upsert',
535
- rowVersion: stored.serverVersion,
555
+ rowVersion: movedVersion,
536
556
  scopes: extracted.scopes,
537
557
  payload,
538
558
  },
@@ -555,7 +575,7 @@ async function applyOperation(
555
575
  row: toValidateRow(table.columns, values),
556
576
  stored: storedRow,
557
577
  storedServerVersion: stored.serverVersion,
558
- nextServerVersion: stored.serverVersion,
578
+ nextServerVersion: movedVersion,
559
579
  },
560
580
  ],
561
581
  staged: {
@@ -635,7 +655,9 @@ async function applyOperation(
635
655
  table: op.table,
636
656
  rowId: op.rowId,
637
657
  op: 'delete',
638
- deletedVersion: stored.serverVersion,
658
+ // §2.2: a delete advances the version by 1; its tombstone records
659
+ // it and the delete floor rises to it.
660
+ rowVersion: stored.serverVersion + 1,
639
661
  scopes: stored.scopes,
640
662
  },
641
663
  ],
@@ -985,11 +1007,11 @@ async function applyOperation(
985
1007
  actorId,
986
1008
  );
987
1009
  if (insertReject !== undefined) return insertReject;
988
- // §6.2: an insert recreates over the tombstone and starts above the
989
- // deleted row's version; without a column-versions blob every column sits
990
- // at that version.
1010
+ // §2.2, §6.2: every insert starts one above the delete floor, over a
1011
+ // tombstone or not; without a column-versions blob every column sits at
1012
+ // that version.
991
1013
  if (tombstone !== undefined) await tx.clearTombstone(op.table, op.rowId);
992
- const serverVersion = (deleted?.serverVersion ?? 0) + 1;
1014
+ const serverVersion = deleteFloor + 1;
993
1015
  const newRow = {
994
1016
  rowId: op.rowId,
995
1017
  serverVersion,
@@ -1579,7 +1601,7 @@ async function enforceReferences(
1579
1601
  table: child.name,
1580
1602
  rowId: childRow.rowId,
1581
1603
  op: 'delete',
1582
- deletedVersion: childRow.serverVersion,
1604
+ rowVersion: childRow.serverVersion + 1,
1583
1605
  scopes: childRow.scopes,
1584
1606
  });
1585
1607
  operations.push({
@@ -1694,16 +1716,28 @@ export interface ProcessedPushCommit {
1694
1716
  readonly replayed: boolean;
1695
1717
  readonly recordedAtMs?: number;
1696
1718
  readonly cacheIdentity?: string;
1719
+ /** The outbox watermark `W` the commit's settlement read (§6.3.2). */
1720
+ readonly watermark?: number;
1721
+ }
1722
+
1723
+ /** A pushed commit's outbox position and the request's watermark (§6.3.2). */
1724
+ export interface PushOutbox extends OutboxSequence {
1725
+ /** `W` after the request's first commit. */
1726
+ readonly watermark: number;
1727
+ /** Present with the request's first commit: steps 1 and 2 run with it. */
1728
+ readonly settlement?: OutboxSettlement;
1697
1729
  }
1698
1730
 
1699
1731
  function processedPushCommit(
1700
1732
  clientCommitId: string,
1701
1733
  stored: StoredPushResult,
1702
1734
  replayed: boolean,
1735
+ watermark?: number,
1703
1736
  ): ProcessedPushCommit {
1704
1737
  return {
1705
1738
  frame: resultFrame(clientCommitId, stored, replayed),
1706
1739
  replayed,
1740
+ ...(watermark !== undefined ? { watermark } : {}),
1707
1741
  ...(stored.recordedAtMs !== undefined
1708
1742
  ? { recordedAtMs: stored.recordedAtMs }
1709
1743
  : {}),
@@ -1744,6 +1778,24 @@ function idempotencyCacheMissFrame(
1744
1778
  };
1745
1779
  }
1746
1780
 
1781
+ /** §6.3: the unpersisted refusal of a commit below its outbox's watermark. */
1782
+ function commitExpiredFrame(clientCommitId: string): PushResultFrame {
1783
+ return {
1784
+ type: 'PUSH_RESULT',
1785
+ clientCommitId,
1786
+ status: 'rejected',
1787
+ results: [
1788
+ {
1789
+ opIndex: 0,
1790
+ status: 'error',
1791
+ code: 'sync.commit_expired',
1792
+ message: 'the commit lies below its outbox watermark (§6.3.2)',
1793
+ retryable: false,
1794
+ },
1795
+ ],
1796
+ };
1797
+ }
1798
+
1747
1799
  export interface AppliedCommitEvent {
1748
1800
  readonly commit: StoredCommit;
1749
1801
  }
@@ -1776,6 +1828,7 @@ export async function processPushCommitWithTrace(
1776
1828
  clientId: string,
1777
1829
  frame: PushCommitFrame,
1778
1830
  wireSchema: CompiledSchema = schema,
1831
+ outbox?: PushOutbox,
1779
1832
  ): Promise<ProcessedPushCommit> {
1780
1833
  // The frame names its rows before the transaction opens, so the admission
1781
1834
  // read carries the commit's read-ahead and the apply issues no separate
@@ -1800,6 +1853,7 @@ export async function processPushCommitWithTrace(
1800
1853
  targets.size === 0
1801
1854
  ? undefined
1802
1855
  : [...targets].map(([table, rowIds]) => ({ table, rowIds })),
1856
+ outbox,
1803
1857
  );
1804
1858
  }
1805
1859
 
@@ -1822,6 +1876,7 @@ export async function processPushOperationsWithTrace(
1822
1876
  readonly table: string;
1823
1877
  readonly rowIds: readonly string[];
1824
1878
  }[],
1879
+ outbox?: PushOutbox,
1825
1880
  ): Promise<ProcessedPushCommit> {
1826
1881
  const { partition } = ctx;
1827
1882
  const createdAtMs = clockOf(ctx)();
@@ -1835,6 +1890,13 @@ export async function processPushOperationsWithTrace(
1835
1890
  const commitValidator = ctx.commitValidator;
1836
1891
  const reactionPlanner = ctx.reactionPlanner;
1837
1892
  const tx = await ctx.storage.begin(partition);
1893
+ // §6.3.2: the settlement's writes commit whatever the commit's outcome.
1894
+ const release = (): Promise<void> =>
1895
+ outbox?.settlement !== undefined ? tx.commit() : tx.rollback();
1896
+ const sequence =
1897
+ outbox === undefined
1898
+ ? undefined
1899
+ : { outboxId: outbox.outboxId, clientSeq: outbox.clientSeq };
1838
1900
  try {
1839
1901
  if (reactionPlanner !== undefined && tx.enqueueReactions === undefined) {
1840
1902
  throw new Error(
@@ -1854,6 +1916,7 @@ export async function processPushOperationsWithTrace(
1854
1916
  clientCommitId,
1855
1917
  schema.version,
1856
1918
  prefetchTargets,
1919
+ outbox?.settlement,
1857
1920
  );
1858
1921
  } catch (error) {
1859
1922
  if (
@@ -1862,7 +1925,7 @@ export async function processPushOperationsWithTrace(
1862
1925
  ) {
1863
1926
  // §6.3: answer the retryable cache-miss for this commit rather than
1864
1927
  // re-applying. Not persisted — a retry may find a readable record.
1865
- await tx.rollback();
1928
+ await release();
1866
1929
  return {
1867
1930
  frame: idempotencyCacheMissFrame(clientCommitId, error),
1868
1931
  replayed: false,
@@ -1879,9 +1942,10 @@ export async function processPushOperationsWithTrace(
1879
1942
  ctx.checkpoints,
1880
1943
  );
1881
1944
  if (refusal !== undefined) {
1882
- await tx.rollback();
1945
+ await release();
1883
1946
  throw serveNotReadyError(refusal);
1884
1947
  }
1948
+ const watermark = prepared.watermark ?? outbox?.watermark;
1885
1949
  if (prepared.persisted !== undefined) {
1886
1950
  const conflict = prepared.persisted.results.find(
1887
1951
  (result) => result.status === 'conflict',
@@ -1895,7 +1959,7 @@ export async function processPushOperationsWithTrace(
1895
1959
  table === undefined ||
1896
1960
  !authorizeWrite(table, authorization.scopes, resolved))
1897
1961
  ) {
1898
- await tx.rollback();
1962
+ await release();
1899
1963
  return processedPushCommit(
1900
1964
  clientCommitId,
1901
1965
  {
@@ -1911,10 +1975,29 @@ export async function processPushOperationsWithTrace(
1911
1975
  ],
1912
1976
  },
1913
1977
  true,
1978
+ watermark,
1914
1979
  );
1915
1980
  }
1916
- await tx.rollback();
1917
- return processedPushCommit(clientCommitId, prepared.persisted, true);
1981
+ await release();
1982
+ return processedPushCommit(
1983
+ clientCommitId,
1984
+ prepared.persisted,
1985
+ true,
1986
+ watermark,
1987
+ );
1988
+ }
1989
+ // §6.3.2 step 3: below the watermark without a retained result.
1990
+ if (
1991
+ outbox !== undefined &&
1992
+ watermark !== undefined &&
1993
+ outbox.clientSeq < watermark
1994
+ ) {
1995
+ await release();
1996
+ return {
1997
+ frame: commitExpiredFrame(clientCommitId),
1998
+ replayed: false,
1999
+ watermark,
2000
+ };
1918
2001
  }
1919
2002
  const results: PushOperationResult[] = [];
1920
2003
  const changes: NewChange[] = [];
@@ -1973,6 +2056,9 @@ export async function processPushOperationsWithTrace(
1973
2056
  // §6.4: the row each touched (table, rowId) held before this commit, so a
1974
2057
  // conflict reports the row as stored once the commit rolls back.
1975
2058
  const storedBeforeCommit = new Map<string, StoredRow | undefined>();
2059
+ // §2.2: inserts start above the floor, which this commit's deletes raise
2060
+ // as they stage.
2061
+ let deleteFloor = prepared.deleteFloor;
1976
2062
  for (let opIndex = 0; opIndex < operations.length; opIndex++) {
1977
2063
  const op = operations[opIndex];
1978
2064
  if (op === undefined) continue;
@@ -1988,11 +2074,17 @@ export async function processPushOperationsWithTrace(
1988
2074
  validators,
1989
2075
  partition,
1990
2076
  ctx.actorId,
2077
+ deleteFloor,
1991
2078
  );
1992
2079
  if (outcome.kind === 'terminate') {
1993
2080
  terminated = outcome.record;
1994
2081
  break;
1995
2082
  }
2083
+ for (const change of outcome.changes) {
2084
+ if (change.op === 'delete') {
2085
+ deleteFloor = Math.max(deleteFloor, change.rowVersion);
2086
+ }
2087
+ }
1996
2088
  const rowKey = `${op.table}\u0000${op.rowId}`;
1997
2089
  if (!storedBeforeCommit.has(rowKey)) {
1998
2090
  storedBeforeCommit.set(rowKey, outcome.staged.stored);
@@ -2122,11 +2214,13 @@ export async function processPushOperationsWithTrace(
2122
2214
  clientId,
2123
2215
  clientCommitId,
2124
2216
  stored,
2217
+ sequence,
2125
2218
  );
2126
2219
  return processedPushCommit(
2127
2220
  clientCommitId,
2128
2221
  canonical,
2129
2222
  canonical.cacheIdentity !== stored.cacheIdentity,
2223
+ watermark,
2130
2224
  );
2131
2225
  }
2132
2226
 
@@ -2155,7 +2249,7 @@ export async function processPushOperationsWithTrace(
2155
2249
  commitSeq,
2156
2250
  results,
2157
2251
  });
2158
- await tx.putPushResult(clientId, clientCommitId, stored);
2252
+ await tx.putPushResult(clientId, clientCommitId, stored, sequence);
2159
2253
  await tx.commit();
2160
2254
  if (ctx.events !== undefined) {
2161
2255
  const atMs = clockOf(ctx)();
@@ -2189,7 +2283,7 @@ export async function processPushOperationsWithTrace(
2189
2283
  changes,
2190
2284
  });
2191
2285
  }
2192
- return processedPushCommit(clientCommitId, stored, false);
2286
+ return processedPushCommit(clientCommitId, stored, false, watermark);
2193
2287
  } catch (error) {
2194
2288
  if (error instanceof StorageConstraintError) {
2195
2289
  const stored = newStoredPushResult(createdAtMs, {
@@ -2209,6 +2303,7 @@ export async function processPushOperationsWithTrace(
2209
2303
  clientId,
2210
2304
  clientCommitId,
2211
2305
  stored,
2306
+ sequence,
2212
2307
  );
2213
2308
  return processedPushCommit(
2214
2309
  clientCommitId,