@syncular/server 0.95.0 → 0.97.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/README.md +6 -5
  2. package/dist/context.d.ts +4 -16
  3. package/dist/context.js +0 -3
  4. package/dist/d1-storage.js +1 -1
  5. package/dist/errors.d.ts +11 -0
  6. package/dist/errors.js +36 -0
  7. package/dist/frame-bytes.js +1 -0
  8. package/dist/handler.d.ts +20 -2
  9. package/dist/handler.js +199 -59
  10. package/dist/postgres-storage.d.ts +3 -2
  11. package/dist/postgres-storage.js +18 -2
  12. package/dist/pull.d.ts +2 -2
  13. package/dist/pull.js +22 -16
  14. package/dist/push.js +41 -77
  15. package/dist/realtime.d.ts +5 -11
  16. package/dist/realtime.js +10 -23
  17. package/dist/relational-rows.d.ts +23 -13
  18. package/dist/relational-rows.js +49 -30
  19. package/dist/schema-window.js +2 -2
  20. package/dist/schema.d.ts +2 -2
  21. package/dist/schema.js +2 -2
  22. package/dist/segment-download.js +11 -4
  23. package/dist/sqlite-dialect.d.ts +3 -1
  24. package/dist/sqlite-dialect.js +37 -6
  25. package/dist/storage-query.d.ts +5 -2
  26. package/dist/storage-query.js +7 -4
  27. package/dist/storage.d.ts +12 -0
  28. package/dist/validate.d.ts +21 -39
  29. package/dist/validate.js +4 -3
  30. package/package.json +2 -2
  31. package/src/context.ts +4 -23
  32. package/src/d1-storage.ts +0 -1
  33. package/src/errors.ts +46 -0
  34. package/src/frame-bytes.ts +1 -0
  35. package/src/handler.ts +257 -73
  36. package/src/postgres-storage.ts +32 -6
  37. package/src/pull.ts +50 -35
  38. package/src/push.ts +51 -129
  39. package/src/realtime.ts +10 -22
  40. package/src/relational-rows.ts +51 -30
  41. package/src/schema-window.ts +2 -2
  42. package/src/schema.ts +4 -4
  43. package/src/segment-download.ts +19 -12
  44. package/src/sqlite-dialect.ts +47 -7
  45. package/src/storage-query.ts +7 -4
  46. package/src/storage.ts +13 -0
  47. package/src/validate.ts +21 -48
package/src/errors.ts CHANGED
@@ -234,6 +234,22 @@ export const ERROR_CATALOG: Readonly<Record<string, ErrorCatalogEntry>> = {
234
234
  recommendedAction: 'forceResync',
235
235
  httpStatus: 404,
236
236
  },
237
+ // §1.6: a response still building at the 300 s build deadline. In-band
238
+ // after its header frames, HTTP 503 before them.
239
+ 'sync.segment_build_timeout': {
240
+ category: 'internal',
241
+ retryable: true,
242
+ recommendedAction: 'retryLater',
243
+ httpStatus: 503,
244
+ },
245
+ // §5.4/§5.5: a SegmentStore call rejected (an object store outage). The
246
+ // HTTP answer carries `Retry-After: 5` (`errorHeaders`).
247
+ 'sync.segment_store_unavailable': {
248
+ category: 'internal',
249
+ retryable: true,
250
+ recommendedAction: 'retryLater',
251
+ httpStatus: 503,
252
+ },
237
253
  'sync.segment_expired': {
238
254
  category: 'not-found',
239
255
  retryable: true,
@@ -356,6 +372,36 @@ export function errorBody(error: SyncError): {
356
372
  };
357
373
  }
358
374
 
375
+ /**
376
+ * HTTP headers of a §10.1 error answer: `sync.segment_store_unavailable`
377
+ * carries a fixed `Retry-After: 5` (§10.2).
378
+ */
379
+ export function errorHeaders(error: SyncError): Record<string, string> {
380
+ return error.code === 'sync.segment_store_unavailable'
381
+ ? { 'retry-after': '5' }
382
+ : {};
383
+ }
384
+
385
+ /**
386
+ * The rejection handler of a `SegmentStore` call (§5.4, §5.5): the store's
387
+ * error goes to `onError`, and the request fails with the retryable
388
+ * `sync.segment_store_unavailable`, its `details.operation` naming the call.
389
+ */
390
+ export function segmentStoreFailure(
391
+ onError: SyncularErrorHandler | undefined,
392
+ route: 'sync' | 'segments',
393
+ operation: 'put' | 'find' | 'get' | 'open',
394
+ ): (error: unknown) => never {
395
+ return (error) => {
396
+ reportError(error, onError, route);
397
+ throw new SyncError(
398
+ 'sync.segment_store_unavailable',
399
+ 'the segment store failed',
400
+ JSON.stringify({ operation }),
401
+ );
402
+ };
403
+ }
404
+
359
405
  /** The adapter surface that caught an exception reported through `onError`. */
360
406
  export type SyncularErrorRoute =
361
407
  | 'sync'
@@ -71,6 +71,7 @@ function wrapperFor(frame: ResponseFrame): {
71
71
  switch (frame.type) {
72
72
  case 'RESP_HEADER':
73
73
  return { frames: [frame], index: 0 };
74
+ case 'KEEPALIVE':
74
75
  case 'LEASE':
75
76
  // §7.3.2: LEASE rides immediately after RESP_HEADER.
76
77
  return { frames: [stubHeader(), frame], index: 1 };
package/src/handler.ts CHANGED
@@ -37,7 +37,7 @@ import {
37
37
  RESOLVER_OUTAGE,
38
38
  touchAuthenticatedPartition,
39
39
  } from './context';
40
- import { reportError, SyncError, syncError } from './errors';
40
+ import { adapterSyncError, reportError, SyncError, syncError } from './errors';
41
41
  import {
42
42
  emitEvent,
43
43
  type PullUnitSummary,
@@ -855,7 +855,8 @@ async function* streamResponse(
855
855
  );
856
856
  }
857
857
  } else {
858
- const horizonSeq = await ctx.storage.getHorizonSeq(ctx.partition);
858
+ const horizons = await ctx.storage.getHorizons(ctx.partition);
859
+ const { seq: horizonSeq, tables: tableHorizons } = horizons;
859
860
  const R = pull.header.cursor;
860
861
  const echo = echoDigest(
861
862
  units.map(({ plan: unit }) => ({
@@ -875,6 +876,20 @@ async function* streamResponse(
875
876
  );
876
877
  // §4.6 replica reset: a live unit with R outside [horizon, max].
877
878
  let liveReset = liveUnits && (R < horizonSeq || R > maxSeq);
879
+ // §4.6 unit reset: `cursor` lies below the table horizon of one of
880
+ // the unit's active pairs outside its RESTRICT tables.
881
+ const belowTable = (
882
+ index: number,
883
+ cursor: number,
884
+ horizons: ReadonlyMap<string, number>,
885
+ ): boolean => {
886
+ const { plan: unit, restrict } = units[index]!;
887
+ return [...unit.pairs.keys()].some(
888
+ (table) =>
889
+ cursor < (horizons.get(table) ?? -1) &&
890
+ restrict?.tables.includes(table) !== true,
891
+ );
892
+ };
878
893
  // Per unit: its section kind and its scan-set bound (§4.5).
879
894
  type UnitState =
880
895
  | { kind: 'revoked' }
@@ -890,12 +905,17 @@ async function* streamResponse(
890
905
  token: string | undefined;
891
906
  };
892
907
  const states: UnitState[] = units.map(
893
- ({ plan: unit, progress, restrict }): UnitState => {
908
+ ({ plan: unit, progress, restrict }, index): UnitState => {
894
909
  if (unit.status === 'revoked') return { kind: 'revoked' };
895
- if (progress === undefined) return { kind: 'live' };
910
+ if (progress === undefined)
911
+ return !liveReset && belowTable(index, R, tableHorizons)
912
+ ? { kind: 'reset' }
913
+ : { kind: 'live' };
896
914
  // §4.6: a kept cursor outside [horizon, max] cannot join.
897
915
  if (restrict !== undefined)
898
- return restrict.cursor < horizonSeq || restrict.cursor > maxSeq
916
+ return restrict.cursor < horizonSeq ||
917
+ restrict.cursor > maxSeq ||
918
+ belowTable(index, restrict.cursor, tableHorizons)
899
919
  ? { kind: 'reset' }
900
920
  : {
901
921
  kind: 'bootstrap',
@@ -910,7 +930,8 @@ async function* streamResponse(
910
930
  progress.cursor >= 0 &&
911
931
  progress.cursor <= maxSeq
912
932
  ) {
913
- return progress.cursor < horizonSeq
933
+ return progress.cursor < horizonSeq ||
934
+ belowTable(index, progress.cursor, tableHorizons)
914
935
  ? { kind: 'reset' }
915
936
  : { kind: 'post', cursor: progress.cursor };
916
937
  }
@@ -945,7 +966,6 @@ async function* streamResponse(
945
966
  // pull holds the encoded bytes of the puts in flight.
946
967
  const puts: Promise<ResponseFrame>[] = [];
947
968
  const inFlight = new Set<Promise<void>>();
948
- const maxPuts = limitsOf(ctx).maxConcurrentSegmentPuts;
949
969
  let failed = false;
950
970
  for (let next = 0; next < bootstrapping.length;) {
951
971
  const window: number[] = [];
@@ -985,7 +1005,7 @@ async function* streamResponse(
985
1005
  },
986
1006
  progress?.bootstrapState,
987
1007
  maxSeq,
988
- horizonSeq,
1008
+ horizons,
989
1009
  plan.logEpoch,
990
1010
  restrict?.cursor,
991
1011
  ),
@@ -1006,7 +1026,9 @@ async function* streamResponse(
1006
1026
  const frame = step.value;
1007
1027
  if (frame.type === 'PENDING_REF') {
1008
1028
  // A rows segment's put runs while later pages are scanned;
1009
- // the next page waits while `maxPuts` puts are in flight.
1029
+ // the next page waits while six puts are in flight: a
1030
+ // Workers invocation holds six connections open at once and
1031
+ // an R2 put uses one, so past six the runtime queues the put.
1010
1032
  const settled: Promise<void> = frame.ref.then(
1011
1033
  () => void inFlight.delete(settled),
1012
1034
  () => {
@@ -1017,7 +1039,7 @@ async function* streamResponse(
1017
1039
  inFlight.add(settled);
1018
1040
  puts.push(frame.ref);
1019
1041
  frames.push(frame.ref);
1020
- while (inFlight.size >= maxPuts) {
1042
+ while (inFlight.size >= 6) {
1021
1043
  await Promise.race(inFlight);
1022
1044
  if (failed) break;
1023
1045
  }
@@ -1058,10 +1080,27 @@ async function* streamResponse(
1058
1080
  let live: LiveWindow | undefined;
1059
1081
  for (;;) {
1060
1082
  const scanned = units.flatMap(({ plan: unit }, index) => {
1061
- const afterSeq = bound(states[index]!);
1062
- return afterSeq === undefined
1063
- ? []
1064
- : [{ index, scopes: unit.scopes, afterSeq, tables: unit.pairs }];
1083
+ const bounded = bound(states[index]!);
1084
+ if (bounded === undefined) return [];
1085
+ // §4.5: a table reaches the unit only above its table horizon,
1086
+ // which lies above the bound only for a RESTRICT table.
1087
+ const byBound = new Map<number, Map<string, ScopeMap>>();
1088
+ for (const [table, scopes] of unit.pairs) {
1089
+ const afterSeq = Math.max(
1090
+ bounded,
1091
+ tableHorizons.get(table) ?? -1,
1092
+ );
1093
+ byBound.set(
1094
+ afterSeq,
1095
+ (byBound.get(afterSeq) ?? new Map()).set(table, scopes),
1096
+ );
1097
+ }
1098
+ return [...byBound].map(([afterSeq, tables]) => ({
1099
+ index,
1100
+ scopes: unit.scopes,
1101
+ afterSeq,
1102
+ tables,
1103
+ }));
1065
1104
  });
1066
1105
  if (liveReset || scanned.length === 0) break;
1067
1106
  live = await ctx.storage.readLiveWindow(ctx.partition, {
@@ -1081,18 +1120,33 @@ async function* streamResponse(
1081
1120
  live = undefined;
1082
1121
  continue;
1083
1122
  }
1084
- const recheck = await ctx.storage.getHorizonSeq(ctx.partition);
1123
+ const { seq: recheck, tables: recheckTables } =
1124
+ await ctx.storage.getHorizons(ctx.partition);
1085
1125
  let expired = false;
1086
1126
  for (const [index, state] of states.entries()) {
1087
- // §4.6: a restricted bootstrap's kept cursor is rechecked too.
1127
+ // §4.6: a restricted bootstrap's kept cursor is rechecked too,
1128
+ // and a table horizon that rose during the round resets it: its
1129
+ // segments were read before the re-key.
1130
+ const restricted = state.kind === 'bootstrap' && state.restricted;
1088
1131
  const afterSeq =
1089
- bound(state) ??
1090
- (state.kind === 'bootstrap' && state.restricted
1091
- ? state.pin
1092
- : undefined);
1093
- if (afterSeq === undefined || afterSeq >= recheck) continue;
1132
+ bound(state) ?? (restricted ? state.pin : undefined);
1133
+ if (
1134
+ afterSeq === undefined ||
1135
+ (afterSeq >= recheck &&
1136
+ !belowTable(index, afterSeq, recheckTables) &&
1137
+ !(
1138
+ restricted &&
1139
+ [...units[index]!.plan.pairs.keys()].some(
1140
+ (table) =>
1141
+ (recheckTables.get(table) ?? -1) >
1142
+ (tableHorizons.get(table) ?? -1),
1143
+ )
1144
+ ))
1145
+ )
1146
+ continue;
1094
1147
  expired = true;
1095
- if (state.kind === 'live') liveReset = true;
1148
+ // A live unit below a table horizon alone takes a unit reset.
1149
+ if (state.kind === 'live' && afterSeq < recheck) liveReset = true;
1096
1150
  else states[index] = { kind: 'reset' };
1097
1151
  }
1098
1152
  if (!expired) break;
@@ -1303,23 +1357,17 @@ async function* streamResponse(
1303
1357
  }
1304
1358
  }
1305
1359
  } catch (error) {
1306
- if (error instanceof SyncError) {
1360
+ // §5.4: a segment store failure leaves the build, so `createStreamCore`
1361
+ // answers it as HTTP 503 before the header frames and in-band after them.
1362
+ if (
1363
+ error instanceof SyncError &&
1364
+ error.code !== 'sync.segment_store_unavailable'
1365
+ ) {
1307
1366
  report.outcome = 'error';
1308
1367
  report.errorCode = error.code;
1309
1368
  await recordRoundFailure(ctx, clientId, error);
1310
1369
  // §1.6: in-band ERROR, then END, nothing else.
1311
- yield encodeResponseFrame(
1312
- {
1313
- type: 'ERROR',
1314
- code: error.code,
1315
- message: error.message,
1316
- category: error.category,
1317
- retryable: error.retryable,
1318
- recommendedAction: error.recommendedAction,
1319
- ...(error.details !== undefined ? { details: error.details } : {}),
1320
- },
1321
- plan.wireVersion,
1322
- );
1370
+ yield errorFrameBytes(error, plan.wireVersion);
1323
1371
  yield END_FRAME_BYTES;
1324
1372
  return;
1325
1373
  }
@@ -1328,15 +1376,43 @@ async function* streamResponse(
1328
1376
  yield END_FRAME_BYTES;
1329
1377
  }
1330
1378
 
1379
+ /** The in-band `ERROR` frame of a failure (§1.6). */
1380
+ function errorFrameBytes(error: SyncError, wireVersion: number): Uint8Array {
1381
+ return encodeResponseFrame(
1382
+ {
1383
+ type: 'ERROR',
1384
+ code: error.code,
1385
+ message: error.message,
1386
+ category: error.category,
1387
+ retryable: error.retryable,
1388
+ recommendedAction: error.recommendedAction,
1389
+ ...(error.details !== undefined ? { details: error.details } : {}),
1390
+ },
1391
+ wireVersion,
1392
+ );
1393
+ }
1394
+
1395
+ /**
1396
+ * A response's chunks, one per encoded frame (§1.4). `live` turns true when
1397
+ * the response goes out before it is built: its header frames and
1398
+ * `KEEPALIVE` frames (§1.6) precede the rest, and a later failure is an
1399
+ * in-band `ERROR`.
1400
+ */
1401
+ export interface SyncResponseStream extends AsyncIterable<Uint8Array> {
1402
+ readonly live: boolean;
1403
+ }
1404
+
1331
1405
  /**
1332
1406
  * Decode and validate the request, then return the streaming response.
1333
1407
  * Request-validation failures (§1.7) throw `SyncError` here, before any
1334
- * bytes are produced; adapters map them to HTTP JSON errors (§1.1).
1408
+ * bytes are produced; adapters map them to HTTP JSON errors (§1.1). The
1409
+ * first chunk arrives once the response is built or its first `KEEPALIVE`
1410
+ * is due (§1.6); a failure before then throws from it, before any byte.
1335
1411
  */
1336
1412
  export async function createSyncResponseStream(
1337
1413
  bytes: Uint8Array,
1338
1414
  ctx: SyncRequestContext,
1339
- ): Promise<AsyncIterable<Uint8Array>> {
1415
+ ): Promise<SyncResponseStream> {
1340
1416
  const sink = ctx.events;
1341
1417
  if (sink === undefined) {
1342
1418
  try {
@@ -1448,7 +1524,7 @@ async function createStreamCore(
1448
1524
  ctx: SyncRequestContext,
1449
1525
  events: SyncularServerEvents | undefined,
1450
1526
  startedAtMs = 0,
1451
- ): Promise<AsyncIterable<Uint8Array>> {
1527
+ ): Promise<SyncResponseStream> {
1452
1528
  let request: RequestMessage;
1453
1529
  try {
1454
1530
  request = decodeRequest(bytes);
@@ -1507,19 +1583,23 @@ async function createStreamCore(
1507
1583
  );
1508
1584
  }
1509
1585
  const report: RequestReport = { outcome: 'ok' };
1510
- // RFC 0007 read-verify-refuse for the streamed read path: the generator
1511
- // builds the whole response (its data reads) before yielding the first
1512
- // frame, then compares the gate token across those reads. Nothing escapes
1513
- // before the verification, and the buffered size is bounded by the request's
1514
- // pull limits and the server's inline segment cap. Every request verifies,
1586
+ // RFC 0007 read-verify-refuse for the streamed read path: the build
1587
+ // buffers the whole response (its data reads), then compares the gate token
1588
+ // across those reads. No push result or section escapes before the
1589
+ // verification, and the buffered size is bounded by the request's pull
1590
+ // limits and the server's inline segment cap. Every request verifies,
1515
1591
  // including a mixed push+pull: a write that already applied when the token
1516
1592
  // changed is durable and replayable under the same commit id / idempotency
1517
1593
  // key, so refusing the request is safe (proved in the serve-gate lane).
1518
- const verified = async function* (): AsyncGenerator<Uint8Array> {
1519
- const buffered: Uint8Array[] = [];
1594
+ const buffered: Uint8Array[] = [];
1595
+ // Set at the build deadline: a build that finishes later stops before the
1596
+ // serve-gate read and persists nothing.
1597
+ let abandoned = false;
1598
+ const built = (async () => {
1520
1599
  for await (const chunk of streamResponse(plan, ctx, served, report)) {
1521
1600
  buffered.push(chunk);
1522
1601
  }
1602
+ if (abandoned) return false;
1523
1603
  const after = await ctx.storage.readServeGate(
1524
1604
  ctx.partition,
1525
1605
  schema.version,
@@ -1533,24 +1613,138 @@ async function createStreamCore(
1533
1613
  if (afterRefusal !== undefined) throw serveNotReadyError(afterRefusal);
1534
1614
  if (changed !== undefined) throw serveNotReadyError(changed);
1535
1615
  await report.persist?.();
1536
- for (const chunk of buffered) yield chunk;
1616
+ return true;
1617
+ })();
1618
+ // The races below observe the failure; this keeps an abandoned stream's
1619
+ // failure from surfacing as unhandled.
1620
+ built.catch(() => {});
1621
+ // §1.6 header frames: the envelope header, RESP_HEADER and its LEASE.
1622
+ const headerChunks = plan.leaseToEmit !== undefined ? 3 : 2;
1623
+ let live = false;
1624
+ const verified = async function* (): AsyncGenerator<Uint8Array> {
1625
+ let timer: ReturnType<typeof setTimeout> | undefined;
1626
+ let elapsedMs = 0;
1627
+ try {
1628
+ for (;;) {
1629
+ let done: boolean;
1630
+ try {
1631
+ done = await Promise.race([
1632
+ built,
1633
+ // §1.6: every 10 s, a third of the native client's 30 s socket
1634
+ // round silence bound (§1.8).
1635
+ new Promise<false>((resolve) => {
1636
+ timer = setTimeout(() => resolve(false), 10_000);
1637
+ }),
1638
+ ]);
1639
+ if (!done) elapsedMs += 10_000;
1640
+ // §1.6 build deadline: 300 s, 2.5 times the slowest cold build
1641
+ // measured in production (119.7 s for 67 segment puts). A build
1642
+ // still running then is abandoned like a failed put.
1643
+ if (elapsedMs === 300_000) {
1644
+ abandoned = true;
1645
+ throw syncError(
1646
+ 'sync.segment_build_timeout',
1647
+ 'the response was not built within the server build deadline',
1648
+ );
1649
+ }
1650
+ } catch (thrown) {
1651
+ if (!live) throw thrown;
1652
+ // The header frames went out: the failure is in-band (§1.6), and
1653
+ // one outside the catalog answers `sync.internal_error`.
1654
+ const error = adapterSyncError(
1655
+ thrown,
1656
+ ctx.onError,
1657
+ 'sync',
1658
+ ctx.mapError,
1659
+ );
1660
+ report.outcome = 'error';
1661
+ report.errorCode = error.code;
1662
+ await recordRoundFailure(ctx, plan.header.clientId, error);
1663
+ yield errorFrameBytes(error, plan.wireVersion);
1664
+ yield END_FRAME_BYTES;
1665
+ return;
1666
+ }
1667
+ clearTimeout(timer);
1668
+ if (done) break;
1669
+ if (!live && buffered.length < headerChunks) continue;
1670
+ if (!live) {
1671
+ live = true;
1672
+ yield* buffered.slice(0, headerChunks);
1673
+ }
1674
+ yield encodeResponseFrame({ type: 'KEEPALIVE' }, plan.wireVersion);
1675
+ }
1676
+ } finally {
1677
+ clearTimeout(timer);
1678
+ }
1679
+ yield* buffered.slice(live ? headerChunks : 0);
1537
1680
  };
1538
- if (events === undefined) {
1539
- return recordedStream(verified(), ctx, plan.header.clientId);
1540
- }
1541
- return recordedStream(
1542
- instrumentedStream(
1543
- verified(),
1544
- ctx,
1545
- events,
1546
- plan,
1547
- report,
1548
- bytes.length,
1549
- startedAtMs,
1550
- ),
1681
+ const stream = recordedStream(
1682
+ events === undefined
1683
+ ? verified()
1684
+ : instrumentedStream(
1685
+ verified(),
1686
+ ctx,
1687
+ events,
1688
+ plan,
1689
+ report,
1690
+ bytes.length,
1691
+ startedAtMs,
1692
+ ),
1551
1693
  ctx,
1552
1694
  plan.header.clientId,
1553
1695
  );
1696
+ return {
1697
+ get live() {
1698
+ return live;
1699
+ },
1700
+ [Symbol.asyncIterator]: () => stream,
1701
+ };
1702
+ }
1703
+
1704
+ /**
1705
+ * The `POST /sync` body (§1.1): the whole response when it was built before
1706
+ * its first `KEEPALIVE` was due, otherwise a stream that carries the header
1707
+ * frames and `KEEPALIVE` frames first (§1.6). Adapters compress only the
1708
+ * whole response (§1.3).
1709
+ */
1710
+ export async function createSyncResponseBody(
1711
+ bytes: Uint8Array,
1712
+ ctx: SyncRequestContext,
1713
+ ): Promise<Uint8Array | ReadableStream<Uint8Array>> {
1714
+ const stream = await createSyncResponseStream(bytes, ctx);
1715
+ const iterator = stream[Symbol.asyncIterator]();
1716
+ const first = await iterator.next();
1717
+ if (stream.live) {
1718
+ return new ReadableStream<Uint8Array>({
1719
+ start(controller) {
1720
+ if (!first.done) controller.enqueue(first.value);
1721
+ },
1722
+ async pull(controller) {
1723
+ const step = await iterator.next();
1724
+ if (step.done) controller.close();
1725
+ else controller.enqueue(step.value);
1726
+ },
1727
+ async cancel() {
1728
+ await iterator.return?.();
1729
+ },
1730
+ });
1731
+ }
1732
+ const chunks: Uint8Array[] = first.done ? [] : [first.value];
1733
+ for (
1734
+ let step = await iterator.next();
1735
+ !step.done;
1736
+ step = await iterator.next()
1737
+ )
1738
+ chunks.push(step.value);
1739
+ const out = new Uint8Array(
1740
+ chunks.reduce((sum, chunk) => sum + chunk.length, 0),
1741
+ );
1742
+ let offset = 0;
1743
+ for (const chunk of chunks) {
1744
+ out.set(chunk, offset);
1745
+ offset += chunk.length;
1746
+ }
1747
+ return out;
1554
1748
  }
1555
1749
 
1556
1750
  /** Counts response bytes and emits `request.handled` when the stream
@@ -1605,18 +1799,8 @@ export async function handleSyncRequest(
1605
1799
  bytes: Uint8Array,
1606
1800
  ctx: SyncRequestContext,
1607
1801
  ): Promise<Uint8Array> {
1608
- const stream = await createSyncResponseStream(bytes, ctx);
1609
- const chunks: Uint8Array[] = [];
1610
- let total = 0;
1611
- for await (const chunk of stream) {
1612
- chunks.push(chunk);
1613
- total += chunk.length;
1614
- }
1615
- const out = new Uint8Array(total);
1616
- let offset = 0;
1617
- for (const chunk of chunks) {
1618
- out.set(chunk, offset);
1619
- offset += chunk.length;
1620
- }
1621
- return out;
1802
+ const body = await createSyncResponseBody(bytes, ctx);
1803
+ return body instanceof Uint8Array
1804
+ ? body
1805
+ : new Uint8Array(await new Response(body).arrayBuffer());
1622
1806
  }
@@ -1,6 +1,10 @@
1
1
  import { validateCommitPruneQuery } from './prune';
2
2
  import { StorageQueryError } from './storage-errors';
3
- import type { CommitPruneQuery, CommitPruneResult } from './storage';
3
+ import type {
4
+ CommitPruneQuery,
5
+ CommitPruneResult,
6
+ PartitionHorizons,
7
+ } from './storage';
4
8
  import {
5
9
  asJson,
6
10
  CLIENT_RECORD_COLUMNS,
@@ -222,6 +226,11 @@ CREATE TABLE IF NOT EXISTS sync_partition_registry(
222
226
  epoch_required BOOLEAN NOT NULL DEFAULT FALSE,
223
227
  last_authenticated_at_ms BIGINT NOT NULL
224
228
  );
229
+ CREATE TABLE IF NOT EXISTS sync_table_horizons(
230
+ partition TEXT NOT NULL, tbl TEXT NOT NULL,
231
+ commit_seq BIGINT NOT NULL,
232
+ PRIMARY KEY(partition, tbl)
233
+ );
225
234
  CREATE TABLE IF NOT EXISTS sync_row_scopes(
226
235
  partition TEXT NOT NULL, tbl TEXT NOT NULL,
227
236
  var TEXT NOT NULL, value TEXT NOT NULL, row_id TEXT NOT NULL,
@@ -2794,6 +2803,20 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
2794
2803
  return rows[0] === undefined ? 0 : asNumber(rows[0].horizon_seq);
2795
2804
  }
2796
2805
 
2806
+ async getHorizons(partition: string): Promise<PartitionHorizons> {
2807
+ const { rows } = await this.#exec.query<{ seq: unknown; tables: string }>(
2808
+ `SELECT coalesce((SELECT horizon_seq FROM sync_partitions WHERE partition=$1),0) AS seq,
2809
+ coalesce((SELECT jsonb_object_agg(tbl, commit_seq) FROM sync_table_horizons WHERE partition=$1),'{}')::text AS tables`,
2810
+ [partition],
2811
+ );
2812
+ return {
2813
+ seq: asNumber(rows[0]?.seq),
2814
+ tables: new Map(
2815
+ Object.entries(asJson<Record<string, number>>(rows[0]?.tables ?? '{}')),
2816
+ ),
2817
+ };
2818
+ }
2819
+
2797
2820
  async setHorizonSeq(partition: string, seq: number): Promise<void> {
2798
2821
  await this.#exec.query(
2799
2822
  `INSERT INTO sync_partitions(partition, horizon_seq) VALUES ($1,$2)
@@ -2845,11 +2868,14 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
2845
2868
  'DELETE FROM sync_changes WHERE partition=$1 AND commit_seq<=$2',
2846
2869
  [partition, horizonSeq],
2847
2870
  );
2848
- // §4.6: tombstones prune in the same pass as the commit log.
2849
- await client.query(
2850
- 'DELETE FROM sync_tombstones WHERE partition=$1 AND commit_seq<=$2',
2851
- [partition, horizonSeq],
2852
- );
2871
+ // §4.6: tombstones and table horizons prune in the same pass as the
2872
+ // commit log.
2873
+ for (const table of ['sync_tombstones', 'sync_table_horizons']) {
2874
+ await client.query(
2875
+ `DELETE FROM ${table} WHERE partition=$1 AND commit_seq<=$2`,
2876
+ [partition, horizonSeq],
2877
+ );
2878
+ }
2853
2879
  return {
2854
2880
  previousHorizonSeq,
2855
2881
  horizonSeq,