@syncular/server 0.59.0 → 0.60.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/README.md +20 -16
  2. package/dist/admin.d.ts +32 -20
  3. package/dist/admin.js +42 -40
  4. package/dist/context.d.ts +6 -8
  5. package/dist/context.js +1 -3
  6. package/dist/d1-storage.js +42 -24
  7. package/dist/errors.js +8 -0
  8. package/dist/events.d.ts +35 -17
  9. package/dist/frame-bytes.js +24 -6
  10. package/dist/handler.js +486 -223
  11. package/dist/local-authority.js +7 -9
  12. package/dist/operations.d.ts +5 -2
  13. package/dist/operations.js +20 -29
  14. package/dist/postgres-storage.d.ts +24 -19
  15. package/dist/postgres-storage.js +249 -168
  16. package/dist/pull.d.ts +68 -53
  17. package/dist/pull.js +258 -287
  18. package/dist/push.d.ts +12 -3
  19. package/dist/push.js +95 -32
  20. package/dist/realtime.d.ts +17 -11
  21. package/dist/realtime.js +65 -90
  22. package/dist/relational-rows.d.ts +0 -57
  23. package/dist/relational-rows.js +0 -97
  24. package/dist/seed.js +10 -15
  25. package/dist/segment-download.d.ts +4 -1
  26. package/dist/segment-download.js +4 -2
  27. package/dist/sqlite-dialect.d.ts +36 -59
  28. package/dist/sqlite-dialect.js +255 -159
  29. package/dist/sqlite-storage.js +24 -20
  30. package/dist/storage-query.d.ts +30 -1
  31. package/dist/storage-query.js +68 -0
  32. package/dist/storage.d.ts +156 -54
  33. package/dist/storage.js +15 -0
  34. package/package.json +2 -2
  35. package/src/admin.ts +87 -59
  36. package/src/context.ts +7 -11
  37. package/src/d1-storage.ts +49 -33
  38. package/src/errors.ts +8 -0
  39. package/src/events.ts +36 -17
  40. package/src/frame-bytes.ts +26 -6
  41. package/src/handler.ts +628 -286
  42. package/src/local-authority.ts +10 -11
  43. package/src/operations.ts +44 -42
  44. package/src/postgres-storage.ts +336 -226
  45. package/src/pull.ts +394 -414
  46. package/src/push.ts +116 -21
  47. package/src/realtime.ts +85 -129
  48. package/src/relational-rows.ts +0 -105
  49. package/src/seed.ts +13 -19
  50. package/src/segment-download.ts +8 -3
  51. package/src/sqlite-dialect.ts +363 -204
  52. package/src/sqlite-storage.ts +45 -25
  53. package/src/storage-query.ts +102 -0
  54. package/src/storage.ts +185 -58
package/src/admin.ts CHANGED
@@ -19,6 +19,8 @@ import {
19
19
  type PushOperationResult,
20
20
  renderRowValues,
21
21
  type ScopeMap,
22
+ unitId,
23
+ type UnitScopes,
22
24
  } from '@syncular/core';
23
25
  import type { BlobStore, BlobStoreStats } from './blob-store';
24
26
  import {
@@ -52,6 +54,7 @@ import type {
52
54
  CheckpointDeclaration,
53
55
  ClientRecord,
54
56
  CommitMetadata,
57
+ PushWatermark,
55
58
  ReactionStatus,
56
59
  StoredPushResult,
57
60
  StoredRow,
@@ -61,13 +64,22 @@ import type {
61
64
  StoredReaction,
62
65
  } from './storage';
63
66
 
67
+ /** One registered unit of a client record. */
68
+ export interface AdminClientUnit {
69
+ /** The unit id (§4.1). */
70
+ readonly id: string;
71
+ readonly scopes: UnitScopes;
72
+ /** The unit's cursor while it catches up; absent for a live unit. */
73
+ readonly catchingUpCursor?: number;
74
+ }
75
+
64
76
  /** A connected/known client as the console sees it (§4.5, §8.1). */
65
77
  export interface AdminClient {
66
78
  readonly clientId: string;
67
79
  readonly actorId: string;
68
80
  /** SSP2 wire version last accepted from this client. */
69
81
  readonly wireVersion: number;
70
- /** Highest codec schema version among its subscriptions, when recorded. */
82
+ /** Schema version of its latest completed round, when recorded. */
71
83
  readonly schemaVersion?: number;
72
84
  readonly cursor: number;
73
85
  /**
@@ -76,11 +88,8 @@ export interface AdminClient {
76
88
  */
77
89
  readonly lag: number;
78
90
  readonly updatedAtMs: number;
79
- readonly subscriptions: readonly {
80
- readonly id: string;
81
- readonly table: string;
82
- readonly scopes: Record<string, readonly string[]>;
83
- }[];
91
+ /** The stored registration list (§4.1); empty when none is stored. */
92
+ readonly units: readonly AdminClientUnit[];
84
93
  /** True when the cursor record was touched within the active window. */
85
94
  readonly active: boolean;
86
95
  /**
@@ -265,11 +274,11 @@ export interface AdminCommitDetail extends CommitMetadata {
265
274
  * The trace verdict, evaluated in this order:
266
275
  * `client-unknown`: no client record for the id.
267
276
  * `row-missing`: no row and no retained change to it.
268
- * `no-subscription`: the client subscribes to nothing on the table.
277
+ * `no-unit`: no registered unit of the client covers the table.
269
278
  * `row-deleted`: the row's newest retained change is a delete the client
270
279
  * has pulled.
271
- * `scope-mismatch`: no subscription requests the row's stored scopes.
272
- * `scope-denied`: a subscription requests them, but the actor's allowed
280
+ * `scope-mismatch`: no covering unit requests the row's stored scopes.
281
+ * `scope-denied`: a covering unit requests them, but the actor's allowed
273
282
  * scopes (the host `resolveScopes`) exclude the row.
274
283
  * `pending-pull`: the client's cursor is below the row's newest change.
275
284
  * `delivered`: the client has pulled the row's newest change.
@@ -277,24 +286,24 @@ export interface AdminCommitDetail extends CommitMetadata {
277
286
  export type AdminTraceVerdict =
278
287
  | 'client-unknown'
279
288
  | 'row-missing'
280
- | 'no-subscription'
289
+ | 'no-unit'
281
290
  | 'row-deleted'
282
291
  | 'scope-mismatch'
283
292
  | 'scope-denied'
284
293
  | 'pending-pull'
285
294
  | 'delivered';
286
295
 
287
- /** One of the client's subscriptions on the traced table. */
288
- export interface AdminTraceSubscription {
296
+ /** One of the client's units covering the traced table. */
297
+ export interface AdminTraceUnit {
289
298
  readonly id: string;
290
- readonly scopes: Record<string, readonly string[]>;
291
- /** The row's stored scopes satisfy the requested scopes. */
299
+ readonly scopes: UnitScopes;
300
+ /** The row's stored scopes carry the unit's values. */
292
301
  readonly requestedMatch: boolean;
293
- /** Each requested variable the row's stored value fails. */
302
+ /** Each unit variable the row's stored value fails. */
294
303
  readonly mismatches: readonly {
295
304
  readonly variable: string;
296
305
  readonly rowValue: string | null;
297
- readonly requested: readonly string[];
306
+ readonly requested: string;
298
307
  }[];
299
308
  /**
300
309
  * The row also satisfies requested ∩ allowed (§3.2). Absent when the
@@ -320,7 +329,7 @@ export interface AdminRowTrace {
320
329
  * when it returned the §7.3.3 outage sentinel. Absent without a resolver.
321
330
  */
322
331
  readonly allowed?: ScopeMap | 'outage';
323
- readonly subscriptions: readonly AdminTraceSubscription[];
332
+ readonly units: readonly AdminTraceUnit[];
324
333
  /** The client's retained push rejections and conflicts, newest first. */
325
334
  readonly rejections: readonly SyncularServerEvent[];
326
335
  }
@@ -392,6 +401,13 @@ export interface AdminWriteLookup {
392
401
  readonly clientCommitId: string;
393
402
  /** One outcome per client that sent this clientCommitId. */
394
403
  readonly outcomes: readonly AdminWriteOutcome[];
404
+ /**
405
+ * With no stored outcome: the §6.3.2 watermarks of the clients whose
406
+ * retained commit log carries this clientCommitId. A commit below an
407
+ * outbox's `throughSeq` lost its result to the watermark ("expired below
408
+ * sequence N").
409
+ */
410
+ readonly watermarks: readonly PushWatermark[];
395
411
  /** Retained ring events carrying this clientCommitId, newest first. */
396
412
  readonly events: readonly SyncularServerEvent[];
397
413
  }
@@ -525,7 +541,7 @@ function required<T>(value: T | undefined, what: string): T {
525
541
  const VERDICT_ORDER: readonly AdminTraceVerdict[] = [
526
542
  'client-unknown',
527
543
  'row-missing',
528
- 'no-subscription',
544
+ 'no-unit',
529
545
  'row-deleted',
530
546
  'scope-mismatch',
531
547
  'scope-denied',
@@ -533,7 +549,7 @@ const VERDICT_ORDER: readonly AdminTraceVerdict[] = [
533
549
  'delivered',
534
550
  ];
535
551
 
536
- /** One known client's subscriptions on the row's table and its verdict. */
552
+ /** One known client's units covering the row's table and its verdict. */
537
553
  function traceVerdict(
538
554
  declared: ReadonlySet<string>,
539
555
  record: ClientRecord,
@@ -542,34 +558,40 @@ function traceVerdict(
542
558
  horizonSeq: number,
543
559
  allowed: ScopeMap | 'outage' | undefined,
544
560
  ): {
545
- readonly subscriptions: AdminTraceSubscription[];
561
+ readonly units: AdminTraceUnit[];
546
562
  readonly verdict: AdminTraceVerdict;
547
563
  } {
548
564
  const rowScopes = row.scopes ?? {};
549
- const subscriptions = record.subscriptions
550
- .filter((sub) => sub.table === row.table)
551
- .map((sub): AdminTraceSubscription => {
552
- const mismatches = Object.entries(sub.scopes)
553
- .filter(([variable, values]) => {
554
- const value = rowScopes[variable];
555
- return value === undefined || !values.includes(value);
556
- })
557
- .map(([variable, values]) => ({
565
+ // §4.1: a unit covers every table that declares all of its variables.
566
+ const units = (record.registration?.units ?? [])
567
+ .filter((scopes) =>
568
+ Object.keys(scopes).every((variable) => declared.has(variable)),
569
+ )
570
+ .map((scopes): AdminTraceUnit => {
571
+ const mismatches = Object.entries(scopes)
572
+ .filter(([variable, value]) => rowScopes[variable] !== value)
573
+ .map(([variable, value]) => ({
558
574
  variable,
559
575
  rowValue: rowScopes[variable] ?? null,
560
- requested: values,
576
+ requested: value,
561
577
  }));
562
578
  const traced = {
563
- id: sub.id,
564
- scopes: sub.scopes,
579
+ id: unitId(scopes),
580
+ scopes,
565
581
  requestedMatch: mismatches.length === 0,
566
582
  mismatches,
567
583
  };
568
584
  if (allowed === undefined || allowed === 'outage') return traced;
569
- const effective = computeEffective(declared, sub.scopes, {
570
- ok: true,
571
- allowed,
572
- });
585
+ const effective = computeEffective(
586
+ declared,
587
+ Object.fromEntries(
588
+ Object.entries(scopes).map(([variable, value]) => [
589
+ variable,
590
+ [value],
591
+ ]),
592
+ ),
593
+ { ok: true, allowed },
594
+ );
573
595
  return {
574
596
  ...traced,
575
597
  effectiveMatch:
@@ -583,20 +605,20 @@ function traceVerdict(
583
605
  const verdict: AdminTraceVerdict =
584
606
  !row.exists && lastChange === undefined
585
607
  ? 'row-missing'
586
- : subscriptions.length === 0
587
- ? 'no-subscription'
608
+ : units.length === 0
609
+ ? 'no-unit'
588
610
  : !row.exists
589
611
  ? pulled
590
612
  ? 'row-deleted'
591
613
  : 'pending-pull'
592
- : !subscriptions.some((sub) => sub.requestedMatch)
614
+ : !units.some((unit) => unit.requestedMatch)
593
615
  ? 'scope-mismatch'
594
- : subscriptions.every((sub) => sub.effectiveMatch === false)
616
+ : units.every((unit) => unit.effectiveMatch === false)
595
617
  ? 'scope-denied'
596
618
  : pulled
597
619
  ? 'delivered'
598
620
  : 'pending-pull';
599
- return { subscriptions, verdict };
621
+ return { units, verdict };
600
622
  }
601
623
 
602
624
  /** The console query surface plus its operator actions. One per process. */
@@ -870,7 +892,7 @@ export class SyncularAdmin {
870
892
 
871
893
  /**
872
894
  * Trace whether one client receives one row, combining the row's stored
873
- * scopes and newest change with the client's subscriptions, cursor, the
895
+ * scopes and newest change with the client's units, cursor, the
874
896
  * actor's allowed scopes, and its retained push failures. See
875
897
  * `AdminTraceVerdict` for the verdict order.
876
898
  */
@@ -903,7 +925,7 @@ export class SyncularAdmin {
903
925
  ...(lastChange !== undefined ? { lastChange } : {}),
904
926
  };
905
927
  if (record === undefined) {
906
- return { ...base, verdict: 'client-unknown', subscriptions: [] };
928
+ return { ...base, verdict: 'client-unknown', units: [] };
907
929
  }
908
930
  const allowed = await this.#allowedScopes(partition, record);
909
931
  return {
@@ -1134,7 +1156,14 @@ export class SyncularAdmin {
1134
1156
  (event) =>
1135
1157
  'clientCommitId' in event && event.clientCommitId === clientCommitId,
1136
1158
  );
1137
- return { clientCommitId, outcomes, events };
1159
+ const watermarks =
1160
+ outcomes.length > 0
1161
+ ? []
1162
+ : await required(
1163
+ this.#storage.listPushWatermarks?.bind(this.#storage),
1164
+ 'storage',
1165
+ )(partition, clientCommitId);
1166
+ return { clientCommitId, outcomes, watermarks, events };
1138
1167
  }
1139
1168
 
1140
1169
  /**
@@ -1624,30 +1653,29 @@ export class SyncularAdmin {
1624
1653
  }
1625
1654
  }
1626
1655
  return (record) => {
1627
- let schemaVersion: number | undefined;
1628
- for (const sub of record.subscriptions) {
1629
- if (
1630
- sub.schemaVersion !== undefined &&
1631
- (schemaVersion === undefined || sub.schemaVersion > schemaVersion)
1632
- ) {
1633
- schemaVersion = sub.schemaVersion;
1634
- }
1635
- }
1636
1656
  const connectedSinceMs = connectedSince.get(record.clientId);
1637
1657
  return {
1638
1658
  clientId: record.clientId,
1639
1659
  actorId: record.actorId,
1640
1660
  wireVersion: record.wireVersion,
1641
- ...(schemaVersion !== undefined ? { schemaVersion } : {}),
1661
+ ...(record.schemaVersion !== undefined
1662
+ ? { schemaVersion: record.schemaVersion }
1663
+ : {}),
1642
1664
  cursor: record.cursor,
1643
1665
  // A never-pulled cursor (-1) has seen nothing: it lags the whole log.
1644
1666
  lag: Math.max(0, maxCommitSeq - Math.max(0, record.cursor)),
1645
1667
  updatedAtMs: record.updatedAtMs,
1646
- subscriptions: record.subscriptions.map((sub) => ({
1647
- id: sub.id,
1648
- table: sub.table,
1649
- scopes: sub.scopes,
1650
- })),
1668
+ units: (record.registration?.units ?? []).map((scopes) => {
1669
+ const id = unitId(scopes);
1670
+ const catchingUp = record.catchingUp.find((unit) => unit.id === id);
1671
+ return {
1672
+ id,
1673
+ scopes,
1674
+ ...(catchingUp !== undefined
1675
+ ? { catchingUpCursor: catchingUp.cursor }
1676
+ : {}),
1677
+ };
1678
+ }),
1651
1679
  active: record.updatedAtMs >= activeFloorMs,
1652
1680
  ...(sockets !== undefined
1653
1681
  ? { connected: connectedSinceMs !== undefined }
package/src/context.ts CHANGED
@@ -58,7 +58,7 @@ export type ResolverOutage = typeof RESOLVER_OUTAGE;
58
58
  /**
59
59
  * Host callback: actor → allowed scopes (§3.2 step 3). Resolved at most
60
60
  * once per request and memoized (§3.4 step 1). `'*'` means "any value for
61
- * this variable". A throwing resolver revokes subscriptions and rejects
61
+ * this variable". A throwing resolver revokes units and rejects
62
62
  * writes with `sync.forbidden` — fail loud, never leak. Returning
63
63
  * `RESOLVER_OUTAGE` instead opts the request into lease authorization
64
64
  * (§7.3.3) — a deliberate outage signal, not a failure.
@@ -87,10 +87,8 @@ export interface ServerLimits {
87
87
  * decoding, and a socket round's assembled request (§8.7).
88
88
  */
89
89
  readonly maxRequestBytes: number;
90
- /** §1.7 cap on `SUBSCRIPTION` frames per request. */
90
+ /** §1.7 cap on `UNIT` frames per request: the units of a traveling list. */
91
91
  readonly maxSubscriptionsPerRequest: number;
92
- /** §1.7 cap on requested scope values, summed over a subscription's keys. */
93
- readonly maxScopeValuesPerSubscription: number;
94
92
  /** Cap on concurrent registered-query watches per watch session. */
95
93
  readonly maxWatchesPerSession: number;
96
94
  }
@@ -101,10 +99,8 @@ export const DEFAULT_LIMITS: ServerLimits = {
101
99
  maxCascadeOperationsPerCommit: 1_000,
102
100
  // The size above which adapters already refuse to buffer a segment body.
103
101
  maxRequestBytes: 16 * 1024 * 1024,
104
- // One subscription per pushable operation: the existing per-request cap.
102
+ // One unit per pushable operation: the existing per-request cap.
105
103
  maxSubscriptionsPerRequest: 500,
106
- // One default snapshot page (§4.2): a longer list wants a coarser scope.
107
- maxScopeValuesPerSubscription: 1_000,
108
104
  // Every commit reruns a session's affected watches one after another.
109
105
  maxWatchesPerSession: 100,
110
106
  };
@@ -113,10 +109,10 @@ export const DEFAULT_LIMITS: ServerLimits = {
113
109
  export interface RealtimeNotifier {
114
110
  notifyCommit(partition: string, commit: StoredCommit): void | Promise<void>;
115
111
  /**
116
- * A pull changed the client's persisted subscription list (§8.7). A hub
117
- * re-registers that client's live sockets, so a client that runs its
118
- * rounds over HTTP while a socket is open receives deltas for the new
119
- * subscriptions.
112
+ * A pull changed the client's registration list or live set (§8.1). A
113
+ * hub re-registers that client's live sockets, so a client that runs its
114
+ * rounds over HTTP while a socket is open receives deltas for the units
115
+ * that joined.
120
116
  */
121
117
  notifySubscriptions?(
122
118
  partition: string,
package/src/d1-storage.ts CHANGED
@@ -78,9 +78,10 @@ import {
78
78
  type SqliteStatementSpec,
79
79
  SqliteFamilyStorage,
80
80
  SqliteFamilyTransaction,
81
- SQLITE_ADDED_COLUMNS,
82
81
  type SqliteTombstoneRecord,
83
82
  sqliteDdlStatements,
83
+ SQLITE_UPGRADE_PROBE,
84
+ sqliteUpgradeDdl,
84
85
  toStoredRow,
85
86
  toTombstone,
86
87
  } from './sqlite-dialect';
@@ -93,6 +94,8 @@ import type {
93
94
  CommitPruneResult,
94
95
  IndexRowScanQuery,
95
96
  NewCommit,
97
+ OutboxSequence,
98
+ OutboxSettlement,
96
99
  PreparedPushApply,
97
100
  RowScanQuery,
98
101
  ScopeValueHistoryQuery,
@@ -104,6 +107,7 @@ import type {
104
107
  StoredRow,
105
108
  Tombstone,
106
109
  } from './storage';
110
+ import { commitDeleteFloor } from './storage';
107
111
  import {
108
112
  isD1ConstraintError,
109
113
  StorageConstraintError,
@@ -528,6 +532,7 @@ class D1Transaction
528
532
  readonly table: string;
529
533
  readonly rowIds: readonly string[];
530
534
  }[] = [],
535
+ outbox?: OutboxSettlement,
531
536
  ): Promise<PreparedPushApply> {
532
537
  this.assertOpen();
533
538
  if (!this.#pushApplySerialized) {
@@ -537,11 +542,33 @@ class D1Transaction
537
542
  }
538
543
  // D1 has no interactive lock. The caller explicitly asserted that every
539
544
  // write for this partition is already serialized (normally by its DO).
545
+ // The settlement's writes sit below the checkpoint, so a rejection keeps
546
+ // them.
547
+ const settled =
548
+ outbox === undefined
549
+ ? undefined
550
+ : await this.settleOutbox(clientId, clientCommitId, outbox);
540
551
  this.#pushApplyCheckpoint = this.#buffer.length;
541
- return {
542
- gate: await this.readServeGate(runningSchemaVersion),
543
- persisted: await this.getPushResult(clientId, clientCommitId),
552
+ // The serve gate and the delete floor (§2.2) in one round trip.
553
+ const read = await this.#db
554
+ .prepare(
555
+ 'SELECT (SELECT schema_version FROM sync_schema_meta WHERE id=1) AS schema_version, (SELECT delete_floor FROM sync_partitions WHERE partition=?) AS delete_floor',
556
+ )
557
+ .bind(this.partition)
558
+ .first<{ schema_version: number | null; delete_floor: number | null }>();
559
+ const gate: ServeGate = {
560
+ storedSchemaVersion: read?.schema_version ?? 0,
561
+ logEpoch: undefined,
562
+ pending: [],
544
563
  };
564
+ const deleteFloor = read?.delete_floor ?? 0;
565
+ return settled === undefined
566
+ ? {
567
+ gate,
568
+ deleteFloor,
569
+ persisted: await this.getPushResult(clientId, clientCommitId),
570
+ }
571
+ : { gate, deleteFloor, ...settled };
545
572
  }
546
573
 
547
574
  advanceCheckpoint(
@@ -573,6 +600,7 @@ class D1Transaction
573
600
  clientId: string,
574
601
  clientCommitId: string,
575
602
  result: StoredPushResult,
603
+ sequence?: OutboxSequence,
576
604
  ): Promise<StoredPushResult> {
577
605
  this.assertOpen();
578
606
  const checkpoint = this.#pushApplyCheckpoint;
@@ -584,7 +612,7 @@ class D1Transaction
584
612
  this.#clearedTombstones.clear();
585
613
  this.#maxCommitSeq = undefined;
586
614
  this.#applicationOpIndexes.clear();
587
- await this.putPushResult(clientId, clientCommitId, result);
615
+ await this.putPushResult(clientId, clientCommitId, result, sequence);
588
616
  await this.commit();
589
617
  // The idempotency re-check found no record under the serialized apply, so
590
618
  // the insert above is the canonical outcome.
@@ -710,8 +738,8 @@ class D1Transaction
710
738
  const commitSeq = this.#maxCommitSeq + 1;
711
739
  this.#maxCommitSeq = commitSeq;
712
740
  this.run(
713
- 'INSERT INTO sync_partitions(partition, max_commit_seq) VALUES (?,?) ON CONFLICT(partition) DO UPDATE SET max_commit_seq=excluded.max_commit_seq',
714
- [p, commitSeq],
741
+ 'INSERT INTO sync_partitions(partition, max_commit_seq, delete_floor) VALUES (?,?,?) ON CONFLICT(partition) DO UPDATE SET max_commit_seq=excluded.max_commit_seq, delete_floor=max(delete_floor, excluded.delete_floor)',
742
+ [p, commitSeq, commitDeleteFloor(commit.changes)],
715
743
  );
716
744
  this.run(
717
745
  'INSERT INTO sync_commits(partition, commit_seq, client_id, client_commit_id, actor_id, created_at_ms) VALUES (?,?,?,?,?,?)',
@@ -794,6 +822,7 @@ const D1_CORE_TABLES = [
794
822
  'sync_changes',
795
823
  'sync_change_scopes',
796
824
  'sync_push_results',
825
+ 'sync_push_watermarks',
797
826
  'sync_reactions',
798
827
  'sync_clients',
799
828
  'sync_blob_refs',
@@ -840,9 +869,9 @@ export class D1ServerStorage
840
869
  protected readonly maxBindParams = 100;
841
870
  /**
842
871
  * Page reads waiting for the end of the current microtask turn. A pull
843
- * starts the window read (or first snapshot page) of every subscription
872
+ * starts the first snapshot page and scope pin of every bootstrapping unit
844
873
  * before awaiting any of them, so they leave as ONE `db.batch` round trip
845
- * instead of one per subscription (SYNCULAR-PULL-ROUNDTRIPS-001).
874
+ * instead of one per unit (SYNCULAR-PULL-ROUNDTRIPS-001).
846
875
  */
847
876
  #pendingReads: PendingReads | undefined;
848
877
 
@@ -924,33 +953,18 @@ export class D1ServerStorage
924
953
  for (const statement of sqliteDdlStatements()) {
925
954
  await this.#db.exec(`${statement.replace(/\s+/g, ' ')};`);
926
955
  }
927
- for (const sql of await this.#addedColumnDdl()) {
956
+ for (const sql of await this.#upgradeDdl()) {
928
957
  await this.#db.exec(`${sql};`);
929
958
  }
930
959
  }
931
960
 
932
- /**
933
- * `ALTER TABLE` statements for the `SQLITE_ADDED_COLUMNS` this database
934
- * lacks, read in one statement.
935
- */
936
- async #addedColumnDdl(): Promise<string[]> {
961
+ /** `sqliteUpgradeDdl` for this database, read in one statement. */
962
+ async #upgradeDdl(): Promise<string[]> {
937
963
  const { results } = await this.#db
938
- .prepare(
939
- 'SELECT t.value AS tbl, c.name AS name FROM json_each(?) AS t, pragma_table_info(t.value) AS c',
940
- )
941
- .bind(
942
- JSON.stringify([
943
- ...new Set(SQLITE_ADDED_COLUMNS.map(([table]) => table)),
944
- ]),
945
- )
964
+ .prepare(SQLITE_UPGRADE_PROBE.sql)
965
+ .bind(...SQLITE_UPGRADE_PROBE.params)
946
966
  .all<{ tbl: string; name: string }>();
947
- return SQLITE_ADDED_COLUMNS.filter(
948
- ([table, name]) =>
949
- !results.some((column) => column.tbl === table && column.name === name),
950
- ).map(
951
- ([table, name, definition]) =>
952
- `ALTER TABLE ${table} ADD COLUMN ${name} ${definition}`,
953
- );
967
+ return sqliteUpgradeDdl(results);
954
968
  }
955
969
 
956
970
  async ensureSchema(
@@ -1070,9 +1084,11 @@ export class D1ServerStorage
1070
1084
  'sqlite',
1071
1085
  );
1072
1086
  }
1073
- // A database from before a core-table column gains it in place.
1087
+ // A database from an earlier release gains the core-table columns and
1088
+ // indexes it lacks in place, and loses the columns wire version 4
1089
+ // deleted.
1074
1090
  statementsExecuted++;
1075
- const added = await this.#addedColumnDdl();
1091
+ const added = await this.#upgradeDdl();
1076
1092
  if (added.length > 0) {
1077
1093
  await batch(added.map((sql) => this.#db.prepare(sql)));
1078
1094
  }
@@ -1211,7 +1227,7 @@ export class D1ServerStorage
1211
1227
  // The column read, the checkpoint pair, and every missing column.
1212
1228
  if (remaining < 4) break;
1213
1229
  statementsExecuted++;
1214
- const added = await this.#addedColumnDdl();
1230
+ const added = await this.#upgradeDdl();
1215
1231
  if (remaining < 3 + added.length) break;
1216
1232
  if (
1217
1233
  !(await save(
package/src/errors.ts CHANGED
@@ -176,6 +176,14 @@ export const ERROR_CATALOG: Readonly<Record<string, ErrorCatalogEntry>> = {
176
176
  recommendedAction: 'retryLater',
177
177
  httpStatus: 500,
178
178
  },
179
+ // §6.3.2: below the outbox watermark with no retained result. A push
180
+ // operation-result error record only, never persisted.
181
+ 'sync.commit_expired': {
182
+ category: 'invalid-request',
183
+ retryable: false,
184
+ recommendedAction: 'resetClientId',
185
+ httpStatus: 400,
186
+ },
179
187
  // §2.4 on a non-D1 backend: the server refuses while a declared backfill
180
188
  // checkpoint is incomplete or the database is newer than the running build.
181
189
  // Formerly host-only; promoted to the §10.2 catalogue so an in-flight
package/src/events.ts CHANGED
@@ -13,7 +13,7 @@
13
13
  * - Timestamps and durations come from the ctx clock where one exists, so
14
14
  * the conformance virtual-clock discipline stays intact.
15
15
  */
16
- import type { ScopeMap, WakeReason } from '@syncular/core';
16
+ import type { FencedRecord, UnitScopes, WakeReason } from '@syncular/core';
17
17
 
18
18
  /** One `POST /sync` request, emitted once per request after the response
19
19
  * bytes are fully produced (or the request was rejected up front). */
@@ -48,7 +48,8 @@ export interface RequestHandledEvent {
48
48
  readonly errorCode?: string;
49
49
  readonly pushCommits: number;
50
50
  readonly pulled: boolean;
51
- readonly subscriptions: number;
51
+ /** Units of the request's effective list (§4.1); 0 without one. */
52
+ readonly units: number;
52
53
  }
53
54
 
54
55
  interface PushEventBase {
@@ -154,7 +155,7 @@ export interface ReactionPruneCompletedEvent {
154
155
  readonly mayHaveMore: boolean;
155
156
  }
156
157
 
157
- /** One emitted segment within a pull subscription section. */
158
+ /** One emitted segment within a unit section. */
158
159
  export interface PullSegmentSummary {
159
160
  readonly mediaType: 'rows' | 'sqlite';
160
161
  readonly delivery: 'inline' | 'ref';
@@ -164,25 +165,40 @@ export interface PullSegmentSummary {
164
165
  readonly rows: number;
165
166
  }
166
167
 
167
- /** One subscription section of a served pull (§1.6). */
168
- export interface PullSubscriptionSummary {
168
+ /** One unit of a served pull's effective list (§4.1). */
169
+ export interface PullUnitSummary {
169
170
  readonly id: string;
170
- readonly table: string;
171
+ readonly scopes: UnitScopes;
171
172
  readonly status: 'active' | 'revoked' | 'reset';
172
- /** `none` for revoked/reset sections (no data half was produced). */
173
- readonly mode: 'bootstrap' | 'incremental' | 'none';
174
- readonly requestedScopes: ScopeMap;
175
- readonly effectiveScopes: ScopeMap;
176
- readonly firstCommitSeq?: number;
177
- readonly lastCommitSeq?: number;
173
+ /**
174
+ * `live` and `catching-up` (a post-bootstrap unit) units take part in the
175
+ * live section; `bootstrap` sections carry segments; `none` for revoked and
176
+ * reset units.
177
+ */
178
+ readonly mode: 'live' | 'catching-up' | 'bootstrap' | 'none';
179
+ /** Whether the response carried a unit section for it (§4.4). */
180
+ readonly section: boolean;
181
+ /** The unit's echo (§4.4). */
182
+ readonly fenced: readonly FencedRecord[];
183
+ /** The cursor the request held for the unit: its `PROGRESS` cursor or R. */
178
184
  readonly fromCursor: number;
179
- readonly nextCursor: number;
185
+ readonly segments: readonly PullSegmentSummary[];
186
+ }
187
+
188
+ /** The live section of a served pull (§4.5). */
189
+ export interface PullLiveSummary {
190
+ readonly status: 'active' | 'reset' | 'unregistered';
191
+ /** The request's replica cursor R. */
192
+ readonly fromCursor: number;
193
+ /** `LIVE_END.throughSeq`. */
194
+ readonly throughSeq: number;
180
195
  readonly commits: number;
181
196
  readonly changes: number;
182
- readonly segments: readonly PullSegmentSummary[];
197
+ readonly firstCommitSeq?: number;
198
+ readonly lastCommitSeq?: number;
183
199
  }
184
200
 
185
- /** The pull half of a request, emitted once after all sections streamed. */
201
+ /** The pull half of a request, emitted once after every section streamed. */
186
202
  export interface PullServedEvent {
187
203
  readonly type: 'pull.served';
188
204
  readonly atMs: number;
@@ -191,7 +207,10 @@ export interface PullServedEvent {
191
207
  readonly clientId: string;
192
208
  readonly acceptedThrough: number;
193
209
  readonly storageMaxCommitSeq: number;
194
- readonly subscriptions: readonly PullSubscriptionSummary[];
210
+ /** The request carried its registration list (§4.1). */
211
+ readonly registrationTraveled: boolean;
212
+ readonly units: readonly PullUnitSummary[];
213
+ readonly live: PullLiveSummary;
195
214
  }
196
215
 
197
216
  /** A direct segment download (§5.5), success or failure. */
@@ -292,7 +311,7 @@ export interface PruneCompletedEvent {
292
311
  }
293
312
 
294
313
  /** The host `resolveScopes` callback threw — the §3.2/§3.4 fail-loud
295
- * path (subscriptions revoke, writes reject, downloads deny). */
314
+ * path (units revoke, writes reject, downloads deny). */
296
315
  export interface ScopesResolveFailedEvent {
297
316
  readonly type: 'scopes.resolve_failed';
298
317
  readonly atMs: number;