@syncular/server 0.59.0 → 0.60.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/README.md +20 -16
  2. package/dist/admin.d.ts +32 -20
  3. package/dist/admin.js +42 -40
  4. package/dist/context.d.ts +6 -8
  5. package/dist/context.js +1 -3
  6. package/dist/d1-storage.js +42 -24
  7. package/dist/errors.js +8 -0
  8. package/dist/events.d.ts +35 -17
  9. package/dist/frame-bytes.js +24 -6
  10. package/dist/handler.js +486 -223
  11. package/dist/local-authority.js +7 -9
  12. package/dist/operations.d.ts +5 -2
  13. package/dist/operations.js +20 -29
  14. package/dist/postgres-storage.d.ts +24 -19
  15. package/dist/postgres-storage.js +249 -168
  16. package/dist/pull.d.ts +68 -53
  17. package/dist/pull.js +258 -287
  18. package/dist/push.d.ts +12 -3
  19. package/dist/push.js +95 -32
  20. package/dist/realtime.d.ts +17 -11
  21. package/dist/realtime.js +65 -90
  22. package/dist/relational-rows.d.ts +0 -57
  23. package/dist/relational-rows.js +4 -100
  24. package/dist/seed.js +10 -15
  25. package/dist/segment-download.d.ts +4 -1
  26. package/dist/segment-download.js +4 -2
  27. package/dist/sqlite-dialect.d.ts +36 -59
  28. package/dist/sqlite-dialect.js +255 -159
  29. package/dist/sqlite-storage.js +24 -20
  30. package/dist/storage-query.d.ts +30 -1
  31. package/dist/storage-query.js +68 -0
  32. package/dist/storage.d.ts +156 -54
  33. package/dist/storage.js +15 -0
  34. package/package.json +2 -2
  35. package/src/admin.ts +87 -59
  36. package/src/context.ts +7 -11
  37. package/src/d1-storage.ts +49 -33
  38. package/src/errors.ts +8 -0
  39. package/src/events.ts +36 -17
  40. package/src/frame-bytes.ts +26 -6
  41. package/src/handler.ts +628 -286
  42. package/src/local-authority.ts +10 -11
  43. package/src/operations.ts +44 -42
  44. package/src/postgres-storage.ts +336 -226
  45. package/src/pull.ts +394 -414
  46. package/src/push.ts +116 -21
  47. package/src/realtime.ts +85 -129
  48. package/src/relational-rows.ts +4 -107
  49. package/src/seed.ts +13 -19
  50. package/src/segment-download.ts +8 -3
  51. package/src/sqlite-dialect.ts +363 -204
  52. package/src/sqlite-storage.ts +45 -25
  53. package/src/storage-query.ts +102 -0
  54. package/src/storage.ts +185 -58
package/README.md CHANGED
@@ -408,7 +408,7 @@ must enumerate every row in one Workspace. Syncular does not currently expose
408
408
  trusted prefix or range scans.
409
409
 
410
410
  This is a trusted `@syncular/server` storage capability, not SSP2: it creates no
411
- scope variable, named-query obligation, subscription descriptor, or client
411
+ scope variable, named-query obligation, unit, or client
412
412
  authority. Never expose table/index/value selection through a client-controlled
413
413
  route. Custom adapters may omit the additive method; authoritative commands
414
414
  must check for it and fail closed. See the public
@@ -571,7 +571,7 @@ context). The demo server wires it behind `SYNCULAR_DEMO_EVENTS=1`.
571
571
 
572
572
  | Event | When | Key fields |
573
573
  | --- | --- | --- |
574
- | `request.handled` | Once per `POST /sync`, after the response bytes are fully produced (or the request was rejected up front) | `kind` (`sync`), `partition`, `actorId`, `durationMs`, `bytesIn`, `bytesOut`, `outcome` (`ok` \| `schema_floor` \| `epoch_acquired` \| `reset` \| `rejected` \| `error`; `epoch_acquired` answers a first round without `logEpoch`, `reset` a `logEpoch` the log no longer matches), `errorCode?`, `clientId?` (absent when the request did not decode), `pushCommits`, `pulled`, `subscriptions` |
574
+ | `request.handled` | Once per `POST /sync`, after the response bytes are fully produced (or the request was rejected up front) | `kind` (`sync`), `partition`, `actorId`, `durationMs`, `bytesIn`, `bytesOut`, `outcome` (`ok` \| `schema_floor` \| `epoch_acquired` \| `reset` \| `rejected` \| `error`; `epoch_acquired` answers a first round without `logEpoch`, `reset` a `logEpoch` the log no longer matches), `errorCode?`, `clientId?` (absent when the request did not decode), `pushCommits`, `pulled`, `units` (units of the request's effective list, 0 without one) |
575
575
  | `push.applied` | A `PUSH_COMMIT` applied, or replayed from the idempotency cache (§2.3) | `clientId`, `clientCommitId`, `operations`, `commitSeq?`, `replay` |
576
576
  | `push.rejected` | A commit rejected (§6.3) | `clientId`, `clientCommitId`, `operations`, `code` (§10.2), `opIndex`, `table?` and `rowId?` of the terminating operation |
577
577
  | `push.conflicted` | A commit terminated by a version conflict (§6.2) | `clientId`, `clientCommitId`, `operations`, `opIndex`, `table?` and `rowId?` of the conflicting operation |
@@ -581,7 +581,7 @@ context). The demo server wires it behind `SYNCULAR_DEMO_EVENTS=1`.
581
581
  | `reaction.completed` | A handler finished and its owner acknowledged | started fields |
582
582
  | `reaction.dead_lettered` | A permanent or exhausted failure was recorded | started fields plus `errorCode` |
583
583
  | `reaction.prune_completed` | One bounded terminal-reaction retention pass finished | `completedBeforeMs`, `deadLetterBeforeMs`, `limit`, `removedCompleted`, `removedDeadLetter`, `mayHaveMore` |
584
- | `pull.served` | Once per served pull half, after all sections streamed | `clientId`, `subscriptions[]`: `{id, table, status, mode` (`bootstrap` \| `incremental` \| `none`)`, fromCursor, nextCursor, commits, changes, segments[]}`; each segment: `{mediaType` (`rows` \| `sqlite`)`, delivery` (`inline` \| `ref`)`, origin` (`built` \| `reused`)`, bytes, rows}` |
584
+ | `pull.served` | Once per served pull half, after all sections streamed | `clientId`, `acceptedThrough`, `storageMaxCommitSeq`, `registrationTraveled` (the request carried its registration list), `units[]`: `{id, scopes, status` (`active` \| `revoked` \| `reset`)`, mode` (`live` \| `catching-up` \| `bootstrap` \| `none`)`, section` (the response carried a unit section)`, fenced, fromCursor, segments[]}`; each segment: `{mediaType` (`rows` \| `sqlite`)`, delivery` (`inline` \| `ref`)`, origin` (`built` \| `reused`)`, bytes, rows}`; `live`: `{status` (`active` \| `reset` \| `unregistered`)`, fromCursor` (the replica cursor)`, throughSeq, commits, changes, firstCommitSeq?, lastCommitSeq?}` |
585
585
  | `segment.downloaded` | Every direct segment download (§5.5), success or failure | `segmentId`, `outcome` (`ok` \| `error`), `errorCode?`, `mediaType?`, `bytes?`, `durationMs` |
586
586
  | `blob.swept` | Every `sweepOrphanBlobs` pass (§5.9.2 orphan GC) | `partition`, `swept` (deleted count), `referenced` (old candidates still referenced), `graceMs` |
587
587
  | `realtime.opened` | A socket registered with the hub and got `hello` (§8.1) | `sessionId`, `clientId`, `registrations`, `cursor`, `latestSeq` |
@@ -645,14 +645,14 @@ design. Row reads need the server `schema` (`fromConfig` passes it).
645
645
  | `tableCounts(partition)` | Row count per schema table. |
646
646
  | `listRows(partition, table, {afterRowId?, limit?, scope?})` | One page of decoded rows ordered by rowId, optionally only rows whose stored `scope` (`{variable, value}`) matches, read through the scope index; `nextAfterRowId` continues. |
647
647
  | `rowDelivery(partition, table, rowId)` | The trace verdict for every known client on one row, sorted by verdict then last seen: who has the row and who does not. |
648
- | `findWrite(partition, clientCommitId)` | Every stored idempotency outcome for that clientCommitId (applied with `commitSeq`, or rejected with per-operation results; conflict bytes as base64) and the ring events that carry it. |
649
- | `listClients(partition, {limit?, sort?})` | `{clients, total}`. Each client: `clientId`, `actorId`, `wireVersion`, `schemaVersion?` (highest among its subscriptions), `cursor`, `lag` (`maxCommitSeq − max(cursor, 0)`), `updatedAtMs`, `subscriptions[]`, `active` (touched within the §4.6 active window), `connected?` and `connectedSinceMs?` (open realtime sockets on this process; present when a hub is wired, which `fromConfig` does when `config.realtime` is a `RealtimeHub`). `sort`: `seen` (default) or `lag`; `limit` default 200. |
648
+ | `findWrite(partition, clientCommitId)` | Every stored idempotency outcome for that clientCommitId (applied with `commitSeq`, or rejected with per-operation results; conflict bytes as base64), the ring events that carry it, and, when no outcome is stored, the §6.3.2 `watermarks[]` (`clientId`, `outboxId`, `throughSeq`) of the clients whose retained log carries the commit: a commit below `throughSeq` lost its result to the watermark ("expired below sequence N"). |
649
+ | `listClients(partition, {limit?, sort?})` | `{clients, total}`. Each client: `clientId`, `actorId`, `wireVersion`, `schemaVersion?` (of its latest completed round), `cursor`, `lag` (`maxCommitSeq − max(cursor, 0)`), `updatedAtMs`, `units[]` (the stored registration list: `id`, `scopes`, `catchingUpCursor?` for a unit still catching up), `active` (touched within the §4.6 active window), `connected?` and `connectedSinceMs?` (open realtime sockets on this process; present when a hub is wired, which `fromConfig` does when `config.realtime` is a `RealtimeHub`). `sort`: `seen` (default) or `lag`; `limit` default 200. |
650
650
  | `clientDetail(partition, clientId, {eventLimit?})` | `{exists, client?, lease?, events}`: the record, its §7.3 lease when a lease store is wired, and its slice of the event tail. |
651
651
  | `listCommits(partition, {afterSeq?, beforeSeq?, limit?, table?, changeSample?})` | Commit metadata, newest first: `commitSeq`, `clientId`, `clientCommitId`, `actorId`, `createdAtMs`, `changeCount`, `tables[]`, and with `changeSample` the first N changes as `sample[]` (`table`, `rowId`, `op`). Pass the oldest `commitSeq` of a page as `beforeSeq` for the next older page. |
652
652
  | `commitDetail(partition, commitSeq)` | The commit's metadata plus `changes[]` (`table`, `rowId`, `op`, `rowVersion?`, `scopes`, `values?` rendered per SPEC §11, `undecodable?` for a payload written under an older column set). Throws `sync.not_found` for a missing or pruned commit. |
653
653
  | `listReactions(partition, {statuses?, types?, limit?})` | Durable reaction lifecycle rows, newest first, including source commit, attempts, lease, completion, and bounded failure information. |
654
654
  | `inspectRow(partition, table, rowId, {historyLimit?})` | `{exists, serverVersion?, scopes?, values?, history}`: the row decoded per SPEC §11 and its retained changes, newest first (`commitSeq`, `op`, `rowVersion?`, `actorId`, `clientId`, `createdAtMs`). Throws `sync.unknown_table` for a table outside the schema. |
655
- | `traceRow(partition, {table, rowId, clientId})` | Whether the client receives the row: a `verdict` (`client-unknown`, `row-missing`, `no-subscription`, `row-deleted`, `scope-mismatch`, `scope-denied`, `pending-pull`, `delivered`, evaluated in that order) with the row, its newest change, the horizon, the client, the actor's `allowed` scopes from `resolveScopes` (`outage` for the §7.3.3 sentinel), each subscription on the table with `requestedMatch`, `mismatches[]` and `effectiveMatch` (requested ∩ allowed, §3.2), and the client's retained push rejections. |
655
+ | `traceRow(partition, {table, rowId, clientId})` | Whether the client receives the row: a `verdict` (`client-unknown`, `row-missing`, `no-unit`, `row-deleted`, `scope-mismatch`, `scope-denied`, `pending-pull`, `delivered`, evaluated in that order) with the row, its newest change, the horizon, the client, the actor's `allowed` scopes from `resolveScopes` (`outage` for the §7.3.3 sentinel), each of the client's units that covers the table with `requestedMatch`, `mismatches[]` and `effectiveMatch` (requested ∩ allowed, §3.2), and the client's retained push rejections. |
656
656
  | `scopeActivity(partition, {variable, value}, {limit?})` | Recent commits touching one scope key, via the §3.1 change-scope index (never a log scan). |
657
657
  | `horizonStatus(partition)` | `{maxCommitSeq, horizonSeq, retainedCommits, activeCursorFloor, recommendedHorizonSeq, recommendation}`: the horizon a prune pass would reach now (§4.6) and `up-to-date` / `prune-recommended`. |
658
658
  | `stats(partition)` | `{segments?, blobs?}` counts/bytes where the stores expose them (segments split rows/sqlite). |
@@ -962,8 +962,8 @@ bytes straight to the object store. **Upload authz is host-authentication only**
962
962
  bytes is not a scope-bearing act, because the content address discloses
963
963
  nothing. The grant signs `content-length` (the declared `byteLength`) and
964
964
  `x-amz-checksum-sha256` (the `blobId` digest as base64), and the client sends
965
- that checksum header, so the object store refuses a PUT of another size or
966
- other bytes. The **size cap is enforced up front** against the declared
965
+ that checksum header, derived from the `blobId` without hashing the body again,
966
+ so the object store refuses a PUT of another size or other bytes. The **size cap is enforced up front** against the declared
967
967
  `byteLength`, before any URL is minted. An already-present blob returns `{ present: true }` (skip
968
968
  the PUT, idempotent §5.9.3). A **single PUT only** — never a multipart or chunk
969
969
  protocol; resumable upload, when it lands, is provider multipart behind this
@@ -1151,14 +1151,16 @@ regardless, and at least the newest `minRetainedCommits` (default 1000)
1151
1151
  commits are always kept. Raise the defaults freely, lower them with care.
1152
1152
 
1153
1153
  **What `sync.cursor_expired` means operationally.** A client whose
1154
- cursor fell behind the horizon gets `SUB_START.status = reset` and
1155
- re-bootstraps from scratch (§4.7). That is correct behavior, not an
1154
+ replica cursor fell behind the horizon gets `LIVE_START.status = reset`
1155
+ (a post-bootstrap unit behind it gets `SUB_START.status = reset`) and
1156
+ re-bootstraps the affected units from scratch (§4.6, §4.7). That is correct behavior, not an
1156
1157
  error — but its *rate* is your pruning health signal. A steady trickle
1157
1158
  means devices returning from >30-day absences (expected). A spike means
1158
1159
  you pruned faster than your fleet syncs: `ageForceMs` or
1159
1160
  `activeWindowMs` is too tight for real usage, and you are paying for it
1160
1161
  in bootstrap load (full re-scans + segment builds), not just in resets.
1161
- Observe it via `pull.served` subscriptions with `status: "reset"`.
1162
+ Observe it via `pull.served` events whose `live.status` or unit `status`
1163
+ is `"reset"`.
1162
1164
 
1163
1165
  **Segment TTL interplay.** Segments are cache entries, not durable state
1164
1166
  (§5.1; default TTL 24 h). A bootstrap that resumes past segment expiry
@@ -1178,7 +1180,7 @@ same table+scope during a storm, your TTL is shorter than the storm.
1178
1180
  (`push.conflicted` is normal offline-first traffic; alert only on
1179
1181
  gross shifts.)
1180
1182
  - `scopes.resolve_failed` — any nonzero rate. This is the fail-loud
1181
- path: every occurrence revokes subscriptions or rejects writes for a
1183
+ path: every occurrence revokes units or rejects writes for a
1182
1184
  real request, and it is almost always a host bug or a dead dependency
1183
1185
  of the resolver.
1184
1186
  - `request.handled` with `outcome: "error"` and `errorCode: "internal"`
@@ -1399,10 +1401,12 @@ rows. The image database and serialized output still consume memory.
1399
1401
  Realtime acknowledgements call `advanceClientCursor(partition, clientId,
1400
1402
  actorId, logEpoch, cursor, updatedAtMs)`. Custom storage adapters must implement
1401
1403
  this atomic update: advance the cursor and activity timestamp to their respective
1402
- maxima, preserve registration fields, and require a matching actor and current
1403
- partition log epoch. Leave missing records unchanged. SQLite, Postgres, and D1
1404
- perform one update without reading or serializing the subscription list. HTTP
1405
- registration keeps its existing cursor and subscription replacement rules.
1404
+ maxima, cap the cursor at the lowest non-negative cursor of the record's
1405
+ catching-up units, preserve registration fields, and require a matching actor and
1406
+ current partition log epoch. Leave missing records unchanged. SQLite, Postgres,
1407
+ and D1 perform one update without reading or serializing the registration list.
1408
+ A completed sync round writes the record through `putClientRecord`, which keeps
1409
+ the stored registration list unless the round's list traveled.
1406
1410
 
1407
1411
  ## Reviewed schema windows
1408
1412
 
package/dist/admin.d.ts CHANGED
@@ -11,7 +11,7 @@
11
11
  * thrown `Error`) when a backend lacks one, so a host wiring an unsupported
12
12
  * store learns immediately rather than getting a silently-empty console.
13
13
  */
14
- import { type ColumnType, type JsonValue, type PushOperationResult, type ScopeMap } from '@syncular/core';
14
+ import { type ColumnType, type JsonValue, type PushOperationResult, type ScopeMap, type UnitScopes } from '@syncular/core';
15
15
  import type { BlobStore, BlobStoreStats } from './blob-store.js';
16
16
  import { type ResolveScopes, type SyncServerConfig } from './context.js';
17
17
  import { type SyncularServerEvent, type SyncularServerEvents } from './events.js';
@@ -21,14 +21,22 @@ import { RealtimeHub } from './realtime.js';
21
21
  import { type RetentionPolicy } from './prune.js';
22
22
  import { type ServerSchema } from './schema.js';
23
23
  import type { SegmentStore, SegmentStoreStats } from './segment-store.js';
24
- import type { CheckpointDeclaration, CommitMetadata, ReactionStatus, StoredPushResult, RowChangeRecord, ScopeCommitActivity, ServerStorage, StoredReaction } from './storage.js';
24
+ import type { CheckpointDeclaration, CommitMetadata, PushWatermark, ReactionStatus, StoredPushResult, RowChangeRecord, ScopeCommitActivity, ServerStorage, StoredReaction } from './storage.js';
25
+ /** One registered unit of a client record. */
26
+ export interface AdminClientUnit {
27
+ /** The unit id (§4.1). */
28
+ readonly id: string;
29
+ readonly scopes: UnitScopes;
30
+ /** The unit's cursor while it catches up; absent for a live unit. */
31
+ readonly catchingUpCursor?: number;
32
+ }
25
33
  /** A connected/known client as the console sees it (§4.5, §8.1). */
26
34
  export interface AdminClient {
27
35
  readonly clientId: string;
28
36
  readonly actorId: string;
29
37
  /** SSP2 wire version last accepted from this client. */
30
38
  readonly wireVersion: number;
31
- /** Highest codec schema version among its subscriptions, when recorded. */
39
+ /** Schema version of its latest completed round, when recorded. */
32
40
  readonly schemaVersion?: number;
33
41
  readonly cursor: number;
34
42
  /**
@@ -37,11 +45,8 @@ export interface AdminClient {
37
45
  */
38
46
  readonly lag: number;
39
47
  readonly updatedAtMs: number;
40
- readonly subscriptions: readonly {
41
- readonly id: string;
42
- readonly table: string;
43
- readonly scopes: Record<string, readonly string[]>;
44
- }[];
48
+ /** The stored registration list (§4.1); empty when none is stored. */
49
+ readonly units: readonly AdminClientUnit[];
45
50
  /** True when the cursor record was touched within the active window. */
46
51
  readonly active: boolean;
47
52
  /**
@@ -211,27 +216,27 @@ export interface AdminCommitDetail extends CommitMetadata {
211
216
  * The trace verdict, evaluated in this order:
212
217
  * `client-unknown`: no client record for the id.
213
218
  * `row-missing`: no row and no retained change to it.
214
- * `no-subscription`: the client subscribes to nothing on the table.
219
+ * `no-unit`: no registered unit of the client covers the table.
215
220
  * `row-deleted`: the row's newest retained change is a delete the client
216
221
  * has pulled.
217
- * `scope-mismatch`: no subscription requests the row's stored scopes.
218
- * `scope-denied`: a subscription requests them, but the actor's allowed
222
+ * `scope-mismatch`: no covering unit requests the row's stored scopes.
223
+ * `scope-denied`: a covering unit requests them, but the actor's allowed
219
224
  * scopes (the host `resolveScopes`) exclude the row.
220
225
  * `pending-pull`: the client's cursor is below the row's newest change.
221
226
  * `delivered`: the client has pulled the row's newest change.
222
227
  */
223
- export type AdminTraceVerdict = 'client-unknown' | 'row-missing' | 'no-subscription' | 'row-deleted' | 'scope-mismatch' | 'scope-denied' | 'pending-pull' | 'delivered';
224
- /** One of the client's subscriptions on the traced table. */
225
- export interface AdminTraceSubscription {
228
+ export type AdminTraceVerdict = 'client-unknown' | 'row-missing' | 'no-unit' | 'row-deleted' | 'scope-mismatch' | 'scope-denied' | 'pending-pull' | 'delivered';
229
+ /** One of the client's units covering the traced table. */
230
+ export interface AdminTraceUnit {
226
231
  readonly id: string;
227
- readonly scopes: Record<string, readonly string[]>;
228
- /** The row's stored scopes satisfy the requested scopes. */
232
+ readonly scopes: UnitScopes;
233
+ /** The row's stored scopes carry the unit's values. */
229
234
  readonly requestedMatch: boolean;
230
- /** Each requested variable the row's stored value fails. */
235
+ /** Each unit variable the row's stored value fails. */
231
236
  readonly mismatches: readonly {
232
237
  readonly variable: string;
233
238
  readonly rowValue: string | null;
234
- readonly requested: readonly string[];
239
+ readonly requested: string;
235
240
  }[];
236
241
  /**
237
242
  * The row also satisfies requested ∩ allowed (§3.2). Absent when the
@@ -256,7 +261,7 @@ export interface AdminRowTrace {
256
261
  * when it returned the §7.3.3 outage sentinel. Absent without a resolver.
257
262
  */
258
263
  readonly allowed?: ScopeMap | 'outage';
259
- readonly subscriptions: readonly AdminTraceSubscription[];
264
+ readonly units: readonly AdminTraceUnit[];
260
265
  /** The client's retained push rejections and conflicts, newest first. */
261
266
  readonly rejections: readonly SyncularServerEvent[];
262
267
  }
@@ -323,6 +328,13 @@ export interface AdminWriteLookup {
323
328
  readonly clientCommitId: string;
324
329
  /** One outcome per client that sent this clientCommitId. */
325
330
  readonly outcomes: readonly AdminWriteOutcome[];
331
+ /**
332
+ * With no stored outcome: the §6.3.2 watermarks of the clients whose
333
+ * retained commit log carries this clientCommitId. A commit below an
334
+ * outbox's `throughSeq` lost its result to the watermark ("expired below
335
+ * sequence N").
336
+ */
337
+ readonly watermarks: readonly PushWatermark[];
326
338
  /** Retained ring events carrying this clientCommitId, newest first. */
327
339
  readonly events: readonly SyncularServerEvent[];
328
340
  }
@@ -464,7 +476,7 @@ export declare class SyncularAdmin {
464
476
  }): Promise<AdminRowInspection>;
465
477
  /**
466
478
  * Trace whether one client receives one row, combining the row's stored
467
- * scopes and newest change with the client's subscriptions, cursor, the
479
+ * scopes and newest change with the client's units, cursor, the
468
480
  * actor's allowed scopes, and its retained push failures. See
469
481
  * `AdminTraceVerdict` for the verdict order.
470
482
  */
package/dist/admin.js CHANGED
@@ -11,7 +11,7 @@
11
11
  * thrown `Error`) when a backend lacks one, so a host wiring an unsupported
12
12
  * store learns immediately rather than getting a silently-empty console.
13
13
  */
14
- import { bytesToBase64, decodeRow, renderRowValues, } from '@syncular/core';
14
+ import { bytesToBase64, decodeRow, renderRowValues, unitId, } from '@syncular/core';
15
15
  import { clockOf, RESOLVER_OUTAGE, } from './context.js';
16
16
  import { SyncError } from './errors.js';
17
17
  import { emitEvent, } from './events.js';
@@ -47,41 +47,39 @@ function required(value, what) {
47
47
  const VERDICT_ORDER = [
48
48
  'client-unknown',
49
49
  'row-missing',
50
- 'no-subscription',
50
+ 'no-unit',
51
51
  'row-deleted',
52
52
  'scope-mismatch',
53
53
  'scope-denied',
54
54
  'pending-pull',
55
55
  'delivered',
56
56
  ];
57
- /** One known client's subscriptions on the row's table and its verdict. */
57
+ /** One known client's units covering the row's table and its verdict. */
58
58
  function traceVerdict(declared, record, row, lastChange, horizonSeq, allowed) {
59
59
  const rowScopes = row.scopes ?? {};
60
- const subscriptions = record.subscriptions
61
- .filter((sub) => sub.table === row.table)
62
- .map((sub) => {
63
- const mismatches = Object.entries(sub.scopes)
64
- .filter(([variable, values]) => {
65
- const value = rowScopes[variable];
66
- return value === undefined || !values.includes(value);
67
- })
68
- .map(([variable, values]) => ({
60
+ // §4.1: a unit covers every table that declares all of its variables.
61
+ const units = (record.registration?.units ?? [])
62
+ .filter((scopes) => Object.keys(scopes).every((variable) => declared.has(variable)))
63
+ .map((scopes) => {
64
+ const mismatches = Object.entries(scopes)
65
+ .filter(([variable, value]) => rowScopes[variable] !== value)
66
+ .map(([variable, value]) => ({
69
67
  variable,
70
68
  rowValue: rowScopes[variable] ?? null,
71
- requested: values,
69
+ requested: value,
72
70
  }));
73
71
  const traced = {
74
- id: sub.id,
75
- scopes: sub.scopes,
72
+ id: unitId(scopes),
73
+ scopes,
76
74
  requestedMatch: mismatches.length === 0,
77
75
  mismatches,
78
76
  };
79
77
  if (allowed === undefined || allowed === 'outage')
80
78
  return traced;
81
- const effective = computeEffective(declared, sub.scopes, {
82
- ok: true,
83
- allowed,
84
- });
79
+ const effective = computeEffective(declared, Object.fromEntries(Object.entries(scopes).map(([variable, value]) => [
80
+ variable,
81
+ [value],
82
+ ])), { ok: true, allowed });
85
83
  return {
86
84
  ...traced,
87
85
  effectiveMatch: effective.status === 'active' &&
@@ -93,20 +91,20 @@ function traceVerdict(declared, record, row, lastChange, horizonSeq, allowed) {
93
91
  const pulled = record.cursor >= (lastChange?.commitSeq ?? horizonSeq);
94
92
  const verdict = !row.exists && lastChange === undefined
95
93
  ? 'row-missing'
96
- : subscriptions.length === 0
97
- ? 'no-subscription'
94
+ : units.length === 0
95
+ ? 'no-unit'
98
96
  : !row.exists
99
97
  ? pulled
100
98
  ? 'row-deleted'
101
99
  : 'pending-pull'
102
- : !subscriptions.some((sub) => sub.requestedMatch)
100
+ : !units.some((unit) => unit.requestedMatch)
103
101
  ? 'scope-mismatch'
104
- : subscriptions.every((sub) => sub.effectiveMatch === false)
102
+ : units.every((unit) => unit.effectiveMatch === false)
105
103
  ? 'scope-denied'
106
104
  : pulled
107
105
  ? 'delivered'
108
106
  : 'pending-pull';
109
- return { subscriptions, verdict };
107
+ return { units, verdict };
110
108
  }
111
109
  /** The console query surface plus its operator actions. One per process. */
112
110
  export class SyncularAdmin {
@@ -322,7 +320,7 @@ export class SyncularAdmin {
322
320
  }
323
321
  /**
324
322
  * Trace whether one client receives one row, combining the row's stored
325
- * scopes and newest change with the client's subscriptions, cursor, the
323
+ * scopes and newest change with the client's units, cursor, the
326
324
  * actor's allowed scopes, and its retained push failures. See
327
325
  * `AdminTraceVerdict` for the verdict order.
328
326
  */
@@ -345,7 +343,7 @@ export class SyncularAdmin {
345
343
  ...(lastChange !== undefined ? { lastChange } : {}),
346
344
  };
347
345
  if (record === undefined) {
348
- return { ...base, verdict: 'client-unknown', subscriptions: [] };
346
+ return { ...base, verdict: 'client-unknown', units: [] };
349
347
  }
350
348
  const allowed = await this.#allowedScopes(partition, record);
351
349
  return {
@@ -503,7 +501,10 @@ export class SyncularAdmin {
503
501
  : entry),
504
502
  }));
505
503
  const events = this.events({ partition }).filter((event) => 'clientCommitId' in event && event.clientCommitId === clientCommitId);
506
- return { clientCommitId, outcomes, events };
504
+ const watermarks = outcomes.length > 0
505
+ ? []
506
+ : await required(this.#storage.listPushWatermarks?.bind(this.#storage), 'storage')(partition, clientCommitId);
507
+ return { clientCommitId, outcomes, watermarks, events };
507
508
  }
508
509
  /**
509
510
  * Resolve an id pasted from a bug report: clients and actors containing
@@ -903,28 +904,29 @@ export class SyncularAdmin {
903
904
  }
904
905
  }
905
906
  return (record) => {
906
- let schemaVersion;
907
- for (const sub of record.subscriptions) {
908
- if (sub.schemaVersion !== undefined &&
909
- (schemaVersion === undefined || sub.schemaVersion > schemaVersion)) {
910
- schemaVersion = sub.schemaVersion;
911
- }
912
- }
913
907
  const connectedSinceMs = connectedSince.get(record.clientId);
914
908
  return {
915
909
  clientId: record.clientId,
916
910
  actorId: record.actorId,
917
911
  wireVersion: record.wireVersion,
918
- ...(schemaVersion !== undefined ? { schemaVersion } : {}),
912
+ ...(record.schemaVersion !== undefined
913
+ ? { schemaVersion: record.schemaVersion }
914
+ : {}),
919
915
  cursor: record.cursor,
920
916
  // A never-pulled cursor (-1) has seen nothing: it lags the whole log.
921
917
  lag: Math.max(0, maxCommitSeq - Math.max(0, record.cursor)),
922
918
  updatedAtMs: record.updatedAtMs,
923
- subscriptions: record.subscriptions.map((sub) => ({
924
- id: sub.id,
925
- table: sub.table,
926
- scopes: sub.scopes,
927
- })),
919
+ units: (record.registration?.units ?? []).map((scopes) => {
920
+ const id = unitId(scopes);
921
+ const catchingUp = record.catchingUp.find((unit) => unit.id === id);
922
+ return {
923
+ id,
924
+ scopes,
925
+ ...(catchingUp !== undefined
926
+ ? { catchingUpCursor: catchingUp.cursor }
927
+ : {}),
928
+ };
929
+ }),
928
930
  active: record.updatedAtMs >= activeFloorMs,
929
931
  ...(sockets !== undefined
930
932
  ? { connected: connectedSinceMs !== undefined }
package/dist/context.d.ts CHANGED
@@ -44,7 +44,7 @@ export type ResolverOutage = typeof RESOLVER_OUTAGE;
44
44
  /**
45
45
  * Host callback: actor → allowed scopes (§3.2 step 3). Resolved at most
46
46
  * once per request and memoized (§3.4 step 1). `'*'` means "any value for
47
- * this variable". A throwing resolver revokes subscriptions and rejects
47
+ * this variable". A throwing resolver revokes units and rejects
48
48
  * writes with `sync.forbidden` — fail loud, never leak. Returning
49
49
  * `RESOLVER_OUTAGE` instead opts the request into lease authorization
50
50
  * (§7.3.3) — a deliberate outage signal, not a failure.
@@ -69,10 +69,8 @@ export interface ServerLimits {
69
69
  * decoding, and a socket round's assembled request (§8.7).
70
70
  */
71
71
  readonly maxRequestBytes: number;
72
- /** §1.7 cap on `SUBSCRIPTION` frames per request. */
72
+ /** §1.7 cap on `UNIT` frames per request: the units of a traveling list. */
73
73
  readonly maxSubscriptionsPerRequest: number;
74
- /** §1.7 cap on requested scope values, summed over a subscription's keys. */
75
- readonly maxScopeValuesPerSubscription: number;
76
74
  /** Cap on concurrent registered-query watches per watch session. */
77
75
  readonly maxWatchesPerSession: number;
78
76
  }
@@ -81,10 +79,10 @@ export declare const DEFAULT_LIMITS: ServerLimits;
81
79
  export interface RealtimeNotifier {
82
80
  notifyCommit(partition: string, commit: StoredCommit): void | Promise<void>;
83
81
  /**
84
- * A pull changed the client's persisted subscription list (§8.7). A hub
85
- * re-registers that client's live sockets, so a client that runs its
86
- * rounds over HTTP while a socket is open receives deltas for the new
87
- * subscriptions.
82
+ * A pull changed the client's registration list or live set (§8.1). A
83
+ * hub re-registers that client's live sockets, so a client that runs its
84
+ * rounds over HTTP while a socket is open receives deltas for the units
85
+ * that joined.
88
86
  */
89
87
  notifySubscriptions?(partition: string, clientId: string): void | Promise<void>;
90
88
  }
package/dist/context.js CHANGED
@@ -17,10 +17,8 @@ export const DEFAULT_LIMITS = {
17
17
  maxCascadeOperationsPerCommit: 1_000,
18
18
  // The size above which adapters already refuse to buffer a segment body.
19
19
  maxRequestBytes: 16 * 1024 * 1024,
20
- // One subscription per pushable operation: the existing per-request cap.
20
+ // One unit per pushable operation: the existing per-request cap.
21
21
  maxSubscriptionsPerRequest: 500,
22
- // One default snapshot page (§4.2): a longer list wants a coarser scope.
23
- maxScopeValuesPerSubscription: 1_000,
24
22
  // Every commit reruns a session's affected watches one after another.
25
23
  maxWatchesPerSession: 100,
26
24
  };
@@ -42,7 +42,8 @@ var _a;
42
42
  import { decodeRow } from '@syncular/core';
43
43
  import { assertAppendOnlyMigration, assertPhysicalColumns, assertStoredLayouts, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, quoteIdent, retiredTableNames, rewritePlan, rewriteRowSql, rewriteValues, SCHEMA_META_DDL_SQLITE, scanRowPageSql, schemaDdl, serverIndexDdl, SQLITE_TABLE_INDEXES_SQL, sqliteIndexSignatures, selectRowSql, selectRowsSql, selectRowsForRewriteSql, tableColumnNames, toSqlValue, } from './relational-rows.js';
44
44
  import { matchesEffective } from './scopes.js';
45
- import { asUint8Array, deserializePushResult, placeholders, SqliteFamilyStorage, SqliteFamilyTransaction, SQLITE_ADDED_COLUMNS, sqliteDdlStatements, toStoredRow, toTombstone, } from './sqlite-dialect.js';
45
+ import { asUint8Array, deserializePushResult, placeholders, SqliteFamilyStorage, SqliteFamilyTransaction, sqliteDdlStatements, SQLITE_UPGRADE_PROBE, sqliteUpgradeDdl, toStoredRow, toTombstone, } from './sqlite-dialect.js';
46
+ import { commitDeleteFloor } from './storage.js';
46
47
  import { isD1ConstraintError, StorageConstraintError, StorageQueryError, } from './storage-errors.js';
47
48
  import { assertScopeIndexedScan, resolveIndexRowScan } from './storage-query.js';
48
49
  /** The rows of one `db.batch` result entry (`{ results }` per statement). */
@@ -316,18 +317,37 @@ class D1Transaction extends SqliteFamilyTransaction {
316
317
  }
317
318
  return queryAuthoritativeOn(this.#db, this.partition, this.#prepareQuery(query));
318
319
  }
319
- async preparePushApply(clientId, clientCommitId, runningSchemaVersion, _targets = []) {
320
+ async preparePushApply(clientId, clientCommitId, runningSchemaVersion, _targets = [], outbox) {
320
321
  this.assertOpen();
321
322
  if (!this.#pushApplySerialized) {
322
323
  throw new Error('D1 push apply requires externally serialized partition writes');
323
324
  }
324
325
  // D1 has no interactive lock. The caller explicitly asserted that every
325
326
  // write for this partition is already serialized (normally by its DO).
327
+ // The settlement's writes sit below the checkpoint, so a rejection keeps
328
+ // them.
329
+ const settled = outbox === undefined
330
+ ? undefined
331
+ : await this.settleOutbox(clientId, clientCommitId, outbox);
326
332
  this.#pushApplyCheckpoint = this.#buffer.length;
327
- return {
328
- gate: await this.readServeGate(runningSchemaVersion),
329
- persisted: await this.getPushResult(clientId, clientCommitId),
333
+ // The serve gate and the delete floor (§2.2) in one round trip.
334
+ const read = await this.#db
335
+ .prepare('SELECT (SELECT schema_version FROM sync_schema_meta WHERE id=1) AS schema_version, (SELECT delete_floor FROM sync_partitions WHERE partition=?) AS delete_floor')
336
+ .bind(this.partition)
337
+ .first();
338
+ const gate = {
339
+ storedSchemaVersion: read?.schema_version ?? 0,
340
+ logEpoch: undefined,
341
+ pending: [],
330
342
  };
343
+ const deleteFloor = read?.delete_floor ?? 0;
344
+ return settled === undefined
345
+ ? {
346
+ gate,
347
+ deleteFloor,
348
+ persisted: await this.getPushResult(clientId, clientCommitId),
349
+ }
350
+ : { gate, deleteFloor, ...settled };
331
351
  }
332
352
  advanceCheckpoint(_name, _ownerEpoch, _watermark, _observedRows, _nowMs) {
333
353
  // D1 installs no fence and declares no checkpoints in this release.
@@ -344,7 +364,7 @@ class D1Transaction extends SqliteFamilyTransaction {
344
364
  pending: [],
345
365
  };
346
366
  }
347
- async commitRejectedPushResult(clientId, clientCommitId, result) {
367
+ async commitRejectedPushResult(clientId, clientCommitId, result, sequence) {
348
368
  this.assertOpen();
349
369
  const checkpoint = this.#pushApplyCheckpoint;
350
370
  if (checkpoint === undefined) {
@@ -355,7 +375,7 @@ class D1Transaction extends SqliteFamilyTransaction {
355
375
  this.#clearedTombstones.clear();
356
376
  this.#maxCommitSeq = undefined;
357
377
  this.#applicationOpIndexes.clear();
358
- await this.putPushResult(clientId, clientCommitId, result);
378
+ await this.putPushResult(clientId, clientCommitId, result, sequence);
359
379
  await this.commit();
360
380
  // The idempotency re-check found no record under the serialized apply, so
361
381
  // the insert above is the canonical outcome.
@@ -447,7 +467,7 @@ class D1Transaction extends SqliteFamilyTransaction {
447
467
  }
448
468
  const commitSeq = this.#maxCommitSeq + 1;
449
469
  this.#maxCommitSeq = commitSeq;
450
- this.run('INSERT INTO sync_partitions(partition, max_commit_seq) VALUES (?,?) ON CONFLICT(partition) DO UPDATE SET max_commit_seq=excluded.max_commit_seq', [p, commitSeq]);
470
+ this.run('INSERT INTO sync_partitions(partition, max_commit_seq, delete_floor) VALUES (?,?,?) ON CONFLICT(partition) DO UPDATE SET max_commit_seq=excluded.max_commit_seq, delete_floor=max(delete_floor, excluded.delete_floor)', [p, commitSeq, commitDeleteFloor(commit.changes)]);
451
471
  this.run('INSERT INTO sync_commits(partition, commit_seq, client_id, client_commit_id, actor_id, created_at_ms) VALUES (?,?,?,?,?,?)', [
452
472
  p,
453
473
  commitSeq,
@@ -505,6 +525,7 @@ const D1_CORE_TABLES = [
505
525
  'sync_changes',
506
526
  'sync_change_scopes',
507
527
  'sync_push_results',
528
+ 'sync_push_watermarks',
508
529
  'sync_reactions',
509
530
  'sync_clients',
510
531
  'sync_blob_refs',
@@ -520,9 +541,9 @@ export class D1ServerStorage extends SqliteFamilyStorage {
520
541
  maxBindParams = 100;
521
542
  /**
522
543
  * Page reads waiting for the end of the current microtask turn. A pull
523
- * starts the window read (or first snapshot page) of every subscription
544
+ * starts the first snapshot page and scope pin of every bootstrapping unit
524
545
  * before awaiting any of them, so they leave as ONE `db.batch` round trip
525
- * instead of one per subscription (SYNCULAR-PULL-ROUNDTRIPS-001).
546
+ * instead of one per unit (SYNCULAR-PULL-ROUNDTRIPS-001).
526
547
  */
527
548
  #pendingReads;
528
549
  constructor(db, options = {}) {
@@ -584,22 +605,17 @@ export class D1ServerStorage extends SqliteFamilyStorage {
584
605
  for (const statement of sqliteDdlStatements()) {
585
606
  await this.#db.exec(`${statement.replace(/\s+/g, ' ')};`);
586
607
  }
587
- for (const sql of await this.#addedColumnDdl()) {
608
+ for (const sql of await this.#upgradeDdl()) {
588
609
  await this.#db.exec(`${sql};`);
589
610
  }
590
611
  }
591
- /**
592
- * `ALTER TABLE` statements for the `SQLITE_ADDED_COLUMNS` this database
593
- * lacks, read in one statement.
594
- */
595
- async #addedColumnDdl() {
612
+ /** `sqliteUpgradeDdl` for this database, read in one statement. */
613
+ async #upgradeDdl() {
596
614
  const { results } = await this.#db
597
- .prepare('SELECT t.value AS tbl, c.name AS name FROM json_each(?) AS t, pragma_table_info(t.value) AS c')
598
- .bind(JSON.stringify([
599
- ...new Set(SQLITE_ADDED_COLUMNS.map(([table]) => table)),
600
- ]))
615
+ .prepare(SQLITE_UPGRADE_PROBE.sql)
616
+ .bind(...SQLITE_UPGRADE_PROBE.params)
601
617
  .all();
602
- return SQLITE_ADDED_COLUMNS.filter(([table, name]) => !results.some((column) => column.tbl === table && column.name === name)).map(([table, name, definition]) => `ALTER TABLE ${table} ADD COLUMN ${name} ${definition}`);
618
+ return sqliteUpgradeDdl(results);
603
619
  }
604
620
  async ensureSchema(schema, checkpoints) {
605
621
  if (checkpoints !== undefined && checkpoints.length > 0) {
@@ -688,9 +704,11 @@ export class D1ServerStorage extends SqliteFamilyStorage {
688
704
  primaryKeyPosition: column.pk,
689
705
  })), 'sqlite');
690
706
  }
691
- // A database from before a core-table column gains it in place.
707
+ // A database from an earlier release gains the core-table columns and
708
+ // indexes it lacks in place, and loses the columns wire version 4
709
+ // deleted.
692
710
  statementsExecuted++;
693
- const added = await this.#addedColumnDdl();
711
+ const added = await this.#upgradeDdl();
694
712
  if (added.length > 0) {
695
713
  await batch(added.map((sql) => this.#db.prepare(sql)));
696
714
  }
@@ -810,7 +828,7 @@ export class D1ServerStorage extends SqliteFamilyStorage {
810
828
  if (remaining < 4)
811
829
  break;
812
830
  statementsExecuted++;
813
- const added = await this.#addedColumnDdl();
831
+ const added = await this.#upgradeDdl();
814
832
  if (remaining < 3 + added.length)
815
833
  break;
816
834
  if (!(await save({ ...state, phase: 'prepare', offset: 0 }, added.map((sql) => this.#db.prepare(sql)))))
package/dist/errors.js CHANGED
@@ -168,6 +168,14 @@ export const ERROR_CATALOG = {
168
168
  recommendedAction: 'retryLater',
169
169
  httpStatus: 500,
170
170
  },
171
+ // §6.3.2: below the outbox watermark with no retained result. A push
172
+ // operation-result error record only, never persisted.
173
+ 'sync.commit_expired': {
174
+ category: 'invalid-request',
175
+ retryable: false,
176
+ recommendedAction: 'resetClientId',
177
+ httpStatus: 400,
178
+ },
171
179
  // §2.4 on a non-D1 backend: the server refuses while a declared backfill
172
180
  // checkpoint is incomplete or the database is newer than the running build.
173
181
  // Formerly host-only; promoted to the §10.2 catalogue so an in-flight