@syncular/server 0.58.0 → 0.60.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/README.md +49 -36
  2. package/dist/admin.d.ts +32 -20
  3. package/dist/admin.js +46 -40
  4. package/dist/blob-handlers.d.ts +25 -31
  5. package/dist/blob-handlers.js +63 -41
  6. package/dist/blob-store.d.ts +42 -24
  7. package/dist/blob-store.js +0 -0
  8. package/dist/bucket-blob-store.d.ts +12 -8
  9. package/dist/bucket-blob-store.js +54 -45
  10. package/dist/bucket-segment-store.js +18 -16
  11. package/dist/content-encoding.d.ts +5 -2
  12. package/dist/content-encoding.js +42 -13
  13. package/dist/context.d.ts +19 -5
  14. package/dist/context.js +19 -1
  15. package/dist/d1-storage.d.ts +14 -98
  16. package/dist/d1-storage.js +249 -1119
  17. package/dist/errors.js +16 -0
  18. package/dist/events.d.ts +36 -18
  19. package/dist/frame-bytes.js +24 -6
  20. package/dist/handler.js +503 -208
  21. package/dist/local-authority.d.ts +2 -0
  22. package/dist/local-authority.js +24 -12
  23. package/dist/object-bucket.d.ts +2 -0
  24. package/dist/operations-realtime.js +69 -50
  25. package/dist/operations.d.ts +5 -2
  26. package/dist/operations.js +20 -29
  27. package/dist/pg-executor-pglite.d.ts +5 -6
  28. package/dist/pg-executor-pglite.js +22 -14
  29. package/dist/pg-executor.d.ts +4 -1
  30. package/dist/postgres-storage.d.ts +24 -19
  31. package/dist/postgres-storage.js +420 -262
  32. package/dist/pull.d.ts +70 -47
  33. package/dist/pull.js +270 -301
  34. package/dist/push.d.ts +12 -3
  35. package/dist/push.js +361 -173
  36. package/dist/realtime.d.ts +85 -39
  37. package/dist/realtime.js +355 -219
  38. package/dist/relational-rows.d.ts +21 -66
  39. package/dist/relational-rows.js +28 -109
  40. package/dist/s3-blob-store.d.ts +11 -12
  41. package/dist/s3-blob-store.js +20 -13
  42. package/dist/s3-object-bucket.d.ts +8 -1
  43. package/dist/s3-object-bucket.js +16 -3
  44. package/dist/schema.d.ts +14 -0
  45. package/dist/schema.js +23 -0
  46. package/dist/scopes.d.ts +6 -2
  47. package/dist/scopes.js +22 -4
  48. package/dist/seed.js +10 -16
  49. package/dist/segment-download.d.ts +4 -1
  50. package/dist/segment-download.js +22 -6
  51. package/dist/signed-url.d.ts +2 -2
  52. package/dist/signed-url.js +9 -2
  53. package/dist/sigv4.d.ts +8 -2
  54. package/dist/sigv4.js +8 -4
  55. package/dist/sqlite-blob-store.d.ts +1 -1
  56. package/dist/sqlite-blob-store.js +23 -22
  57. package/dist/sqlite-dialect.d.ts +164 -84
  58. package/dist/sqlite-dialect.js +1145 -150
  59. package/dist/sqlite-lease-store.js +13 -18
  60. package/dist/sqlite-storage.d.ts +9 -62
  61. package/dist/sqlite-storage.js +303 -1123
  62. package/dist/storage-query.d.ts +30 -1
  63. package/dist/storage-query.js +68 -0
  64. package/dist/storage.d.ts +185 -63
  65. package/dist/storage.js +15 -0
  66. package/package.json +2 -2
  67. package/src/admin.ts +99 -57
  68. package/src/blob-handlers.ts +91 -72
  69. package/src/blob-store.ts +0 -0
  70. package/src/bucket-blob-store.ts +75 -48
  71. package/src/bucket-segment-store.ts +20 -15
  72. package/src/content-encoding.ts +48 -16
  73. package/src/context.ts +33 -6
  74. package/src/d1-storage.ts +350 -1700
  75. package/src/errors.ts +16 -0
  76. package/src/events.ts +37 -18
  77. package/src/frame-bytes.ts +26 -6
  78. package/src/handler.ts +654 -261
  79. package/src/local-authority.ts +27 -14
  80. package/src/object-bucket.ts +2 -0
  81. package/src/operations-realtime.ts +85 -61
  82. package/src/operations.ts +44 -42
  83. package/src/pg-executor-pglite.ts +27 -23
  84. package/src/pg-executor.ts +5 -1
  85. package/src/postgres-storage.ts +557 -354
  86. package/src/pull.ts +406 -440
  87. package/src/push.ts +466 -220
  88. package/src/realtime.ts +447 -290
  89. package/src/relational-rows.ts +43 -121
  90. package/src/s3-blob-store.ts +23 -13
  91. package/src/s3-object-bucket.ts +20 -2
  92. package/src/schema.ts +31 -0
  93. package/src/scopes.ts +23 -3
  94. package/src/seed.ts +13 -20
  95. package/src/segment-download.ts +33 -7
  96. package/src/signed-url.ts +10 -4
  97. package/src/sigv4.ts +17 -4
  98. package/src/sqlite-blob-store.ts +27 -26
  99. package/src/sqlite-dialect.ts +1851 -199
  100. package/src/sqlite-lease-store.ts +12 -23
  101. package/src/sqlite-storage.ts +390 -1821
  102. package/src/storage-query.ts +102 -0
  103. package/src/storage.ts +215 -67
package/README.md CHANGED
@@ -357,7 +357,7 @@ revision (`catalog-v1` to `catalog-v2`) and rerun it; do not delete the database
357
357
  or unrelated rows. This does not apply to application commands. After an
358
358
  unknown command outcome, reuse the original idempotency key because changing it
359
359
  can execute the operation twice. The full inspection and recovery recipe is in
360
- the public [server guide](https://syncular.dev/guide-server/#seeding-data).
360
+ the public [server guide](https://syncular.dev/server-operations/#seeding-data).
361
361
 
362
362
  One extra rule applies when the seed actor changes. A Syncular `clientId` is
363
363
  bound to its first actor inside a partition. Keep the client ID stable for seed
@@ -408,11 +408,11 @@ must enumerate every row in one Workspace. Syncular does not currently expose
408
408
  trusted prefix or range scans.
409
409
 
410
410
  This is a trusted `@syncular/server` storage capability, not SSP2: it creates no
411
- scope variable, named-query obligation, subscription descriptor, or client
411
+ scope variable, named-query obligation, unit, or client
412
412
  authority. Never expose table/index/value selection through a client-controlled
413
413
  route. Custom adapters may omit the additive method; authoritative commands
414
414
  must check for it and fail closed. See the public
415
- [storage lookup guide](https://syncular.dev/server-storage/#choosing-the-right-row-lookup)
415
+ [storage lookup guide](https://syncular.dev/server-storage-reference/#advanced-row-lookups-for-trusted-server-code)
416
416
  for a user-scoped key-grant table revoked through a Workspace index and for the
417
417
  atomic reverse-index/queue fallback required by ordered or derived lookups.
418
418
 
@@ -571,7 +571,7 @@ context). The demo server wires it behind `SYNCULAR_DEMO_EVENTS=1`.
571
571
 
572
572
  | Event | When | Key fields |
573
573
  | --- | --- | --- |
574
- | `request.handled` | Once per `POST /sync`, after the response bytes are fully produced (or the request was rejected up front) | `kind` (`sync`), `partition`, `actorId`, `durationMs`, `bytesIn`, `bytesOut`, `outcome` (`ok` \| `schema_floor` \| `epoch_acquired` \| `reset` \| `rejected` \| `error`; `epoch_acquired` answers a first round without `logEpoch`, `reset` a `logEpoch` the log no longer matches), `errorCode?`, `clientId?` (absent when the request did not decode), `pushCommits`, `pulled`, `subscriptions` |
574
+ | `request.handled` | Once per `POST /sync`, after the response bytes are fully produced (or the request was rejected up front) | `kind` (`sync`), `partition`, `actorId`, `durationMs`, `bytesIn`, `bytesOut`, `outcome` (`ok` \| `schema_floor` \| `epoch_acquired` \| `reset` \| `rejected` \| `error`; `epoch_acquired` answers a first round without `logEpoch`, `reset` a `logEpoch` the log no longer matches), `errorCode?`, `clientId?` (absent when the request did not decode), `pushCommits`, `pulled`, `units` (units of the request's effective list, 0 without one) |
575
575
  | `push.applied` | A `PUSH_COMMIT` applied, or replayed from the idempotency cache (§2.3) | `clientId`, `clientCommitId`, `operations`, `commitSeq?`, `replay` |
576
576
  | `push.rejected` | A commit rejected (§6.3) | `clientId`, `clientCommitId`, `operations`, `code` (§10.2), `opIndex`, `table?` and `rowId?` of the terminating operation |
577
577
  | `push.conflicted` | A commit terminated by a version conflict (§6.2) | `clientId`, `clientCommitId`, `operations`, `opIndex`, `table?` and `rowId?` of the conflicting operation |
@@ -581,9 +581,9 @@ context). The demo server wires it behind `SYNCULAR_DEMO_EVENTS=1`.
581
581
  | `reaction.completed` | A handler finished and its owner acknowledged | started fields |
582
582
  | `reaction.dead_lettered` | A permanent or exhausted failure was recorded | started fields plus `errorCode` |
583
583
  | `reaction.prune_completed` | One bounded terminal-reaction retention pass finished | `completedBeforeMs`, `deadLetterBeforeMs`, `limit`, `removedCompleted`, `removedDeadLetter`, `mayHaveMore` |
584
- | `pull.served` | Once per served pull half, after all sections streamed | `clientId`, `subscriptions[]`: `{id, table, status, mode` (`bootstrap` \| `incremental` \| `none`)`, fromCursor, nextCursor, commits, changes, segments[]}`; each segment: `{mediaType` (`rows` \| `sqlite`)`, delivery` (`inline` \| `ref`)`, origin` (`built` \| `reused`)`, bytes, rows}` |
584
+ | `pull.served` | Once per served pull half, after all sections streamed | `clientId`, `acceptedThrough`, `storageMaxCommitSeq`, `registrationTraveled` (the request carried its registration list), `units[]`: `{id, scopes, status` (`active` \| `revoked` \| `reset`)`, mode` (`live` \| `catching-up` \| `bootstrap` \| `none`)`, section` (the response carried a unit section)`, fenced, fromCursor, segments[]}`; each segment: `{mediaType` (`rows` \| `sqlite`)`, delivery` (`inline` \| `ref`)`, origin` (`built` \| `reused`)`, bytes, rows}`; `live`: `{status` (`active` \| `reset` \| `unregistered`)`, fromCursor` (the replica cursor)`, throughSeq, commits, changes, firstCommitSeq?, lastCommitSeq?}` |
585
585
  | `segment.downloaded` | Every direct segment download (§5.5), success or failure | `segmentId`, `outcome` (`ok` \| `error`), `errorCode?`, `mediaType?`, `bytes?`, `durationMs` |
586
- | `blob.swept` | Every `sweepOrphanBlobs` pass (§5.9.2 orphan GC) | `partition`, `swept` (deleted count), `referenced` (keep-set size), `graceMs` |
586
+ | `blob.swept` | Every `sweepOrphanBlobs` pass (§5.9.2 orphan GC) | `partition`, `swept` (deleted count), `referenced` (old candidates still referenced), `graceMs` |
587
587
  | `realtime.opened` | A socket registered with the hub and got `hello` (§8.1) | `sessionId`, `clientId`, `registrations`, `cursor`, `latestSeq` |
588
588
  | `realtime.closed` | A session left the hub (once per session) | `sessionId`, `durationMs` |
589
589
  | `realtime.delta` | A delta message pushed over the socket (§8.2) | `sessionId`, `commitSeq`, `bytes`, `changes` |
@@ -645,14 +645,14 @@ design. Row reads need the server `schema` (`fromConfig` passes it).
645
645
  | `tableCounts(partition)` | Row count per schema table. |
646
646
  | `listRows(partition, table, {afterRowId?, limit?, scope?})` | One page of decoded rows ordered by rowId, optionally only rows whose stored `scope` (`{variable, value}`) matches, read through the scope index; `nextAfterRowId` continues. |
647
647
  | `rowDelivery(partition, table, rowId)` | The trace verdict for every known client on one row, sorted by verdict then last seen: who has the row and who does not. |
648
- | `findWrite(partition, clientCommitId)` | Every stored idempotency outcome for that clientCommitId (applied with `commitSeq`, or rejected with per-operation results; conflict bytes as base64) and the ring events that carry it. |
649
- | `listClients(partition, {limit?, sort?})` | `{clients, total}`. Each client: `clientId`, `actorId`, `wireVersion`, `schemaVersion?` (highest among its subscriptions), `cursor`, `lag` (`maxCommitSeq − max(cursor, 0)`), `updatedAtMs`, `subscriptions[]`, `active` (touched within the §4.6 active window), `connected?` and `connectedSinceMs?` (open realtime sockets on this process; present when a hub is wired, which `fromConfig` does when `config.realtime` is a `RealtimeHub`). `sort`: `seen` (default) or `lag`; `limit` default 200. |
648
+ | `findWrite(partition, clientCommitId)` | Every stored idempotency outcome for that clientCommitId (applied with `commitSeq`, or rejected with per-operation results; conflict bytes as base64), the ring events that carry it, and, when no outcome is stored, the §6.3.2 `watermarks[]` (`clientId`, `outboxId`, `throughSeq`) of the clients whose retained log carries the commit: a commit below `throughSeq` lost its result to the watermark ("expired below sequence N"). |
649
+ | `listClients(partition, {limit?, sort?})` | `{clients, total}`. Each client: `clientId`, `actorId`, `wireVersion`, `schemaVersion?` (of its latest completed round), `cursor`, `lag` (`maxCommitSeq − max(cursor, 0)`), `updatedAtMs`, `units[]` (the stored registration list: `id`, `scopes`, `catchingUpCursor?` for a unit still catching up), `active` (touched within the §4.6 active window), `connected?` and `connectedSinceMs?` (open realtime sockets on this process; present when a hub is wired, which `fromConfig` does when `config.realtime` is a `RealtimeHub`). `sort`: `seen` (default) or `lag`; `limit` default 200. |
650
650
  | `clientDetail(partition, clientId, {eventLimit?})` | `{exists, client?, lease?, events}`: the record, its §7.3 lease when a lease store is wired, and its slice of the event tail. |
651
651
  | `listCommits(partition, {afterSeq?, beforeSeq?, limit?, table?, changeSample?})` | Commit metadata, newest first: `commitSeq`, `clientId`, `clientCommitId`, `actorId`, `createdAtMs`, `changeCount`, `tables[]`, and with `changeSample` the first N changes as `sample[]` (`table`, `rowId`, `op`). Pass the oldest `commitSeq` of a page as `beforeSeq` for the next older page. |
652
652
  | `commitDetail(partition, commitSeq)` | The commit's metadata plus `changes[]` (`table`, `rowId`, `op`, `rowVersion?`, `scopes`, `values?` rendered per SPEC §11, `undecodable?` for a payload written under an older column set). Throws `sync.not_found` for a missing or pruned commit. |
653
653
  | `listReactions(partition, {statuses?, types?, limit?})` | Durable reaction lifecycle rows, newest first, including source commit, attempts, lease, completion, and bounded failure information. |
654
654
  | `inspectRow(partition, table, rowId, {historyLimit?})` | `{exists, serverVersion?, scopes?, values?, history}`: the row decoded per SPEC §11 and its retained changes, newest first (`commitSeq`, `op`, `rowVersion?`, `actorId`, `clientId`, `createdAtMs`). Throws `sync.unknown_table` for a table outside the schema. |
655
- | `traceRow(partition, {table, rowId, clientId})` | Whether the client receives the row: a `verdict` (`client-unknown`, `row-missing`, `no-subscription`, `row-deleted`, `scope-mismatch`, `scope-denied`, `pending-pull`, `delivered`, evaluated in that order) with the row, its newest change, the horizon, the client, the actor's `allowed` scopes from `resolveScopes` (`outage` for the §7.3.3 sentinel), each subscription on the table with `requestedMatch`, `mismatches[]` and `effectiveMatch` (requested ∩ allowed, §3.2), and the client's retained push rejections. |
655
+ | `traceRow(partition, {table, rowId, clientId})` | Whether the client receives the row: a `verdict` (`client-unknown`, `row-missing`, `no-unit`, `row-deleted`, `scope-mismatch`, `scope-denied`, `pending-pull`, `delivered`, evaluated in that order) with the row, its newest change, the horizon, the client, the actor's `allowed` scopes from `resolveScopes` (`outage` for the §7.3.3 sentinel), each of the client's units that covers the table with `requestedMatch`, `mismatches[]` and `effectiveMatch` (requested ∩ allowed, §3.2), and the client's retained push rejections. |
656
656
  | `scopeActivity(partition, {variable, value}, {limit?})` | Recent commits touching one scope key, via the §3.1 change-scope index (never a log scan). |
657
657
  | `horizonStatus(partition)` | `{maxCommitSeq, horizonSeq, retainedCommits, activeCursorFloor, recommendedHorizonSeq, recommendation}`: the horizon a prune pass would reach now (§4.6) and `up-to-date` / `prune-recommended`. |
658
658
  | `stats(partition)` | `{segments?, blobs?}` counts/bytes where the stores expose them (segments split rows/sqlite). |
@@ -959,12 +959,12 @@ and mints a fresh URL) — never a fall-through.
959
959
  `{ byteLength, mediaType? }`) mints a single presigned PUT; the client PUTs
960
960
  bytes straight to the object store. **Upload authz is host-authentication only**
961
961
  (any authenticated actor may obtain a grant within the size cap) — uploading
962
- bytes is not a scope-bearing act, because the content address discloses nothing
963
- and integrity is enforced at *reference time* (the §5.9.6 push existence check
964
- + every download's content-address verify), never by a store-side hash
965
- recompute. The **size cap is enforced up front** against the declared
966
- `byteLength`, before any URL is minted (the object-store hop cannot re-check the
967
- streamed byte count). An already-present blob returns `{ present: true }` (skip
962
+ bytes is not a scope-bearing act, because the content address discloses
963
+ nothing. The grant signs `content-length` (the declared `byteLength`) and
964
+ `x-amz-checksum-sha256` (the `blobId` digest as base64), and the client sends
965
+ that checksum header, derived from the `blobId` without hashing the body again,
966
+ so the object store refuses a PUT of another size or other bytes. The **size cap is enforced up front** against the declared
967
+ `byteLength`, before any URL is minted. An already-present blob returns `{ present: true }` (skip
968
968
  the PUT, idempotent §5.9.3). A **single PUT only** — never a multipart or chunk
969
969
  protocol; resumable upload, when it lands, is provider multipart behind this
970
970
  same grant. Absent config ⇒ the client streams through the direct
@@ -1006,8 +1006,11 @@ Nothing reclaims blobs automatically — the host schedules the sweep, the blob
1006
1006
  analogue of `pruneCommitLog`. `sweepOrphanBlobs(storage, blobStore, partition,
1007
1007
  { graceMs })` reads the live keep-set from the §5.9.4 reference index
1008
1008
  (`storage.listReferencedBlobIds`) and deletes every blob that is **both**
1009
- unreferenced **and** older than the grace period. It emits one `blob.swept`
1010
- ops event (`{ swept, referenced, graceMs }`) and returns the deleted ids.
1009
+ unreferenced **and** older than the grace period. Before each delete it reads
1010
+ the index again, so a reference that lands during the sweep keeps its blob. It
1011
+ emits one `blob.swept` ops event (`{ swept, referenced, graceMs }`, where
1012
+ `referenced` counts old candidates a row still references) and returns the
1013
+ deleted ids.
1011
1014
 
1012
1015
  ```ts
1013
1016
  import { sweepOrphanBlobs } from '@syncular/server';
@@ -1028,8 +1031,10 @@ arrived. So the grace period must comfortably exceed any sane upload→push
1028
1031
  latency — the **default is 24 h**, deliberately far above any push window.
1029
1032
  Lower it only if you fully understand your clients' outbox latency; there is
1030
1033
  no upside to a tight grace and a real data-loss risk. The sweep compares
1031
- against the blob's **upload time** (`createdAtMs` from object metadata), and an
1032
- idempotent re-upload does **not** reset that clock.
1034
+ against the blob's **latest upload time**: `createdAtMs` from object metadata,
1035
+ or the object's last-modified time for an object a presigned PUT wrote. An
1036
+ object with neither is skipped, and an idempotent re-upload rewrites the
1037
+ object with the re-upload time, which restarts its grace period.
1033
1038
 
1034
1039
  `sweepOrphanBlobs` **requires** `storage.listReferencedBlobIds` (the §5.9.4
1035
1040
  reference index — SQLite, D1, and Postgres all implement it). Against a
@@ -1058,11 +1063,10 @@ row-derived authorization check passes** (`handleBlobDownload` resolves the
1058
1063
  actor's scopes and tests the referencing rows first; a `blobId` is never a
1059
1064
  bearer capability minted from the id alone). The signed object key embeds the
1060
1065
  `blobId`, TTL SHOULD be ≤ 15 min (default 900 s), and the URL is a short-lived
1061
- grant to exactly those immutable bytes. The issued `url`/`urlExpiresAtMs` ride
1062
- additively on `BlobDownloadResult` alongside the bytes; the mounted §5.9.5
1063
- direct-download endpoint stays the default serving path. Client consumption of
1064
- the presigned URL (following it instead of streaming bytes through the sync
1065
- server) is a later rung — the server-side issuance ships now.
1066
+ grant to exactly those immutable bytes. When `blobSignedUrls` is set, the
1067
+ download answers `{ url, urlExpiresAtMs }` and no bytes, and the reference
1068
+ clients fetch the URL and verify the content address (see "Presigned blobs,
1069
+ end to end" above).
1066
1070
 
1067
1071
  ### Native HMAC vs delegated presign (§5.4)
1068
1072
 
@@ -1147,14 +1151,16 @@ regardless, and at least the newest `minRetainedCommits` (default 1000)
1147
1151
  commits are always kept. Raise the defaults freely, lower them with care.
1148
1152
 
1149
1153
  **What `sync.cursor_expired` means operationally.** A client whose
1150
- cursor fell behind the horizon gets `SUB_START.status = reset` and
1151
- re-bootstraps from scratch (§4.7). That is correct behavior, not an
1154
+ replica cursor fell behind the horizon gets `LIVE_START.status = reset`
1155
+ (a post-bootstrap unit behind it gets `SUB_START.status = reset`) and
1156
+ re-bootstraps the affected units from scratch (§4.6, §4.7). That is correct behavior, not an
1152
1157
  error — but its *rate* is your pruning health signal. A steady trickle
1153
1158
  means devices returning from >30-day absences (expected). A spike means
1154
1159
  you pruned faster than your fleet syncs: `ageForceMs` or
1155
1160
  `activeWindowMs` is too tight for real usage, and you are paying for it
1156
1161
  in bootstrap load (full re-scans + segment builds), not just in resets.
1157
- Observe it via `pull.served` subscriptions with `status: "reset"`.
1162
+ Observe it via `pull.served` events whose `live.status` or unit `status`
1163
+ is `"reset"`.
1158
1164
 
1159
1165
  **Segment TTL interplay.** Segments are cache entries, not durable state
1160
1166
  (§5.1; default TTL 24 h). A bootstrap that resumes past segment expiry
@@ -1174,7 +1180,7 @@ same table+scope during a storm, your TTL is shorter than the storm.
1174
1180
  (`push.conflicted` is normal offline-first traffic; alert only on
1175
1181
  gross shifts.)
1176
1182
  - `scopes.resolve_failed` — any nonzero rate. This is the fail-loud
1177
- path: every occurrence revokes subscriptions or rejects writes for a
1183
+ path: every occurrence revokes units or rejects writes for a
1178
1184
  real request, and it is almost always a host bug or a dead dependency
1179
1185
  of the resolver.
1180
1186
  - `request.handled` with `outcome: "error"` and `errorCode: "internal"`
@@ -1235,7 +1241,7 @@ function bunSqlExecutor(sql: import('bun').SQL): PgExecutor {
1235
1241
  const over = (h: any): PgQueryable => ({
1236
1242
  async query(text, params) {
1237
1243
  const rows = await h.unsafe(text, params ? [...params] : []);
1238
- return { rows, rowCount: rows.length };
1244
+ return { rows };
1239
1245
  },
1240
1246
  });
1241
1247
  return {
@@ -1283,6 +1289,11 @@ function pgPoolExecutor(pool: Pool): PgExecutor {
1283
1289
  }
1284
1290
  ```
1285
1291
 
1292
+ **Result shape.** `query` returns `{ rows }`. The storage never reads an
1293
+ affected-row count: every write whose outcome matters ends in `RETURNING`
1294
+ and counts the returned rows, because drivers disagree on the count of a
1295
+ statement without result rows.
1296
+
1286
1297
  **Type-parser note.** `commit_seq`/`server_version` are `int8`. Drivers
1287
1298
  decode `int8` differently (node-postgres → `string`, Bun.sql → `bigint`,
1288
1299
  pglite → `number`); the storage layer coerces every sequence read through
@@ -1367,9 +1378,9 @@ SYNCULAR_PG_URL=postgres://user:pass@localhost:5432/db bun run bench
1367
1378
  ```
1368
1379
 
1369
1380
  Custom storage adapters must implement
1370
- `getActiveClientCursorFloor(partition, cutoffMs)`. Return the minimum cursor
1371
- whose `updatedAtMs >= cutoffMs`, or `null` when no client qualifies. Preserve
1372
- negative bootstrap cursors. Pruning and admin horizon status use this scalar
1381
+ `getActiveClientCursorFloor(partition, cutoffMs)`. Return the minimum
1382
+ non-negative cursor whose `updatedAtMs >= cutoffMs`, or `null` when no client
1383
+ qualifies. A bootstrapping client at cursor -1 holds no cursor. Pruning and admin horizon status use this scalar
1373
1384
  aggregate; `listClientCursors` remains the explicit listing interface.
1374
1385
 
1375
1386
  SQLite image builders now return `Promise<Uint8Array>` and receive
@@ -1390,10 +1401,12 @@ rows. The image database and serialized output still consume memory.
1390
1401
  Realtime acknowledgements call `advanceClientCursor(partition, clientId,
1391
1402
  actorId, logEpoch, cursor, updatedAtMs)`. Custom storage adapters must implement
1392
1403
  this atomic update: advance the cursor and activity timestamp to their respective
1393
- maxima, preserve registration fields, and require a matching actor and current
1394
- partition log epoch. Leave missing records unchanged. SQLite, Postgres, and D1
1395
- perform one update without reading or serializing the subscription list. HTTP
1396
- registration keeps its existing cursor and subscription replacement rules.
1404
+ maxima, cap the cursor at the lowest non-negative cursor of the record's
1405
+ catching-up units, preserve registration fields, and require a matching actor and
1406
+ current partition log epoch. Leave missing records unchanged. SQLite, Postgres,
1407
+ and D1 perform one update without reading or serializing the registration list.
1408
+ A completed sync round writes the record through `putClientRecord`, which keeps
1409
+ the stored registration list unless the round's list traveled.
1397
1410
 
1398
1411
  ## Reviewed schema windows
1399
1412
 
package/dist/admin.d.ts CHANGED
@@ -11,7 +11,7 @@
11
11
  * thrown `Error`) when a backend lacks one, so a host wiring an unsupported
12
12
  * store learns immediately rather than getting a silently-empty console.
13
13
  */
14
- import { type ColumnType, type JsonValue, type PushOperationResult, type ScopeMap } from '@syncular/core';
14
+ import { type ColumnType, type JsonValue, type PushOperationResult, type ScopeMap, type UnitScopes } from '@syncular/core';
15
15
  import type { BlobStore, BlobStoreStats } from './blob-store.js';
16
16
  import { type ResolveScopes, type SyncServerConfig } from './context.js';
17
17
  import { type SyncularServerEvent, type SyncularServerEvents } from './events.js';
@@ -21,14 +21,22 @@ import { RealtimeHub } from './realtime.js';
21
21
  import { type RetentionPolicy } from './prune.js';
22
22
  import { type ServerSchema } from './schema.js';
23
23
  import type { SegmentStore, SegmentStoreStats } from './segment-store.js';
24
- import type { CheckpointDeclaration, CommitMetadata, ReactionStatus, StoredPushResult, RowChangeRecord, ScopeCommitActivity, ServerStorage, StoredReaction } from './storage.js';
24
+ import type { CheckpointDeclaration, CommitMetadata, PushWatermark, ReactionStatus, StoredPushResult, RowChangeRecord, ScopeCommitActivity, ServerStorage, StoredReaction } from './storage.js';
25
+ /** One registered unit of a client record. */
26
+ export interface AdminClientUnit {
27
+ /** The unit id (§4.1). */
28
+ readonly id: string;
29
+ readonly scopes: UnitScopes;
30
+ /** The unit's cursor while it catches up; absent for a live unit. */
31
+ readonly catchingUpCursor?: number;
32
+ }
25
33
  /** A connected/known client as the console sees it (§4.5, §8.1). */
26
34
  export interface AdminClient {
27
35
  readonly clientId: string;
28
36
  readonly actorId: string;
29
37
  /** SSP2 wire version last accepted from this client. */
30
38
  readonly wireVersion: number;
31
- /** Highest codec schema version among its subscriptions, when recorded. */
39
+ /** Schema version of its latest completed round, when recorded. */
32
40
  readonly schemaVersion?: number;
33
41
  readonly cursor: number;
34
42
  /**
@@ -37,11 +45,8 @@ export interface AdminClient {
37
45
  */
38
46
  readonly lag: number;
39
47
  readonly updatedAtMs: number;
40
- readonly subscriptions: readonly {
41
- readonly id: string;
42
- readonly table: string;
43
- readonly scopes: Record<string, readonly string[]>;
44
- }[];
48
+ /** The stored registration list (§4.1); empty when none is stored. */
49
+ readonly units: readonly AdminClientUnit[];
45
50
  /** True when the cursor record was touched within the active window. */
46
51
  readonly active: boolean;
47
52
  /**
@@ -211,27 +216,27 @@ export interface AdminCommitDetail extends CommitMetadata {
211
216
  * The trace verdict, evaluated in this order:
212
217
  * `client-unknown`: no client record for the id.
213
218
  * `row-missing`: no row and no retained change to it.
214
- * `no-subscription`: the client subscribes to nothing on the table.
219
+ * `no-unit`: no registered unit of the client covers the table.
215
220
  * `row-deleted`: the row's newest retained change is a delete the client
216
221
  * has pulled.
217
- * `scope-mismatch`: no subscription requests the row's stored scopes.
218
- * `scope-denied`: a subscription requests them, but the actor's allowed
222
+ * `scope-mismatch`: no covering unit requests the row's stored scopes.
223
+ * `scope-denied`: a covering unit requests them, but the actor's allowed
219
224
  * scopes (the host `resolveScopes`) exclude the row.
220
225
  * `pending-pull`: the client's cursor is below the row's newest change.
221
226
  * `delivered`: the client has pulled the row's newest change.
222
227
  */
223
- export type AdminTraceVerdict = 'client-unknown' | 'row-missing' | 'no-subscription' | 'row-deleted' | 'scope-mismatch' | 'scope-denied' | 'pending-pull' | 'delivered';
224
- /** One of the client's subscriptions on the traced table. */
225
- export interface AdminTraceSubscription {
228
+ export type AdminTraceVerdict = 'client-unknown' | 'row-missing' | 'no-unit' | 'row-deleted' | 'scope-mismatch' | 'scope-denied' | 'pending-pull' | 'delivered';
229
+ /** One of the client's units covering the traced table. */
230
+ export interface AdminTraceUnit {
226
231
  readonly id: string;
227
- readonly scopes: Record<string, readonly string[]>;
228
- /** The row's stored scopes satisfy the requested scopes. */
232
+ readonly scopes: UnitScopes;
233
+ /** The row's stored scopes carry the unit's values. */
229
234
  readonly requestedMatch: boolean;
230
- /** Each requested variable the row's stored value fails. */
235
+ /** Each unit variable the row's stored value fails. */
231
236
  readonly mismatches: readonly {
232
237
  readonly variable: string;
233
238
  readonly rowValue: string | null;
234
- readonly requested: readonly string[];
239
+ readonly requested: string;
235
240
  }[];
236
241
  /**
237
242
  * The row also satisfies requested ∩ allowed (§3.2). Absent when the
@@ -256,7 +261,7 @@ export interface AdminRowTrace {
256
261
  * when it returned the §7.3.3 outage sentinel. Absent without a resolver.
257
262
  */
258
263
  readonly allowed?: ScopeMap | 'outage';
259
- readonly subscriptions: readonly AdminTraceSubscription[];
264
+ readonly units: readonly AdminTraceUnit[];
260
265
  /** The client's retained push rejections and conflicts, newest first. */
261
266
  readonly rejections: readonly SyncularServerEvent[];
262
267
  }
@@ -323,6 +328,13 @@ export interface AdminWriteLookup {
323
328
  readonly clientCommitId: string;
324
329
  /** One outcome per client that sent this clientCommitId. */
325
330
  readonly outcomes: readonly AdminWriteOutcome[];
331
+ /**
332
+ * With no stored outcome: the §6.3.2 watermarks of the clients whose
333
+ * retained commit log carries this clientCommitId. A commit below an
334
+ * outbox's `throughSeq` lost its result to the watermark ("expired below
335
+ * sequence N").
336
+ */
337
+ readonly watermarks: readonly PushWatermark[];
326
338
  /** Retained ring events carrying this clientCommitId, newest first. */
327
339
  readonly events: readonly SyncularServerEvent[];
328
340
  }
@@ -464,7 +476,7 @@ export declare class SyncularAdmin {
464
476
  }): Promise<AdminRowInspection>;
465
477
  /**
466
478
  * Trace whether one client receives one row, combining the row's stored
467
- * scopes and newest change with the client's subscriptions, cursor, the
479
+ * scopes and newest change with the client's units, cursor, the
468
480
  * actor's allowed scopes, and its retained push failures. See
469
481
  * `AdminTraceVerdict` for the verdict order.
470
482
  */
package/dist/admin.js CHANGED
@@ -11,7 +11,7 @@
11
11
  * thrown `Error`) when a backend lacks one, so a host wiring an unsupported
12
12
  * store learns immediately rather than getting a silently-empty console.
13
13
  */
14
- import { bytesToBase64, decodeRow, renderRowValues, } from '@syncular/core';
14
+ import { bytesToBase64, decodeRow, renderRowValues, unitId, } from '@syncular/core';
15
15
  import { clockOf, RESOLVER_OUTAGE, } from './context.js';
16
16
  import { SyncError } from './errors.js';
17
17
  import { emitEvent, } from './events.js';
@@ -47,38 +47,39 @@ function required(value, what) {
47
47
  const VERDICT_ORDER = [
48
48
  'client-unknown',
49
49
  'row-missing',
50
- 'no-subscription',
50
+ 'no-unit',
51
51
  'row-deleted',
52
52
  'scope-mismatch',
53
53
  'scope-denied',
54
54
  'pending-pull',
55
55
  'delivered',
56
56
  ];
57
- /** One known client's subscriptions on the row's table and its verdict. */
58
- function traceVerdict(record, row, lastChange, horizonSeq, allowed) {
57
+ /** One known client's units covering the row's table and its verdict. */
58
+ function traceVerdict(declared, record, row, lastChange, horizonSeq, allowed) {
59
59
  const rowScopes = row.scopes ?? {};
60
- const subscriptions = record.subscriptions
61
- .filter((sub) => sub.table === row.table)
62
- .map((sub) => {
63
- const mismatches = Object.entries(sub.scopes)
64
- .filter(([variable, values]) => {
65
- const value = rowScopes[variable];
66
- return value === undefined || !values.includes(value);
67
- })
68
- .map(([variable, values]) => ({
60
+ // §4.1: a unit covers every table that declares all of its variables.
61
+ const units = (record.registration?.units ?? [])
62
+ .filter((scopes) => Object.keys(scopes).every((variable) => declared.has(variable)))
63
+ .map((scopes) => {
64
+ const mismatches = Object.entries(scopes)
65
+ .filter(([variable, value]) => rowScopes[variable] !== value)
66
+ .map(([variable, value]) => ({
69
67
  variable,
70
68
  rowValue: rowScopes[variable] ?? null,
71
- requested: values,
69
+ requested: value,
72
70
  }));
73
71
  const traced = {
74
- id: sub.id,
75
- scopes: sub.scopes,
72
+ id: unitId(scopes),
73
+ scopes,
76
74
  requestedMatch: mismatches.length === 0,
77
75
  mismatches,
78
76
  };
79
77
  if (allowed === undefined || allowed === 'outage')
80
78
  return traced;
81
- const effective = computeEffective(sub.scopes, { ok: true, allowed });
79
+ const effective = computeEffective(declared, Object.fromEntries(Object.entries(scopes).map(([variable, value]) => [
80
+ variable,
81
+ [value],
82
+ ])), { ok: true, allowed });
82
83
  return {
83
84
  ...traced,
84
85
  effectiveMatch: effective.status === 'active' &&
@@ -90,20 +91,20 @@ function traceVerdict(record, row, lastChange, horizonSeq, allowed) {
90
91
  const pulled = record.cursor >= (lastChange?.commitSeq ?? horizonSeq);
91
92
  const verdict = !row.exists && lastChange === undefined
92
93
  ? 'row-missing'
93
- : subscriptions.length === 0
94
- ? 'no-subscription'
94
+ : units.length === 0
95
+ ? 'no-unit'
95
96
  : !row.exists
96
97
  ? pulled
97
98
  ? 'row-deleted'
98
99
  : 'pending-pull'
99
- : !subscriptions.some((sub) => sub.requestedMatch)
100
+ : !units.some((unit) => unit.requestedMatch)
100
101
  ? 'scope-mismatch'
101
- : subscriptions.every((sub) => sub.effectiveMatch === false)
102
+ : units.every((unit) => unit.effectiveMatch === false)
102
103
  ? 'scope-denied'
103
104
  : pulled
104
105
  ? 'delivered'
105
106
  : 'pending-pull';
106
- return { subscriptions, verdict };
107
+ return { units, verdict };
107
108
  }
108
109
  /** The console query surface plus its operator actions. One per process. */
109
110
  export class SyncularAdmin {
@@ -319,7 +320,7 @@ export class SyncularAdmin {
319
320
  }
320
321
  /**
321
322
  * Trace whether one client receives one row, combining the row's stored
322
- * scopes and newest change with the client's subscriptions, cursor, the
323
+ * scopes and newest change with the client's units, cursor, the
323
324
  * actor's allowed scopes, and its retained push failures. See
324
325
  * `AdminTraceVerdict` for the verdict order.
325
326
  */
@@ -342,14 +343,14 @@ export class SyncularAdmin {
342
343
  ...(lastChange !== undefined ? { lastChange } : {}),
343
344
  };
344
345
  if (record === undefined) {
345
- return { ...base, verdict: 'client-unknown', subscriptions: [] };
346
+ return { ...base, verdict: 'client-unknown', units: [] };
346
347
  }
347
348
  const allowed = await this.#allowedScopes(partition, record);
348
349
  return {
349
350
  ...base,
350
351
  client: (await this.#clientView(partition))(record),
351
352
  ...(allowed !== undefined ? { allowed } : {}),
352
- ...traceVerdict(record, row, lastChange, horizonSeq, allowed),
353
+ ...traceVerdict(this.#requireSchema().tables.get(table)?.declaredVariables ?? new Set(), record, row, lastChange, horizonSeq, allowed),
353
354
  };
354
355
  }
355
356
  /**
@@ -364,6 +365,7 @@ export class SyncularAdmin {
364
365
  const lastChange = row.history[0];
365
366
  const horizonSeq = await this.#storage.getHorizonSeq(partition);
366
367
  const view = await this.#clientView(partition);
368
+ const declared = this.#requireSchema().tables.get(table)?.declaredVariables ?? new Set();
367
369
  const allowedByActor = new Map();
368
370
  const clients = [];
369
371
  for (const record of await list(partition)) {
@@ -372,7 +374,7 @@ export class SyncularAdmin {
372
374
  }
373
375
  clients.push({
374
376
  client: view(record),
375
- verdict: traceVerdict(record, row, lastChange, horizonSeq, allowedByActor.get(record.actorId)).verdict,
377
+ verdict: traceVerdict(declared, record, row, lastChange, horizonSeq, allowedByActor.get(record.actorId)).verdict,
376
378
  });
377
379
  }
378
380
  clients.sort((a, b) => VERDICT_ORDER.indexOf(a.verdict) - VERDICT_ORDER.indexOf(b.verdict) ||
@@ -499,7 +501,10 @@ export class SyncularAdmin {
499
501
  : entry),
500
502
  }));
501
503
  const events = this.events({ partition }).filter((event) => 'clientCommitId' in event && event.clientCommitId === clientCommitId);
502
- return { clientCommitId, outcomes, events };
504
+ const watermarks = outcomes.length > 0
505
+ ? []
506
+ : await required(this.#storage.listPushWatermarks?.bind(this.#storage), 'storage')(partition, clientCommitId);
507
+ return { clientCommitId, outcomes, watermarks, events };
503
508
  }
504
509
  /**
505
510
  * Resolve an id pasted from a bug report: clients and actors containing
@@ -899,28 +904,29 @@ export class SyncularAdmin {
899
904
  }
900
905
  }
901
906
  return (record) => {
902
- let schemaVersion;
903
- for (const sub of record.subscriptions) {
904
- if (sub.schemaVersion !== undefined &&
905
- (schemaVersion === undefined || sub.schemaVersion > schemaVersion)) {
906
- schemaVersion = sub.schemaVersion;
907
- }
908
- }
909
907
  const connectedSinceMs = connectedSince.get(record.clientId);
910
908
  return {
911
909
  clientId: record.clientId,
912
910
  actorId: record.actorId,
913
911
  wireVersion: record.wireVersion,
914
- ...(schemaVersion !== undefined ? { schemaVersion } : {}),
912
+ ...(record.schemaVersion !== undefined
913
+ ? { schemaVersion: record.schemaVersion }
914
+ : {}),
915
915
  cursor: record.cursor,
916
916
  // A never-pulled cursor (-1) has seen nothing: it lags the whole log.
917
917
  lag: Math.max(0, maxCommitSeq - Math.max(0, record.cursor)),
918
918
  updatedAtMs: record.updatedAtMs,
919
- subscriptions: record.subscriptions.map((sub) => ({
920
- id: sub.id,
921
- table: sub.table,
922
- scopes: sub.scopes,
923
- })),
919
+ units: (record.registration?.units ?? []).map((scopes) => {
920
+ const id = unitId(scopes);
921
+ const catchingUp = record.catchingUp.find((unit) => unit.id === id);
922
+ return {
923
+ id,
924
+ scopes,
925
+ ...(catchingUp !== undefined
926
+ ? { catchingUpCursor: catchingUp.cursor }
927
+ : {}),
928
+ };
929
+ }),
924
930
  active: record.updatedAtMs >= activeFloorMs,
925
931
  ...(sockets !== undefined
926
932
  ? { connected: connectedSinceMs !== undefined }
@@ -1,29 +1,20 @@
1
1
  import type { SyncRequestContext } from './context.js';
2
- export interface BlobUploadRequest {
3
- readonly blobId: string;
4
- readonly bytes: Uint8Array;
5
- /** Advisory MIME type from the upload `Content-Type`, if any. */
6
- readonly mediaType?: string;
7
- }
8
- export interface BlobDownloadResult {
9
- /**
10
- * The blob bytes, when served inline. **Absent** when the host configured
11
- * `blobSignedUrls` and a `url` was issued: the sync server exits the egress
12
- * path and the client fetches `url` directly (§5.9.5 always-issue).
13
- */
14
- readonly bytes?: Uint8Array;
2
+ /**
3
+ * A §5.9.5 download: the stored bytes as a stream with their length (inline
4
+ * delivery), or, when the host configured `blobSignedUrls`, a provider-signed
5
+ * GET URL issued after the row-derived authorization check. The client
6
+ * fetches a URL directly (no host auth), verifies the content address, and on
7
+ * failure re-requests this endpoint (§5.9.5 recovery rule).
8
+ */
9
+ export type BlobDownloadResult = {
15
10
  readonly headers: Record<string, string>;
16
- /**
17
- * §5.9.5 delegated presign (always-issue): a provider-signed GET URL for
18
- * the bytes, issued only after the row-derived authorization check passed.
19
- * Present iff the host configured `blobSignedUrls`; when present, `bytes` is
20
- * absent and the client MUST fetch the URL directly (no host auth), verify
21
- * the content address, and on failure re-request this endpoint — never fall
22
- * through (§5.9.5 recovery rule).
23
- */
24
- readonly url?: string;
25
- readonly urlExpiresAtMs?: number;
26
- }
11
+ readonly body: ReadableStream<Uint8Array<ArrayBuffer>>;
12
+ readonly byteLength: number;
13
+ } | {
14
+ readonly headers: Record<string, string>;
15
+ readonly url: string;
16
+ readonly urlExpiresAtMs: number;
17
+ };
27
18
  /** `POST <mount>/blobs/{blobId}/upload-grant` request body (§5.9.3). */
28
19
  export interface BlobUploadGrantRequest {
29
20
  readonly blobId: string;
@@ -45,8 +36,13 @@ export interface BlobUploadGrantResult {
45
36
  readonly urlExpiresAtMs?: number;
46
37
  readonly present?: boolean;
47
38
  }
48
- /** `PUT <mount>/blobs/{blobId}` (§5.9.3). Host auth is the adapter's job. */
49
- export declare function handleBlobUpload(ctx: SyncRequestContext, request: BlobUploadRequest): Promise<void>;
39
+ /**
40
+ * `PUT <mount>/blobs/{blobId}` (§5.9.3). Host auth is the adapter's job. The
41
+ * size cap applies to the declared `Content-Length` before any byte is read
42
+ * and again to the counted decoded bytes while reading, so an oversized body
43
+ * is never buffered whole.
44
+ */
45
+ export declare function handleBlobUpload(ctx: SyncRequestContext, blobId: string, request: Request): Promise<void>;
50
46
  /**
51
47
  * `GET <mount>/blobs/{blobId}` (§5.9.5). Authorization derives from the
52
48
  * referencing rows: the actor may download iff at least one row in the
@@ -58,11 +54,9 @@ export declare function handleBlobDownload(ctx: SyncRequestContext, blobId: stri
58
54
  * auth is the adapter's job — uploading is host-auth-only, not scope-bearing,
59
55
  * so any authenticated actor may obtain a grant within the size cap (§5.9.3).
60
56
  *
61
- * The size cap is enforced HERE, up front, against the declared `byteLength`
62
- * (the object-store hop cannot re-check the streamed byte count). Integrity is
63
- * NOT checked here: the object store places bytes at the content-addressed key,
64
- * and the address is verified at reference time (§5.9.6 push existence) and on
65
- * every download (§5.9.5/§5.1). Returns a presigned single PUT, or a
57
+ * The size cap is enforced HERE, up front, against the declared `byteLength`;
58
+ * the presigned PUT binds that length and the content address, so the object
59
+ * store rejects any other bytes (§5.9.3). Returns a presigned single PUT, or a
66
60
  * `present` marker if the blob already exists (idempotent §5.9.3), or an empty
67
61
  * result when no presigned-upload store is configured (client streams direct).
68
62
  */