@ultimat3/realtime 21.0.0 → 22.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. package/CLAUDE.md +302 -1009
  2. package/README.md +130 -26
  3. package/package.json +4 -4
  4. package/src/changefeed.ts +7 -1
  5. package/src/channel-authz.ts +23 -4
  6. package/src/channel-decl.ts +16 -5
  7. package/src/channel-describe.ts +7 -5
  8. package/src/channel-logs.ts +19 -1
  9. package/src/channel-records.ts +8 -0
  10. package/src/client-channels.ts +75 -5
  11. package/src/client.ts +14 -2
  12. package/src/cursor.ts +5 -0
  13. package/src/errors.ts +43 -1
  14. package/src/idb-fake.ts +24 -4
  15. package/src/idb-types.ts +7 -0
  16. package/src/index.ts +1 -1
  17. package/src/live-definition.ts +5 -1
  18. package/src/live-fanout.ts +51 -2
  19. package/src/live-query.ts +11 -0
  20. package/src/live-replicator.ts +160 -0
  21. package/src/local-store-idb.ts +89 -15
  22. package/src/matcher-bridge.ts +5 -0
  23. package/src/nats-fake.ts +10 -1
  24. package/src/nats-jetstream.ts +36 -14
  25. package/src/nats-transport.ts +2 -2
  26. package/src/offline-queue.ts +76 -21
  27. package/src/page-outbox.ts +80 -10
  28. package/src/page-socket.ts +39 -8
  29. package/src/pg-entity-row.ts +37 -184
  30. package/src/pg-identifier.ts +23 -0
  31. package/src/pg-preflight.ts +32 -45
  32. package/src/pg-publication.ts +95 -0
  33. package/src/pg-replication.ts +21 -7
  34. package/src/pg-socket.ts +139 -53
  35. package/src/pg-tls.ts +124 -0
  36. package/src/pg-wire.ts +65 -16
  37. package/src/policy-fake.ts +14 -0
  38. package/src/query-window.ts +35 -21
  39. package/src/replication-errors.ts +29 -16
  40. package/src/replicator.ts +13 -3
  41. package/src/server.ts +8 -3
  42. package/src/socket-drops.ts +30 -0
  43. package/src/socket-engine.ts +15 -3
  44. package/src/socket-host.ts +103 -4
  45. package/src/socket-idle.ts +21 -0
  46. package/src/socket.ts +41 -38
  47. package/src/subscriber-gate.ts +92 -3
  48. package/src/sync-node-contract.ts +6 -0
  49. package/src/sync-node.ts +3 -7
  50. package/src/sync-origin.ts +33 -0
  51. package/src/sync-upgrade.ts +33 -9
  52. package/src/thundering-herd.ts +21 -11
  53. package/src/transport-env.ts +55 -14
  54. package/src/use-mutation.ts +13 -0
  55. package/src/use-query.ts +10 -5
package/README.md CHANGED
@@ -69,22 +69,53 @@ class — stay on `.`.
69
69
 
70
70
  | Concern | Entry | Export |
71
71
  |---|---|---|
72
- | tier 1 | `./server` | `topic`, `ChannelHub`, `PresenceRegistry`, `SyncSocket`, `SocketRegistry` |
72
+ | tier 1 | `./server` | `ChannelHub`, `PresenceRegistry`, `SyncSocket`, `SocketRegistry` |
73
73
  | tier 2 | `./server` | `LiveQueryRegistry`, `InMemoryChangeFeed`, `PgLogicalReplicationFeed`, `selectChangeFeed`, `createReplicator`, `PgAdvisoryLock`, `matcherFor` |
74
74
  | replication | `./server` | `parsePgUrl`, `bunPgStream`, `PgOutputDecoder`, `entityRow`, `changeLsn`, `commitPositionOf` |
75
+ | the in-process change source | `./server` | `startLiveReplicator` — a repository's own writes as `ChangeEvent`s, for the embedded database `x dev` runs on (PGlite has no walsender). Moved from `@ultimat3/testing`, which no longer re-exports it |
75
76
  | fanout | `./server` | `Transport`, `InProcessTransport`, `NatsTransport`, `selectTransport`, `subjectMatches` |
76
77
  | the bus, behind `NatsTransport` | `./server` | the port — `NatsClient`, `NatsMessage`, `NatsSubscription`, `NatsConnect`, `NatsTarget`, `parseNatsUrl` — plus `openNatsClient` (the `nats` adapter), `NatsKvSet`, `ensureKvBucket`, `kvGet`/`kvLast`/`kvWrite`, `assertBucket`, `encodeToken`/`decodeToken`, and `FakeNatsBroker`/`fakeNatsConnect` for tests |
77
- | reconnect | both | `LiveCursor`, `resumeFrom`, `shouldResnapshot`, `defaultReconnectBudget`, `backoffDelay`, `Scheduler`, `timeoutScheduler` on `.`; `RingChangeBuffer`, `drainPlan`, `AcceptBudget`, `reconnectFrame` on `./server` — the node's half of the reconnect is the node's |
78
+ | reconnect | both | `LiveCursor`, `resumeFrom`, `shouldResnapshot`, `defaultReconnectBudget`, `Scheduler`, `timeoutScheduler` on `.`; `RingChangeBuffer`, `drainPlan`, `AcceptBudget`, `reconnectFrame` on `./server` — the node's half of the reconnect is the node's |
78
79
  | the page's record store | `.` | `RecordStore` — one record per `type:key`, synced truth plus the optimistic overlay — `recordKey`, `LocalTx`, `RowWindows`, `applyPatches`/`orderAfterPatches` |
79
80
  | the outbox | `.` | `OfflineQueue`, `MemoryQueueStore` — replayed over HTTP from plan 101 slice 12. The conflict vocabulary is `ConflictPolicy` from `@ultimat3/core`; realtime declares none |
80
81
  | wire | `.` | `PROTOCOL_VERSION` (3), `encode`, `decode`, `Frame` |
81
82
  | the node | `./server` | `createSyncNode` / `listenSyncNode` (`sync` role) |
82
83
  | a socket's identity | `./server` | `SyncAuthenticator`, `SyncGrant`, `GrantBook`, `sweepGrants`, `DEFAULT_REAUTH_INTERVAL_MS` |
83
84
  | hooks | `.` | `useQuery`, `useRecord`, `useMutation`, `useMutationQueue`, `useConnection`, `useChannel`, `usePresence`, `hasPageSocket`, `installRealtime` |
84
- | channels | `.` | `channel`, `topic`, `readPresence`, the channel frame types |
85
+ | channels | `.` | `channel`, `channelRef`, `ChannelHandle`, `topic`, `readPresence`, the channel frame types |
85
86
  | offline | `.` | `pageOutbox`, `recordPersister`, `persistedTypes`, `openLocalStore`, `pageLocalStore`, `MemoryLocalStore` |
86
87
  | the socket's worker | `./sync-worker` | the SharedWorker entry — no exports |
87
88
 
89
+ ## `channelRef` — a channel an island can hold
90
+
91
+ A channel has two halves, and they live in two files so a browser chunk never bundles an entity or
92
+ a policy. `channelRef(name, { params, catchUp })` is the **client** half: the name, the ordered
93
+ params and the catch-up read — nothing else. The server declares the records and who may join on
94
+ **the same ref** with `channel(ref, { … })`, so the name and the params are written once.
95
+
96
+ ```ts
97
+ // app/posts/channel-ref.ts — imported by islands
98
+ import { channelRef } from '@ultimat3/realtime';
99
+
100
+ export const ORG_POSTS = channelRef('org-posts', {
101
+ params: ['orgId'],
102
+ catchUp: { name: 'orgPosts' }, // the query a client re-runs on `replay-gap` or a new epoch
103
+ });
104
+
105
+ // The one topic spelling both halves use.
106
+ ORG_POSTS.topic({ orgId: 'org_1' }); // 'org-posts.org_1'
107
+ ```
108
+
109
+ The server half, in its own file, names the entity and the policy on the same ref —
110
+ `channel(ORG_POSTS, { records: [posts], policy: feedRead })` — and an island subscribes with
111
+ `useChannel(ORG_POSTS, { orgId })`.
112
+
113
+ `ref.topic(params)` is the one spelling of the topic both halves use (`org-posts.<orgId>`), and
114
+ `bun run channel-literals` refuses a hand-built topic anywhere else. A bad name or a repeated param
115
+ is `X_CHANNEL_DECLARATION_INVALID` at declaration. `catchUp` is read on every access, so a query
116
+ whose name `registerQueries()` stamps at boot is picked up. The reference app's
117
+ [`app/posts/channel-ref.ts`](../../examples/dummy/apps/web/app/posts/channel-ref.ts) is the idiom.
118
+
88
119
  ## The hooks
89
120
 
90
121
  One record store and one socket per **page**, on `globalThis`: every island is its own bundle, so a
@@ -285,10 +316,26 @@ never stumbled into.
285
316
  On drain, `drainPlan()` gives every client its own jittered slot in a spread window and the node
286
317
  sends a `reconnect` frame carrying that delay — clients redistribute instead of stampeding.
287
318
  `AcceptBudget` is the receiving node's token bucket, and a refusal always carries a retry delay,
288
- because refusing without one just moves the herd next door.
319
+ because refusing without one just moves the herd next door. **A token is reserved before `authenticate`
320
+ and refunded on every exit that takes no socket** (22.1.0): a 401, an authenticator that throws, a
321
+ shed after it, an upgrade that did not take. Reserved first, so a reconnect herd reaches the token
322
+ service bounded by the burst; refunded, because spent-and-kept, one client dialling with no
323
+ credential drained the bucket and every signed-in reconnect behind it was shed. Not per client IP:
324
+ behind an ingress every dial has the ingress's address, and a forwarded header is the caller's own
325
+ claim.
326
+
327
+ **A socket from a foreign page is refused** — `403 X_SOCKET_ORIGIN_REFUSED`, before `authenticate`
328
+ and before the budget. A websocket carries the session cookie and no CORS applies to it, so a page on
329
+ a sibling host (same-site, which `SameSite=Lax` does not stop) could open one as its visitor. The
330
+ rule is `@ultimat3/core`'s `proveSameOrigin`, the one `@ultimat3/http`'s CSRF check asks, with two
331
+ admissions of the node's own (`sync-origin.ts`): no `Origin` header (RFC 6455 has every browser send
332
+ one, so its absence is not a browser), and the node's own host name at any port or scheme (cookies
333
+ are not port-isolated, and the Compose rung serves the page on `:3000` and the node on `:3001`). A
334
+ page on another host is admitted by `createSyncNode({ allowedOrigins })` — the CLI passes
335
+ `APP_URL`'s origin.
289
336
 
290
337
  The client dials itself back. A closed socket arms one timer — the node's delay when a `reconnect`
291
- frame assigned one, otherwise `backoffDelay()` — and that timer calls `connect()`, which re-subscribes
338
+ frame assigned one, otherwise `@ultimat3/core`'s `backoffDelay()` on the client's `BackoffPolicy` — and that timer calls `connect()`, which re-subscribes
292
339
  every registration **and re-announces every topic**. Topic membership is state on the node's socket
293
340
  and `hello` carries none of it, so without that half a channel goes silent from the first reconnect
294
341
  onwards while its handler is still installed — and its presence membership is swept, because
@@ -337,7 +384,8 @@ wire twice by a reconnect that raced an ack.
337
384
  | Backpressure **declines**, it does not fail | over `MAX_BUFFERED_BYTES` (1 MiB, the node's `backpressureLimit` at the other end of the same socket) the sender throws `X_TRANSPORT_UNAVAILABLE`, the mutation stays pending and the next drain resumes there. `ClientSocket.bufferedAmount` is optional; a socket that does not report it is treated as never backed up |
338
385
  | Delivery is therefore at least once | every mutation carries an idempotency key — the `key` argument, or `<mutator>:<uuid>` — and the resend carries the same one |
339
386
  | A lost connection **cancels the pass it interrupted** | the lane orders passes against each other, but a socket death is not a pass and cannot reach one parked inside `send`. `requeueInflight()` bumps a connection epoch; a pass whose epoch went stale returns and leaves the rest `pending`. Without it the parked pass resumed and marked everything behind it `inflight` for a dead socket — never re-sent (`inflight` is not sendable) and never acked |
340
- | The store is handed a **snapshot**, never the live entries | `QueueStore.save` is a durable write and may await before it reads; given the array itself it persists a status that was never true when it was called |
387
+ | The store is handed **snapshots, by key**, never the live entries or the whole queue | `QueueStore.write` is a durable write and may await before it reads; given an entry itself it persists a status that was never true when it was called. By key, because two tabs of one user share the store and a whole-queue save let the last tab erase the other's write |
388
+ | One tab drains at a time, and drains what every tab queued | the outbox's replay runs under the Web Lock `ultimate-outbox:<principal>` and re-reads the queue first, so a write another tab queued is sent, in order, and a stored `inflight` from a closed page goes back to `pending` |
341
389
 
342
390
  ### Limits, stated plainly
343
391
 
@@ -474,7 +522,11 @@ wire twice by a reconnect that raced an ack.
474
522
  (SCRAM-SHA-256, in-band TLS, CopyBoth), no driver dependency. It preflights `wal_level`, the
475
523
  publication, every entity's replica identity and the slot — in that order, because the identity
476
524
  check is worthless once the slot exists — creates the slot when there is none, and confirms the
477
- slot as it goes so the WAL does not grow without bound. `InMemoryChangeFeed` + `InProcessTransport` remain the
525
+ slot as it goes so the WAL does not grow without bound. **The publication is ensured, not merely
526
+ checked** (`pg-publication.ts`): missing, it is created `FOR TABLE` every entity table; present,
527
+ it gains the entity tables it lacks (`ALTER PUBLICATION … ADD TABLE`) and loses none. `FOR TABLE`
528
+ because a table's owner may publish it without a superuser; a role that may not is refused with
529
+ `X_REPLICATION_FAILED` and the statement to run as one that may. `InMemoryChangeFeed` + `InProcessTransport` remain the
478
530
  defaults for `x dev` and every test.
479
531
  - **`selectChangeFeed(env, { entities })` decides which feed a boot installs** — same law
480
532
  `selectMailDriver` follows: an unset variable means the embedded default. It returns `{ feed,
@@ -487,15 +539,38 @@ wire twice by a reconnect that raced an ack.
487
539
  wrong database's WAL would be silently wrong forever. `REPLICATION_SLOT` (default `x_replicator`)
488
540
  and `REPLICATION_PUBLICATION` (default `x_changes`) name the slot and publication, both checked
489
541
  against `[a-z_][a-z0-9_]*` before they reach a replication command.
542
+ - **TLS follows libpq's `sslmode`** (`pg-tls.ts`, 22.1.0): `disable`; `allow`, `prefer` (the
543
+ default) and `require` encrypt and verify **nothing**; `verify-ca` checks the chain;
544
+ `verify-full` the chain and the host name. `sslrootcert=<path>` is the only trust anchor when
545
+ set (and turns `require` into `verify-ca`, as libpq does); `sslrootcert=system` means the runtime
546
+ store and `verify-full`. With neither, the runtime store is used — it honours
547
+ `NODE_EXTRA_CA_CERTS`; libpq's `~/.postgresql/root.crt` is never read. `allow` is served as
548
+ `prefer` (TLS offered first), never cleartext first. The runtime never rejects on its own
549
+ (`rejectUnauthorized: false`); the handshake's report is judged per mode, and a failure is
550
+ `X_REPLICATION_TLS` naming the check. Until 22.1.0 `prefer` verified — every private-CA server
551
+ (CNPG) failed as a refused write — and the raw socket kept feeding ciphertext to the reader
552
+ after the upgrade, so a trusted CA still hung the stream.
553
+ - **`REPLICATION` is a cluster-wide grant.** A `replication=database` session may also run
554
+ `BASE_BACKUP` and `START_REPLICATION PHYSICAL` with no database check, so on a shared Postgres
555
+ cluster the role could copy every database, `pg_authid` included, drop other slots and exhaust
556
+ the walsenders. Run the replicator against a cluster dedicated to the app, or give it its own role
557
+ through `REPLICATION_URL` with `pg_hba.conf`'s `replication` lines restricted to it — never grant
558
+ `REPLICATION` to an app role on a shared cluster. Every fix line that hands the grant over says
559
+ so (`REPLICATION_GRANT_WARNING`, `pg-wire.ts`) and links
560
+ [`docs/ops/01-kubernetes.md`](../../docs/ops/01-kubernetes.md#replication-is-a-cluster-wide-grant).
490
561
  - **`PgAdvisoryLock` is the production `AdvisoryLock`** — `SELECT
491
562
  pg_try_advisory_lock(hashtext('x:replicator:<slot>'))` on its own session. Session-scoped, so a
492
563
  crashed replicator releases it automatically: no lease renewal, no fencing token, no split brain.
493
564
  `InMemoryAdvisoryLock` remains the single-process default for `x dev` and tests.
494
- - **`selectTransport(env)` decides which transport a boot fans out on** — the same law again, and
495
- the only place that reads `NATS_URL`. It returns `{ transport, mode, detail, bucket,
496
- presenceTtlMs, connect }`: unset → `InProcessTransport` and `mode: 'embedded'`, set → a
497
- `NatsTransport` on the KV bucket `NATS_KV_BUCKET` names (default `x_presence`, so two apps on one
498
- cluster do not share one presence namespace), validated here rather than on first connect.
565
+ - **`selectTransport(env, realtime)` decides which transport a boot fans out on** — and since
566
+ 22.0.0 the CONFIG decides it, not the environment: `realtime` is `app.config.ts`'s
567
+ `{ transport, urlEnv }`. It returns `{ transport, mode, detail, bucket, presenceTtlMs, connect }`:
568
+ `'memory'` → `InProcessTransport` and `mode: 'embedded'`; `'nats'` → a `NatsTransport` dialling
569
+ the variable `urlEnv` names, on the KV bucket `NATS_KV_BUCKET` names (default `x_presence`, so
570
+ two apps on one cluster do not share one presence namespace), validated here rather than on
571
+ first connect. Both mismatches refuse with `X_CONFIG_INVALID`: `'nats'` with that variable unset,
572
+ and `'memory'` with `NATS_URL` (or the named variable) set — an operator who set one expected
573
+ fanout across nodes. Until 22.0.0 `NATS_URL` alone decided and both keys were read by nothing.
499
574
  `presenceTtlMs` comes back with it because the bucket's whole-stream age limit was derived from
500
575
  it — a `PresenceRegistry` given a different number would report members leaving that never left.
501
576
  Selection is pure; `connect()` is the dial, so an unreachable bus fails at boot.
@@ -517,18 +592,17 @@ wire twice by a reconnect that raced an ack.
517
592
  *transactions* in commit order, so per-record WAL positions are not monotonic across them. The
518
593
  pair sorts in delivery order and is byte-identical on replay, which is what turns at-least-once
519
594
  redelivery into a drop instead of a duplicate.
520
- - **A live query needs `REPLICA IDENTITY FULL`, and the replicator now says so** (`As of
521
- 2026-08-19`). Deciding whether a row *left* a result set needs the old values; with the default
522
- identity a delete replicates only the key columns, and `toRow` accepts that tuple because it only
523
- requires a text `id`. `preflight` asks `pg_class.relreplident` for every entity in the list — the
524
- fourth question it asks, and **before** `pg_create_logical_replication_slot`, since changing the
525
- identity after a slot exists does not reach the rows that slot will decode. It is a **coded
526
- warning**, `X_LIVE_REPLICA_IDENTITY`, whose `fix:` is the `ALTER TABLE <t> REPLICA IDENTITY FULL;`
527
- per named table — not a throw, because every app on the default identity would otherwise stop
528
- booting, which is worse than the partial rows. `ReplicationStreamStats.partialBefore` is the
529
- running half: one per change delivered off a relation that is not FULL, so the decisions it
530
- actually cost are countable rather than silent. A hard refusal at `x verify` time is the
531
- follow-up.
595
+ - **A keyed table does not need `REPLICA IDENTITY FULL`; a table with NO identity is warned**
596
+ (`As of 2026-09-23`). Under DEFAULT an update carries no old tuple and a delete only the key, and
597
+ neither decides a live query: the shared window holds the whole row, so a row leaving the result
598
+ set is decided from the window and a delete by `holds(id)` — proved on real WAL by
599
+ `pg-identity-window.live.test.ts` (out of the filter, into it, delete, a row never held).
600
+ `preflight`'s fourth question, **before** `pg_create_logical_replication_slot`, names only the
601
+ entity tables with no identity (`NOTHING`, or `DEFAULT` with no primary key), whose `UPDATE` and
602
+ `DELETE` Postgres refuses once published: a **coded warning**, `X_LIVE_REPLICA_IDENTITY`, fix
603
+ `ALTER TABLE <t> REPLICA IDENTITY FULL;` per table — not a throw. Until 22.1.0 it named every
604
+ table not on FULL, on every boot. `ReplicationStreamStats.partialBefore` still counts changes off
605
+ a non-FULL relation — a volume figure, not a correctness one.
532
606
  - The record store is **per page**, in memory, and it is not a query cache: it answers "what is
533
607
  record X now", never "have I run this query before". Nothing evicts by time or size — a record
534
608
  lives as long as something holds it. Persisting it (IndexedDB) is plan 101 slice 12.
@@ -548,8 +622,9 @@ wire twice by a reconnect that raced an ack.
548
622
  `X_NOT_IMPLEMENTED` and `X_TIMEOUT` are **borrowed** from `@ultimat3/core`, which owns and titles
549
623
  them — `REALTIME_BORROWED_ERROR_CODES`. Everything else on that list is realtime's own.
550
624
 
551
- Topics deny by default: a topic with no matching guard is forbidden. An authz hole is not a config
552
- option someone forgot to set.
625
+ Topics deny by default: a topic no `channel()` declares is forbidden, and a `channel()` with no
626
+ `policy` is refused at declaration (`X_CHANNEL_DECLARATION_INVALID`) — a public channel writes
627
+ `policy: allow('public')`. An authz hole is not a config option someone forgot to set.
553
628
 
554
629
  An upgrade `authenticate` refuses is `X_SOCKET_UNAUTHENTICATED` (401) and one it *could not decide*
555
630
  is `X_SOCKET_AUTH_UNAVAILABLE` (503). Two codes, because the two have opposite instructions: the
@@ -568,3 +643,32 @@ The fix is `x queries list --json`, and the name the client sent is echoed back
568
643
  never is.
569
644
 
570
645
  `As of 2026-07`: tiers 1–2 target v1, tier 3 targets v2.
646
+
647
+ ### Error classes
648
+
649
+ Every error class `src/index.ts` exports, for `instanceof` inside one process. Across a wire or
650
+ a job boundary the class is gone and the `code` is what survives — match on that.
651
+
652
+ | Class | Code | Declared in |
653
+ |---|---|---|
654
+ | `CursorStaleError` (extends `RealtimeError`) | `X_CURSOR_STALE` | `src/page-errors.ts` |
655
+ | `FrameRateLimitError` (extends `RealtimeError`) | `X_FRAME_RATE_LIMIT` | `src/errors.ts` |
656
+ | `LiveQueryUnknownError` (extends `RealtimeError`) | `X_LIVE_QUERY_UNKNOWN` | `src/errors.ts` |
657
+ | `LiveRowUnidentifiedError` (extends `RealtimeError`) | `X_LIVE_ROW_UNIDENTIFIED` | `src/errors.ts` |
658
+ | `NotImplementedError` (extends `RealtimeError`) | `X_NOT_IMPLEMENTED` | `src/errors.ts` |
659
+ | `ProtocolVersionError` (extends `RealtimeError`) | `X_PROTOCOL_VERSION` | `src/page-errors.ts` |
660
+ | `RealtimeError` | any `RealtimeErrorCode` — `REALTIME_ERROR_CODES`; the base of every other class here, thrown directly for a code none of them covers | `src/realtime-error.ts` |
661
+ | `RealtimeUninstalledError` (extends `RealtimeError`) | `X_REALTIME_UNINSTALLED` | `src/page-errors.ts` |
662
+ | `RebaseConflictError` (extends `RealtimeError`) | `X_REBASE_CONFLICT` | `src/page-errors.ts` |
663
+ | `RecordRejectedError` (extends `RealtimeError`) | `X_RECORD_REJECTED` | `src/page-errors.ts` |
664
+ | `ReplicaIdentityError` (extends `RealtimeError`) | `X_LIVE_REPLICA_IDENTITY` | `src/replication-errors.ts` |
665
+ | `ReplicationFailedError` (extends `RealtimeError`) | `X_REPLICATION_FAILED` | `src/replication-errors.ts` |
666
+ | `ReplicationProtocolError` (extends `RealtimeError`) | `X_REPLICATION_PROTOCOL` | `src/replication-errors.ts` |
667
+ | `ReplicatorSlotHeldError` (extends `RealtimeError`) | `X_REPLICATOR_SLOT_HELD` | `src/replication-errors.ts` |
668
+ | `ServerRenderLiveError` (extends `RealtimeError`) | `X_LIVE_SERVER_RENDER` | `src/page-errors.ts` |
669
+ | `SubscriptionLimitError` (extends `RealtimeError`) | `X_SUBSCRIPTION_LIMIT` | `src/errors.ts` |
670
+ | `SyncUnconfiguredError` (extends `RealtimeError`) | `X_SYNC_UNCONFIGURED` | `src/page-errors.ts` |
671
+ | `TopicForbiddenError` (extends `RealtimeError`) | `X_TOPIC_FORBIDDEN` | `src/errors.ts` |
672
+ | `TransportProtocolError` (extends `RealtimeError`) | `X_TRANSPORT_PROTOCOL` | `src/errors.ts` |
673
+ | `TransportUnavailableError` (extends `RealtimeError`) | `X_TRANSPORT_UNAVAILABLE` | `src/errors.ts` |
674
+ | `WindowReadTimeoutError` (extends `RealtimeError`) | `X_TIMEOUT` | `src/errors.ts` |
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@ultimat3/realtime",
3
- "version": "21.0.0",
3
+ "version": "22.1.0",
4
4
  "description": "Three-tier realtime: channels, live queries, local-first sync — one protocol, one mutator shape",
5
5
  "license": "MIT",
6
6
  "type": "module",
@@ -38,9 +38,9 @@
38
38
  "test": "bun test"
39
39
  },
40
40
  "dependencies": {
41
- "@ultimat3/core": "21.0.0",
42
- "@ultimat3/entity": "21.0.0",
43
- "@ultimat3/query": "21.0.0",
41
+ "@ultimat3/core": "22.1.0",
42
+ "@ultimat3/entity": "22.1.0",
43
+ "@ultimat3/query": "22.1.0",
44
44
  "nats": "2.29.3"
45
45
  }
46
46
  }
package/src/changefeed.ts CHANGED
@@ -11,7 +11,13 @@ import type { PgTarget } from './pg-socket';
11
11
  import type { PgStream } from './pg-wire';
12
12
  import type { Rng } from './thundering-herd';
13
13
 
14
- export type ChangeOp = 'insert' | 'update' | 'delete';
14
+ /**
15
+ * `truncate` carries no row — `before` and `after` are both `null` — and means every row of the
16
+ * relation is gone. A window cannot be patched from it, only re-read; a channel's members are told
17
+ * to re-read (`replay-gap`). It was decoded and dropped, so with the recommended `FOR ALL TABLES`
18
+ * publication every window and every client kept the truncated rows forever.
19
+ */
20
+ export type ChangeOp = 'insert' | 'update' | 'delete' | 'truncate';
15
21
 
16
22
  export interface ChangeEvent<R extends Row = Row> {
17
23
  /** Entity name, not table name — the matcher's dependency sets are declared in entity terms. */
@@ -2,7 +2,7 @@
2
2
  // the channel's own loader answered. Through `@ultimat3/query`'s `guard`, the package's one authz
3
3
  // seam — the same call `policy-gate.ts` makes for a live query.
4
4
 
5
- import type { Actor, Ctx } from '@ultimat3/core';
5
+ import { type Actor, type Ctx, createContext, runWithContext } from '@ultimat3/core';
6
6
  import { guard, QueryDeniedError } from '@ultimat3/query';
7
7
  import type { Channel } from './channel-decl';
8
8
  import { TopicForbiddenError } from './errors';
@@ -18,10 +18,17 @@ export async function authorizeChannel(
18
18
  topic: string,
19
19
  params: Readonly<Record<string, string>>,
20
20
  ): Promise<void> {
21
- if (channel.policy === undefined) return;
22
- const row = channel.row === undefined ? null : await channel.row({ params, ctx });
21
+ // The loader and the rule run AS the subscriber, never as the node. Under the node's context — no
22
+ // actor, no tenant — a repository read inside the loader was scoped by nothing but what the
23
+ // loader happened to name, and `@ultimat3/entity`'s tenancy seam had no actor to hold it to.
24
+ // Services are rebuilt for this actor by `createContext`, the rule `withChildContext` follows.
25
+ const scoped = actor === null ? ctx : subscriberContext(ctx, actor);
26
+ const row =
27
+ channel.row === undefined
28
+ ? null
29
+ : await runWithContext(scoped, async () => await channel.row?.({ params, ctx: scoped }));
23
30
  try {
24
- guard(channel.policy, { actor, input: params, row, ctx, query: channel.name }, 'live');
31
+ guard(channel.policy, { actor, input: params, row, ctx: scoped, query: channel.name }, 'live');
25
32
  } catch (error) {
26
33
  if (!(error instanceof QueryDeniedError)) throw error;
27
34
  throw new TopicForbiddenError({
@@ -31,3 +38,15 @@ export async function authorizeChannel(
31
38
  });
32
39
  }
33
40
  }
41
+
42
+ /** The node's context, re-made for one subscriber: same deploy, role and clock, their actor. */
43
+ function subscriberContext(node: Ctx, actor: Actor): Ctx {
44
+ return createContext({
45
+ actor,
46
+ role: node.role,
47
+ buildId: node.buildId,
48
+ clock: node.clock,
49
+ locale: node.locale,
50
+ tz: node.tz,
51
+ });
52
+ }
@@ -43,10 +43,12 @@ function projectionFor(name: string, entity: ChannelEntity): RecordProjection {
43
43
  export interface ChannelServerInit {
44
44
  /**
45
45
  * Evaluated on subscribe: the params are the input, and `row` is what the loader below answered
46
- * (`null` without one). Omitted = any socket may join. Records are NOT gated per row — the topic's
46
+ * (`null` without one). REQUIRED, as on an `action` and a `query`: it was optional, and an omitted
47
+ * one let any socket — anonymous included — join with any param and receive every committed row.
48
+ * A public channel says so with `allow('public')`. Records are NOT gated per row — the topic's
47
49
  * params are the scope, so a channel that must hide rows is declared narrower.
48
50
  */
49
- readonly policy?: QueryPolicy;
51
+ readonly policy: QueryPolicy;
50
52
  /**
51
53
  * The subject the policy decides about, loaded by the SURFACE before the rule runs — the same
52
54
  * split an action's `row:` makes, because a rule is synchronous and membership is a read.
@@ -69,7 +71,7 @@ export interface Channel<K extends string = string> {
69
71
  readonly kind: 'channel';
70
72
  readonly name: string;
71
73
  readonly params: readonly K[];
72
- readonly policy: QueryPolicy | undefined;
74
+ readonly policy: QueryPolicy;
73
75
  readonly row: ChannelRowLoader | undefined;
74
76
  readonly catchUp: string;
75
77
  readonly records: readonly RecordProjection[];
@@ -87,7 +89,7 @@ export interface Channel<K extends string = string> {
87
89
  * Registers the declaration (`channel-registry.ts`) — a second one of the same name is refused.
88
90
  *
89
91
  * The params-matching rule, decided here once: a row belongs to the topic whose params equal the
90
- * row's own properties of those names. So `channel('org-feed', { params: ['orgId'], records:
92
+ * row's own properties of those names. So `channel('org-feed', { params: ['orgId'], policy, records:
91
93
  * [posts] })` carries every `posts` row on `org-feed.<row.orgId>`, and every listed entity must have
92
94
  * a column named after every param — refused at declaration, where the author is.
93
95
  */
@@ -103,6 +105,15 @@ export function channel<K extends string>(
103
105
  const ref =
104
106
  typeof nameOrRef === 'string' ? channelRef(nameOrRef, init as ChannelInit<K>) : nameOrRef;
105
107
  const name = ref.name;
108
+ // The type requires it; this is the refusal a JS caller, or a cast, reaches.
109
+ const policy = (init as Partial<ChannelServerInit>).policy;
110
+ if (policy === undefined) {
111
+ refuseChannel(
112
+ name,
113
+ 'declares no policy, so any socket, anonymous included, could join and receive every row',
114
+ "add policy: can('<resource>:read') to the channel() call, or policy: allow('public') for a channel anyone may join",
115
+ );
116
+ }
106
117
  const records = (init.records ?? []).map((entity) => {
107
118
  const projection = projectionFor(name, entity);
108
119
  for (const param of ref.params) {
@@ -121,7 +132,7 @@ export function channel<K extends string>(
121
132
  kind: 'channel' as const,
122
133
  name,
123
134
  params: ref.params,
124
- policy: init.policy,
135
+ policy,
125
136
  row: init.row,
126
137
  get catchUp(): string {
127
138
  return ref.catchUp;
@@ -13,8 +13,11 @@ export interface ChannelDescription {
13
13
  /** Record types (entity names) the channel carries, sorted. */
14
14
  readonly records: readonly string[];
15
15
  readonly events: boolean;
16
- /** The policy's display label, `null` for a channel any socket may join. */
17
- readonly policy: string | null;
16
+ /**
17
+ * The policy's display label. Never absent since 22.0.0 — `channel()` requires a policy, and a
18
+ * channel any socket may join reads `allow('public')`'s label, said out loud.
19
+ */
20
+ readonly policy: string;
18
21
  /** Every permission the policy asserts, flattened — what a report matches a grant against. */
19
22
  readonly permissions: readonly string[];
20
23
  }
@@ -26,8 +29,7 @@ export function describeChannels(): readonly ChannelDescription[] {
26
29
  catchUp: declared.catchUp,
27
30
  records: declared.records.map((projection) => projection.type).sort(),
28
31
  events: declared.events,
29
- policy: declared.policy === undefined ? null : policyCapability(declared.policy),
30
- permissions:
31
- declared.policy === undefined ? [] : [...policyPermissions(declared.policy)].sort(),
32
+ policy: policyCapability(declared.policy),
33
+ permissions: [...policyPermissions(declared.policy)].sort(),
32
34
  }));
33
35
  }
@@ -5,7 +5,7 @@
5
5
  import { logger, renderThrowable, uuid } from '@ultimat3/core';
6
6
  import type { ChangeEvent } from './changefeed';
7
7
  import type { Channel } from './channel-decl';
8
- import { updatesFor } from './channel-records';
8
+ import { carriesTable, updatesFor } from './channel-records';
9
9
  import { renderRecords } from './channel-render';
10
10
  import { ChannelRing } from './channel-ring';
11
11
  import type { ChannelSince, ReplayGapFrame } from './channel-wire';
@@ -59,6 +59,7 @@ export class ChannelLogs {
59
59
  * the change is somebody else's too, and one malformed image must not stop the rest.
60
60
  */
61
61
  deliverChange(channels: Iterable<Channel>, change: ChangeEvent): number {
62
+ if (change.op === 'truncate') return this.#truncated(change.entity);
62
63
  let frames = 0;
63
64
  let updates: ReturnType<typeof updatesFor>;
64
65
  try {
@@ -80,6 +81,23 @@ export class ChannelLogs {
80
81
  return frames;
81
82
  }
82
83
 
84
+ /**
85
+ * Every row of `table` is gone, and no frame can say which ones a member holds. Each open topic
86
+ * of a channel carrying it starts a NEW epoch — nothing in the old ring may be replayed onto a
87
+ * table that no longer has those rows — and every member is told `replay-gap` at it, which the
88
+ * client answers by re-running the channel's catch-up read.
89
+ */
90
+ #truncated(table: string): number {
91
+ let announced = 0;
92
+ for (const [topic, open] of [...this.#byTopic]) {
93
+ if (!carriesTable(open.target.channel, table)) continue;
94
+ this.#byTopic.delete(topic);
95
+ const ring = this.open(topic, open.target);
96
+ announced += this.#sockets.announceGap(topic, ring.epoch);
97
+ }
98
+ return announced;
99
+ }
100
+
83
101
  /**
84
102
  * A (re)subscribe `since` a position: every frame after it from the ring, or — when the ring
85
103
  * cannot prove it holds them all, or the epoch moved — one `replay-gap`.
@@ -29,6 +29,9 @@ export interface TopicUpdate {
29
29
  */
30
30
  export function updatesFor(channels: Iterable<Channel>, change: ChangeEvent): TopicUpdate[] {
31
31
  const updates: TopicUpdate[] = [];
32
+ // A truncate names no row, so it routes to no topic here: `ChannelLogs` announces a gap on every
33
+ // open topic of every channel carrying the relation instead (`truncatedTopics`).
34
+ if (change.op === 'truncate') return updates;
32
35
  for (const channel of channels) {
33
36
  const projection = channel.records.find((candidate) => candidate.table === change.entity);
34
37
  if (projection === undefined) continue;
@@ -77,3 +80,8 @@ function removal(
77
80
  row,
78
81
  };
79
82
  }
83
+
84
+ /** The channels whose records a truncate of `table` wiped: every one listing that relation. */
85
+ export function carriesTable(channel: Channel, table: string): boolean {
86
+ return channel.records.some((projection) => projection.table === table);
87
+ }
@@ -38,6 +38,11 @@ export interface ChannelBookDeps {
38
38
  /** Re-run the channel's catch-up read; its records land in the store through the transport. */
39
39
  catchUp(query: string, params: Readonly<Record<string, string>>): Promise<unknown>;
40
40
  report(error: unknown): void;
41
+ /**
42
+ * When a failed catch-up tries again on its own, on the client's reconnect curve. Absent, a
43
+ * failed read waits for the next open of the socket or the next gap.
44
+ */
45
+ readonly retry?: ((attempt: number, run: () => void) => () => void) | undefined;
41
46
  }
42
47
 
43
48
  interface Entry {
@@ -57,6 +62,17 @@ interface Entry {
57
62
  readonly above: Set<number>;
58
63
  /** Frames that arrived while the catch-up read was in flight, applied after it lands. */
59
64
  buffered: ChannelRecordsFrame[] | null;
65
+ /**
66
+ * A gap announced while a read was in flight: one more read follows it, in this epoch (or the
67
+ * current one when the gap named none). A flag, never a count — the `coalesceReloads` shape.
68
+ */
69
+ again: { readonly epoch: string | undefined } | null;
70
+ /** The read is not in flight: it failed, and waits for a retry with `buffered` still held. */
71
+ waiting: boolean;
72
+ /** Consecutive failed reads — the retry curve's attempt number. */
73
+ failures: number;
74
+ /** Disarms the scheduled retry. */
75
+ disarm: (() => void) | null;
60
76
  }
61
77
 
62
78
  export interface ChannelMembership extends Disposable {
@@ -97,6 +113,10 @@ export class ChannelBook {
97
113
  contiguous: null,
98
114
  above: new Set(),
99
115
  buffered: null,
116
+ again: null,
117
+ waiting: false,
118
+ failures: 0,
119
+ disarm: null,
100
120
  };
101
121
  this.#byTopic.set(topic, entry);
102
122
  if (this.#deps.connected()) this.#deps.send(this.#frame(entry, 'add', true));
@@ -110,6 +130,8 @@ export class ChannelBook {
110
130
  held.holders.delete(handlers);
111
131
  if (held.holders.size > 0) return;
112
132
  this.#byTopic.delete(topic);
133
+ held.disarm?.();
134
+ held.disarm = null;
113
135
  this.#deps.send(this.#frame(held, 'drop', false));
114
136
  };
115
137
  return {
@@ -130,6 +152,12 @@ export class ChannelBook {
130
152
  /** Every membership again, on a new socket — resuming from each cursor. */
131
153
  resubscribe(): void {
132
154
  for (const entry of this.#byTopic.values()) {
155
+ // A catch-up that failed retries on the new socket: the read is what it was waiting for.
156
+ if (entry.waiting) {
157
+ this.#deps.send(this.#frame(entry, 'add', true));
158
+ this.#read(entry);
159
+ continue;
160
+ }
133
161
  this.#set(entry, 'joining');
134
162
  this.#deps.send(this.#frame(entry, 'add', true));
135
163
  }
@@ -225,32 +253,74 @@ export class ChannelBook {
225
253
  /**
226
254
  * Re-read the channel through its catch-up query, holding every frame that arrives meanwhile;
227
255
  * then apply those, in order, over the read. The cursor restarts in the new epoch.
256
+ *
257
+ * A gap that lands DURING the read is not dropped: it earns one more read after this one (it
258
+ * was ignored, and the hole it announced was never repaired).
228
259
  */
229
260
  #catchUp(entry: Entry, pending: ChannelRecordsFrame[], epoch?: string): void {
230
261
  if (entry.buffered !== null) {
231
262
  entry.buffered.push(...pending);
263
+ if (pending.length === 0) entry.again = { epoch: epoch ?? entry.again?.epoch };
264
+ // No read in flight — the last one failed — so this gap is the retry, now.
265
+ if (entry.waiting) this.#read(entry);
232
266
  return;
233
267
  }
234
268
  entry.buffered = [...pending];
235
- entry.epoch = epoch ?? pending[0]?.epoch ?? entry.epoch;
269
+ this.#restart(entry, epoch ?? pending[0]?.epoch ?? entry.epoch);
270
+ this.#read(entry);
271
+ }
272
+
273
+ /** The cursor, reset into `epoch`: every seq before the read is the read's to answer. */
274
+ #restart(entry: Entry, epoch: string | null): void {
275
+ entry.epoch = epoch;
236
276
  entry.contiguous = null;
237
277
  entry.above.clear();
278
+ }
279
+
280
+ #read(entry: Entry): void {
281
+ entry.waiting = false;
282
+ entry.error = undefined;
283
+ entry.disarm?.();
284
+ entry.disarm = null;
238
285
  this.#set(entry, 'catching-up');
239
286
  this.#deps.catchUp(entry.catchUp, entry.params).then(
240
287
  () => this.#drain(entry),
241
288
  (error: unknown) => {
242
289
  this.#deps.report(error);
243
- this.#drain(entry);
290
+ // NOT live: the frames held behind the read would land over a store it never refreshed.
291
+ // Failed, the error exposed, and the frames still held for the retry — which the next
292
+ // open of the socket runs (`resubscribe`), or the next gap.
293
+ entry.waiting = true;
294
+ entry.error = error;
295
+ entry.failures += 1;
296
+ if (this.#byTopic.get(entry.topic) !== entry) return;
297
+ this.#set(entry, 'failed');
298
+ entry.disarm =
299
+ this.#deps.retry?.(entry.failures, () => {
300
+ entry.disarm = null;
301
+ if (entry.waiting && this.#byTopic.get(entry.topic) === entry) this.#read(entry);
302
+ }) ?? null;
244
303
  },
245
304
  );
246
305
  }
247
306
 
248
307
  #drain(entry: Entry): void {
249
308
  const held = entry.buffered ?? [];
309
+ // A newer epoch arrived while the read was in flight (the node restarted mid-read): the read
310
+ // answered for the OLD one, so it is re-read in the new one, keeping only that epoch's frames.
311
+ // `#drain` used to discard them, and the rows they carried with them.
312
+ const newer = held.find((frame) => frame.epoch !== entry.epoch);
313
+ if (entry.again !== null || newer !== undefined) {
314
+ const epoch = newer?.epoch ?? entry.again?.epoch ?? entry.epoch;
315
+ entry.again = null;
316
+ entry.buffered = held.filter((frame) => frame.epoch === epoch);
317
+ this.#restart(entry, epoch);
318
+ this.#read(entry);
319
+ return;
320
+ }
250
321
  entry.buffered = null;
251
- // Frames of an epoch other than the one caught up to predate it; the read already holds them.
252
- const current = held.filter((frame) => frame.epoch === entry.epoch);
253
- current.sort((a, b) => a.seq - b.seq);
322
+ entry.failures = 0;
323
+ const current = [...held].sort((a, b) => a.seq - b.seq);
254
324
  for (const frame of current) {
255
325
  if (entry.contiguous === null) entry.contiguous = frame.seq - 1;
256
326
  this.#apply(entry, frame);
package/src/client.ts CHANGED
@@ -19,7 +19,7 @@ import type { JsonObject, JsonValue } from './json';
19
19
  import { type LiveState, type Registration, RowWindows, unnamedType } from './live-rows';
20
20
  import { RecordStore } from './record-store';
21
21
  import { decode, encode, type Frame, PROTOCOL_VERSION } from './sync-protocol';
22
- import { backoffDelay, browserBackoff, timeoutScheduler } from './thundering-herd';
22
+ import { browserBackoff, policyDelay, timeoutScheduler } from './thundering-herd';
23
23
 
24
24
  export type {
25
25
  ClientSocket,
@@ -78,6 +78,17 @@ export class LiveClient {
78
78
  connected: () => this.#connected,
79
79
  catchUp: options.catchUp,
80
80
  report: (error) => this.#onError(error),
81
+ // A failed catch-up retries on the reconnect curve this client already dials on.
82
+ retry: (attempt, run) =>
83
+ (this.#options.scheduler ?? timeoutScheduler)(
84
+ run,
85
+ // `attempt` is the book's failure count, 1 on the first failure: core's count as is.
86
+ policyDelay(
87
+ this.#options.backoff ?? browserBackoff,
88
+ attempt,
89
+ this.#options.rng ?? Math.random,
90
+ ),
91
+ ),
81
92
  });
82
93
  this.#heartbeat = new Heartbeat({
83
94
  intervalMs: finiteOption(
@@ -369,7 +380,8 @@ export class LiveClient {
369
380
  this.#cancelReconnect();
370
381
  const rng = this.#options.rng ?? Math.random;
371
382
  const delay =
372
- serverDelayMs ?? backoffDelay(this.#attempt, this.#options.backoff ?? browserBackoff, rng);
383
+ // `#attempt` counts reconnects already scheduled, from 0; the wait being armed is the next one.
384
+ serverDelayMs ?? policyDelay(this.#options.backoff ?? browserBackoff, this.#attempt + 1, rng);
373
385
  this.#attempt += 1;
374
386
  this.#setStatus({ reconnectAt: this.#clock.now().getTime() + delay });
375
387
  const schedule = this.#options.scheduler ?? timeoutScheduler;