@ultimat3/realtime 21.0.0 → 22.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +293 -1009
- package/README.md +78 -12
- package/package.json +4 -4
- package/src/changefeed.ts +7 -1
- package/src/channel-authz.ts +23 -4
- package/src/channel-decl.ts +16 -5
- package/src/channel-describe.ts +7 -5
- package/src/channel-logs.ts +19 -1
- package/src/channel-records.ts +8 -0
- package/src/client-channels.ts +75 -5
- package/src/client.ts +14 -2
- package/src/cursor.ts +5 -0
- package/src/errors.ts +21 -0
- package/src/idb-fake.ts +24 -4
- package/src/idb-types.ts +7 -0
- package/src/index.ts +0 -1
- package/src/live-definition.ts +5 -1
- package/src/live-fanout.ts +51 -2
- package/src/live-query.ts +11 -0
- package/src/live-replicator.ts +160 -0
- package/src/local-store-idb.ts +89 -15
- package/src/matcher-bridge.ts +5 -0
- package/src/nats-fake.ts +10 -1
- package/src/nats-jetstream.ts +36 -14
- package/src/nats-transport.ts +2 -2
- package/src/offline-queue.ts +76 -21
- package/src/page-outbox.ts +80 -10
- package/src/page-socket.ts +39 -8
- package/src/pg-entity-row.ts +37 -184
- package/src/pg-preflight.ts +24 -2
- package/src/pg-replication.ts +19 -6
- package/src/pg-wire.ts +51 -15
- package/src/policy-fake.ts +14 -0
- package/src/query-window.ts +35 -21
- package/src/replicator.ts +13 -3
- package/src/server.ts +8 -3
- package/src/socket-drops.ts +30 -0
- package/src/socket-engine.ts +15 -3
- package/src/socket-host.ts +103 -4
- package/src/socket-idle.ts +21 -0
- package/src/socket.ts +41 -38
- package/src/subscriber-gate.ts +92 -3
- package/src/sync-node.ts +2 -7
- package/src/thundering-herd.ts +12 -11
- package/src/transport-env.ts +55 -14
- package/src/use-mutation.ts +13 -0
- package/src/use-query.ts +10 -5
package/README.md
CHANGED
|
@@ -69,22 +69,53 @@ class — stay on `.`.
|
|
|
69
69
|
|
|
70
70
|
| Concern | Entry | Export |
|
|
71
71
|
|---|---|---|
|
|
72
|
-
| tier 1 | `./server` | `
|
|
72
|
+
| tier 1 | `./server` | `ChannelHub`, `PresenceRegistry`, `SyncSocket`, `SocketRegistry` |
|
|
73
73
|
| tier 2 | `./server` | `LiveQueryRegistry`, `InMemoryChangeFeed`, `PgLogicalReplicationFeed`, `selectChangeFeed`, `createReplicator`, `PgAdvisoryLock`, `matcherFor` |
|
|
74
74
|
| replication | `./server` | `parsePgUrl`, `bunPgStream`, `PgOutputDecoder`, `entityRow`, `changeLsn`, `commitPositionOf` |
|
|
75
|
+
| the in-process change source | `./server` | `startLiveReplicator` — a repository's own writes as `ChangeEvent`s, for the embedded database `x dev` runs on (PGlite has no walsender). Moved from `@ultimat3/testing`, which no longer re-exports it |
|
|
75
76
|
| fanout | `./server` | `Transport`, `InProcessTransport`, `NatsTransport`, `selectTransport`, `subjectMatches` |
|
|
76
77
|
| the bus, behind `NatsTransport` | `./server` | the port — `NatsClient`, `NatsMessage`, `NatsSubscription`, `NatsConnect`, `NatsTarget`, `parseNatsUrl` — plus `openNatsClient` (the `nats` adapter), `NatsKvSet`, `ensureKvBucket`, `kvGet`/`kvLast`/`kvWrite`, `assertBucket`, `encodeToken`/`decodeToken`, and `FakeNatsBroker`/`fakeNatsConnect` for tests |
|
|
77
|
-
| reconnect | both | `LiveCursor`, `resumeFrom`, `shouldResnapshot`, `defaultReconnectBudget`, `
|
|
78
|
+
| reconnect | both | `LiveCursor`, `resumeFrom`, `shouldResnapshot`, `defaultReconnectBudget`, `Scheduler`, `timeoutScheduler` on `.`; `RingChangeBuffer`, `drainPlan`, `AcceptBudget`, `reconnectFrame` on `./server` — the node's half of the reconnect is the node's |
|
|
78
79
|
| the page's record store | `.` | `RecordStore` — one record per `type:key`, synced truth plus the optimistic overlay — `recordKey`, `LocalTx`, `RowWindows`, `applyPatches`/`orderAfterPatches` |
|
|
79
80
|
| the outbox | `.` | `OfflineQueue`, `MemoryQueueStore` — replayed over HTTP from plan 101 slice 12. The conflict vocabulary is `ConflictPolicy` from `@ultimat3/core`; realtime declares none |
|
|
80
81
|
| wire | `.` | `PROTOCOL_VERSION` (3), `encode`, `decode`, `Frame` |
|
|
81
82
|
| the node | `./server` | `createSyncNode` / `listenSyncNode` (`sync` role) |
|
|
82
83
|
| a socket's identity | `./server` | `SyncAuthenticator`, `SyncGrant`, `GrantBook`, `sweepGrants`, `DEFAULT_REAUTH_INTERVAL_MS` |
|
|
83
84
|
| hooks | `.` | `useQuery`, `useRecord`, `useMutation`, `useMutationQueue`, `useConnection`, `useChannel`, `usePresence`, `hasPageSocket`, `installRealtime` |
|
|
84
|
-
| channels | `.` | `channel`, `topic`, `readPresence`, the channel frame types |
|
|
85
|
+
| channels | `.` | `channel`, `channelRef`, `ChannelHandle`, `topic`, `readPresence`, the channel frame types |
|
|
85
86
|
| offline | `.` | `pageOutbox`, `recordPersister`, `persistedTypes`, `openLocalStore`, `pageLocalStore`, `MemoryLocalStore` |
|
|
86
87
|
| the socket's worker | `./sync-worker` | the SharedWorker entry — no exports |
|
|
87
88
|
|
|
89
|
+
## `channelRef` — a channel an island can hold
|
|
90
|
+
|
|
91
|
+
A channel has two halves, and they live in two files so a browser chunk never bundles an entity or
|
|
92
|
+
a policy. `channelRef(name, { params, catchUp })` is the **client** half: the name, the ordered
|
|
93
|
+
params and the catch-up read — nothing else. The server declares the records and who may join on
|
|
94
|
+
**the same ref** with `channel(ref, { … })`, so the name and the params are written once.
|
|
95
|
+
|
|
96
|
+
```ts
|
|
97
|
+
// app/posts/channel-ref.ts — imported by islands
|
|
98
|
+
import { channelRef } from '@ultimat3/realtime';
|
|
99
|
+
|
|
100
|
+
export const ORG_POSTS = channelRef('org-posts', {
|
|
101
|
+
params: ['orgId'],
|
|
102
|
+
catchUp: { name: 'orgPosts' }, // the query a client re-runs on `replay-gap` or a new epoch
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
// The one topic spelling both halves use.
|
|
106
|
+
ORG_POSTS.topic({ orgId: 'org_1' }); // 'org-posts.org_1'
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
The server half, in its own file, names the entity and the policy on the same ref —
|
|
110
|
+
`channel(ORG_POSTS, { records: [posts], policy: feedRead })` — and an island subscribes with
|
|
111
|
+
`useChannel(ORG_POSTS, { orgId })`.
|
|
112
|
+
|
|
113
|
+
`ref.topic(params)` is the one spelling of the topic both halves use (`org-posts.<orgId>`), and
|
|
114
|
+
`bun run channel-literals` refuses a hand-built topic anywhere else. A bad name or a repeated param
|
|
115
|
+
is `X_CHANNEL_DECLARATION_INVALID` at declaration. `catchUp` is read on every access, so a query
|
|
116
|
+
whose name `registerQueries()` stamps at boot is picked up. The reference app's
|
|
117
|
+
[`app/posts/channel-ref.ts`](../../examples/dummy/apps/web/app/posts/channel-ref.ts) is the idiom.
|
|
118
|
+
|
|
88
119
|
## The hooks
|
|
89
120
|
|
|
90
121
|
One record store and one socket per **page**, on `globalThis`: every island is its own bundle, so a
|
|
@@ -288,7 +319,7 @@ sends a `reconnect` frame carrying that delay — clients redistribute instead o
|
|
|
288
319
|
because refusing without one just moves the herd next door.
|
|
289
320
|
|
|
290
321
|
The client dials itself back. A closed socket arms one timer — the node's delay when a `reconnect`
|
|
291
|
-
frame assigned one, otherwise `backoffDelay()` — and that timer calls `connect()`, which re-subscribes
|
|
322
|
+
frame assigned one, otherwise `@ultimat3/core`'s `backoffDelay()` on the client's `BackoffPolicy` — and that timer calls `connect()`, which re-subscribes
|
|
292
323
|
every registration **and re-announces every topic**. Topic membership is state on the node's socket
|
|
293
324
|
and `hello` carries none of it, so without that half a channel goes silent from the first reconnect
|
|
294
325
|
onwards while its handler is still installed — and its presence membership is swept, because
|
|
@@ -337,7 +368,8 @@ wire twice by a reconnect that raced an ack.
|
|
|
337
368
|
| Backpressure **declines**, it does not fail | over `MAX_BUFFERED_BYTES` (1 MiB, the node's `backpressureLimit` at the other end of the same socket) the sender throws `X_TRANSPORT_UNAVAILABLE`, the mutation stays pending and the next drain resumes there. `ClientSocket.bufferedAmount` is optional; a socket that does not report it is treated as never backed up |
|
|
338
369
|
| Delivery is therefore at least once | every mutation carries an idempotency key — the `key` argument, or `<mutator>:<uuid>` — and the resend carries the same one |
|
|
339
370
|
| A lost connection **cancels the pass it interrupted** | the lane orders passes against each other, but a socket death is not a pass and cannot reach one parked inside `send`. `requeueInflight()` bumps a connection epoch; a pass whose epoch went stale returns and leaves the rest `pending`. Without it the parked pass resumed and marked everything behind it `inflight` for a dead socket — never re-sent (`inflight` is not sendable) and never acked |
|
|
340
|
-
| The store is handed
|
|
371
|
+
| The store is handed **snapshots, by key**, never the live entries or the whole queue | `QueueStore.write` is a durable write and may await before it reads; given an entry itself it persists a status that was never true when it was called. By key, because two tabs of one user share the store and a whole-queue save let the last tab erase the other's write |
|
|
372
|
+
| One tab drains at a time, and drains what every tab queued | the outbox's replay runs under the Web Lock `ultimate-outbox:<principal>` and re-reads the queue first, so a write another tab queued is sent, in order, and a stored `inflight` from a closed page goes back to `pending` |
|
|
341
373
|
|
|
342
374
|
### Limits, stated plainly
|
|
343
375
|
|
|
@@ -491,11 +523,15 @@ wire twice by a reconnect that raced an ack.
|
|
|
491
523
|
pg_try_advisory_lock(hashtext('x:replicator:<slot>'))` on its own session. Session-scoped, so a
|
|
492
524
|
crashed replicator releases it automatically: no lease renewal, no fencing token, no split brain.
|
|
493
525
|
`InMemoryAdvisoryLock` remains the single-process default for `x dev` and tests.
|
|
494
|
-
- **`selectTransport(env)` decides which transport a boot fans out on** —
|
|
495
|
-
the
|
|
496
|
-
|
|
497
|
-
`
|
|
498
|
-
|
|
526
|
+
- **`selectTransport(env, realtime)` decides which transport a boot fans out on** — and since
|
|
527
|
+
22.0.0 the CONFIG decides it, not the environment: `realtime` is `app.config.ts`'s
|
|
528
|
+
`{ transport, urlEnv }`. It returns `{ transport, mode, detail, bucket, presenceTtlMs, connect }`:
|
|
529
|
+
`'memory'` → `InProcessTransport` and `mode: 'embedded'`; `'nats'` → a `NatsTransport` dialling
|
|
530
|
+
the variable `urlEnv` names, on the KV bucket `NATS_KV_BUCKET` names (default `x_presence`, so
|
|
531
|
+
two apps on one cluster do not share one presence namespace), validated here rather than on
|
|
532
|
+
first connect. Both mismatches refuse with `X_CONFIG_INVALID`: `'nats'` with that variable unset,
|
|
533
|
+
and `'memory'` with `NATS_URL` (or the named variable) set — an operator who set one expected
|
|
534
|
+
fanout across nodes. Until 22.0.0 `NATS_URL` alone decided and both keys were read by nothing.
|
|
499
535
|
`presenceTtlMs` comes back with it because the bucket's whole-stream age limit was derived from
|
|
500
536
|
it — a `PresenceRegistry` given a different number would report members leaving that never left.
|
|
501
537
|
Selection is pure; `connect()` is the dial, so an unreachable bus fails at boot.
|
|
@@ -548,8 +584,9 @@ wire twice by a reconnect that raced an ack.
|
|
|
548
584
|
`X_NOT_IMPLEMENTED` and `X_TIMEOUT` are **borrowed** from `@ultimat3/core`, which owns and titles
|
|
549
585
|
them — `REALTIME_BORROWED_ERROR_CODES`. Everything else on that list is realtime's own.
|
|
550
586
|
|
|
551
|
-
Topics deny by default: a topic
|
|
552
|
-
|
|
587
|
+
Topics deny by default: a topic no `channel()` declares is forbidden, and a `channel()` with no
|
|
588
|
+
`policy` is refused at declaration (`X_CHANNEL_DECLARATION_INVALID`) — a public channel writes
|
|
589
|
+
`policy: allow('public')`. An authz hole is not a config option someone forgot to set.
|
|
553
590
|
|
|
554
591
|
An upgrade `authenticate` refuses is `X_SOCKET_UNAUTHENTICATED` (401) and one it *could not decide*
|
|
555
592
|
is `X_SOCKET_AUTH_UNAVAILABLE` (503). Two codes, because the two have opposite instructions: the
|
|
@@ -568,3 +605,32 @@ The fix is `x queries list --json`, and the name the client sent is echoed back
|
|
|
568
605
|
never is.
|
|
569
606
|
|
|
570
607
|
`As of 2026-07`: tiers 1–2 target v1, tier 3 targets v2.
|
|
608
|
+
|
|
609
|
+
### Error classes
|
|
610
|
+
|
|
611
|
+
Every error class `src/index.ts` exports, for `instanceof` inside one process. Across a wire or
|
|
612
|
+
a job boundary the class is gone and the `code` is what survives — match on that.
|
|
613
|
+
|
|
614
|
+
| Class | Code | Declared in |
|
|
615
|
+
|---|---|---|
|
|
616
|
+
| `CursorStaleError` (extends `RealtimeError`) | `X_CURSOR_STALE` | `src/page-errors.ts` |
|
|
617
|
+
| `FrameRateLimitError` (extends `RealtimeError`) | `X_FRAME_RATE_LIMIT` | `src/errors.ts` |
|
|
618
|
+
| `LiveQueryUnknownError` (extends `RealtimeError`) | `X_LIVE_QUERY_UNKNOWN` | `src/errors.ts` |
|
|
619
|
+
| `LiveRowUnidentifiedError` (extends `RealtimeError`) | `X_LIVE_ROW_UNIDENTIFIED` | `src/errors.ts` |
|
|
620
|
+
| `NotImplementedError` (extends `RealtimeError`) | `X_NOT_IMPLEMENTED` | `src/errors.ts` |
|
|
621
|
+
| `ProtocolVersionError` (extends `RealtimeError`) | `X_PROTOCOL_VERSION` | `src/page-errors.ts` |
|
|
622
|
+
| `RealtimeError` | any `RealtimeErrorCode` — `REALTIME_ERROR_CODES`; the base of every other class here, thrown directly for a code none of them covers | `src/realtime-error.ts` |
|
|
623
|
+
| `RealtimeUninstalledError` (extends `RealtimeError`) | `X_REALTIME_UNINSTALLED` | `src/page-errors.ts` |
|
|
624
|
+
| `RebaseConflictError` (extends `RealtimeError`) | `X_REBASE_CONFLICT` | `src/page-errors.ts` |
|
|
625
|
+
| `RecordRejectedError` (extends `RealtimeError`) | `X_RECORD_REJECTED` | `src/page-errors.ts` |
|
|
626
|
+
| `ReplicaIdentityError` (extends `RealtimeError`) | `X_LIVE_REPLICA_IDENTITY` | `src/replication-errors.ts` |
|
|
627
|
+
| `ReplicationFailedError` (extends `RealtimeError`) | `X_REPLICATION_FAILED` | `src/replication-errors.ts` |
|
|
628
|
+
| `ReplicationProtocolError` (extends `RealtimeError`) | `X_REPLICATION_PROTOCOL` | `src/replication-errors.ts` |
|
|
629
|
+
| `ReplicatorSlotHeldError` (extends `RealtimeError`) | `X_REPLICATOR_SLOT_HELD` | `src/replication-errors.ts` |
|
|
630
|
+
| `ServerRenderLiveError` (extends `RealtimeError`) | `X_LIVE_SERVER_RENDER` | `src/page-errors.ts` |
|
|
631
|
+
| `SubscriptionLimitError` (extends `RealtimeError`) | `X_SUBSCRIPTION_LIMIT` | `src/errors.ts` |
|
|
632
|
+
| `SyncUnconfiguredError` (extends `RealtimeError`) | `X_SYNC_UNCONFIGURED` | `src/page-errors.ts` |
|
|
633
|
+
| `TopicForbiddenError` (extends `RealtimeError`) | `X_TOPIC_FORBIDDEN` | `src/errors.ts` |
|
|
634
|
+
| `TransportProtocolError` (extends `RealtimeError`) | `X_TRANSPORT_PROTOCOL` | `src/errors.ts` |
|
|
635
|
+
| `TransportUnavailableError` (extends `RealtimeError`) | `X_TRANSPORT_UNAVAILABLE` | `src/errors.ts` |
|
|
636
|
+
| `WindowReadTimeoutError` (extends `RealtimeError`) | `X_TIMEOUT` | `src/errors.ts` |
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ultimat3/realtime",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "22.0.0",
|
|
4
4
|
"description": "Three-tier realtime: channels, live queries, local-first sync — one protocol, one mutator shape",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -38,9 +38,9 @@
|
|
|
38
38
|
"test": "bun test"
|
|
39
39
|
},
|
|
40
40
|
"dependencies": {
|
|
41
|
-
"@ultimat3/core": "
|
|
42
|
-
"@ultimat3/entity": "
|
|
43
|
-
"@ultimat3/query": "
|
|
41
|
+
"@ultimat3/core": "22.0.0",
|
|
42
|
+
"@ultimat3/entity": "22.0.0",
|
|
43
|
+
"@ultimat3/query": "22.0.0",
|
|
44
44
|
"nats": "2.29.3"
|
|
45
45
|
}
|
|
46
46
|
}
|
package/src/changefeed.ts
CHANGED
|
@@ -11,7 +11,13 @@ import type { PgTarget } from './pg-socket';
|
|
|
11
11
|
import type { PgStream } from './pg-wire';
|
|
12
12
|
import type { Rng } from './thundering-herd';
|
|
13
13
|
|
|
14
|
-
|
|
14
|
+
/**
|
|
15
|
+
* `truncate` carries no row — `before` and `after` are both `null` — and means every row of the
|
|
16
|
+
* relation is gone. A window cannot be patched from it, only re-read; a channel's members are told
|
|
17
|
+
* to re-read (`replay-gap`). It was decoded and dropped, so with the recommended `FOR ALL TABLES`
|
|
18
|
+
* publication every window and every client kept the truncated rows forever.
|
|
19
|
+
*/
|
|
20
|
+
export type ChangeOp = 'insert' | 'update' | 'delete' | 'truncate';
|
|
15
21
|
|
|
16
22
|
export interface ChangeEvent<R extends Row = Row> {
|
|
17
23
|
/** Entity name, not table name — the matcher's dependency sets are declared in entity terms. */
|
package/src/channel-authz.ts
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// the channel's own loader answered. Through `@ultimat3/query`'s `guard`, the package's one authz
|
|
3
3
|
// seam — the same call `policy-gate.ts` makes for a live query.
|
|
4
4
|
|
|
5
|
-
import type
|
|
5
|
+
import { type Actor, type Ctx, createContext, runWithContext } from '@ultimat3/core';
|
|
6
6
|
import { guard, QueryDeniedError } from '@ultimat3/query';
|
|
7
7
|
import type { Channel } from './channel-decl';
|
|
8
8
|
import { TopicForbiddenError } from './errors';
|
|
@@ -18,10 +18,17 @@ export async function authorizeChannel(
|
|
|
18
18
|
topic: string,
|
|
19
19
|
params: Readonly<Record<string, string>>,
|
|
20
20
|
): Promise<void> {
|
|
21
|
-
|
|
22
|
-
|
|
21
|
+
// The loader and the rule run AS the subscriber, never as the node. Under the node's context — no
|
|
22
|
+
// actor, no tenant — a repository read inside the loader was scoped by nothing but what the
|
|
23
|
+
// loader happened to name, and `@ultimat3/entity`'s tenancy seam had no actor to hold it to.
|
|
24
|
+
// Services are rebuilt for this actor by `createContext`, the rule `withChildContext` follows.
|
|
25
|
+
const scoped = actor === null ? ctx : subscriberContext(ctx, actor);
|
|
26
|
+
const row =
|
|
27
|
+
channel.row === undefined
|
|
28
|
+
? null
|
|
29
|
+
: await runWithContext(scoped, async () => await channel.row?.({ params, ctx: scoped }));
|
|
23
30
|
try {
|
|
24
|
-
guard(channel.policy, { actor, input: params, row, ctx, query: channel.name }, 'live');
|
|
31
|
+
guard(channel.policy, { actor, input: params, row, ctx: scoped, query: channel.name }, 'live');
|
|
25
32
|
} catch (error) {
|
|
26
33
|
if (!(error instanceof QueryDeniedError)) throw error;
|
|
27
34
|
throw new TopicForbiddenError({
|
|
@@ -31,3 +38,15 @@ export async function authorizeChannel(
|
|
|
31
38
|
});
|
|
32
39
|
}
|
|
33
40
|
}
|
|
41
|
+
|
|
42
|
+
/** The node's context, re-made for one subscriber: same deploy, role and clock, their actor. */
|
|
43
|
+
function subscriberContext(node: Ctx, actor: Actor): Ctx {
|
|
44
|
+
return createContext({
|
|
45
|
+
actor,
|
|
46
|
+
role: node.role,
|
|
47
|
+
buildId: node.buildId,
|
|
48
|
+
clock: node.clock,
|
|
49
|
+
locale: node.locale,
|
|
50
|
+
tz: node.tz,
|
|
51
|
+
});
|
|
52
|
+
}
|
package/src/channel-decl.ts
CHANGED
|
@@ -43,10 +43,12 @@ function projectionFor(name: string, entity: ChannelEntity): RecordProjection {
|
|
|
43
43
|
export interface ChannelServerInit {
|
|
44
44
|
/**
|
|
45
45
|
* Evaluated on subscribe: the params are the input, and `row` is what the loader below answered
|
|
46
|
-
* (`null` without one).
|
|
46
|
+
* (`null` without one). REQUIRED, as on an `action` and a `query`: it was optional, and an omitted
|
|
47
|
+
* one let any socket — anonymous included — join with any param and receive every committed row.
|
|
48
|
+
* A public channel says so with `allow('public')`. Records are NOT gated per row — the topic's
|
|
47
49
|
* params are the scope, so a channel that must hide rows is declared narrower.
|
|
48
50
|
*/
|
|
49
|
-
readonly policy
|
|
51
|
+
readonly policy: QueryPolicy;
|
|
50
52
|
/**
|
|
51
53
|
* The subject the policy decides about, loaded by the SURFACE before the rule runs — the same
|
|
52
54
|
* split an action's `row:` makes, because a rule is synchronous and membership is a read.
|
|
@@ -69,7 +71,7 @@ export interface Channel<K extends string = string> {
|
|
|
69
71
|
readonly kind: 'channel';
|
|
70
72
|
readonly name: string;
|
|
71
73
|
readonly params: readonly K[];
|
|
72
|
-
readonly policy: QueryPolicy
|
|
74
|
+
readonly policy: QueryPolicy;
|
|
73
75
|
readonly row: ChannelRowLoader | undefined;
|
|
74
76
|
readonly catchUp: string;
|
|
75
77
|
readonly records: readonly RecordProjection[];
|
|
@@ -87,7 +89,7 @@ export interface Channel<K extends string = string> {
|
|
|
87
89
|
* Registers the declaration (`channel-registry.ts`) — a second one of the same name is refused.
|
|
88
90
|
*
|
|
89
91
|
* The params-matching rule, decided here once: a row belongs to the topic whose params equal the
|
|
90
|
-
* row's own properties of those names. So `channel('org-feed', { params: ['orgId'], records:
|
|
92
|
+
* row's own properties of those names. So `channel('org-feed', { params: ['orgId'], policy, records:
|
|
91
93
|
* [posts] })` carries every `posts` row on `org-feed.<row.orgId>`, and every listed entity must have
|
|
92
94
|
* a column named after every param — refused at declaration, where the author is.
|
|
93
95
|
*/
|
|
@@ -103,6 +105,15 @@ export function channel<K extends string>(
|
|
|
103
105
|
const ref =
|
|
104
106
|
typeof nameOrRef === 'string' ? channelRef(nameOrRef, init as ChannelInit<K>) : nameOrRef;
|
|
105
107
|
const name = ref.name;
|
|
108
|
+
// The type requires it; this is the refusal a JS caller, or a cast, reaches.
|
|
109
|
+
const policy = (init as Partial<ChannelServerInit>).policy;
|
|
110
|
+
if (policy === undefined) {
|
|
111
|
+
refuseChannel(
|
|
112
|
+
name,
|
|
113
|
+
'declares no policy, so any socket, anonymous included, could join and receive every row',
|
|
114
|
+
"add policy: can('<resource>:read') to the channel() call, or policy: allow('public') for a channel anyone may join",
|
|
115
|
+
);
|
|
116
|
+
}
|
|
106
117
|
const records = (init.records ?? []).map((entity) => {
|
|
107
118
|
const projection = projectionFor(name, entity);
|
|
108
119
|
for (const param of ref.params) {
|
|
@@ -121,7 +132,7 @@ export function channel<K extends string>(
|
|
|
121
132
|
kind: 'channel' as const,
|
|
122
133
|
name,
|
|
123
134
|
params: ref.params,
|
|
124
|
-
policy
|
|
135
|
+
policy,
|
|
125
136
|
row: init.row,
|
|
126
137
|
get catchUp(): string {
|
|
127
138
|
return ref.catchUp;
|
package/src/channel-describe.ts
CHANGED
|
@@ -13,8 +13,11 @@ export interface ChannelDescription {
|
|
|
13
13
|
/** Record types (entity names) the channel carries, sorted. */
|
|
14
14
|
readonly records: readonly string[];
|
|
15
15
|
readonly events: boolean;
|
|
16
|
-
/**
|
|
17
|
-
|
|
16
|
+
/**
|
|
17
|
+
* The policy's display label. Never absent since 22.0.0 — `channel()` requires a policy, and a
|
|
18
|
+
* channel any socket may join reads `allow('public')`'s label, said out loud.
|
|
19
|
+
*/
|
|
20
|
+
readonly policy: string;
|
|
18
21
|
/** Every permission the policy asserts, flattened — what a report matches a grant against. */
|
|
19
22
|
readonly permissions: readonly string[];
|
|
20
23
|
}
|
|
@@ -26,8 +29,7 @@ export function describeChannels(): readonly ChannelDescription[] {
|
|
|
26
29
|
catchUp: declared.catchUp,
|
|
27
30
|
records: declared.records.map((projection) => projection.type).sort(),
|
|
28
31
|
events: declared.events,
|
|
29
|
-
policy:
|
|
30
|
-
permissions:
|
|
31
|
-
declared.policy === undefined ? [] : [...policyPermissions(declared.policy)].sort(),
|
|
32
|
+
policy: policyCapability(declared.policy),
|
|
33
|
+
permissions: [...policyPermissions(declared.policy)].sort(),
|
|
32
34
|
}));
|
|
33
35
|
}
|
package/src/channel-logs.ts
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
import { logger, renderThrowable, uuid } from '@ultimat3/core';
|
|
6
6
|
import type { ChangeEvent } from './changefeed';
|
|
7
7
|
import type { Channel } from './channel-decl';
|
|
8
|
-
import { updatesFor } from './channel-records';
|
|
8
|
+
import { carriesTable, updatesFor } from './channel-records';
|
|
9
9
|
import { renderRecords } from './channel-render';
|
|
10
10
|
import { ChannelRing } from './channel-ring';
|
|
11
11
|
import type { ChannelSince, ReplayGapFrame } from './channel-wire';
|
|
@@ -59,6 +59,7 @@ export class ChannelLogs {
|
|
|
59
59
|
* the change is somebody else's too, and one malformed image must not stop the rest.
|
|
60
60
|
*/
|
|
61
61
|
deliverChange(channels: Iterable<Channel>, change: ChangeEvent): number {
|
|
62
|
+
if (change.op === 'truncate') return this.#truncated(change.entity);
|
|
62
63
|
let frames = 0;
|
|
63
64
|
let updates: ReturnType<typeof updatesFor>;
|
|
64
65
|
try {
|
|
@@ -80,6 +81,23 @@ export class ChannelLogs {
|
|
|
80
81
|
return frames;
|
|
81
82
|
}
|
|
82
83
|
|
|
84
|
+
/**
|
|
85
|
+
* Every row of `table` is gone, and no frame can say which ones a member holds. Each open topic
|
|
86
|
+
* of a channel carrying it starts a NEW epoch — nothing in the old ring may be replayed onto a
|
|
87
|
+
* table that no longer has those rows — and every member is told `replay-gap` at it, which the
|
|
88
|
+
* client answers by re-running the channel's catch-up read.
|
|
89
|
+
*/
|
|
90
|
+
#truncated(table: string): number {
|
|
91
|
+
let announced = 0;
|
|
92
|
+
for (const [topic, open] of [...this.#byTopic]) {
|
|
93
|
+
if (!carriesTable(open.target.channel, table)) continue;
|
|
94
|
+
this.#byTopic.delete(topic);
|
|
95
|
+
const ring = this.open(topic, open.target);
|
|
96
|
+
announced += this.#sockets.announceGap(topic, ring.epoch);
|
|
97
|
+
}
|
|
98
|
+
return announced;
|
|
99
|
+
}
|
|
100
|
+
|
|
83
101
|
/**
|
|
84
102
|
* A (re)subscribe `since` a position: every frame after it from the ring, or — when the ring
|
|
85
103
|
* cannot prove it holds them all, or the epoch moved — one `replay-gap`.
|
package/src/channel-records.ts
CHANGED
|
@@ -29,6 +29,9 @@ export interface TopicUpdate {
|
|
|
29
29
|
*/
|
|
30
30
|
export function updatesFor(channels: Iterable<Channel>, change: ChangeEvent): TopicUpdate[] {
|
|
31
31
|
const updates: TopicUpdate[] = [];
|
|
32
|
+
// A truncate names no row, so it routes to no topic here: `ChannelLogs` announces a gap on every
|
|
33
|
+
// open topic of every channel carrying the relation instead (`truncatedTopics`).
|
|
34
|
+
if (change.op === 'truncate') return updates;
|
|
32
35
|
for (const channel of channels) {
|
|
33
36
|
const projection = channel.records.find((candidate) => candidate.table === change.entity);
|
|
34
37
|
if (projection === undefined) continue;
|
|
@@ -77,3 +80,8 @@ function removal(
|
|
|
77
80
|
row,
|
|
78
81
|
};
|
|
79
82
|
}
|
|
83
|
+
|
|
84
|
+
/** The channels whose records a truncate of `table` wiped: every one listing that relation. */
|
|
85
|
+
export function carriesTable(channel: Channel, table: string): boolean {
|
|
86
|
+
return channel.records.some((projection) => projection.table === table);
|
|
87
|
+
}
|
package/src/client-channels.ts
CHANGED
|
@@ -38,6 +38,11 @@ export interface ChannelBookDeps {
|
|
|
38
38
|
/** Re-run the channel's catch-up read; its records land in the store through the transport. */
|
|
39
39
|
catchUp(query: string, params: Readonly<Record<string, string>>): Promise<unknown>;
|
|
40
40
|
report(error: unknown): void;
|
|
41
|
+
/**
|
|
42
|
+
* When a failed catch-up tries again on its own, on the client's reconnect curve. Absent, a
|
|
43
|
+
* failed read waits for the next open of the socket or the next gap.
|
|
44
|
+
*/
|
|
45
|
+
readonly retry?: ((attempt: number, run: () => void) => () => void) | undefined;
|
|
41
46
|
}
|
|
42
47
|
|
|
43
48
|
interface Entry {
|
|
@@ -57,6 +62,17 @@ interface Entry {
|
|
|
57
62
|
readonly above: Set<number>;
|
|
58
63
|
/** Frames that arrived while the catch-up read was in flight, applied after it lands. */
|
|
59
64
|
buffered: ChannelRecordsFrame[] | null;
|
|
65
|
+
/**
|
|
66
|
+
* A gap announced while a read was in flight: one more read follows it, in this epoch (or the
|
|
67
|
+
* current one when the gap named none). A flag, never a count — the `coalesceReloads` shape.
|
|
68
|
+
*/
|
|
69
|
+
again: { readonly epoch: string | undefined } | null;
|
|
70
|
+
/** The read is not in flight: it failed, and waits for a retry with `buffered` still held. */
|
|
71
|
+
waiting: boolean;
|
|
72
|
+
/** Consecutive failed reads — the retry curve's attempt number. */
|
|
73
|
+
failures: number;
|
|
74
|
+
/** Disarms the scheduled retry. */
|
|
75
|
+
disarm: (() => void) | null;
|
|
60
76
|
}
|
|
61
77
|
|
|
62
78
|
export interface ChannelMembership extends Disposable {
|
|
@@ -97,6 +113,10 @@ export class ChannelBook {
|
|
|
97
113
|
contiguous: null,
|
|
98
114
|
above: new Set(),
|
|
99
115
|
buffered: null,
|
|
116
|
+
again: null,
|
|
117
|
+
waiting: false,
|
|
118
|
+
failures: 0,
|
|
119
|
+
disarm: null,
|
|
100
120
|
};
|
|
101
121
|
this.#byTopic.set(topic, entry);
|
|
102
122
|
if (this.#deps.connected()) this.#deps.send(this.#frame(entry, 'add', true));
|
|
@@ -110,6 +130,8 @@ export class ChannelBook {
|
|
|
110
130
|
held.holders.delete(handlers);
|
|
111
131
|
if (held.holders.size > 0) return;
|
|
112
132
|
this.#byTopic.delete(topic);
|
|
133
|
+
held.disarm?.();
|
|
134
|
+
held.disarm = null;
|
|
113
135
|
this.#deps.send(this.#frame(held, 'drop', false));
|
|
114
136
|
};
|
|
115
137
|
return {
|
|
@@ -130,6 +152,12 @@ export class ChannelBook {
|
|
|
130
152
|
/** Every membership again, on a new socket — resuming from each cursor. */
|
|
131
153
|
resubscribe(): void {
|
|
132
154
|
for (const entry of this.#byTopic.values()) {
|
|
155
|
+
// A catch-up that failed retries on the new socket: the read is what it was waiting for.
|
|
156
|
+
if (entry.waiting) {
|
|
157
|
+
this.#deps.send(this.#frame(entry, 'add', true));
|
|
158
|
+
this.#read(entry);
|
|
159
|
+
continue;
|
|
160
|
+
}
|
|
133
161
|
this.#set(entry, 'joining');
|
|
134
162
|
this.#deps.send(this.#frame(entry, 'add', true));
|
|
135
163
|
}
|
|
@@ -225,32 +253,74 @@ export class ChannelBook {
|
|
|
225
253
|
/**
|
|
226
254
|
* Re-read the channel through its catch-up query, holding every frame that arrives meanwhile;
|
|
227
255
|
* then apply those, in order, over the read. The cursor restarts in the new epoch.
|
|
256
|
+
*
|
|
257
|
+
* A gap that lands DURING the read is not dropped: it earns one more read after this one (it
|
|
258
|
+
* was ignored, and the hole it announced was never repaired).
|
|
228
259
|
*/
|
|
229
260
|
#catchUp(entry: Entry, pending: ChannelRecordsFrame[], epoch?: string): void {
|
|
230
261
|
if (entry.buffered !== null) {
|
|
231
262
|
entry.buffered.push(...pending);
|
|
263
|
+
if (pending.length === 0) entry.again = { epoch: epoch ?? entry.again?.epoch };
|
|
264
|
+
// No read in flight — the last one failed — so this gap is the retry, now.
|
|
265
|
+
if (entry.waiting) this.#read(entry);
|
|
232
266
|
return;
|
|
233
267
|
}
|
|
234
268
|
entry.buffered = [...pending];
|
|
235
|
-
entry
|
|
269
|
+
this.#restart(entry, epoch ?? pending[0]?.epoch ?? entry.epoch);
|
|
270
|
+
this.#read(entry);
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
/** The cursor, reset into `epoch`: every seq before the read is the read's to answer. */
|
|
274
|
+
#restart(entry: Entry, epoch: string | null): void {
|
|
275
|
+
entry.epoch = epoch;
|
|
236
276
|
entry.contiguous = null;
|
|
237
277
|
entry.above.clear();
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
#read(entry: Entry): void {
|
|
281
|
+
entry.waiting = false;
|
|
282
|
+
entry.error = undefined;
|
|
283
|
+
entry.disarm?.();
|
|
284
|
+
entry.disarm = null;
|
|
238
285
|
this.#set(entry, 'catching-up');
|
|
239
286
|
this.#deps.catchUp(entry.catchUp, entry.params).then(
|
|
240
287
|
() => this.#drain(entry),
|
|
241
288
|
(error: unknown) => {
|
|
242
289
|
this.#deps.report(error);
|
|
243
|
-
|
|
290
|
+
// NOT live: the frames held behind the read would land over a store it never refreshed.
|
|
291
|
+
// Failed, the error exposed, and the frames still held for the retry — which the next
|
|
292
|
+
// open of the socket runs (`resubscribe`), or the next gap.
|
|
293
|
+
entry.waiting = true;
|
|
294
|
+
entry.error = error;
|
|
295
|
+
entry.failures += 1;
|
|
296
|
+
if (this.#byTopic.get(entry.topic) !== entry) return;
|
|
297
|
+
this.#set(entry, 'failed');
|
|
298
|
+
entry.disarm =
|
|
299
|
+
this.#deps.retry?.(entry.failures, () => {
|
|
300
|
+
entry.disarm = null;
|
|
301
|
+
if (entry.waiting && this.#byTopic.get(entry.topic) === entry) this.#read(entry);
|
|
302
|
+
}) ?? null;
|
|
244
303
|
},
|
|
245
304
|
);
|
|
246
305
|
}
|
|
247
306
|
|
|
248
307
|
#drain(entry: Entry): void {
|
|
249
308
|
const held = entry.buffered ?? [];
|
|
309
|
+
// A newer epoch arrived while the read was in flight (the node restarted mid-read): the read
|
|
310
|
+
// answered for the OLD one, so it is re-read in the new one, keeping only that epoch's frames.
|
|
311
|
+
// `#drain` used to discard them, and the rows they carried with them.
|
|
312
|
+
const newer = held.find((frame) => frame.epoch !== entry.epoch);
|
|
313
|
+
if (entry.again !== null || newer !== undefined) {
|
|
314
|
+
const epoch = newer?.epoch ?? entry.again?.epoch ?? entry.epoch;
|
|
315
|
+
entry.again = null;
|
|
316
|
+
entry.buffered = held.filter((frame) => frame.epoch === epoch);
|
|
317
|
+
this.#restart(entry, epoch);
|
|
318
|
+
this.#read(entry);
|
|
319
|
+
return;
|
|
320
|
+
}
|
|
250
321
|
entry.buffered = null;
|
|
251
|
-
|
|
252
|
-
const current = held.
|
|
253
|
-
current.sort((a, b) => a.seq - b.seq);
|
|
322
|
+
entry.failures = 0;
|
|
323
|
+
const current = [...held].sort((a, b) => a.seq - b.seq);
|
|
254
324
|
for (const frame of current) {
|
|
255
325
|
if (entry.contiguous === null) entry.contiguous = frame.seq - 1;
|
|
256
326
|
this.#apply(entry, frame);
|
package/src/client.ts
CHANGED
|
@@ -19,7 +19,7 @@ import type { JsonObject, JsonValue } from './json';
|
|
|
19
19
|
import { type LiveState, type Registration, RowWindows, unnamedType } from './live-rows';
|
|
20
20
|
import { RecordStore } from './record-store';
|
|
21
21
|
import { decode, encode, type Frame, PROTOCOL_VERSION } from './sync-protocol';
|
|
22
|
-
import {
|
|
22
|
+
import { browserBackoff, policyDelay, timeoutScheduler } from './thundering-herd';
|
|
23
23
|
|
|
24
24
|
export type {
|
|
25
25
|
ClientSocket,
|
|
@@ -78,6 +78,17 @@ export class LiveClient {
|
|
|
78
78
|
connected: () => this.#connected,
|
|
79
79
|
catchUp: options.catchUp,
|
|
80
80
|
report: (error) => this.#onError(error),
|
|
81
|
+
// A failed catch-up retries on the reconnect curve this client already dials on.
|
|
82
|
+
retry: (attempt, run) =>
|
|
83
|
+
(this.#options.scheduler ?? timeoutScheduler)(
|
|
84
|
+
run,
|
|
85
|
+
// `attempt` is the book's failure count, 1 on the first failure: core's count as is.
|
|
86
|
+
policyDelay(
|
|
87
|
+
this.#options.backoff ?? browserBackoff,
|
|
88
|
+
attempt,
|
|
89
|
+
this.#options.rng ?? Math.random,
|
|
90
|
+
),
|
|
91
|
+
),
|
|
81
92
|
});
|
|
82
93
|
this.#heartbeat = new Heartbeat({
|
|
83
94
|
intervalMs: finiteOption(
|
|
@@ -369,7 +380,8 @@ export class LiveClient {
|
|
|
369
380
|
this.#cancelReconnect();
|
|
370
381
|
const rng = this.#options.rng ?? Math.random;
|
|
371
382
|
const delay =
|
|
372
|
-
|
|
383
|
+
// `#attempt` counts reconnects already scheduled, from 0; the wait being armed is the next one.
|
|
384
|
+
serverDelayMs ?? policyDelay(this.#options.backoff ?? browserBackoff, this.#attempt + 1, rng);
|
|
373
385
|
this.#attempt += 1;
|
|
374
386
|
this.#setStatus({ reconnectAt: this.#clock.now().getTime() + delay });
|
|
375
387
|
const schedule = this.#options.scheduler ?? timeoutScheduler;
|
package/src/cursor.ts
CHANGED
|
@@ -161,6 +161,11 @@ export function advance(
|
|
|
161
161
|
lsn: string,
|
|
162
162
|
now: number,
|
|
163
163
|
): LiveCursor {
|
|
164
|
+
// An update moves no id in or out, and it is the common change: the ids are reused as they are,
|
|
165
|
+
// so a fan-out to N subscribers of a W-row window is not N rebuilds of a W-id set per change.
|
|
166
|
+
if (!patches.some((patch) => patch.op !== 'update')) {
|
|
167
|
+
return { qid: cursor.qid, lsn, ids: cursor.ids, at: now };
|
|
168
|
+
}
|
|
164
169
|
const ids = new Set(cursor.ids);
|
|
165
170
|
for (const patch of patches) {
|
|
166
171
|
if (patch.op === 'delete') ids.delete(patch.id);
|
package/src/errors.ts
CHANGED
|
@@ -33,6 +33,7 @@ export const REALTIME_OWNED_ERROR_CODES = [
|
|
|
33
33
|
'X_LIVE_REPLICA_IDENTITY',
|
|
34
34
|
'X_SOCKET_UNAUTHENTICATED',
|
|
35
35
|
'X_SOCKET_AUTH_UNAVAILABLE',
|
|
36
|
+
'X_REALTIME_TOPOLOGY',
|
|
36
37
|
] as const;
|
|
37
38
|
|
|
38
39
|
/**
|
|
@@ -140,6 +141,7 @@ export const REALTIME_ERROR_TITLES: Readonly<Record<RealtimeOwnedErrorCode, stri
|
|
|
140
141
|
X_LIVE_REPLICA_IDENTITY: 'a replicated table sends a key-only row on delete',
|
|
141
142
|
X_SOCKET_UNAUTHENTICATED: 'the sync upgrade carried no credential this app accepts',
|
|
142
143
|
X_SOCKET_AUTH_UNAVAILABLE: 'the sync node could not decide who a connecting socket is',
|
|
144
|
+
X_REALTIME_TOPOLOGY: 'a sync node boots on a real database with no reachable change feed',
|
|
143
145
|
};
|
|
144
146
|
|
|
145
147
|
// One unconditional call, so a second package claiming one of realtime's codes throws
|
|
@@ -255,6 +257,25 @@ export class TransportUnavailableError extends RealtimeError {
|
|
|
255
257
|
}
|
|
256
258
|
}
|
|
257
259
|
|
|
260
|
+
/**
|
|
261
|
+
* A `sync` node that can hear no change: a real database, the in-process bus, and no replicator in
|
|
262
|
+
* this process. A replicator in another process publishes into ITS in-process bus, so every live
|
|
263
|
+
* query and channel here is silent, with no error on either side. Refused at boot, where the
|
|
264
|
+
* topology is known, rather than discovered as a live feature that never updates.
|
|
265
|
+
*/
|
|
266
|
+
export class RealtimeTopologyError extends RealtimeError {
|
|
267
|
+
constructor() {
|
|
268
|
+
super({
|
|
269
|
+
code: 'X_REALTIME_TOPOLOGY',
|
|
270
|
+
cause:
|
|
271
|
+
'role sync runs on an external database over the in-process transport with no replicator in this process, so no committed change can reach it',
|
|
272
|
+
// Since 22.0.0 NATS_URL alone selects nothing: `realtime.transport` does, and a set NATS_URL
|
|
273
|
+
// under `'memory'` is refused, so the fix has to name both halves.
|
|
274
|
+
fix: "set realtime: { transport: 'nats', urlEnv: 'NATS_URL' } in app.config.ts and NATS_URL for every realtime role (web, sync, replicator), or run ROLE=sync with the replicator in one process: x dev --role sync,replicator",
|
|
275
|
+
});
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
|
|
258
279
|
/**
|
|
259
280
|
* The bytes on the bus socket are not the protocol we speak: an unknown NATS verb, a header block
|
|
260
281
|
* that is not `NATS/1.0`, a JetStream reply in a shape the API never produces. Always a version or
|