@ultimat3/realtime 27.6.1 → 27.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +7 -6
- package/README.md +18 -3
- package/package.json +5 -5
- package/src/client-channels.ts +23 -1
- package/src/detach.ts +11 -8
- package/src/nats-client.ts +5 -0
- package/src/nats-dial-wait.ts +61 -0
- package/src/nats-lib-client.ts +56 -5
- package/src/nats-reports.ts +80 -0
- package/src/nats-transport.ts +134 -43
- package/src/presence.ts +7 -0
- package/src/replicator.ts +51 -2
- package/src/server.ts +2 -0
- package/src/sync-bus-handlers.ts +4 -0
- package/src/sync-bus-outage.ts +62 -0
- package/src/sync-frames.ts +3 -0
- package/src/sync-node.ts +7 -7
- package/src/transport-env.ts +64 -2
package/CLAUDE.md
CHANGED
|
@@ -119,12 +119,13 @@ Tier 3 package. Channels, live queries, local-first sync. One protocol for all t
|
|
|
119
119
|
## The node, the bus and presence
|
|
120
120
|
|
|
121
121
|
- `sync` is stateless: no sticky sessions, nothing on a socket survives a restart.
|
|
122
|
-
- **`selectTransport(env, realtime)` is the one place
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
122
|
+
- **`selectTransport(env, realtime, { use })` is the one place config + env become a bus** (bucket
|
|
123
|
+
and presence TTL come back). `'nats'` refuses (`X_CONFIG_INVALID`) an unset `urlEnv` variable;
|
|
124
|
+
`'memory'` a set one. `use`: `'sockets'`/`'feed'` await a 15 s dial, then
|
|
125
|
+
`X_TRANSPORT_UNAVAILABLE`; `'publish'` dials in background. Never `waitOnFirstConnect`
|
|
126
|
+
(retries forever). A publish with no live connection is refused.
|
|
127
|
+
- **`nats` is imported only by `nats-lib-client.ts`**; its reconnect re-subscribes. When it GIVES
|
|
128
|
+
UP (`onClosed`) `NatsTransport` re-dials on backoff and re-binds.
|
|
128
129
|
- **Nothing leaves `NatsTransport` uncoded** — `#translating` wraps `publish`/`subscribe` refusals
|
|
129
130
|
as `X_TRANSPORT_UNAVAILABLE`. `#ensure` reuses a mid-reconnect client on purpose;
|
|
130
131
|
`Transport.onReconnect(listener)` (required) announces each recovery.
|
package/README.md
CHANGED
|
@@ -128,11 +128,11 @@ leaves `joining` on the node's first answer; nothing waits for app traffic.
|
|
|
128
128
|
|
|
129
129
|
| State | Entered when | Left when |
|
|
130
130
|
|---|---|---|
|
|
131
|
-
| `joining` | the subscribe is sent (on hold, and again on every reconnect) | the node's answer: a `records` channel's `replay-gap` (a fresh seat) or the `records` frame at a current cursor; an `events` channel's roster — sent on every seat, an empty one from a node with no presence (24.0.0) |
|
|
131
|
+
| `joining` | the subscribe is sent (on hold, and again on every reconnect); or the node refused it with `X_TRANSPORT_UNAVAILABLE` — its bus is away, which is its condition and not its verdict. The heartbeat re-asks every interval | the node's answer: a `records` channel's `replay-gap` (a fresh seat) or the `records` frame at a current cursor; an `events` channel's roster — sent on every seat, an empty one from a node with no presence (24.0.0) |
|
|
132
132
|
| `catching-up` | a `replay-gap`, or a frame in a new epoch: the `catchUp` query is re-read, frames held behind it | the read lands → `live` (the held frames applied in order) |
|
|
133
133
|
| `live` | as above | the socket drops → `offline`; a gap → `catching-up` |
|
|
134
134
|
| `offline` | the page socket closed | it reopens → `joining` |
|
|
135
|
-
| `failed` | the node
|
|
135
|
+
| `failed` | the node **decided** against the subscribe — a policy denial, an undeclared channel, a cap (`error()` is the coded refusal; terminal until the socket reopens) — or a catch-up read failed (retried on the client's curve, and on the next open) | a successful retry → `live` |
|
|
136
136
|
|
|
137
137
|
A channel declared with **neither** `records` nor `events` carries nothing, so nothing answers its
|
|
138
138
|
join and it reads `joining` for as long as it is held.
|
|
@@ -800,7 +800,22 @@ principal.
|
|
|
800
800
|
fanout across nodes. Until 22.0.0 `NATS_URL` alone decided and both keys were read by nothing.
|
|
801
801
|
`presenceTtlMs` comes back with it because the bucket's whole-stream age limit was derived from
|
|
802
802
|
it — a `PresenceRegistry` given a different number would report members leaving that never left.
|
|
803
|
-
Selection is pure; `connect()` is the dial
|
|
803
|
+
Selection is pure; `connect()` is the dial. **What the process does with the bus decides whether
|
|
804
|
+
that dial is waited for** — `selectTransport(env, realtime, { use })`, `BusUse`:
|
|
805
|
+
`'sockets'` (a `sync` node; the default) awaits it and the presence bucket, retried on backoff for
|
|
806
|
+
`connectWithinMs` (default `BUS_CONNECT_WAIT_MS`, 15 s) and then `X_TRANSPORT_UNAVAILABLE`;
|
|
807
|
+
`'feed'` (the replicator) awaits it and asks JetStream for nothing; `'publish'` (`web`, `worker`,
|
|
808
|
+
`scheduler`) starts it in the background and resolves at once. `selection.state()` answers
|
|
809
|
+
`'up' | 'connecting'`.
|
|
810
|
+
- **`NatsTransport` has two ways to open.** `connect({ withinMs, fix })` is for a node that cannot
|
|
811
|
+
work without the bus. `connectInBackground()` is for a process that only publishes: retried on the
|
|
812
|
+
transport's own backoff until it lands or `close()` runs, announced through `onReconnect`, and
|
|
813
|
+
until then every `publish`/`subscribe` rejects at once with `X_TRANSPORT_UNAVAILABLE` — never
|
|
814
|
+
parked, never queued. A publish made while the client is **reconnecting** is refused too: the
|
|
815
|
+
`nats` client would buffer it without bound and replay it late. `presenceBucket: 'first-use'`
|
|
816
|
+
asserts the KV bucket when `shared` is first read instead of at the dial, so a publisher needs no
|
|
817
|
+
JetStream. The first dial is ONE attempt of the library's — `waitOnFirstConnect` is not set,
|
|
818
|
+
because under it nats@2.29.3 retries a failed first dial forever and the promise never settles.
|
|
804
819
|
- **`NatsTransport` runs on the official `nats` client** — `nats@2.29.3`, pinned exact, admitted at
|
|
805
820
|
this transport seam and nowhere else
|
|
806
821
|
([`docs/idea/18-build-vs-wrap.md`](../../docs/idea/18-build-vs-wrap.md)). The package reaches it
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ultimat3/realtime",
|
|
3
|
-
"version": "27.
|
|
3
|
+
"version": "27.8.0",
|
|
4
4
|
"description": "Three-tier realtime: channels, live queries, local-first sync — one protocol, one mutator shape",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -40,10 +40,10 @@
|
|
|
40
40
|
"test": "bun test"
|
|
41
41
|
},
|
|
42
42
|
"dependencies": {
|
|
43
|
-
"@ultimat3/core": "27.
|
|
44
|
-
"@ultimat3/entity": "27.
|
|
45
|
-
"@ultimat3/policy": "27.
|
|
46
|
-
"@ultimat3/query": "27.
|
|
43
|
+
"@ultimat3/core": "27.8.0",
|
|
44
|
+
"@ultimat3/entity": "27.8.0",
|
|
45
|
+
"@ultimat3/policy": "27.8.0",
|
|
46
|
+
"@ultimat3/query": "27.8.0",
|
|
47
47
|
"nats": "2.29.3"
|
|
48
48
|
}
|
|
49
49
|
}
|
package/src/client-channels.ts
CHANGED
|
@@ -174,10 +174,21 @@ export class ChannelBook {
|
|
|
174
174
|
}
|
|
175
175
|
}
|
|
176
176
|
|
|
177
|
-
/**
|
|
177
|
+
/**
|
|
178
|
+
* A refusal `ack` naming the sid a channel is subscribed under. `false` when no membership
|
|
179
|
+
* here answers to it.
|
|
180
|
+
*/
|
|
178
181
|
refused(sid: string, error: unknown): boolean {
|
|
179
182
|
const entry = [...this.#byTopic.values()].find((candidate) => sidOf(candidate) === sid);
|
|
180
183
|
if (entry === undefined) return false;
|
|
184
|
+
if (isTransientRefusal(error)) {
|
|
185
|
+
// Not a decision: the node's bus is away, so nothing reaches this seat (or there is none
|
|
186
|
+
// yet). `joining` — asked, unanswered — and never `failed`: the beat re-asks every interval
|
|
187
|
+
// and the node's first answer (a roster, a `records` frame, a `replay-gap`) ends it. A read
|
|
188
|
+
// in flight and a terminal refusal keep their own state.
|
|
189
|
+
if (entry.state === 'live') this.#set(entry, 'joining');
|
|
190
|
+
return true;
|
|
191
|
+
}
|
|
181
192
|
entry.error = error;
|
|
182
193
|
this.#set(entry, 'failed');
|
|
183
194
|
return true;
|
|
@@ -361,6 +372,17 @@ export class ChannelBook {
|
|
|
361
372
|
}
|
|
362
373
|
}
|
|
363
374
|
|
|
375
|
+
/**
|
|
376
|
+
* The one refusal that is the NODE's condition and not its verdict: its bus is away
|
|
377
|
+
* (`X_TRANSPORT_UNAVAILABLE`). Everything else an `ack` carries — a policy denial, an unknown
|
|
378
|
+
* channel, a cap — is a decision and terminal. Read off `unknown`: a refusal is the node's data.
|
|
379
|
+
*/
|
|
380
|
+
export function isTransientRefusal(error: unknown): boolean {
|
|
381
|
+
return (
|
|
382
|
+
(error as { readonly code?: unknown } | null | undefined)?.code === 'X_TRANSPORT_UNAVAILABLE'
|
|
383
|
+
);
|
|
384
|
+
}
|
|
385
|
+
|
|
364
386
|
/** A channel membership's sid is its topic behind this — what the socket engine routes on too. */
|
|
365
387
|
export const CHANNEL_SID_PREFIX = 'channel:';
|
|
366
388
|
|
package/src/detach.ts
CHANGED
|
@@ -15,13 +15,16 @@ import { logger, renderThrowable, reportError } from '@ultimat3/core';
|
|
|
15
15
|
* with nothing above it to answer. `channel.ts` already imports it for the same reason.
|
|
16
16
|
*/
|
|
17
17
|
export function detach(work: Promise<unknown>, operation: string, at?: string): void {
|
|
18
|
-
void work.catch((error: unknown) =>
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
18
|
+
void work.catch((error: unknown) => reportDetached(error, operation, at));
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/** What `detach` says of a failure — also `sync-bus-outage.ts`'s answer to one that is no outage. */
|
|
22
|
+
export function reportDetached(error: unknown, operation: string, at?: string): void {
|
|
23
|
+
logger.error(`${operation} failed`, {
|
|
24
|
+
...(at === undefined ? {} : { at }),
|
|
25
|
+
error: renderThrowable(error),
|
|
26
26
|
});
|
|
27
|
+
// Nobody is awaiting this, so the log is the only trace it leaves — and a log is not a signal
|
|
28
|
+
// anyone is paged on. The bus is this node's dependency, never the client's.
|
|
29
|
+
reportError(error, { source: 'realtime', scope: { operation } });
|
|
27
30
|
}
|
package/src/nats-client.ts
CHANGED
|
@@ -71,6 +71,11 @@ export interface NatsClientOptions {
|
|
|
71
71
|
/** Our jitter policy, asked once per attempt — the herd is ours to spread, not the library's. */
|
|
72
72
|
readonly reconnectDelay?: (() => number) | undefined;
|
|
73
73
|
readonly requestTimeoutMs?: number | undefined;
|
|
74
|
+
/**
|
|
75
|
+
* How long ONE dial may sit before it is an attempt that failed. The library's own default is
|
|
76
|
+
* 20 s, which a server that accepts the socket and says nothing spends in full.
|
|
77
|
+
*/
|
|
78
|
+
readonly connectTimeoutMs?: number | undefined;
|
|
74
79
|
/** A background failure: a dropped connection, a server error, an exhausted reconnect. */
|
|
75
80
|
readonly onError?: ((error: unknown) => void) | undefined;
|
|
76
81
|
/** The library re-established the connection. The cluster behind it may be a different one. */
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
// Single responsibility: how long a dial is waited for — one attempt's connect timeout, and the
|
|
2
|
+
// bounded wait a process that cannot work without the bus puts on the whole retry loop.
|
|
3
|
+
|
|
4
|
+
import { finiteOption } from '@ultimat3/core';
|
|
5
|
+
import { TransportUnavailableError } from './errors';
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* One dial's connect timeout. Well under `BUS_CONNECT_WAIT_MS`: a black-holed server must fail
|
|
9
|
+
* several attempts inside a `sync` node's boot wait, so its refusal carries the attempt's own
|
|
10
|
+
* reason rather than an empty one, and a background dial keeps its backoff instead of the
|
|
11
|
+
* library's 20 s.
|
|
12
|
+
*/
|
|
13
|
+
export const DIAL_TIMEOUT_MS = 4_000;
|
|
14
|
+
|
|
15
|
+
export interface NatsConnectWait {
|
|
16
|
+
/** Refuse with `X_TRANSPORT_UNAVAILABLE` once this long has passed without a connection. */
|
|
17
|
+
readonly withinMs?: number | undefined;
|
|
18
|
+
/** The `fix:` of that refusal — the caller knows which role is waiting and what it needs. */
|
|
19
|
+
readonly fix?: string | undefined;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export interface DialWait {
|
|
23
|
+
readonly transport: string;
|
|
24
|
+
/** `host:port`, never the url: the url carries the credential. */
|
|
25
|
+
readonly server: string;
|
|
26
|
+
readonly withinMs: number;
|
|
27
|
+
readonly fix: string | undefined;
|
|
28
|
+
/** The last attempt's own refusal, read when the wait runs out. */
|
|
29
|
+
readonly lastFailure: () => string | undefined;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* `landing`, or a coded refusal once `withinMs` has passed. A real timer, never an injected clock:
|
|
34
|
+
* the wait bounds a container's boot, which the kubelet counts in real seconds. The loser is not
|
|
35
|
+
* cancelled — a dial cannot be — so the caller's `close()` is what ends it.
|
|
36
|
+
*/
|
|
37
|
+
export async function landWithin(
|
|
38
|
+
landing: Promise<void> | undefined,
|
|
39
|
+
wait: DialWait,
|
|
40
|
+
): Promise<void> {
|
|
41
|
+
const withinMs = finiteOption('createNatsTransport', 'connect.withinMs', wait.withinMs);
|
|
42
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
43
|
+
const late = new Promise<never>((_, reject) => {
|
|
44
|
+
timer = setTimeout(() => {
|
|
45
|
+
const failure = wait.lastFailure();
|
|
46
|
+
const last = failure === undefined ? '' : ` — last attempt: ${failure}`;
|
|
47
|
+
reject(
|
|
48
|
+
new TransportUnavailableError({
|
|
49
|
+
transport: wait.transport,
|
|
50
|
+
reason: `${wait.server} did not accept a connection within ${withinMs}ms${last}`,
|
|
51
|
+
...(wait.fix === undefined ? {} : { fix: wait.fix }),
|
|
52
|
+
}),
|
|
53
|
+
);
|
|
54
|
+
}, withinMs);
|
|
55
|
+
});
|
|
56
|
+
try {
|
|
57
|
+
await Promise.race([landing, late]);
|
|
58
|
+
} finally {
|
|
59
|
+
clearTimeout(timer);
|
|
60
|
+
}
|
|
61
|
+
}
|
package/src/nats-lib-client.ts
CHANGED
|
@@ -92,7 +92,7 @@ class LibNatsClient implements NatsClient {
|
|
|
92
92
|
}
|
|
93
93
|
|
|
94
94
|
get connected(): boolean {
|
|
95
|
-
return this.#connected && !this.#connection.isClosed();
|
|
95
|
+
return this.#connected && !this.#connection.isClosed() && protocolUp(this.#connection);
|
|
96
96
|
}
|
|
97
97
|
|
|
98
98
|
publish(subject: string, payload: Uint8Array): void {
|
|
@@ -118,6 +118,7 @@ class LibNatsClient implements NatsClient {
|
|
|
118
118
|
payload: Uint8Array,
|
|
119
119
|
options: NatsRequestOptions = {},
|
|
120
120
|
): Promise<NatsMessage> {
|
|
121
|
+
this.#assertLive(subject);
|
|
121
122
|
const built = headersOf(options.headers);
|
|
122
123
|
try {
|
|
123
124
|
const reply = await this.#connection.request(subject, payload, {
|
|
@@ -140,6 +141,7 @@ class LibNatsClient implements NatsClient {
|
|
|
140
141
|
payload: Uint8Array,
|
|
141
142
|
options: NatsRequestManyOptions,
|
|
142
143
|
): Promise<readonly NatsMessage[]> {
|
|
144
|
+
this.#assertLive(subject);
|
|
143
145
|
const collected: NatsMessage[] = [];
|
|
144
146
|
try {
|
|
145
147
|
const replies = await this.#connection.requestMany(subject, payload, {
|
|
@@ -162,6 +164,21 @@ class LibNatsClient implements NatsClient {
|
|
|
162
164
|
await this.#connection.close();
|
|
163
165
|
}
|
|
164
166
|
|
|
167
|
+
/**
|
|
168
|
+
* A request has an answer to wait for, so one made with no live connection is refused HERE.
|
|
169
|
+
* The library takes it instead — buffered with its publishes, sent if the reconnect lands and
|
|
170
|
+
* timed out if it does not — so every presence write on a `sync` node whose bus was away held
|
|
171
|
+
* its socket's frame lane for the whole request timeout before saying what was already known.
|
|
172
|
+
* The fake has always refused it (`nats-fake.ts`, `#assertLive`); this is the client agreeing.
|
|
173
|
+
*/
|
|
174
|
+
#assertLive(subject: string): void {
|
|
175
|
+
if (this.connected) return;
|
|
176
|
+
throw unavailable(
|
|
177
|
+
this.#target,
|
|
178
|
+
`${subject} was not asked: the client is reconnecting to the bus`,
|
|
179
|
+
);
|
|
180
|
+
}
|
|
181
|
+
|
|
165
182
|
/**
|
|
166
183
|
* The library's own status stream is the only place a drop or a recovery is announced. Nothing
|
|
167
184
|
* awaits it, so it can neither throw nor end the process: a drop reports and flips `connected`,
|
|
@@ -211,6 +228,29 @@ class LibNatsClient implements NatsClient {
|
|
|
211
228
|
}
|
|
212
229
|
}
|
|
213
230
|
|
|
231
|
+
/**
|
|
232
|
+
* The library's own flag, read where it is set. `#connected` above follows the status ITERATOR,
|
|
233
|
+
* which delivers a disconnect a tick or more after the socket dropped; in between the library
|
|
234
|
+
* already knows (`protocol.connected` is cleared in the socket's close handler) and already
|
|
235
|
+
* buffers every publish. nats@2.29.3 has no option to forbid that buffer, so the flag is the only
|
|
236
|
+
* way to refuse a publish it would queue. It is not on the typed surface: a library that moves it
|
|
237
|
+
* answers `true` here and the status iterator is the reading again — the behaviour before this.
|
|
238
|
+
*/
|
|
239
|
+
const protocolUp = (connection: NatsConnection): boolean => {
|
|
240
|
+
const protocol: unknown = Reflect.get(connection, 'protocol');
|
|
241
|
+
if (typeof protocol !== 'object' || protocol === null) return true;
|
|
242
|
+
return Reflect.get(protocol, 'connected') !== false;
|
|
243
|
+
};
|
|
244
|
+
|
|
245
|
+
/**
|
|
246
|
+
* How often the client pings, and how many may go unanswered. The library's defaults are two
|
|
247
|
+
* minutes and two: a connection whose packets are silently dropped stays "connected" — and keeps
|
|
248
|
+
* buffering publishes — for up to six minutes. 10 s and two is noticed within half a minute, for
|
|
249
|
+
* one 6-byte frame every ten seconds on a connection that is otherwise idle.
|
|
250
|
+
*/
|
|
251
|
+
export const PING_INTERVAL_MS = 10_000;
|
|
252
|
+
export const MAX_PINGS_OUT = 2;
|
|
253
|
+
|
|
214
254
|
/** `QueuedIterator.stop()` is the library's, and missing from the `AsyncIterable` it is typed as. */
|
|
215
255
|
const stopStream = (stream: unknown): void => {
|
|
216
256
|
if (typeof stream !== 'object' || stream === null || !('stop' in stream)) return;
|
|
@@ -219,9 +259,14 @@ const stopStream = (stream: unknown): void => {
|
|
|
219
259
|
};
|
|
220
260
|
|
|
221
261
|
/**
|
|
222
|
-
* The production `NatsConnect
|
|
223
|
-
*
|
|
224
|
-
*
|
|
262
|
+
* The production `NatsConnect`: ONE first dial, refused at once when the server is not there.
|
|
263
|
+
*
|
|
264
|
+
* Deliberately not `waitOnFirstConnect`. Until 2026-10 it was set, on the reading that the first
|
|
265
|
+
* dial "retries on the same budget as a later loss" — it does not: in nats@2.29.3 a failed first
|
|
266
|
+
* dial under that option `continue`s past the attempt counter, so it retries FOREVER, and the
|
|
267
|
+
* promise this returns never settles while the server is down. Every role awaited it at boot, so
|
|
268
|
+
* a pod that restarted during a NATS outage hung before it bound a socket. The retry is the
|
|
269
|
+
* transport's (`NatsTransport.#redial`), on our backoff, where `close()` can end it.
|
|
225
270
|
*/
|
|
226
271
|
export const openNatsClient = async (options: NatsClientOptions): Promise<NatsClient> => {
|
|
227
272
|
const target = parseNatsUrl(options.url);
|
|
@@ -235,11 +280,17 @@ export const openNatsClient = async (options: NatsClientOptions): Promise<NatsCl
|
|
|
235
280
|
options.maxReconnectAttempts === undefined
|
|
236
281
|
? undefined
|
|
237
282
|
: finiteOption('the nats client', 'maxReconnectAttempts', options.maxReconnectAttempts);
|
|
283
|
+
const connectTimeoutMs =
|
|
284
|
+
options.connectTimeoutMs === undefined
|
|
285
|
+
? undefined
|
|
286
|
+
: finiteOption('the nats client', 'connectTimeoutMs', options.connectTimeoutMs);
|
|
238
287
|
try {
|
|
239
288
|
const connection = await connect({
|
|
240
289
|
servers: [`${target.host}:${target.port}`],
|
|
241
290
|
name: options.name ?? 'ultimate',
|
|
242
|
-
|
|
291
|
+
pingInterval: PING_INTERVAL_MS,
|
|
292
|
+
maxPingOut: MAX_PINGS_OUT,
|
|
293
|
+
...(connectTimeoutMs === undefined ? {} : { timeout: connectTimeoutMs }),
|
|
243
294
|
...(maxReconnectAttempts === undefined ? {} : { maxReconnectAttempts }),
|
|
244
295
|
...(options.reconnectDelay === undefined
|
|
245
296
|
? {}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
// Single responsibility: where a `NatsTransport`'s background failures go, and how often. A FAULT
|
|
2
|
+
// (a subscriber that threw, a re-subscribe the server refused) is said every time. An OUTAGE (a
|
|
3
|
+
// drop, a refused dial, a client the library gave up on) is one fact however long it lasts: said
|
|
4
|
+
// on its 1st, 2nd, 4th, 8th, … failure and once more when the bus answers again.
|
|
5
|
+
|
|
6
|
+
import { isOutageMilestone, isUltimateError, logger, renderThrowable } from '@ultimat3/core';
|
|
7
|
+
|
|
8
|
+
export interface NatsReportsOptions {
|
|
9
|
+
readonly transport: string;
|
|
10
|
+
/** The app's own sink. Absent, or throwing, the logger line is written instead. */
|
|
11
|
+
readonly onError: ((error: unknown, subject: string) => void) | undefined;
|
|
12
|
+
/**
|
|
13
|
+
* `'error'` for a process that cannot do its job without the bus (a `sync` node, a replicator);
|
|
14
|
+
* `'warn'` for one that only publishes — degraded, and still serving.
|
|
15
|
+
*/
|
|
16
|
+
readonly outageLevel: 'warn' | 'error';
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export class NatsReports {
|
|
20
|
+
readonly #options: NatsReportsOptions;
|
|
21
|
+
/** Failures of the bus itself since it last answered. */
|
|
22
|
+
#failures = 0;
|
|
23
|
+
|
|
24
|
+
constructor(options: NatsReportsOptions) {
|
|
25
|
+
this.#options = options;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Every background failure lands here or in `outage`. Dropping it when the caller passed no
|
|
30
|
+
* handler is what turns "no changes arrive" into a debugging session with nothing to read, so
|
|
31
|
+
* the default emits rather than swallows.
|
|
32
|
+
*/
|
|
33
|
+
fault(error: unknown, subject: string): void {
|
|
34
|
+
this.#say(error, subject, 'error');
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* One more failure of the bus itself, thinned and at the level its owner chose. `always` is for
|
|
39
|
+
* a TRANSITION rather than a retry — the library giving up on a client happens once per client,
|
|
40
|
+
* and it is the line that says the transport's own re-dial took over.
|
|
41
|
+
*/
|
|
42
|
+
outage(error: unknown, always = false): void {
|
|
43
|
+
this.#failures += 1;
|
|
44
|
+
if (!always && !isOutageMilestone(this.#failures)) return;
|
|
45
|
+
this.#say(error, this.#options.transport, this.#options.outageLevel, this.#failures);
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** The bus answers again: one line saying after how many failures, and the count starts over. */
|
|
49
|
+
recovered(): void {
|
|
50
|
+
const after = this.#failures;
|
|
51
|
+
if (after === 0) return;
|
|
52
|
+
this.#failures = 0;
|
|
53
|
+
logger.info('nats transport recovered', { transport: this.#options.transport, after });
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
#say(error: unknown, subject: string, level: 'warn' | 'error', failures?: number): void {
|
|
57
|
+
const handler = this.#options.onError;
|
|
58
|
+
if (handler !== undefined) {
|
|
59
|
+
// Total: the handler is injected, and its throw would land in whatever called this — the
|
|
60
|
+
// dial loop (started with `void`: an unhandled rejection and no more retries), the
|
|
61
|
+
// library's status loop, a subscriber's delivery. A reporter that cannot report is not a
|
|
62
|
+
// reason to stop dialling; the fault falls through to the logger line below.
|
|
63
|
+
try {
|
|
64
|
+
handler(error, subject);
|
|
65
|
+
return;
|
|
66
|
+
} catch (thrown) {
|
|
67
|
+
logger.error('nats transport onError threw', { error: renderThrowable(thrown) });
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
logger[level]('nats transport error', {
|
|
71
|
+
transport: this.#options.transport,
|
|
72
|
+
subject,
|
|
73
|
+
...(failures === undefined ? {} : { failures }),
|
|
74
|
+
code: isUltimateError(error) ? error.code : undefined,
|
|
75
|
+
// `renderThrowable`, never `String(error)`: this is a reporter, and a throwable that fights
|
|
76
|
+
// being read makes the report the thing that throws.
|
|
77
|
+
error: renderThrowable(error),
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
}
|
package/src/nats-transport.ts
CHANGED
|
@@ -3,12 +3,16 @@
|
|
|
3
3
|
// re-establishing subscriptions, which is what makes a `sync` node stateless. The one loss it does
|
|
4
4
|
// not own is its own end: a client closed for good once its reconnect budget is spent is replaced
|
|
5
5
|
// here, on our backoff, and every kept subscription is bound again on the new one.
|
|
6
|
+
//
|
|
7
|
+
// Two ways to open it, because two kinds of process hold one. A node that SERVES from the bus
|
|
8
|
+
// awaits `connect()` and refuses to boot without it; a process that only PUBLISHES calls
|
|
9
|
+
// `connectInBackground()` and boots regardless — its publishes are refused, typed and at once,
|
|
10
|
+
// until the dial lands.
|
|
6
11
|
|
|
7
12
|
import {
|
|
8
13
|
type Clock,
|
|
9
14
|
finiteOption,
|
|
10
15
|
isUltimateError,
|
|
11
|
-
logger,
|
|
12
16
|
renderThrowable,
|
|
13
17
|
systemClock,
|
|
14
18
|
} from '@ultimat3/core';
|
|
@@ -16,9 +20,11 @@ import { TransportUnavailableError } from './errors';
|
|
|
16
20
|
import type { Transport, TransportHandler, TransportSet, TransportSubscription } from './fanout';
|
|
17
21
|
import type { NatsClient, NatsConnect } from './nats-client';
|
|
18
22
|
import { parseNatsUrl } from './nats-client';
|
|
23
|
+
import { DIAL_TIMEOUT_MS, landWithin, type NatsConnectWait } from './nats-dial-wait';
|
|
19
24
|
import { ensureKvBucket } from './nats-jetstream';
|
|
20
25
|
import { NatsKvSet } from './nats-kv';
|
|
21
26
|
import { openNatsClient } from './nats-open';
|
|
27
|
+
import { NatsReports } from './nats-reports';
|
|
22
28
|
import { NatsSubscriptions } from './nats-subscriptions';
|
|
23
29
|
import { type BackoffPolicy, defaultBackoff, policyDelay, type Rng } from './thundering-herd';
|
|
24
30
|
|
|
@@ -39,6 +45,20 @@ export interface NatsTransportOptions {
|
|
|
39
45
|
readonly rng?: Rng;
|
|
40
46
|
/** Injected so the whole transport — reconnect included — runs in a test with no network. */
|
|
41
47
|
readonly connect?: NatsConnect;
|
|
48
|
+
/**
|
|
49
|
+
* When the presence bucket is asserted. `'dial'` (the default) is a node that serves presence: no
|
|
50
|
+
* JetStream, no connection. `'first-use'` is a process that only publishes — it never reads
|
|
51
|
+
* `shared`, so a server without JetStream, or without the bucket, must not cost it the bus.
|
|
52
|
+
*/
|
|
53
|
+
readonly presenceBucket?: 'dial' | 'first-use';
|
|
54
|
+
/** One dial's connect timeout. Default `DIAL_TIMEOUT_MS`. */
|
|
55
|
+
readonly dialTimeoutMs?: number;
|
|
56
|
+
/**
|
|
57
|
+
* The level an OUTAGE is logged at. `'error'` (the default) is a process that cannot do its job
|
|
58
|
+
* without the bus — a `sync` node, a replicator. `'warn'` is one that only publishes: degraded,
|
|
59
|
+
* and still serving. A subscriber that throws is an `error` under either.
|
|
60
|
+
*/
|
|
61
|
+
readonly outageLevel?: 'warn' | 'error';
|
|
42
62
|
}
|
|
43
63
|
|
|
44
64
|
const DEFAULT_ATTEMPTS = 10;
|
|
@@ -57,9 +77,20 @@ export class NatsTransport implements Transport {
|
|
|
57
77
|
#dialing: Promise<NatsClient> | undefined;
|
|
58
78
|
#retries = 0;
|
|
59
79
|
#closed = false;
|
|
60
|
-
/**
|
|
80
|
+
/** False until the first dial lands: what a refusal says while a background dial is in flight. */
|
|
81
|
+
#everUp = false;
|
|
82
|
+
readonly #lazyBucket: boolean;
|
|
83
|
+
readonly #dialTimeoutMs: number;
|
|
84
|
+
/** Under `'first-use'`: the clients whose bucket has been asserted since they last connected. */
|
|
85
|
+
readonly #bucketed = new WeakSet<NatsClient>();
|
|
86
|
+
/** Set while no client is held and one is being dialled in the background; the attempt in flight. */
|
|
61
87
|
#redialing: number | undefined;
|
|
88
|
+
#redialLoop: Promise<void> | undefined;
|
|
89
|
+
/** The last background dial's own refusal, for the cause of a `connect({ withinMs })` that gave up. */
|
|
90
|
+
#lastFailure: string | undefined;
|
|
62
91
|
#wakeRedial: (() => void) | undefined;
|
|
92
|
+
/** Where background failures go: a fault every time, an outage thinned (`nats-reports.ts`). */
|
|
93
|
+
readonly #reports: NatsReports;
|
|
63
94
|
readonly #subscriptions = new NatsSubscriptions();
|
|
64
95
|
readonly #reconnectListeners = new Set<() => void>();
|
|
65
96
|
|
|
@@ -68,6 +99,11 @@ export class NatsTransport implements Transport {
|
|
|
68
99
|
// a container that reports itself healthy on one is a container nothing will ever page about.
|
|
69
100
|
parseNatsUrl(options.url);
|
|
70
101
|
this.#options = options;
|
|
102
|
+
this.#reports = new NatsReports({
|
|
103
|
+
transport: this.name,
|
|
104
|
+
onError: options.onError,
|
|
105
|
+
outageLevel: options.outageLevel ?? 'error',
|
|
106
|
+
});
|
|
71
107
|
this.#connect = options.connect ?? openNatsClient;
|
|
72
108
|
this.#backoff = options.backoff ?? defaultBackoff;
|
|
73
109
|
this.#attempts = finiteOption(
|
|
@@ -76,16 +112,51 @@ export class NatsTransport implements Transport {
|
|
|
76
112
|
options.maxReconnectAttempts ?? DEFAULT_ATTEMPTS,
|
|
77
113
|
);
|
|
78
114
|
this.#rng = options.rng ?? Math.random;
|
|
115
|
+
this.#dialTimeoutMs = finiteOption(
|
|
116
|
+
'createNatsTransport',
|
|
117
|
+
'dialTimeoutMs',
|
|
118
|
+
options.dialTimeoutMs ?? DIAL_TIMEOUT_MS,
|
|
119
|
+
);
|
|
120
|
+
this.#lazyBucket = options.presenceBucket === 'first-use';
|
|
79
121
|
this.shared = new NatsKvSet({
|
|
80
|
-
client: () => this.#
|
|
122
|
+
client: () => this.#presenceClient(),
|
|
81
123
|
bucket: options.bucket,
|
|
82
124
|
clock: options.clock ?? systemClock,
|
|
83
125
|
});
|
|
84
126
|
}
|
|
85
127
|
|
|
86
|
-
/**
|
|
87
|
-
|
|
88
|
-
|
|
128
|
+
/**
|
|
129
|
+
* Fail fast at boot rather than on the first change: `/readyz` is meant to catch a dead bus.
|
|
130
|
+
* Bare, it is ONE dial. With `withinMs` the dial is retried on our backoff for that long — a
|
|
131
|
+
* node that raced the bus into readiness recovers on its own — and then refused, naming the
|
|
132
|
+
* server, the wait and the last attempt's own failure. The retry is not stopped by the refusal;
|
|
133
|
+
* `close()` stops it, and that is what a failed boot calls.
|
|
134
|
+
*/
|
|
135
|
+
async connect(wait: NatsConnectWait = {}): Promise<void> {
|
|
136
|
+
if (wait.withinMs === undefined) {
|
|
137
|
+
await this.#ensure();
|
|
138
|
+
return;
|
|
139
|
+
}
|
|
140
|
+
const target = parseNatsUrl(this.#options.url);
|
|
141
|
+
await landWithin(this.#client === undefined ? this.#redial() : undefined, {
|
|
142
|
+
transport: this.name,
|
|
143
|
+
server: `${target.host}:${target.port}`,
|
|
144
|
+
withinMs: wait.withinMs,
|
|
145
|
+
fix: wait.fix,
|
|
146
|
+
lastFailure: () => this.#lastFailure,
|
|
147
|
+
});
|
|
148
|
+
// The loop also ends when the transport is closed under it.
|
|
149
|
+
if (this.#client === undefined) await this.#ensure();
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/**
|
|
153
|
+
* Dial without being waited for: retried on our backoff until it lands or `close()` runs, and
|
|
154
|
+
* announced to the `onReconnect` listeners when it does. Until then every publish and subscribe
|
|
155
|
+
* is refused at once (`#ensure`) — never parked behind the dial, never queued.
|
|
156
|
+
*/
|
|
157
|
+
connectInBackground(): void {
|
|
158
|
+
if (this.#closed || this.#client !== undefined) return;
|
|
159
|
+
void this.#redial();
|
|
89
160
|
}
|
|
90
161
|
|
|
91
162
|
get connected(): boolean {
|
|
@@ -94,6 +165,16 @@ export class NatsTransport implements Transport {
|
|
|
94
165
|
|
|
95
166
|
async publish(subject: string, payload: string): Promise<void> {
|
|
96
167
|
const client = await this.#ensure();
|
|
168
|
+
// The library BUFFERS a publish made while it reconnects, without bound, and replays the lot
|
|
169
|
+
// when the server is back: a queue nobody sized, delivering events whose only meaning was
|
|
170
|
+
// "re-read now" minutes late. A subscriber is told of the gap (`onReconnect`); the publisher
|
|
171
|
+
// is told here.
|
|
172
|
+
if (!client.connected) {
|
|
173
|
+
throw new TransportUnavailableError({
|
|
174
|
+
transport: this.name,
|
|
175
|
+
reason: `publish to ${subject} was refused: the client is reconnecting to the bus`,
|
|
176
|
+
});
|
|
177
|
+
}
|
|
97
178
|
// `client.publish` is synchronous and refuses locally: a bad subject, a payload over the
|
|
98
179
|
// server's `max_payload`, a connection torn down between the `#ensure` and this line. Those
|
|
99
180
|
// are the LIBRARY's errors — or an app-supplied `connect`'s — so they arrive uncoded, and
|
|
@@ -118,7 +199,7 @@ export class NatsTransport implements Transport {
|
|
|
118
199
|
try {
|
|
119
200
|
handler(decoder.decode(message.payload), message.subject);
|
|
120
201
|
} catch (error) {
|
|
121
|
-
this.#
|
|
202
|
+
this.#reports.fault(error, message.subject);
|
|
122
203
|
}
|
|
123
204
|
},
|
|
124
205
|
(call) => this.#translating(`subscribe to ${subject}`, call),
|
|
@@ -167,7 +248,9 @@ export class NatsTransport implements Transport {
|
|
|
167
248
|
return Promise.reject(
|
|
168
249
|
new TransportUnavailableError({
|
|
169
250
|
transport: this.name,
|
|
170
|
-
reason:
|
|
251
|
+
reason: this.#everUp
|
|
252
|
+
? `the connection closed once its reconnect budget was spent, and re-dial attempt ${this.#redialing} is in progress`
|
|
253
|
+
: `the bus has not connected yet, and dial attempt ${this.#redialing} is in progress`,
|
|
171
254
|
}),
|
|
172
255
|
);
|
|
173
256
|
}
|
|
@@ -197,10 +280,11 @@ export class NatsTransport implements Transport {
|
|
|
197
280
|
url: this.#options.url,
|
|
198
281
|
name: 'ultimate',
|
|
199
282
|
maxReconnectAttempts: this.#attempts,
|
|
283
|
+
connectTimeoutMs: this.#dialTimeoutMs,
|
|
200
284
|
// The library retries; the spread is ours, so a cluster restart does not bring every node
|
|
201
285
|
// back on the same millisecond.
|
|
202
286
|
reconnectDelay: () => policyDelay(this.#backoff, ++this.#retries, this.#rng),
|
|
203
|
-
onError: (error) => this.#
|
|
287
|
+
onError: (error) => this.#reports.outage(error),
|
|
204
288
|
onReconnect: () => this.#recovered(),
|
|
205
289
|
onClosed: () => {
|
|
206
290
|
if (opened === undefined) closedEarly = true;
|
|
@@ -208,7 +292,7 @@ export class NatsTransport implements Transport {
|
|
|
208
292
|
},
|
|
209
293
|
});
|
|
210
294
|
try {
|
|
211
|
-
await this.#ensureBucket(client);
|
|
295
|
+
if (!this.#lazyBucket) await this.#ensureBucket(client);
|
|
212
296
|
if (closedEarly) {
|
|
213
297
|
throw new TransportUnavailableError({
|
|
214
298
|
transport: this.name,
|
|
@@ -229,10 +313,21 @@ export class NatsTransport implements Transport {
|
|
|
229
313
|
}
|
|
230
314
|
opened = client;
|
|
231
315
|
this.#client = client;
|
|
316
|
+
this.#everUp = true;
|
|
232
317
|
this.#retries = 0;
|
|
233
318
|
return client;
|
|
234
319
|
}
|
|
235
320
|
|
|
321
|
+
/** `shared`'s client: under `'first-use'` this is where the bucket is asserted, once per client. */
|
|
322
|
+
async #presenceClient(): Promise<NatsClient> {
|
|
323
|
+
const client = await this.#ensure();
|
|
324
|
+
if (this.#lazyBucket && !this.#bucketed.has(client)) {
|
|
325
|
+
await this.#ensureBucket(client);
|
|
326
|
+
this.#bucketed.add(client);
|
|
327
|
+
}
|
|
328
|
+
return client;
|
|
329
|
+
}
|
|
330
|
+
|
|
236
331
|
/**
|
|
237
332
|
* The library spent its reconnect budget and closed the client. Handing it out again is a node
|
|
238
333
|
* that refuses every publish until somebody restarts it — and the liveness probe never asks the
|
|
@@ -242,32 +337,39 @@ export class NatsTransport implements Transport {
|
|
|
242
337
|
if (this.#closed || this.#client !== client) return;
|
|
243
338
|
this.#client = undefined;
|
|
244
339
|
this.#subscriptions.orphan();
|
|
245
|
-
this.#
|
|
340
|
+
this.#reports.outage(
|
|
246
341
|
new TransportUnavailableError({
|
|
247
342
|
transport: this.name,
|
|
248
343
|
reason: 'the connection closed once its reconnect budget was spent; re-dialling',
|
|
249
344
|
}),
|
|
250
|
-
|
|
345
|
+
true,
|
|
251
346
|
);
|
|
252
347
|
void this.#redial();
|
|
253
348
|
}
|
|
254
349
|
|
|
255
350
|
/**
|
|
256
|
-
*
|
|
257
|
-
*
|
|
258
|
-
*
|
|
259
|
-
*
|
|
351
|
+
* Dial until a client is up or the transport is closed — the first dial of a process that does
|
|
352
|
+
* not wait for the bus, and the replacement of a client the library gave up on. One loop however
|
|
353
|
+
* many callers ask. Bounded by our backoff between attempts (capped at the policy's `maxMs`), so
|
|
354
|
+
* a long outage costs one dial per interval rather than a spin. A dial that lands re-binds every
|
|
355
|
+
* kept subscription and is announced: changes published in the gap are gone.
|
|
260
356
|
*/
|
|
261
|
-
|
|
262
|
-
|
|
357
|
+
#redial(): Promise<void> {
|
|
358
|
+
this.#redialLoop ??= this.#redialUntilUp();
|
|
359
|
+
return this.#redialLoop;
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
async #redialUntilUp(): Promise<void> {
|
|
263
363
|
try {
|
|
264
364
|
for (let attempt = 1; !this.#closed && this.#client === undefined; attempt += 1) {
|
|
265
365
|
this.#redialing = attempt;
|
|
266
366
|
try {
|
|
267
367
|
const client = await this.#dialOnce();
|
|
368
|
+
this.#lastFailure = undefined;
|
|
369
|
+
this.#reports.recovered();
|
|
268
370
|
// Coded like a first subscribe's refusal; the subscription stays kept for the next client.
|
|
269
371
|
for (const failure of this.#subscriptions.bindAll(client)) {
|
|
270
|
-
this.#
|
|
372
|
+
this.#reports.fault(
|
|
271
373
|
this.#coded(`re-subscribe to ${failure.subject}`, failure.error),
|
|
272
374
|
failure.subject,
|
|
273
375
|
);
|
|
@@ -275,12 +377,19 @@ export class NatsTransport implements Transport {
|
|
|
275
377
|
this.#announce();
|
|
276
378
|
} catch (error) {
|
|
277
379
|
if (this.#closed) return;
|
|
278
|
-
this.#
|
|
380
|
+
this.#lastFailure = renderThrowable(error);
|
|
381
|
+
// Thinned (`#outage`): said at once and then ever more rarely, rather than once per
|
|
382
|
+
// dial per pod for as long as it lasts.
|
|
383
|
+
this.#reports.outage(error);
|
|
279
384
|
await this.#pause(policyDelay(this.#backoff, attempt, this.#rng));
|
|
280
385
|
}
|
|
281
386
|
}
|
|
282
387
|
} finally {
|
|
388
|
+
// Both, HERE, in the loop's own last synchronous step — never in a reaction to its promise.
|
|
389
|
+
// A reaction runs a microtask later, and a client lost in that gap (`#lost`) was handed the
|
|
390
|
+
// finished loop back: nothing dialled again and its subscriptions stayed orphaned.
|
|
283
391
|
this.#redialing = undefined;
|
|
392
|
+
this.#redialLoop = undefined;
|
|
284
393
|
}
|
|
285
394
|
}
|
|
286
395
|
|
|
@@ -318,8 +427,11 @@ export class NatsTransport implements Transport {
|
|
|
318
427
|
this.#retries = 0;
|
|
319
428
|
const client = this.#client;
|
|
320
429
|
if (client === undefined) return;
|
|
430
|
+
this.#reports.recovered();
|
|
321
431
|
// Not awaited: the library calls this from its status loop. The failure has one place to go.
|
|
322
|
-
|
|
432
|
+
// A lazy bucket is forgotten instead, and asserted again by whoever next reads `shared`.
|
|
433
|
+
if (this.#lazyBucket) this.#bucketed.delete(client);
|
|
434
|
+
else void this.#ensureBucket(client).catch((error: unknown) => this.#reports.outage(error));
|
|
323
435
|
this.#announce();
|
|
324
436
|
}
|
|
325
437
|
|
|
@@ -328,7 +440,7 @@ export class NatsTransport implements Transport {
|
|
|
328
440
|
try {
|
|
329
441
|
listener();
|
|
330
442
|
} catch (error) {
|
|
331
|
-
this.#
|
|
443
|
+
this.#reports.fault(error, this.name);
|
|
332
444
|
}
|
|
333
445
|
}
|
|
334
446
|
}
|
|
@@ -355,25 +467,4 @@ export class NatsTransport implements Transport {
|
|
|
355
467
|
reason: `${what} was refused: ${renderThrowable(error)}`,
|
|
356
468
|
});
|
|
357
469
|
}
|
|
358
|
-
|
|
359
|
-
/**
|
|
360
|
-
* Every background failure lands here — a lost connection, a throwing subscriber, an exhausted
|
|
361
|
-
* reconnect. Dropping it when the caller passed no handler is what turns "no changes arrive"
|
|
362
|
-
* into a debugging session with nothing to read, so the default emits rather than swallows.
|
|
363
|
-
*/
|
|
364
|
-
#report(error: unknown, subject: string): void {
|
|
365
|
-
const handler = this.#options.onError;
|
|
366
|
-
if (handler !== undefined) {
|
|
367
|
-
handler(error, subject);
|
|
368
|
-
return;
|
|
369
|
-
}
|
|
370
|
-
logger.error('nats transport error', {
|
|
371
|
-
transport: this.name,
|
|
372
|
-
subject,
|
|
373
|
-
code: isUltimateError(error) ? error.code : undefined,
|
|
374
|
-
// `renderThrowable`, never `String(error)`: this is a reporter, and a throwable that fights
|
|
375
|
-
// being read makes the report the thing that throws.
|
|
376
|
-
error: renderThrowable(error),
|
|
377
|
-
});
|
|
378
|
-
}
|
|
379
470
|
}
|
package/src/presence.ts
CHANGED
|
@@ -93,6 +93,13 @@ export class PresenceRegistry {
|
|
|
93
93
|
return this.#ttlMs;
|
|
94
94
|
}
|
|
95
95
|
|
|
96
|
+
/** Rooms with a member this node has seen — what the next `sweepAll` will read the bus for. */
|
|
97
|
+
get rooms(): number {
|
|
98
|
+
let rooms = 0;
|
|
99
|
+
for (const members of this.#seen.values()) if (members.size > 0) rooms += 1;
|
|
100
|
+
return rooms;
|
|
101
|
+
}
|
|
102
|
+
|
|
96
103
|
/** Recommended client heartbeat interval: one lost beat must not read as a leave. */
|
|
97
104
|
get heartbeatMs(): number {
|
|
98
105
|
return Math.max(1_000, Math.floor(this.#ttlMs / 3));
|
package/src/replicator.ts
CHANGED
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
// its feed and reports `/readyz` false. A stream that ENDS — or a lock that is LOST — is this file's
|
|
12
12
|
// to answer: `running` goes false, everything is let go, and the process asks again on a backoff.
|
|
13
13
|
|
|
14
|
-
import { logger, renderThrowable, uuidV7, withSpan } from '@ultimat3/core';
|
|
14
|
+
import { isUltimateError, logger, renderThrowable, uuidV7, withSpan } from '@ultimat3/core';
|
|
15
15
|
import type { AdvisoryLock } from './advisory-lock';
|
|
16
16
|
import type { ChangeEvent, ChangeFeed } from './changefeed';
|
|
17
17
|
import { ReplicationFailedError } from './errors';
|
|
@@ -100,6 +100,18 @@ interface Refusal {
|
|
|
100
100
|
*/
|
|
101
101
|
export const STOP_DEADLINE_MS = 5_000;
|
|
102
102
|
|
|
103
|
+
/**
|
|
104
|
+
* How long a publish refused because the bus is AWAY is waited out, in the handler, before the run
|
|
105
|
+
* ends: three more attempts over two seconds. A NATS restart is a second or two, and ending the
|
|
106
|
+
* run for it costs a stream stop, an unlock, a backoff and a re-read from `lastLsn` — with
|
|
107
|
+
* `/readyz` false throughout. Longer than this is an outage, and the takeover loop's to handle.
|
|
108
|
+
*/
|
|
109
|
+
export const PUBLISH_RETRY_DELAYS_MS: readonly number[] = [100, 400, 1_500];
|
|
110
|
+
|
|
111
|
+
/** The bus is unreachable — as opposed to a change it will never take (a payload over the limit). */
|
|
112
|
+
const busAway = (thrown: unknown): boolean =>
|
|
113
|
+
isUltimateError(thrown) && thrown.code === 'X_TRANSPORT_UNAVAILABLE';
|
|
114
|
+
|
|
103
115
|
const fencedOut = (reason: string): ReplicationFailedError =>
|
|
104
116
|
new ReplicationFailedError({
|
|
105
117
|
stage: 'stream',
|
|
@@ -131,6 +143,8 @@ export function changeFeedReplicator(options: ReplicatorOptions): Replicator {
|
|
|
131
143
|
/** The run whose stream and lock still speak for this replicator. */
|
|
132
144
|
let current: Run | undefined;
|
|
133
145
|
let cancelRetry: (() => void) | undefined;
|
|
146
|
+
/** Ends the publish retry's wait in flight, if any — see `publishThroughBlip`. */
|
|
147
|
+
let endBlipWait: (() => void) | undefined;
|
|
134
148
|
/** Stops listening for the lock being lost. Held only while this replicator holds the lock. */
|
|
135
149
|
let unwatchLock: (() => void) | undefined;
|
|
136
150
|
const unwatch = (): void => {
|
|
@@ -180,6 +194,39 @@ export function changeFeedReplicator(options: ReplicatorOptions): Replicator {
|
|
|
180
194
|
return outcome.how === 'failed' ? outcome.error : undefined;
|
|
181
195
|
};
|
|
182
196
|
|
|
197
|
+
/**
|
|
198
|
+
* One envelope, published — retried on `PUBLISH_RETRY_DELAYS_MS` while the bus is away. The SAME
|
|
199
|
+
* envelope each time, so a consumer sees one `seq`. Fenced after every wait: a run that ended
|
|
200
|
+
* meanwhile publishes nothing, for the reason the handler below gives.
|
|
201
|
+
*/
|
|
202
|
+
const publishThroughBlip = async (run: Run, subject: string, envelope: string): Promise<void> => {
|
|
203
|
+
for (let retry = 0; ; retry += 1) {
|
|
204
|
+
try {
|
|
205
|
+
await options.transport.publish(subject, envelope);
|
|
206
|
+
return;
|
|
207
|
+
} catch (thrown) {
|
|
208
|
+
const delay = PUBLISH_RETRY_DELAYS_MS[retry];
|
|
209
|
+
if (delay === undefined || !busAway(thrown)) throw thrown;
|
|
210
|
+
await new Promise<void>((resolve) => {
|
|
211
|
+
const cancel = schedule(() => {
|
|
212
|
+
endBlipWait = undefined;
|
|
213
|
+
resolve();
|
|
214
|
+
}, delay);
|
|
215
|
+
// Held so `stop()` can end the wait: the timer is cleared AND the wait settles, into the
|
|
216
|
+
// fence below. A stop that only waited it out held the feed's handler for up to 1.5 s.
|
|
217
|
+
endBlipWait = (): void => {
|
|
218
|
+
endBlipWait = undefined;
|
|
219
|
+
cancel();
|
|
220
|
+
resolve();
|
|
221
|
+
};
|
|
222
|
+
});
|
|
223
|
+
if (current !== run || run.over !== null) {
|
|
224
|
+
throw fencedOut(run.over ?? 'a newer run started');
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
};
|
|
229
|
+
|
|
183
230
|
const changeHandler =
|
|
184
231
|
(run: Run) =>
|
|
185
232
|
async (raw: ChangeEvent): Promise<void> => {
|
|
@@ -200,7 +247,7 @@ export function changeFeedReplicator(options: ReplicatorOptions): Replicator {
|
|
|
200
247
|
await withSpan('realtime.replicate', async () => {
|
|
201
248
|
seq += 1;
|
|
202
249
|
try {
|
|
203
|
-
await
|
|
250
|
+
await publishThroughBlip(run, subjectOf(change), encodeEnvelope(change, seq, producer));
|
|
204
251
|
} catch (thrown) {
|
|
205
252
|
const error = renderThrowable(thrown);
|
|
206
253
|
const same = refusal !== null && refusal.lsn === change.lsn;
|
|
@@ -391,6 +438,8 @@ export function changeFeedReplicator(options: ReplicatorOptions): Replicator {
|
|
|
391
438
|
pumping = false;
|
|
392
439
|
if (current !== undefined) current.over ??= 'the replicator was stopped';
|
|
393
440
|
current = undefined;
|
|
441
|
+
// After the run is marked over, so the wait it ends lands in the fence and publishes nothing.
|
|
442
|
+
endBlipWait?.();
|
|
394
443
|
unwatch();
|
|
395
444
|
// Whatever the feed said: a lock kept because the stream would not close politely is a
|
|
396
445
|
// slot no standby can take. Its refusal is still the one the caller is owed.
|
package/src/server.ts
CHANGED
package/src/sync-bus-handlers.ts
CHANGED
|
@@ -14,12 +14,15 @@ import {
|
|
|
14
14
|
parseEnvelope,
|
|
15
15
|
type SeqGapDetector,
|
|
16
16
|
} from './replicator-envelope';
|
|
17
|
+
import type { BusOutage } from './sync-bus-outage';
|
|
17
18
|
|
|
18
19
|
export interface SyncBus {
|
|
19
20
|
readonly registry: LiveQueryRegistry;
|
|
20
21
|
readonly hub: ChannelHub;
|
|
21
22
|
readonly gaps: SeqGapDetector;
|
|
22
23
|
readonly producers: ProducerKinds;
|
|
24
|
+
/** The node's outage: a bus that reconnected has answered. */
|
|
25
|
+
readonly outage?: Pick<BusOutage, 'answered'> | undefined;
|
|
23
26
|
}
|
|
24
27
|
|
|
25
28
|
/**
|
|
@@ -123,6 +126,7 @@ export function changeHandler(bus: SyncBus): (payload: string) => void {
|
|
|
123
126
|
*/
|
|
124
127
|
export function reconnectHandler(bus: SyncBus): () => void {
|
|
125
128
|
return () => {
|
|
129
|
+
bus.outage?.answered();
|
|
126
130
|
bus.gaps.forget();
|
|
127
131
|
const marked = bus.registry.invalidate();
|
|
128
132
|
const channelGaps = bus.hub.invalidate();
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
// Single responsibility: what a `sync` node says while its bus is away. Without the bus the node
|
|
2
|
+
// cannot do its job — that is an ERROR — but it is ONE fact however many sockets beat into it and
|
|
3
|
+
// however many sweeps fail: said on failures 1, 2, 4, 8, … and once more when the bus answers.
|
|
4
|
+
// Until 2026-10-10 it was `presence.sweep failed` every sweep and one monitor event per refused beat.
|
|
5
|
+
|
|
6
|
+
import { outageLog, renderThrowable, reportError } from '@ultimat3/core';
|
|
7
|
+
import { reportDetached } from './detach';
|
|
8
|
+
import type { PresenceRegistry } from './presence';
|
|
9
|
+
|
|
10
|
+
/** The line a node writes while its bus is away; `<event> recovered` when it is back. */
|
|
11
|
+
export const SYNC_BUS_UNAVAILABLE = 'sync.bus_unavailable';
|
|
12
|
+
|
|
13
|
+
export interface BusOutage {
|
|
14
|
+
/**
|
|
15
|
+
* `true` when `error` is the bus being away — counted, and logged + reported only on the
|
|
16
|
+
* outage's milestones. `false` for anything else, which stays the caller's to report.
|
|
17
|
+
*/
|
|
18
|
+
refused(error: unknown, operation: string, at?: string): boolean;
|
|
19
|
+
/** The bus answered: the outage, if there was one, is over. */
|
|
20
|
+
answered(): void;
|
|
21
|
+
/** `detach` for work that reaches the bus: an outage is thinned, any other failure is not. */
|
|
22
|
+
detach(work: Promise<unknown>, operation: string, at?: string): void;
|
|
23
|
+
/** One presence sweep. A pass that read at least one room is the bus answering. */
|
|
24
|
+
sweep(presence: PresenceRegistry): void;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
const codeOf = (error: unknown): unknown =>
|
|
28
|
+
typeof error === 'object' && error !== null ? (error as { readonly code?: unknown }).code : null;
|
|
29
|
+
|
|
30
|
+
export function busOutage(): BusOutage {
|
|
31
|
+
const log = outageLog({ event: SYNC_BUS_UNAVAILABLE, level: 'error' });
|
|
32
|
+
const refused = (error: unknown, operation: string, at?: string): boolean => {
|
|
33
|
+
if (codeOf(error) !== 'X_TRANSPORT_UNAVAILABLE') return false;
|
|
34
|
+
const said = log.failed({
|
|
35
|
+
operation,
|
|
36
|
+
...(at === undefined ? {} : { at }),
|
|
37
|
+
error: renderThrowable(error),
|
|
38
|
+
});
|
|
39
|
+
// The monitor is thinned with the log: one outage is not one event per socket per beat.
|
|
40
|
+
if (said) reportError(error, { source: 'realtime', scope: { operation } });
|
|
41
|
+
return true;
|
|
42
|
+
};
|
|
43
|
+
const detached = (work: Promise<unknown>, operation: string, at?: string): void => {
|
|
44
|
+
void work.catch((error: unknown) => {
|
|
45
|
+
if (!refused(error, operation, at)) reportDetached(error, operation, at);
|
|
46
|
+
});
|
|
47
|
+
};
|
|
48
|
+
return {
|
|
49
|
+
refused,
|
|
50
|
+
answered: () => log.recovered(),
|
|
51
|
+
detach: detached,
|
|
52
|
+
sweep(presence) {
|
|
53
|
+
const rooms = presence.rooms;
|
|
54
|
+
detached(
|
|
55
|
+
presence.sweepAll().then(() => {
|
|
56
|
+
if (rooms > 0) log.recovered();
|
|
57
|
+
}),
|
|
58
|
+
'presence.sweep',
|
|
59
|
+
);
|
|
60
|
+
},
|
|
61
|
+
};
|
|
62
|
+
}
|
package/src/sync-frames.ts
CHANGED
|
@@ -22,6 +22,8 @@ export interface FrameRouterOptions {
|
|
|
22
22
|
readonly heartbeatMs?: number | undefined;
|
|
23
23
|
/** Shared with the node, which names a seat's sid when a re-auth drops it. */
|
|
24
24
|
readonly channelSids?: ChannelSids | undefined;
|
|
25
|
+
/** Told when a presence read or write landed: the bus answered (`sync-bus-outage.ts`). */
|
|
26
|
+
readonly busAnswered?: (() => void) | undefined;
|
|
25
27
|
}
|
|
26
28
|
|
|
27
29
|
/**
|
|
@@ -73,6 +75,7 @@ export function frameRouter(options: FrameRouterOptions): FrameRouter {
|
|
|
73
75
|
: beat && (await presence.heartbeat(name, socket.id))
|
|
74
76
|
? await presence.roster(name)
|
|
75
77
|
: await presence.join(name, { id: socket.id, actorId: socket.actorId });
|
|
78
|
+
if (presence) options.busAnswered?.();
|
|
76
79
|
// Read, though nothing here can repair it: the next beat re-sends the set, and the log is the
|
|
77
80
|
// only trace a dropped one leaves anywhere.
|
|
78
81
|
if (!socket.send(presenceFrame(name, 'sync', roster.members, roster.total))) {
|
package/src/sync-node.ts
CHANGED
|
@@ -27,6 +27,7 @@ import { idleSweepPeriodMs } from './socket-idle';
|
|
|
27
27
|
import { actorChangeHandler } from './sync-actor-change';
|
|
28
28
|
import { GrantBook, sweepGrants } from './sync-auth';
|
|
29
29
|
import { changeHandler, ProducerKinds, reconnectHandler } from './sync-bus-handlers';
|
|
30
|
+
import { busOutage } from './sync-bus-outage';
|
|
30
31
|
import { ackRefOf, frameRouter } from './sync-frames';
|
|
31
32
|
import {
|
|
32
33
|
clientHeartbeatMs,
|
|
@@ -73,6 +74,7 @@ export function syncNode(options: SyncNodeOptions): SyncNode {
|
|
|
73
74
|
const gaps = new SeqGapDetector();
|
|
74
75
|
// Kept across restarts of the bus subscription: a table's producer kind is a fact of the fleet.
|
|
75
76
|
const producers = new ProducerKinds();
|
|
77
|
+
const outage = busOutage();
|
|
76
78
|
const channelSids = new ChannelSids();
|
|
77
79
|
let reconnects: (() => void) | null = null;
|
|
78
80
|
/** The re-auth pass in flight, shared by every tick that lands while it runs. */
|
|
@@ -145,7 +147,7 @@ export function syncNode(options: SyncNodeOptions): SyncNode {
|
|
|
145
147
|
const leave = presence.leave(name, socket.id);
|
|
146
148
|
// Detached as well as returned: `detach` attaches the reporting catch, so a caller that
|
|
147
149
|
// awaits this later is awaiting a promise whose rejection is already handled.
|
|
148
|
-
detach(leave, 'presence.leave', name);
|
|
150
|
+
outage.detach(leave, 'presence.leave', name);
|
|
149
151
|
leaves.push(leave);
|
|
150
152
|
}
|
|
151
153
|
}
|
|
@@ -237,13 +239,14 @@ export function syncNode(options: SyncNodeOptions): SyncNode {
|
|
|
237
239
|
buildId: options.buildId,
|
|
238
240
|
presence,
|
|
239
241
|
channelSids,
|
|
242
|
+
busAnswered: outage.answered,
|
|
240
243
|
heartbeatMs: clientHeartbeatMs(presence?.heartbeatMs, sockets.idleTimeoutMs),
|
|
241
244
|
});
|
|
242
245
|
|
|
243
246
|
/** Everything `start()` does, fenced on the generation it began under — see `started`. */
|
|
244
247
|
const begin = async (): Promise<void> => {
|
|
245
248
|
const run = generation;
|
|
246
|
-
const bus = { registry: options.registry, hub: options.hub, gaps, producers };
|
|
249
|
+
const bus = { registry: options.registry, hub: options.hub, gaps, producers, outage };
|
|
247
250
|
const subscription = await options.transport.subscribe(CHANGE_SUBJECT_ALL, changeHandler(bus));
|
|
248
251
|
// A stop or drain ran inside that await: what it released did not include this yet.
|
|
249
252
|
if (run !== generation) {
|
|
@@ -255,10 +258,7 @@ export function syncNode(options: SyncNodeOptions): SyncNode {
|
|
|
255
258
|
// One pass per heartbeat window: a member is swept only once it has actually missed its
|
|
256
259
|
// window, and the interval never holds the process open — shutdown is the drain's job.
|
|
257
260
|
if (presence) {
|
|
258
|
-
sweeping = setInterval(
|
|
259
|
-
() => detach(presence.sweepAll(), 'presence.sweep'),
|
|
260
|
-
presence.heartbeatMs,
|
|
261
|
-
);
|
|
261
|
+
sweeping = setInterval(() => outage.sweep(presence), presence.heartbeatMs);
|
|
262
262
|
sweeping.unref();
|
|
263
263
|
}
|
|
264
264
|
// The half-open connection Bun's own `idleTimeout` renews through its ping/pong: a client
|
|
@@ -409,7 +409,7 @@ export function syncNode(options: SyncNodeOptions): SyncNode {
|
|
|
409
409
|
} catch (error) {
|
|
410
410
|
// The ack frame tells the client what it did wrong; the monitor only hears about what
|
|
411
411
|
// this node did wrong. Same rule the HTTP pipeline applies at `status >= 500`.
|
|
412
|
-
if (!isClientFault(error)) {
|
|
412
|
+
if (!outage.refused(error, 'sync.frame') && !isClientFault(error)) {
|
|
413
413
|
reportError(error, {
|
|
414
414
|
source: 'realtime',
|
|
415
415
|
scope: { operation: 'sync.frame', extra: { socketId: socket.id } },
|
package/src/transport-env.ts
CHANGED
|
@@ -11,6 +11,7 @@ import { InProcessTransport } from './fanout';
|
|
|
11
11
|
import type { NatsConnect } from './nats-client';
|
|
12
12
|
import { assertBucket } from './nats-jetstream';
|
|
13
13
|
import { NatsTransport } from './nats-transport';
|
|
14
|
+
import type { BackoffPolicy } from './thundering-herd';
|
|
14
15
|
|
|
15
16
|
/**
|
|
16
17
|
* The keys read here, and nothing else. Named once so docs and tests cannot drift from the code.
|
|
@@ -36,6 +37,27 @@ export const DEFAULT_PRESENCE_TTL_MS = 30_000;
|
|
|
36
37
|
|
|
37
38
|
export type TransportEnvironment = Readonly<Record<string, string | undefined>>;
|
|
38
39
|
|
|
40
|
+
/**
|
|
41
|
+
* What this process does with the bus, which decides whether its boot waits for it:
|
|
42
|
+
*
|
|
43
|
+
* - `'sockets'` — it serves sockets from the bus (a `sync` node). The dial and the presence bucket
|
|
44
|
+
* are awaited, and a bus that is not there within `connectWithinMs` refuses the boot.
|
|
45
|
+
* - `'feed'` — it feeds those nodes (the replicator). The dial is awaited; it reads no presence,
|
|
46
|
+
* so it asks JetStream for nothing.
|
|
47
|
+
* - `'publish'` — it only publishes channel events and cache invalidations (`web`, `worker`,
|
|
48
|
+
* `scheduler`). The dial runs in the background and the boot does not wait: both kinds of
|
|
49
|
+
* message mean "re-read", and a page that cannot say it is still a page.
|
|
50
|
+
*/
|
|
51
|
+
export type BusUse = 'sockets' | 'feed' | 'publish';
|
|
52
|
+
|
|
53
|
+
/** How long a process that cannot work without the bus waits for it before refusing its boot. */
|
|
54
|
+
export const BUS_CONNECT_WAIT_MS = 15_000;
|
|
55
|
+
|
|
56
|
+
const NEEDS: Record<Exclude<BusUse, 'publish'>, string> = {
|
|
57
|
+
sockets: 'serves sockets from the bus',
|
|
58
|
+
feed: 'feeds the sync nodes over the bus',
|
|
59
|
+
};
|
|
60
|
+
|
|
39
61
|
export interface TransportSelection {
|
|
40
62
|
readonly transport: Transport;
|
|
41
63
|
/** `embedded` fans out in this process only; `external` reaches every node on the bus. */
|
|
@@ -50,12 +72,18 @@ export interface TransportSelection {
|
|
|
50
72
|
readonly bucket: string | null;
|
|
51
73
|
/** What `PresenceRegistry` must be given, so its TTL and the bucket's cannot disagree. */
|
|
52
74
|
readonly presenceTtlMs: number;
|
|
75
|
+
/** What the process does with the bus, as `connect()` and `state()` obey it. */
|
|
76
|
+
readonly use: BusUse;
|
|
53
77
|
/**
|
|
54
78
|
* Dial now rather than on the first change nobody receives. Selection itself stays pure — it
|
|
55
79
|
* parses env and constructs, it does not touch a socket — so a boot can order its dials.
|
|
56
|
-
* Embedded resolves immediately: there is nothing to reach.
|
|
80
|
+
* Embedded resolves immediately: there is nothing to reach. Under `use: 'publish'` it STARTS the
|
|
81
|
+
* dial and resolves at once; under the other two it resolves connected or rejects with
|
|
82
|
+
* `X_TRANSPORT_UNAVAILABLE` inside `connectWithinMs`.
|
|
57
83
|
*/
|
|
58
84
|
connect(): Promise<void>;
|
|
85
|
+
/** `'connecting'` while a NATS bus is not reachable; the in-process bus is always `'up'`. */
|
|
86
|
+
state(): 'up' | 'connecting';
|
|
59
87
|
}
|
|
60
88
|
|
|
61
89
|
export interface SelectTransportOptions {
|
|
@@ -63,6 +91,14 @@ export interface SelectTransportOptions {
|
|
|
63
91
|
readonly clock?: Clock | undefined;
|
|
64
92
|
/** Injected so a boot — reconnect included — can be proven with no network. */
|
|
65
93
|
readonly connect?: NatsConnect | undefined;
|
|
94
|
+
/** Default `'sockets'`: a caller that does not say is held to the strictest reading. */
|
|
95
|
+
readonly use?: BusUse | undefined;
|
|
96
|
+
/** Default `BUS_CONNECT_WAIT_MS`. Read under `'sockets'` and `'feed'` only. */
|
|
97
|
+
readonly connectWithinMs?: number | undefined;
|
|
98
|
+
/** The dial's retry curve; the transport's own default when unset. */
|
|
99
|
+
readonly backoff?: BackoffPolicy | undefined;
|
|
100
|
+
/** Where the transport's background failures go; its own logger line when unset. */
|
|
101
|
+
readonly onError?: ((error: unknown, subject: string) => void) | undefined;
|
|
66
102
|
}
|
|
67
103
|
|
|
68
104
|
const nonEmpty = (value: string | undefined): string | undefined =>
|
|
@@ -92,6 +128,7 @@ export function selectTransport(
|
|
|
92
128
|
'presenceTtlMs',
|
|
93
129
|
options.presenceTtlMs ?? DEFAULT_PRESENCE_TTL_MS,
|
|
94
130
|
);
|
|
131
|
+
const use = options.use ?? 'sockets';
|
|
95
132
|
|
|
96
133
|
if (topology.transport === 'memory') {
|
|
97
134
|
refuseStrayBusUrl(env, topology);
|
|
@@ -104,7 +141,9 @@ export function selectTransport(
|
|
|
104
141
|
detail: `in-process fanout — set realtime.transport 'nats' in ${CONFIG_FILE} and NATS_URL to reach the other nodes`,
|
|
105
142
|
bucket: null,
|
|
106
143
|
presenceTtlMs,
|
|
144
|
+
use,
|
|
107
145
|
connect: () => Promise.resolve(),
|
|
146
|
+
state: () => 'up',
|
|
108
147
|
};
|
|
109
148
|
}
|
|
110
149
|
|
|
@@ -124,16 +163,39 @@ export function selectTransport(
|
|
|
124
163
|
url,
|
|
125
164
|
bucket,
|
|
126
165
|
presenceTtlMs,
|
|
166
|
+
// Only a node that serves presence needs the bucket to exist before it can start.
|
|
167
|
+
presenceBucket: use === 'sockets' ? 'dial' : 'first-use',
|
|
168
|
+
// A publisher without the bus is degraded and still serves; a node that serves FROM it is not.
|
|
169
|
+
outageLevel: use === 'publish' ? 'warn' : 'error',
|
|
127
170
|
...(options.clock === undefined ? {} : { clock: options.clock }),
|
|
128
171
|
...(options.connect === undefined ? {} : { connect: options.connect }),
|
|
172
|
+
...(options.backoff === undefined ? {} : { backoff: options.backoff }),
|
|
173
|
+
...(options.onError === undefined ? {} : { onError: options.onError }),
|
|
129
174
|
});
|
|
175
|
+
const withinMs = finiteOption(
|
|
176
|
+
'the transport env',
|
|
177
|
+
'connectWithinMs',
|
|
178
|
+
options.connectWithinMs ?? BUS_CONNECT_WAIT_MS,
|
|
179
|
+
);
|
|
130
180
|
return {
|
|
131
181
|
transport,
|
|
132
182
|
mode: 'external',
|
|
133
183
|
detail: urlEnv,
|
|
134
184
|
bucket,
|
|
135
185
|
presenceTtlMs,
|
|
136
|
-
|
|
186
|
+
use,
|
|
187
|
+
connect: async () => {
|
|
188
|
+
if (use === 'publish') {
|
|
189
|
+
transport.connectInBackground();
|
|
190
|
+
return;
|
|
191
|
+
}
|
|
192
|
+
await transport.connect({
|
|
193
|
+
withinMs,
|
|
194
|
+
// The variable, never the url behind it. `-js` only where the bucket is asked for.
|
|
195
|
+
fix: `x doctor — then check ${urlEnv} names a reachable nats-server${use === 'sockets' ? ' started with -js (JetStream)' : ''} — this process ${NEEDS[use]} and cannot start without it; the web, worker and scheduler roles boot without one and connect in the background`,
|
|
196
|
+
});
|
|
197
|
+
},
|
|
198
|
+
state: () => (transport.connected ? 'up' : 'connecting'),
|
|
137
199
|
};
|
|
138
200
|
}
|
|
139
201
|
|