@ultimat3/realtime 27.7.0 → 27.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/package.json +5 -5
- package/src/client-channels.ts +23 -1
- package/src/detach.ts +11 -8
- package/src/nats-lib-client.ts +17 -0
- package/src/nats-reports.ts +80 -0
- package/src/nats-transport.ts +26 -40
- package/src/presence.ts +7 -0
- package/src/sync-bus-handlers.ts +4 -0
- package/src/sync-bus-outage.ts +62 -0
- package/src/sync-frames.ts +3 -0
- package/src/sync-node.ts +7 -7
- package/src/transport-env.ts +2 -0
package/README.md
CHANGED
|
@@ -128,11 +128,11 @@ leaves `joining` on the node's first answer; nothing waits for app traffic.
|
|
|
128
128
|
|
|
129
129
|
| State | Entered when | Left when |
|
|
130
130
|
|---|---|---|
|
|
131
|
-
| `joining` | the subscribe is sent (on hold, and again on every reconnect) | the node's answer: a `records` channel's `replay-gap` (a fresh seat) or the `records` frame at a current cursor; an `events` channel's roster — sent on every seat, an empty one from a node with no presence (24.0.0) |
|
|
131
|
+
| `joining` | the subscribe is sent (on hold, and again on every reconnect); or the node refused it with `X_TRANSPORT_UNAVAILABLE` — its bus is away, which is its condition and not its verdict. The heartbeat re-asks every interval | the node's answer: a `records` channel's `replay-gap` (a fresh seat) or the `records` frame at a current cursor; an `events` channel's roster — sent on every seat, an empty one from a node with no presence (24.0.0) |
|
|
132
132
|
| `catching-up` | a `replay-gap`, or a frame in a new epoch: the `catchUp` query is re-read, frames held behind it | the read lands → `live` (the held frames applied in order) |
|
|
133
133
|
| `live` | as above | the socket drops → `offline`; a gap → `catching-up` |
|
|
134
134
|
| `offline` | the page socket closed | it reopens → `joining` |
|
|
135
|
-
| `failed` | the node
|
|
135
|
+
| `failed` | the node **decided** against the subscribe — a policy denial, an undeclared channel, a cap (`error()` is the coded refusal; terminal until the socket reopens) — or a catch-up read failed (retried on the client's curve, and on the next open) | a successful retry → `live` |
|
|
136
136
|
|
|
137
137
|
A channel declared with **neither** `records` nor `events` carries nothing, so nothing answers its
|
|
138
138
|
join and it reads `joining` for as long as it is held.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ultimat3/realtime",
|
|
3
|
-
"version": "27.
|
|
3
|
+
"version": "27.8.0",
|
|
4
4
|
"description": "Three-tier realtime: channels, live queries, local-first sync — one protocol, one mutator shape",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -40,10 +40,10 @@
|
|
|
40
40
|
"test": "bun test"
|
|
41
41
|
},
|
|
42
42
|
"dependencies": {
|
|
43
|
-
"@ultimat3/core": "27.
|
|
44
|
-
"@ultimat3/entity": "27.
|
|
45
|
-
"@ultimat3/policy": "27.
|
|
46
|
-
"@ultimat3/query": "27.
|
|
43
|
+
"@ultimat3/core": "27.8.0",
|
|
44
|
+
"@ultimat3/entity": "27.8.0",
|
|
45
|
+
"@ultimat3/policy": "27.8.0",
|
|
46
|
+
"@ultimat3/query": "27.8.0",
|
|
47
47
|
"nats": "2.29.3"
|
|
48
48
|
}
|
|
49
49
|
}
|
package/src/client-channels.ts
CHANGED
|
@@ -174,10 +174,21 @@ export class ChannelBook {
|
|
|
174
174
|
}
|
|
175
175
|
}
|
|
176
176
|
|
|
177
|
-
/**
|
|
177
|
+
/**
|
|
178
|
+
* A refusal `ack` naming the sid a channel is subscribed under. `false` when no membership
|
|
179
|
+
* here answers to it.
|
|
180
|
+
*/
|
|
178
181
|
refused(sid: string, error: unknown): boolean {
|
|
179
182
|
const entry = [...this.#byTopic.values()].find((candidate) => sidOf(candidate) === sid);
|
|
180
183
|
if (entry === undefined) return false;
|
|
184
|
+
if (isTransientRefusal(error)) {
|
|
185
|
+
// Not a decision: the node's bus is away, so nothing reaches this seat (or there is none
|
|
186
|
+
// yet). `joining` — asked, unanswered — and never `failed`: the beat re-asks every interval
|
|
187
|
+
// and the node's first answer (a roster, a `records` frame, a `replay-gap`) ends it. A read
|
|
188
|
+
// in flight and a terminal refusal keep their own state.
|
|
189
|
+
if (entry.state === 'live') this.#set(entry, 'joining');
|
|
190
|
+
return true;
|
|
191
|
+
}
|
|
181
192
|
entry.error = error;
|
|
182
193
|
this.#set(entry, 'failed');
|
|
183
194
|
return true;
|
|
@@ -361,6 +372,17 @@ export class ChannelBook {
|
|
|
361
372
|
}
|
|
362
373
|
}
|
|
363
374
|
|
|
375
|
+
/**
|
|
376
|
+
* The one refusal that is the NODE's condition and not its verdict: its bus is away
|
|
377
|
+
* (`X_TRANSPORT_UNAVAILABLE`). Everything else an `ack` carries — a policy denial, an unknown
|
|
378
|
+
* channel, a cap — is a decision and terminal. Read off `unknown`: a refusal is the node's data.
|
|
379
|
+
*/
|
|
380
|
+
export function isTransientRefusal(error: unknown): boolean {
|
|
381
|
+
return (
|
|
382
|
+
(error as { readonly code?: unknown } | null | undefined)?.code === 'X_TRANSPORT_UNAVAILABLE'
|
|
383
|
+
);
|
|
384
|
+
}
|
|
385
|
+
|
|
364
386
|
/** A channel membership's sid is its topic behind this — what the socket engine routes on too. */
|
|
365
387
|
export const CHANNEL_SID_PREFIX = 'channel:';
|
|
366
388
|
|
package/src/detach.ts
CHANGED
|
@@ -15,13 +15,16 @@ import { logger, renderThrowable, reportError } from '@ultimat3/core';
|
|
|
15
15
|
* with nothing above it to answer. `channel.ts` already imports it for the same reason.
|
|
16
16
|
*/
|
|
17
17
|
export function detach(work: Promise<unknown>, operation: string, at?: string): void {
|
|
18
|
-
void work.catch((error: unknown) =>
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
18
|
+
void work.catch((error: unknown) => reportDetached(error, operation, at));
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/** What `detach` says of a failure — also `sync-bus-outage.ts`'s answer to one that is no outage. */
|
|
22
|
+
export function reportDetached(error: unknown, operation: string, at?: string): void {
|
|
23
|
+
logger.error(`${operation} failed`, {
|
|
24
|
+
...(at === undefined ? {} : { at }),
|
|
25
|
+
error: renderThrowable(error),
|
|
26
26
|
});
|
|
27
|
+
// Nobody is awaiting this, so the log is the only trace it leaves — and a log is not a signal
|
|
28
|
+
// anyone is paged on. The bus is this node's dependency, never the client's.
|
|
29
|
+
reportError(error, { source: 'realtime', scope: { operation } });
|
|
27
30
|
}
|
package/src/nats-lib-client.ts
CHANGED
|
@@ -118,6 +118,7 @@ class LibNatsClient implements NatsClient {
|
|
|
118
118
|
payload: Uint8Array,
|
|
119
119
|
options: NatsRequestOptions = {},
|
|
120
120
|
): Promise<NatsMessage> {
|
|
121
|
+
this.#assertLive(subject);
|
|
121
122
|
const built = headersOf(options.headers);
|
|
122
123
|
try {
|
|
123
124
|
const reply = await this.#connection.request(subject, payload, {
|
|
@@ -140,6 +141,7 @@ class LibNatsClient implements NatsClient {
|
|
|
140
141
|
payload: Uint8Array,
|
|
141
142
|
options: NatsRequestManyOptions,
|
|
142
143
|
): Promise<readonly NatsMessage[]> {
|
|
144
|
+
this.#assertLive(subject);
|
|
143
145
|
const collected: NatsMessage[] = [];
|
|
144
146
|
try {
|
|
145
147
|
const replies = await this.#connection.requestMany(subject, payload, {
|
|
@@ -162,6 +164,21 @@ class LibNatsClient implements NatsClient {
|
|
|
162
164
|
await this.#connection.close();
|
|
163
165
|
}
|
|
164
166
|
|
|
167
|
+
/**
|
|
168
|
+
* A request has an answer to wait for, so one made with no live connection is refused HERE.
|
|
169
|
+
* The library takes it instead — buffered with its publishes, sent if the reconnect lands and
|
|
170
|
+
* timed out if it does not — so every presence write on a `sync` node whose bus was away held
|
|
171
|
+
* its socket's frame lane for the whole request timeout before saying what was already known.
|
|
172
|
+
* The fake has always refused it (`nats-fake.ts`, `#assertLive`); this is the client agreeing.
|
|
173
|
+
*/
|
|
174
|
+
#assertLive(subject: string): void {
|
|
175
|
+
if (this.connected) return;
|
|
176
|
+
throw unavailable(
|
|
177
|
+
this.#target,
|
|
178
|
+
`${subject} was not asked: the client is reconnecting to the bus`,
|
|
179
|
+
);
|
|
180
|
+
}
|
|
181
|
+
|
|
165
182
|
/**
|
|
166
183
|
* The library's own status stream is the only place a drop or a recovery is announced. Nothing
|
|
167
184
|
* awaits it, so it can neither throw nor end the process: a drop reports and flips `connected`,
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
// Single responsibility: where a `NatsTransport`'s background failures go, and how often. A FAULT
|
|
2
|
+
// (a subscriber that threw, a re-subscribe the server refused) is said every time. An OUTAGE (a
|
|
3
|
+
// drop, a refused dial, a client the library gave up on) is one fact however long it lasts: said
|
|
4
|
+
// on its 1st, 2nd, 4th, 8th, … failure and once more when the bus answers again.
|
|
5
|
+
|
|
6
|
+
import { isOutageMilestone, isUltimateError, logger, renderThrowable } from '@ultimat3/core';
|
|
7
|
+
|
|
8
|
+
export interface NatsReportsOptions {
|
|
9
|
+
readonly transport: string;
|
|
10
|
+
/** The app's own sink. Absent, or throwing, the logger line is written instead. */
|
|
11
|
+
readonly onError: ((error: unknown, subject: string) => void) | undefined;
|
|
12
|
+
/**
|
|
13
|
+
* `'error'` for a process that cannot do its job without the bus (a `sync` node, a replicator);
|
|
14
|
+
* `'warn'` for one that only publishes — degraded, and still serving.
|
|
15
|
+
*/
|
|
16
|
+
readonly outageLevel: 'warn' | 'error';
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export class NatsReports {
|
|
20
|
+
readonly #options: NatsReportsOptions;
|
|
21
|
+
/** Failures of the bus itself since it last answered. */
|
|
22
|
+
#failures = 0;
|
|
23
|
+
|
|
24
|
+
constructor(options: NatsReportsOptions) {
|
|
25
|
+
this.#options = options;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Every background failure lands here or in `outage`. Dropping it when the caller passed no
|
|
30
|
+
* handler is what turns "no changes arrive" into a debugging session with nothing to read, so
|
|
31
|
+
* the default emits rather than swallows.
|
|
32
|
+
*/
|
|
33
|
+
fault(error: unknown, subject: string): void {
|
|
34
|
+
this.#say(error, subject, 'error');
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* One more failure of the bus itself, thinned and at the level its owner chose. `always` is for
|
|
39
|
+
* a TRANSITION rather than a retry — the library giving up on a client happens once per client,
|
|
40
|
+
* and it is the line that says the transport's own re-dial took over.
|
|
41
|
+
*/
|
|
42
|
+
outage(error: unknown, always = false): void {
|
|
43
|
+
this.#failures += 1;
|
|
44
|
+
if (!always && !isOutageMilestone(this.#failures)) return;
|
|
45
|
+
this.#say(error, this.#options.transport, this.#options.outageLevel, this.#failures);
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** The bus answers again: one line saying after how many failures, and the count starts over. */
|
|
49
|
+
recovered(): void {
|
|
50
|
+
const after = this.#failures;
|
|
51
|
+
if (after === 0) return;
|
|
52
|
+
this.#failures = 0;
|
|
53
|
+
logger.info('nats transport recovered', { transport: this.#options.transport, after });
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
#say(error: unknown, subject: string, level: 'warn' | 'error', failures?: number): void {
|
|
57
|
+
const handler = this.#options.onError;
|
|
58
|
+
if (handler !== undefined) {
|
|
59
|
+
// Total: the handler is injected, and its throw would land in whatever called this — the
|
|
60
|
+
// dial loop (started with `void`: an unhandled rejection and no more retries), the
|
|
61
|
+
// library's status loop, a subscriber's delivery. A reporter that cannot report is not a
|
|
62
|
+
// reason to stop dialling; the fault falls through to the logger line below.
|
|
63
|
+
try {
|
|
64
|
+
handler(error, subject);
|
|
65
|
+
return;
|
|
66
|
+
} catch (thrown) {
|
|
67
|
+
logger.error('nats transport onError threw', { error: renderThrowable(thrown) });
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
logger[level]('nats transport error', {
|
|
71
|
+
transport: this.#options.transport,
|
|
72
|
+
subject,
|
|
73
|
+
...(failures === undefined ? {} : { failures }),
|
|
74
|
+
code: isUltimateError(error) ? error.code : undefined,
|
|
75
|
+
// `renderThrowable`, never `String(error)`: this is a reporter, and a throwable that fights
|
|
76
|
+
// being read makes the report the thing that throws.
|
|
77
|
+
error: renderThrowable(error),
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
}
|
package/src/nats-transport.ts
CHANGED
|
@@ -13,7 +13,6 @@ import {
|
|
|
13
13
|
type Clock,
|
|
14
14
|
finiteOption,
|
|
15
15
|
isUltimateError,
|
|
16
|
-
logger,
|
|
17
16
|
renderThrowable,
|
|
18
17
|
systemClock,
|
|
19
18
|
} from '@ultimat3/core';
|
|
@@ -25,6 +24,7 @@ import { DIAL_TIMEOUT_MS, landWithin, type NatsConnectWait } from './nats-dial-w
|
|
|
25
24
|
import { ensureKvBucket } from './nats-jetstream';
|
|
26
25
|
import { NatsKvSet } from './nats-kv';
|
|
27
26
|
import { openNatsClient } from './nats-open';
|
|
27
|
+
import { NatsReports } from './nats-reports';
|
|
28
28
|
import { NatsSubscriptions } from './nats-subscriptions';
|
|
29
29
|
import { type BackoffPolicy, defaultBackoff, policyDelay, type Rng } from './thundering-herd';
|
|
30
30
|
|
|
@@ -53,6 +53,12 @@ export interface NatsTransportOptions {
|
|
|
53
53
|
readonly presenceBucket?: 'dial' | 'first-use';
|
|
54
54
|
/** One dial's connect timeout. Default `DIAL_TIMEOUT_MS`. */
|
|
55
55
|
readonly dialTimeoutMs?: number;
|
|
56
|
+
/**
|
|
57
|
+
* The level an OUTAGE is logged at. `'error'` (the default) is a process that cannot do its job
|
|
58
|
+
* without the bus — a `sync` node, a replicator. `'warn'` is one that only publishes: degraded,
|
|
59
|
+
* and still serving. A subscriber that throws is an `error` under either.
|
|
60
|
+
*/
|
|
61
|
+
readonly outageLevel?: 'warn' | 'error';
|
|
56
62
|
}
|
|
57
63
|
|
|
58
64
|
const DEFAULT_ATTEMPTS = 10;
|
|
@@ -83,6 +89,8 @@ export class NatsTransport implements Transport {
|
|
|
83
89
|
/** The last background dial's own refusal, for the cause of a `connect({ withinMs })` that gave up. */
|
|
84
90
|
#lastFailure: string | undefined;
|
|
85
91
|
#wakeRedial: (() => void) | undefined;
|
|
92
|
+
/** Where background failures go: a fault every time, an outage thinned (`nats-reports.ts`). */
|
|
93
|
+
readonly #reports: NatsReports;
|
|
86
94
|
readonly #subscriptions = new NatsSubscriptions();
|
|
87
95
|
readonly #reconnectListeners = new Set<() => void>();
|
|
88
96
|
|
|
@@ -91,6 +99,11 @@ export class NatsTransport implements Transport {
|
|
|
91
99
|
// a container that reports itself healthy on one is a container nothing will ever page about.
|
|
92
100
|
parseNatsUrl(options.url);
|
|
93
101
|
this.#options = options;
|
|
102
|
+
this.#reports = new NatsReports({
|
|
103
|
+
transport: this.name,
|
|
104
|
+
onError: options.onError,
|
|
105
|
+
outageLevel: options.outageLevel ?? 'error',
|
|
106
|
+
});
|
|
94
107
|
this.#connect = options.connect ?? openNatsClient;
|
|
95
108
|
this.#backoff = options.backoff ?? defaultBackoff;
|
|
96
109
|
this.#attempts = finiteOption(
|
|
@@ -186,7 +199,7 @@ export class NatsTransport implements Transport {
|
|
|
186
199
|
try {
|
|
187
200
|
handler(decoder.decode(message.payload), message.subject);
|
|
188
201
|
} catch (error) {
|
|
189
|
-
this.#
|
|
202
|
+
this.#reports.fault(error, message.subject);
|
|
190
203
|
}
|
|
191
204
|
},
|
|
192
205
|
(call) => this.#translating(`subscribe to ${subject}`, call),
|
|
@@ -271,7 +284,7 @@ export class NatsTransport implements Transport {
|
|
|
271
284
|
// The library retries; the spread is ours, so a cluster restart does not bring every node
|
|
272
285
|
// back on the same millisecond.
|
|
273
286
|
reconnectDelay: () => policyDelay(this.#backoff, ++this.#retries, this.#rng),
|
|
274
|
-
onError: (error) => this.#
|
|
287
|
+
onError: (error) => this.#reports.outage(error),
|
|
275
288
|
onReconnect: () => this.#recovered(),
|
|
276
289
|
onClosed: () => {
|
|
277
290
|
if (opened === undefined) closedEarly = true;
|
|
@@ -324,12 +337,12 @@ export class NatsTransport implements Transport {
|
|
|
324
337
|
if (this.#closed || this.#client !== client) return;
|
|
325
338
|
this.#client = undefined;
|
|
326
339
|
this.#subscriptions.orphan();
|
|
327
|
-
this.#
|
|
340
|
+
this.#reports.outage(
|
|
328
341
|
new TransportUnavailableError({
|
|
329
342
|
transport: this.name,
|
|
330
343
|
reason: 'the connection closed once its reconnect budget was spent; re-dialling',
|
|
331
344
|
}),
|
|
332
|
-
|
|
345
|
+
true,
|
|
333
346
|
);
|
|
334
347
|
void this.#redial();
|
|
335
348
|
}
|
|
@@ -353,9 +366,10 @@ export class NatsTransport implements Transport {
|
|
|
353
366
|
try {
|
|
354
367
|
const client = await this.#dialOnce();
|
|
355
368
|
this.#lastFailure = undefined;
|
|
369
|
+
this.#reports.recovered();
|
|
356
370
|
// Coded like a first subscribe's refusal; the subscription stays kept for the next client.
|
|
357
371
|
for (const failure of this.#subscriptions.bindAll(client)) {
|
|
358
|
-
this.#
|
|
372
|
+
this.#reports.fault(
|
|
359
373
|
this.#coded(`re-subscribe to ${failure.subject}`, failure.error),
|
|
360
374
|
failure.subject,
|
|
361
375
|
);
|
|
@@ -364,9 +378,9 @@ export class NatsTransport implements Transport {
|
|
|
364
378
|
} catch (error) {
|
|
365
379
|
if (this.#closed) return;
|
|
366
380
|
this.#lastFailure = renderThrowable(error);
|
|
367
|
-
//
|
|
368
|
-
//
|
|
369
|
-
|
|
381
|
+
// Thinned (`#outage`): said at once and then ever more rarely, rather than once per
|
|
382
|
+
// dial per pod for as long as it lasts.
|
|
383
|
+
this.#reports.outage(error);
|
|
370
384
|
await this.#pause(policyDelay(this.#backoff, attempt, this.#rng));
|
|
371
385
|
}
|
|
372
386
|
}
|
|
@@ -413,10 +427,11 @@ export class NatsTransport implements Transport {
|
|
|
413
427
|
this.#retries = 0;
|
|
414
428
|
const client = this.#client;
|
|
415
429
|
if (client === undefined) return;
|
|
430
|
+
this.#reports.recovered();
|
|
416
431
|
// Not awaited: the library calls this from its status loop. The failure has one place to go.
|
|
417
432
|
// A lazy bucket is forgotten instead, and asserted again by whoever next reads `shared`.
|
|
418
433
|
if (this.#lazyBucket) this.#bucketed.delete(client);
|
|
419
|
-
else void this.#ensureBucket(client).catch((error: unknown) => this.#
|
|
434
|
+
else void this.#ensureBucket(client).catch((error: unknown) => this.#reports.outage(error));
|
|
420
435
|
this.#announce();
|
|
421
436
|
}
|
|
422
437
|
|
|
@@ -425,7 +440,7 @@ export class NatsTransport implements Transport {
|
|
|
425
440
|
try {
|
|
426
441
|
listener();
|
|
427
442
|
} catch (error) {
|
|
428
|
-
this.#
|
|
443
|
+
this.#reports.fault(error, this.name);
|
|
429
444
|
}
|
|
430
445
|
}
|
|
431
446
|
}
|
|
@@ -452,33 +467,4 @@ export class NatsTransport implements Transport {
|
|
|
452
467
|
reason: `${what} was refused: ${renderThrowable(error)}`,
|
|
453
468
|
});
|
|
454
469
|
}
|
|
455
|
-
|
|
456
|
-
/**
|
|
457
|
-
* Every background failure lands here — a lost connection, a throwing subscriber, an exhausted
|
|
458
|
-
* reconnect. Dropping it when the caller passed no handler is what turns "no changes arrive"
|
|
459
|
-
* into a debugging session with nothing to read, so the default emits rather than swallows.
|
|
460
|
-
*/
|
|
461
|
-
#report(error: unknown, subject: string): void {
|
|
462
|
-
const handler = this.#options.onError;
|
|
463
|
-
if (handler !== undefined) {
|
|
464
|
-
// Total: the handler is injected, and its throw would land in whatever called this — the
|
|
465
|
-
// dial loop (started with `void`: an unhandled rejection and no more retries), the
|
|
466
|
-
// library's status loop, a subscriber's delivery. A reporter that cannot report is not a
|
|
467
|
-
// reason to stop dialling; the fault falls through to the logger line below.
|
|
468
|
-
try {
|
|
469
|
-
handler(error, subject);
|
|
470
|
-
return;
|
|
471
|
-
} catch (thrown) {
|
|
472
|
-
logger.error('nats transport onError threw', { error: renderThrowable(thrown) });
|
|
473
|
-
}
|
|
474
|
-
}
|
|
475
|
-
logger.error('nats transport error', {
|
|
476
|
-
transport: this.name,
|
|
477
|
-
subject,
|
|
478
|
-
code: isUltimateError(error) ? error.code : undefined,
|
|
479
|
-
// `renderThrowable`, never `String(error)`: this is a reporter, and a throwable that fights
|
|
480
|
-
// being read makes the report the thing that throws.
|
|
481
|
-
error: renderThrowable(error),
|
|
482
|
-
});
|
|
483
|
-
}
|
|
484
470
|
}
|
package/src/presence.ts
CHANGED
|
@@ -93,6 +93,13 @@ export class PresenceRegistry {
|
|
|
93
93
|
return this.#ttlMs;
|
|
94
94
|
}
|
|
95
95
|
|
|
96
|
+
/** Rooms with a member this node has seen — what the next `sweepAll` will read the bus for. */
|
|
97
|
+
get rooms(): number {
|
|
98
|
+
let rooms = 0;
|
|
99
|
+
for (const members of this.#seen.values()) if (members.size > 0) rooms += 1;
|
|
100
|
+
return rooms;
|
|
101
|
+
}
|
|
102
|
+
|
|
96
103
|
/** Recommended client heartbeat interval: one lost beat must not read as a leave. */
|
|
97
104
|
get heartbeatMs(): number {
|
|
98
105
|
return Math.max(1_000, Math.floor(this.#ttlMs / 3));
|
package/src/sync-bus-handlers.ts
CHANGED
|
@@ -14,12 +14,15 @@ import {
|
|
|
14
14
|
parseEnvelope,
|
|
15
15
|
type SeqGapDetector,
|
|
16
16
|
} from './replicator-envelope';
|
|
17
|
+
import type { BusOutage } from './sync-bus-outage';
|
|
17
18
|
|
|
18
19
|
export interface SyncBus {
|
|
19
20
|
readonly registry: LiveQueryRegistry;
|
|
20
21
|
readonly hub: ChannelHub;
|
|
21
22
|
readonly gaps: SeqGapDetector;
|
|
22
23
|
readonly producers: ProducerKinds;
|
|
24
|
+
/** The node's outage: a bus that reconnected has answered. */
|
|
25
|
+
readonly outage?: Pick<BusOutage, 'answered'> | undefined;
|
|
23
26
|
}
|
|
24
27
|
|
|
25
28
|
/**
|
|
@@ -123,6 +126,7 @@ export function changeHandler(bus: SyncBus): (payload: string) => void {
|
|
|
123
126
|
*/
|
|
124
127
|
export function reconnectHandler(bus: SyncBus): () => void {
|
|
125
128
|
return () => {
|
|
129
|
+
bus.outage?.answered();
|
|
126
130
|
bus.gaps.forget();
|
|
127
131
|
const marked = bus.registry.invalidate();
|
|
128
132
|
const channelGaps = bus.hub.invalidate();
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
// Single responsibility: what a `sync` node says while its bus is away. Without the bus the node
|
|
2
|
+
// cannot do its job — that is an ERROR — but it is ONE fact however many sockets beat into it and
|
|
3
|
+
// however many sweeps fail: said on failures 1, 2, 4, 8, … and once more when the bus answers.
|
|
4
|
+
// Until 2026-10-10 it was `presence.sweep failed` every sweep and one monitor event per refused beat.
|
|
5
|
+
|
|
6
|
+
import { outageLog, renderThrowable, reportError } from '@ultimat3/core';
|
|
7
|
+
import { reportDetached } from './detach';
|
|
8
|
+
import type { PresenceRegistry } from './presence';
|
|
9
|
+
|
|
10
|
+
/** The line a node writes while its bus is away; `<event> recovered` when it is back. */
|
|
11
|
+
export const SYNC_BUS_UNAVAILABLE = 'sync.bus_unavailable';
|
|
12
|
+
|
|
13
|
+
export interface BusOutage {
|
|
14
|
+
/**
|
|
15
|
+
* `true` when `error` is the bus being away — counted, and logged + reported only on the
|
|
16
|
+
* outage's milestones. `false` for anything else, which stays the caller's to report.
|
|
17
|
+
*/
|
|
18
|
+
refused(error: unknown, operation: string, at?: string): boolean;
|
|
19
|
+
/** The bus answered: the outage, if there was one, is over. */
|
|
20
|
+
answered(): void;
|
|
21
|
+
/** `detach` for work that reaches the bus: an outage is thinned, any other failure is not. */
|
|
22
|
+
detach(work: Promise<unknown>, operation: string, at?: string): void;
|
|
23
|
+
/** One presence sweep. A pass that read at least one room is the bus answering. */
|
|
24
|
+
sweep(presence: PresenceRegistry): void;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
const codeOf = (error: unknown): unknown =>
|
|
28
|
+
typeof error === 'object' && error !== null ? (error as { readonly code?: unknown }).code : null;
|
|
29
|
+
|
|
30
|
+
export function busOutage(): BusOutage {
|
|
31
|
+
const log = outageLog({ event: SYNC_BUS_UNAVAILABLE, level: 'error' });
|
|
32
|
+
const refused = (error: unknown, operation: string, at?: string): boolean => {
|
|
33
|
+
if (codeOf(error) !== 'X_TRANSPORT_UNAVAILABLE') return false;
|
|
34
|
+
const said = log.failed({
|
|
35
|
+
operation,
|
|
36
|
+
...(at === undefined ? {} : { at }),
|
|
37
|
+
error: renderThrowable(error),
|
|
38
|
+
});
|
|
39
|
+
// The monitor is thinned with the log: one outage is not one event per socket per beat.
|
|
40
|
+
if (said) reportError(error, { source: 'realtime', scope: { operation } });
|
|
41
|
+
return true;
|
|
42
|
+
};
|
|
43
|
+
const detached = (work: Promise<unknown>, operation: string, at?: string): void => {
|
|
44
|
+
void work.catch((error: unknown) => {
|
|
45
|
+
if (!refused(error, operation, at)) reportDetached(error, operation, at);
|
|
46
|
+
});
|
|
47
|
+
};
|
|
48
|
+
return {
|
|
49
|
+
refused,
|
|
50
|
+
answered: () => log.recovered(),
|
|
51
|
+
detach: detached,
|
|
52
|
+
sweep(presence) {
|
|
53
|
+
const rooms = presence.rooms;
|
|
54
|
+
detached(
|
|
55
|
+
presence.sweepAll().then(() => {
|
|
56
|
+
if (rooms > 0) log.recovered();
|
|
57
|
+
}),
|
|
58
|
+
'presence.sweep',
|
|
59
|
+
);
|
|
60
|
+
},
|
|
61
|
+
};
|
|
62
|
+
}
|
package/src/sync-frames.ts
CHANGED
|
@@ -22,6 +22,8 @@ export interface FrameRouterOptions {
|
|
|
22
22
|
readonly heartbeatMs?: number | undefined;
|
|
23
23
|
/** Shared with the node, which names a seat's sid when a re-auth drops it. */
|
|
24
24
|
readonly channelSids?: ChannelSids | undefined;
|
|
25
|
+
/** Told when a presence read or write landed: the bus answered (`sync-bus-outage.ts`). */
|
|
26
|
+
readonly busAnswered?: (() => void) | undefined;
|
|
25
27
|
}
|
|
26
28
|
|
|
27
29
|
/**
|
|
@@ -73,6 +75,7 @@ export function frameRouter(options: FrameRouterOptions): FrameRouter {
|
|
|
73
75
|
: beat && (await presence.heartbeat(name, socket.id))
|
|
74
76
|
? await presence.roster(name)
|
|
75
77
|
: await presence.join(name, { id: socket.id, actorId: socket.actorId });
|
|
78
|
+
if (presence) options.busAnswered?.();
|
|
76
79
|
// Read, though nothing here can repair it: the next beat re-sends the set, and the log is the
|
|
77
80
|
// only trace a dropped one leaves anywhere.
|
|
78
81
|
if (!socket.send(presenceFrame(name, 'sync', roster.members, roster.total))) {
|
package/src/sync-node.ts
CHANGED
|
@@ -27,6 +27,7 @@ import { idleSweepPeriodMs } from './socket-idle';
|
|
|
27
27
|
import { actorChangeHandler } from './sync-actor-change';
|
|
28
28
|
import { GrantBook, sweepGrants } from './sync-auth';
|
|
29
29
|
import { changeHandler, ProducerKinds, reconnectHandler } from './sync-bus-handlers';
|
|
30
|
+
import { busOutage } from './sync-bus-outage';
|
|
30
31
|
import { ackRefOf, frameRouter } from './sync-frames';
|
|
31
32
|
import {
|
|
32
33
|
clientHeartbeatMs,
|
|
@@ -73,6 +74,7 @@ export function syncNode(options: SyncNodeOptions): SyncNode {
|
|
|
73
74
|
const gaps = new SeqGapDetector();
|
|
74
75
|
// Kept across restarts of the bus subscription: a table's producer kind is a fact of the fleet.
|
|
75
76
|
const producers = new ProducerKinds();
|
|
77
|
+
const outage = busOutage();
|
|
76
78
|
const channelSids = new ChannelSids();
|
|
77
79
|
let reconnects: (() => void) | null = null;
|
|
78
80
|
/** The re-auth pass in flight, shared by every tick that lands while it runs. */
|
|
@@ -145,7 +147,7 @@ export function syncNode(options: SyncNodeOptions): SyncNode {
|
|
|
145
147
|
const leave = presence.leave(name, socket.id);
|
|
146
148
|
// Detached as well as returned: `detach` attaches the reporting catch, so a caller that
|
|
147
149
|
// awaits this later is awaiting a promise whose rejection is already handled.
|
|
148
|
-
detach(leave, 'presence.leave', name);
|
|
150
|
+
outage.detach(leave, 'presence.leave', name);
|
|
149
151
|
leaves.push(leave);
|
|
150
152
|
}
|
|
151
153
|
}
|
|
@@ -237,13 +239,14 @@ export function syncNode(options: SyncNodeOptions): SyncNode {
|
|
|
237
239
|
buildId: options.buildId,
|
|
238
240
|
presence,
|
|
239
241
|
channelSids,
|
|
242
|
+
busAnswered: outage.answered,
|
|
240
243
|
heartbeatMs: clientHeartbeatMs(presence?.heartbeatMs, sockets.idleTimeoutMs),
|
|
241
244
|
});
|
|
242
245
|
|
|
243
246
|
/** Everything `start()` does, fenced on the generation it began under — see `started`. */
|
|
244
247
|
const begin = async (): Promise<void> => {
|
|
245
248
|
const run = generation;
|
|
246
|
-
const bus = { registry: options.registry, hub: options.hub, gaps, producers };
|
|
249
|
+
const bus = { registry: options.registry, hub: options.hub, gaps, producers, outage };
|
|
247
250
|
const subscription = await options.transport.subscribe(CHANGE_SUBJECT_ALL, changeHandler(bus));
|
|
248
251
|
// A stop or drain ran inside that await: what it released did not include this yet.
|
|
249
252
|
if (run !== generation) {
|
|
@@ -255,10 +258,7 @@ export function syncNode(options: SyncNodeOptions): SyncNode {
|
|
|
255
258
|
// One pass per heartbeat window: a member is swept only once it has actually missed its
|
|
256
259
|
// window, and the interval never holds the process open — shutdown is the drain's job.
|
|
257
260
|
if (presence) {
|
|
258
|
-
sweeping = setInterval(
|
|
259
|
-
() => detach(presence.sweepAll(), 'presence.sweep'),
|
|
260
|
-
presence.heartbeatMs,
|
|
261
|
-
);
|
|
261
|
+
sweeping = setInterval(() => outage.sweep(presence), presence.heartbeatMs);
|
|
262
262
|
sweeping.unref();
|
|
263
263
|
}
|
|
264
264
|
// The half-open connection Bun's own `idleTimeout` renews through its ping/pong: a client
|
|
@@ -409,7 +409,7 @@ export function syncNode(options: SyncNodeOptions): SyncNode {
|
|
|
409
409
|
} catch (error) {
|
|
410
410
|
// The ack frame tells the client what it did wrong; the monitor only hears about what
|
|
411
411
|
// this node did wrong. Same rule the HTTP pipeline applies at `status >= 500`.
|
|
412
|
-
if (!isClientFault(error)) {
|
|
412
|
+
if (!outage.refused(error, 'sync.frame') && !isClientFault(error)) {
|
|
413
413
|
reportError(error, {
|
|
414
414
|
source: 'realtime',
|
|
415
415
|
scope: { operation: 'sync.frame', extra: { socketId: socket.id } },
|
package/src/transport-env.ts
CHANGED
|
@@ -165,6 +165,8 @@ export function selectTransport(
|
|
|
165
165
|
presenceTtlMs,
|
|
166
166
|
// Only a node that serves presence needs the bucket to exist before it can start.
|
|
167
167
|
presenceBucket: use === 'sockets' ? 'dial' : 'first-use',
|
|
168
|
+
// A publisher without the bus is degraded and still serves; a node that serves FROM it is not.
|
|
169
|
+
outageLevel: use === 'publish' ? 'warn' : 'error',
|
|
168
170
|
...(options.clock === undefined ? {} : { clock: options.clock }),
|
|
169
171
|
...(options.connect === undefined ? {} : { connect: options.connect }),
|
|
170
172
|
...(options.backoff === undefined ? {} : { backoff: options.backoff }),
|