@kronos-ts/axon-server 0.4.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/axon-server-event-store.d.ts +0 -16
- package/dist/axon-server-event-store.d.ts.map +1 -1
- package/dist/axon-server-event-store.js +73 -45
- package/dist/axon-server-event-store.js.map +1 -1
- package/dist/axon-server-snapshotting-event-store.d.ts +30 -0
- package/dist/axon-server-snapshotting-event-store.d.ts.map +1 -0
- package/dist/axon-server-snapshotting-event-store.js +154 -0
- package/dist/axon-server-snapshotting-event-store.js.map +1 -0
- package/dist/axon-server.d.ts +46 -65
- package/dist/axon-server.d.ts.map +1 -1
- package/dist/axon-server.js +490 -317
- package/dist/axon-server.js.map +1 -1
- package/dist/bounded-read.d.ts +19 -0
- package/dist/bounded-read.d.ts.map +1 -0
- package/dist/bounded-read.js +39 -0
- package/dist/bounded-read.js.map +1 -0
- package/dist/connection-manager.d.ts +2 -2
- package/dist/connection-manager.d.ts.map +1 -1
- package/dist/connection.d.ts +36 -21
- package/dist/connection.d.ts.map +1 -1
- package/dist/connection.js +84 -45
- package/dist/connection.js.map +1 -1
- package/dist/context-view.d.ts +2 -2
- package/dist/context-view.d.ts.map +1 -1
- package/dist/control-plane.d.ts +14 -24
- package/dist/control-plane.d.ts.map +1 -1
- package/dist/control-plane.js +6 -37
- package/dist/control-plane.js.map +1 -1
- package/dist/event-processor-info.d.ts +7 -25
- package/dist/event-processor-info.d.ts.map +1 -1
- package/dist/event-processor-info.js +16 -20
- package/dist/event-processor-info.js.map +1 -1
- package/dist/flow-controlled-sender.d.ts +2 -2
- package/dist/flow-controlled-sender.d.ts.map +1 -1
- package/dist/flow-controlled-sender.js +37 -17
- package/dist/flow-controlled-sender.js.map +1 -1
- package/dist/generated/command.d.ts +20 -20
- package/dist/generated/command.d.ts.map +1 -1
- package/dist/generated/common.d.ts +20 -20
- package/dist/generated/common.d.ts.map +1 -1
- package/dist/generated/control.d.ts +36 -40
- package/dist/generated/control.d.ts.map +1 -1
- package/dist/generated/dcb.d.ts +88 -94
- package/dist/generated/dcb.d.ts.map +1 -1
- package/dist/generated/event.d.ts +81 -84
- package/dist/generated/event.d.ts.map +1 -1
- package/dist/generated/google/protobuf/empty.d.ts +3 -4
- package/dist/generated/google/protobuf/empty.d.ts.map +1 -1
- package/dist/generated/query.d.ts +40 -40
- package/dist/generated/query.d.ts.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/message-size.d.ts +2 -2
- package/dist/message-size.d.ts.map +1 -1
- package/dist/outbound-stream.d.ts +7 -11
- package/dist/outbound-stream.d.ts.map +1 -1
- package/dist/outbound-stream.js +65 -16
- package/dist/outbound-stream.js.map +1 -1
- package/dist/platform-service.d.ts +14 -5
- package/dist/platform-service.d.ts.map +1 -1
- package/dist/platform-service.js +70 -10
- package/dist/platform-service.js.map +1 -1
- package/dist/resilience.d.ts +62 -0
- package/dist/resilience.d.ts.map +1 -0
- package/dist/resilience.js +103 -0
- package/dist/resilience.js.map +1 -0
- package/dist/shutdown-latch.d.ts +5 -4
- package/dist/shutdown-latch.d.ts.map +1 -1
- package/dist/shutdown-latch.js +20 -1
- package/dist/shutdown-latch.js.map +1 -1
- package/dist/stream-recovery.d.ts +9 -0
- package/dist/stream-recovery.d.ts.map +1 -0
- package/dist/stream-recovery.js +70 -0
- package/dist/stream-recovery.js.map +1 -0
- package/package.json +3 -3
- package/src/axon-server-event-store.ts +77 -47
- package/src/axon-server-snapshotting-event-store.ts +194 -0
- package/src/axon-server.ts +472 -374
- package/src/bounded-read.ts +43 -0
- package/src/connection-manager.ts +1 -1
- package/src/connection.ts +93 -59
- package/src/context-view.ts +1 -1
- package/src/control-plane.ts +22 -66
- package/src/event-processor-info.ts +23 -44
- package/src/flow-controlled-sender.ts +34 -15
- package/src/generated/command.ts +10 -10
- package/src/generated/common.ts +10 -10
- package/src/generated/control.ts +20 -20
- package/src/generated/dcb.ts +47 -47
- package/src/generated/event.ts +42 -42
- package/src/generated/google/protobuf/empty.ts +2 -2
- package/src/generated/query.ts +20 -20
- package/src/index.ts +1 -2
- package/src/message-size.ts +1 -1
- package/src/outbound-stream.ts +58 -28
- package/src/platform-service.ts +77 -13
- package/src/resilience.ts +149 -0
- package/src/shutdown-latch.ts +16 -3
- package/src/stream-recovery.ts +72 -0
- package/dist/axon-server-snapshot-store.d.ts +0 -12
- package/dist/axon-server-snapshot-store.d.ts.map +0 -1
- package/dist/axon-server-snapshot-store.js +0 -82
- package/dist/axon-server-snapshot-store.js.map +0 -1
- package/src/axon-server-snapshot-store.ts +0 -110
package/src/platform-service.ts
CHANGED
|
@@ -35,7 +35,7 @@ export type InstructionHandler = (instruction: PlatformInstruction) => void | Pr
|
|
|
35
35
|
* - Heartbeat protocol to detect dead connections
|
|
36
36
|
* - Receives server-initiated instructions (pause, resume, split, merge segments)
|
|
37
37
|
*/
|
|
38
|
-
export
|
|
38
|
+
export type PlatformConnection = {
|
|
39
39
|
/**
|
|
40
40
|
* DATA PATH. Open the platform stream, register this client, and arm the
|
|
41
41
|
* heartbeat that calls `connection.reconnect()` when the server stops
|
|
@@ -104,8 +104,17 @@ export interface PlatformConnection {
|
|
|
104
104
|
subscriptionsAcked(): Promise<boolean>
|
|
105
105
|
}
|
|
106
106
|
|
|
107
|
-
export
|
|
108
|
-
/**
|
|
107
|
+
export type PlatformServiceOptions = {
|
|
108
|
+
/**
|
|
109
|
+
* Heartbeat interval in ms. Default: 2500.
|
|
110
|
+
*
|
|
111
|
+
* Axon Server's `client-heartbeat-timeout` defaults to 5000 ms and is
|
|
112
|
+
* refreshed ONLY by heartbeats the client sends (`HeartbeatMonitor`), checked
|
|
113
|
+
* every second. A 10 s cadence — Axon Framework's own default — survives
|
|
114
|
+
* there only because the framework also answers every heartbeat the SERVER
|
|
115
|
+
* sends; this client does both, and beats at half the server's window so a
|
|
116
|
+
* paused runner still fits.
|
|
117
|
+
*/
|
|
109
118
|
heartbeatIntervalMs?: number
|
|
110
119
|
/** Heartbeat timeout in ms. If no response within this window, reconnect. Default: 7500 */
|
|
111
120
|
heartbeatTimeoutMs?: number
|
|
@@ -133,7 +142,7 @@ export function platformConnection(
|
|
|
133
142
|
connection: AxonServerConnection,
|
|
134
143
|
options?: PlatformServiceOptions,
|
|
135
144
|
): PlatformConnection {
|
|
136
|
-
const heartbeatIntervalMs = options?.heartbeatIntervalMs ??
|
|
145
|
+
const heartbeatIntervalMs = options?.heartbeatIntervalMs ?? 2500
|
|
137
146
|
const heartbeatTimeoutMs = options?.heartbeatTimeoutMs ?? 7500
|
|
138
147
|
const processorsNotificationRateMs = options?.processorsNotificationRateMs ?? 500
|
|
139
148
|
const processorsNotificationInitialDelayMs = options?.processorsNotificationInitialDelayMs ?? 5000
|
|
@@ -154,12 +163,22 @@ export function platformConnection(
|
|
|
154
163
|
*/
|
|
155
164
|
const pendingInstructions: PlatformInstruction[] = []
|
|
156
165
|
let isConnected = false
|
|
166
|
+
let monitoringArmed = false
|
|
167
|
+
let recoveryTimer: ReturnType<typeof setTimeout> | undefined
|
|
157
168
|
let heartbeatTimer: ReturnType<typeof setInterval> | null = null
|
|
158
169
|
let heartbeatTimeoutTimer: ReturnType<typeof setTimeout> | null = null
|
|
170
|
+
let processorStatusInitialTimer: ReturnType<typeof setTimeout> | undefined
|
|
159
171
|
let processorStatusTimer: ReturnType<typeof setInterval> | null = null
|
|
160
172
|
/** Guards against arming the status-report timer twice — see `startProcessorStatusReporting`. */
|
|
161
173
|
let processorStatusArmed = false
|
|
162
174
|
let lastHeartbeatResponse = Date.now()
|
|
175
|
+
// Whether this server has EVER sent a heartbeat. Axon Server only beats at
|
|
176
|
+
// clients whose registered framework version it recognises (4.2.1+); a client
|
|
177
|
+
// it does not beat at cannot be judged by the silence — that judgement fell
|
|
178
|
+
// every window and tore down a healthy stream, every 10 s, in production.
|
|
179
|
+
// Dead-channel detection for a server that never beats is gRPC keepalive's
|
|
180
|
+
// job (see `connection.ts`), not this timer's.
|
|
181
|
+
let serverBeats = false
|
|
163
182
|
let outbound: ReturnType<typeof outboundStream<PlatformInboundInstruction>> | null = null
|
|
164
183
|
/**
|
|
165
184
|
* Latches once Axon Server sends its first inbound message after
|
|
@@ -176,13 +195,37 @@ export function platformConnection(
|
|
|
176
195
|
grpcMetadata.set("AxonIQ-Access-Token", connection.config.token)
|
|
177
196
|
}
|
|
178
197
|
|
|
179
|
-
|
|
198
|
+
function scheduleRecovery() {
|
|
199
|
+
if (!monitoringArmed || recoveryTimer || connection.state === "closed" || connection.state === "reconnecting") return
|
|
200
|
+
recoveryTimer = setTimeout(() => {
|
|
201
|
+
recoveryTimer = undefined
|
|
202
|
+
if (!monitoringArmed || connection.state === "closed") return
|
|
203
|
+
void connection.reconnect().catch((error) => console.error("Platform recovery failed", error))
|
|
204
|
+
}, connection.config.reconnectIntervalMs ?? 2000)
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
connection.onDisconnect?.(() => {
|
|
208
|
+
isConnected = false
|
|
209
|
+
outbound?.close()
|
|
210
|
+
})
|
|
211
|
+
|
|
212
|
+
connection.onReconnect(() => {
|
|
213
|
+
if (!monitoringArmed) return
|
|
214
|
+
clearTimeout(recoveryTimer)
|
|
215
|
+
recoveryTimer = undefined
|
|
216
|
+
isConnected = false
|
|
217
|
+
openPlatformStream()
|
|
218
|
+
})
|
|
219
|
+
|
|
220
|
+
async function processInboundInstructions(inbound: AsyncIterable<any>, stream: typeof outbound) {
|
|
180
221
|
try {
|
|
181
222
|
for await (const message of inbound) {
|
|
223
|
+
if (stream !== outbound || !monitoringArmed) return
|
|
182
224
|
// First inbound message after start() = the platform has accepted our
|
|
183
225
|
// registration and is talking back. Latch the ack flag (mirror of
|
|
184
226
|
// kronosdb Plan 09-03 / D-102 — replaces the legacy 1s sleep).
|
|
185
227
|
acked = true
|
|
228
|
+
if (message.requestReconnect) scheduleRecovery()
|
|
186
229
|
// Parse instruction type
|
|
187
230
|
const instruction = parseInstruction(message)
|
|
188
231
|
if (instruction) {
|
|
@@ -200,16 +243,27 @@ export function platformConnection(
|
|
|
200
243
|
}
|
|
201
244
|
}
|
|
202
245
|
|
|
203
|
-
//
|
|
246
|
+
// A heartbeat from the server is BOTH the liveness signal this side
|
|
247
|
+
// tracks and a request: Axon Server measures client inactivity by the
|
|
248
|
+
// heartbeats it receives, so every one it sends is answered at once
|
|
249
|
+
// (what Axon Framework's connector does). Without the echo, the only
|
|
250
|
+
// activity the server ever sees is the periodic beat below.
|
|
204
251
|
if (message.heartbeat) {
|
|
252
|
+
serverBeats = true
|
|
205
253
|
lastHeartbeatResponse = Date.now()
|
|
254
|
+
outbound?.send({ heartbeat: { clientId: connection.config.clientId }, instructionId: "" })
|
|
206
255
|
}
|
|
207
256
|
}
|
|
208
257
|
} catch (err) {
|
|
209
|
-
if (isConnected) {
|
|
258
|
+
if (stream === outbound && monitoringArmed && isConnected) {
|
|
210
259
|
console.error("Platform stream error:", err)
|
|
211
260
|
isConnected = false
|
|
212
261
|
}
|
|
262
|
+
} finally {
|
|
263
|
+
if (stream === outbound && monitoringArmed) {
|
|
264
|
+
isConnected = false
|
|
265
|
+
scheduleRecovery()
|
|
266
|
+
}
|
|
213
267
|
}
|
|
214
268
|
}
|
|
215
269
|
|
|
@@ -266,10 +320,13 @@ export function platformConnection(
|
|
|
266
320
|
|
|
267
321
|
heartbeatTimer = setInterval(() => {
|
|
268
322
|
if (!isConnected || !outbound) return
|
|
323
|
+
// Not before the server has acknowledged the registration: a heartbeat
|
|
324
|
+
// from a stream it has not filed yet is an error on its side, not a beat.
|
|
325
|
+
if (!acked) return
|
|
269
326
|
|
|
270
|
-
//
|
|
327
|
+
// Judge silence only from a server that has beaten before.
|
|
271
328
|
const timeSinceLastResponse = Date.now() - lastHeartbeatResponse
|
|
272
|
-
if (timeSinceLastResponse > heartbeatTimeoutMs) {
|
|
329
|
+
if (serverBeats && timeSinceLastResponse > heartbeatTimeoutMs) {
|
|
273
330
|
console.warn(
|
|
274
331
|
`Platform heartbeat timeout: no response in ${timeSinceLastResponse}ms ` +
|
|
275
332
|
`(threshold: ${heartbeatTimeoutMs}ms). Marking connection as lost.`,
|
|
@@ -302,9 +359,9 @@ export function platformConnection(
|
|
|
302
359
|
if (processorStatusTimer) clearInterval(processorStatusTimer)
|
|
303
360
|
|
|
304
361
|
// Initial delay before first report
|
|
305
|
-
setTimeout(() => {
|
|
306
|
-
if (!
|
|
307
|
-
reportProcessorStatus()
|
|
362
|
+
processorStatusInitialTimer = setTimeout(() => {
|
|
363
|
+
if (!monitoringArmed) return
|
|
364
|
+
if (isConnected) void reportProcessorStatus()
|
|
308
365
|
|
|
309
366
|
// Then report at the configured rate
|
|
310
367
|
processorStatusTimer = setInterval(() => {
|
|
@@ -352,6 +409,7 @@ export function platformConnection(
|
|
|
352
409
|
|
|
353
410
|
// Re-arm the ack latch so a stop/start cycle correctly re-waits.
|
|
354
411
|
acked = false
|
|
412
|
+
serverBeats = false
|
|
355
413
|
outbound = outboundStream<PlatformInboundInstruction>()
|
|
356
414
|
|
|
357
415
|
// Register with Axon Server
|
|
@@ -375,7 +433,7 @@ export function platformConnection(
|
|
|
375
433
|
// timeout, which is what the command/query buses hang their stream
|
|
376
434
|
// re-establishment off. It belongs to every service, administered or not.
|
|
377
435
|
startHeartbeat()
|
|
378
|
-
processInboundInstructions(inbound)
|
|
436
|
+
void processInboundInstructions(inbound, outbound)
|
|
379
437
|
|
|
380
438
|
// Axon Server's PlatformService does NOT proactively emit an inbound
|
|
381
439
|
// frame in response to `register` — the stream is held open silently
|
|
@@ -399,15 +457,21 @@ export function platformConnection(
|
|
|
399
457
|
|
|
400
458
|
return {
|
|
401
459
|
async armConnectionMonitoring() {
|
|
460
|
+
monitoringArmed = true
|
|
402
461
|
openPlatformStream()
|
|
403
462
|
},
|
|
404
463
|
|
|
405
464
|
async start() {
|
|
465
|
+
monitoringArmed = true
|
|
406
466
|
openPlatformStream()
|
|
407
467
|
startProcessorStatusReporting()
|
|
408
468
|
},
|
|
409
469
|
|
|
410
470
|
stop() {
|
|
471
|
+
monitoringArmed = false
|
|
472
|
+
clearTimeout(processorStatusInitialTimer)
|
|
473
|
+
clearTimeout(recoveryTimer)
|
|
474
|
+
recoveryTimer = undefined
|
|
411
475
|
isConnected = false
|
|
412
476
|
// A stopped stream's un-routed backlog is stale — do not replay it if a
|
|
413
477
|
// handler registers later.
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Connection resilience — PACKAGE-PRIVATE.
|
|
3
|
+
*
|
|
4
|
+
* Exponential backoff with full jitter, per-event attempt caps, terminal-error
|
|
5
|
+
* short-circuit, and a warn-then-continue health probe. Nothing here is
|
|
6
|
+
* exported from this package's barrel.
|
|
7
|
+
*
|
|
8
|
+
* It used to live in `@kronos-ts/core`, and it does not belong there. Core is
|
|
9
|
+
* only what cannot be a helper, and this is a helper by every test: it touches
|
|
10
|
+
* no core shape, no message, no unit of work — it is `setTimeout` and a loop
|
|
11
|
+
* over a function anybody could have written. Its only consumers were the
|
|
12
|
+
* transports and the pool, core's own `src/` never called it, and a helper
|
|
13
|
+
* lives with its users. Same reasoning that moved each persistence package's
|
|
14
|
+
* transaction glue out of core, applied one folder up.
|
|
15
|
+
*
|
|
16
|
+
* Each consuming package keeps its own copy, carrying only what it uses, so
|
|
17
|
+
* each can also state its own retry policy without negotiating with the others.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
/** Classified retry sites used by transport extensions (D-97). */
|
|
21
|
+
export type RetryEvent =
|
|
22
|
+
| "initial-connect"
|
|
23
|
+
| "reconnect"
|
|
24
|
+
| "per-operation"
|
|
25
|
+
| "health-check"
|
|
26
|
+
|
|
27
|
+
export type ResilienceConfig = {
|
|
28
|
+
/** Initial backoff delay (ms). Default: 100. */
|
|
29
|
+
initialDelayMs: number
|
|
30
|
+
/** Cap for backoff (ms). Default: 30_000. */
|
|
31
|
+
maxDelayMs: number
|
|
32
|
+
/** Max retry attempts before giving up. Per-event defaults applied if not provided. */
|
|
33
|
+
maxAttempts: number
|
|
34
|
+
/** Base for exponential growth. Default: 2. */
|
|
35
|
+
multiplier: number
|
|
36
|
+
/** Health-check threshold (ms) before warn-then-continue. Default: 5000. */
|
|
37
|
+
healthCheckThresholdMs: number
|
|
38
|
+
/** Optional logger. Defaults to console.warn. */
|
|
39
|
+
log?: (msg: string) => void
|
|
40
|
+
/** Per-attempt classification: returns false to short-circuit (terminal error). */
|
|
41
|
+
isRetryable?: (err: unknown) => boolean
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
const DEFAULTS: Omit<ResilienceConfig, "log" | "isRetryable"> = {
|
|
45
|
+
initialDelayMs: 100,
|
|
46
|
+
maxDelayMs: 30_000,
|
|
47
|
+
maxAttempts: 10,
|
|
48
|
+
multiplier: 2,
|
|
49
|
+
healthCheckThresholdMs: 5_000,
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
const ATTEMPTS_BY_EVENT: Record<RetryEvent, number> = {
|
|
53
|
+
"initial-connect": 10,
|
|
54
|
+
"reconnect": 30,
|
|
55
|
+
"per-operation": 5,
|
|
56
|
+
"health-check": 3,
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function resolveConfig(
|
|
60
|
+
event: RetryEvent,
|
|
61
|
+
partial?: Partial<ResilienceConfig>,
|
|
62
|
+
): ResilienceConfig {
|
|
63
|
+
return {
|
|
64
|
+
...DEFAULTS,
|
|
65
|
+
maxAttempts: ATTEMPTS_BY_EVENT[event],
|
|
66
|
+
...partial,
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
function fullJitterDelay(attempt: number, cfg: ResilienceConfig): number {
|
|
71
|
+
const exponential = Math.min(
|
|
72
|
+
cfg.initialDelayMs * Math.pow(cfg.multiplier, attempt),
|
|
73
|
+
cfg.maxDelayMs,
|
|
74
|
+
)
|
|
75
|
+
return Math.random() * exponential
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* Run `fn` with exponential-backoff full-jitter retries.
|
|
80
|
+
*
|
|
81
|
+
* Resolves with the first successful result, or rejects with the last error
|
|
82
|
+
* after `maxAttempts` exhausted. If `isRetryable(err)` returns false on any
|
|
83
|
+
* attempt, the error is rethrown immediately (terminal error short-circuit).
|
|
84
|
+
*
|
|
85
|
+
* D-101: opts may carry caller-supplied resilience overrides — this is the
|
|
86
|
+
* per-extension knob path (`kronosDbConnection({ resilience: { ... } })`).
|
|
87
|
+
*/
|
|
88
|
+
export async function withRetry<T>(
|
|
89
|
+
fn: () => Promise<T>,
|
|
90
|
+
opts: { event: RetryEvent } & Partial<ResilienceConfig>,
|
|
91
|
+
): Promise<T> {
|
|
92
|
+
const cfg = resolveConfig(opts.event, opts)
|
|
93
|
+
let lastErr: unknown
|
|
94
|
+
for (let attempt = 0; attempt < cfg.maxAttempts; attempt++) {
|
|
95
|
+
try {
|
|
96
|
+
return await fn()
|
|
97
|
+
} catch (err) {
|
|
98
|
+
lastErr = err
|
|
99
|
+
if (cfg.isRetryable && !cfg.isRetryable(err)) throw err
|
|
100
|
+
if (attempt === cfg.maxAttempts - 1) break
|
|
101
|
+
const delay = fullJitterDelay(attempt, cfg)
|
|
102
|
+
cfg.log?.(
|
|
103
|
+
`[kronos:resilience] ${opts.event} attempt ${attempt + 1}/${cfg.maxAttempts} ` +
|
|
104
|
+
`failed; retrying in ${Math.round(delay)}ms — ${
|
|
105
|
+
(err as Error)?.message ?? String(err)
|
|
106
|
+
}`,
|
|
107
|
+
)
|
|
108
|
+
await new Promise((r) => setTimeout(r, delay))
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
throw lastErr
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* Run a health-check probe with warn-then-continue semantics (D-100).
|
|
116
|
+
*
|
|
117
|
+
* Resolves WITHOUT throwing on success OR after `thresholdMs` elapses with
|
|
118
|
+
* the probe still pending — production loops should NEVER stall on a flaky
|
|
119
|
+
* health probe. Failures (rejection or threshold breach) emit a single warn
|
|
120
|
+
* via `log` and otherwise resolve normally.
|
|
121
|
+
*/
|
|
122
|
+
export async function healthCheck(
|
|
123
|
+
fn: () => Promise<void>,
|
|
124
|
+
opts: { thresholdMs?: number; log?: (msg: string) => void } = {},
|
|
125
|
+
): Promise<void> {
|
|
126
|
+
const threshold = opts.thresholdMs ?? DEFAULTS.healthCheckThresholdMs
|
|
127
|
+
const log = opts.log ?? ((m: string) => console.warn(m))
|
|
128
|
+
let timer: ReturnType<typeof setTimeout> | undefined
|
|
129
|
+
try {
|
|
130
|
+
await Promise.race([
|
|
131
|
+
fn(),
|
|
132
|
+
new Promise<never>((_, reject) => {
|
|
133
|
+
timer = setTimeout(
|
|
134
|
+
() => reject(new Error("health-check threshold exceeded")),
|
|
135
|
+
threshold,
|
|
136
|
+
)
|
|
137
|
+
}),
|
|
138
|
+
])
|
|
139
|
+
} catch (err) {
|
|
140
|
+
log(
|
|
141
|
+
`[kronos:resilience] health-check soft-failed (${
|
|
142
|
+
(err as Error)?.message ?? String(err)
|
|
143
|
+
}); continuing`,
|
|
144
|
+
)
|
|
145
|
+
// warn-then-continue: do NOT rethrow.
|
|
146
|
+
} finally {
|
|
147
|
+
if (timer) clearTimeout(timer)
|
|
148
|
+
}
|
|
149
|
+
}
|
package/src/shutdown-latch.ts
CHANGED
|
@@ -10,11 +10,12 @@
|
|
|
10
10
|
*
|
|
11
11
|
* This is the TypeScript equivalent of AF5's ShutdownLatch pattern.
|
|
12
12
|
*/
|
|
13
|
-
export
|
|
13
|
+
export type ShutdownLatch = {
|
|
14
14
|
/**
|
|
15
15
|
* Register an in-flight activity. Throws if shutdown is in progress.
|
|
16
16
|
* Call `end()` on the returned handle when the activity completes.
|
|
17
17
|
*/
|
|
18
|
+
onShutdown(callback: () => void): () => void
|
|
18
19
|
registerActivity(): ActivityHandle
|
|
19
20
|
|
|
20
21
|
/**
|
|
@@ -31,7 +32,7 @@ export interface ShutdownLatch {
|
|
|
31
32
|
readonly activeCount: number
|
|
32
33
|
}
|
|
33
34
|
|
|
34
|
-
export
|
|
35
|
+
export type ActivityHandle = {
|
|
35
36
|
/** Mark this activity as complete. */
|
|
36
37
|
end(): void
|
|
37
38
|
}
|
|
@@ -44,8 +45,10 @@ export class ShutdownInProgressError extends Error {
|
|
|
44
45
|
}
|
|
45
46
|
|
|
46
47
|
export function shutdownLatch(): ShutdownLatch {
|
|
48
|
+
const callbacks = new Set<() => void>()
|
|
47
49
|
let activeCount = 0
|
|
48
50
|
let shuttingDown = false
|
|
51
|
+
let drainPromise: Promise<void> | undefined
|
|
49
52
|
let drainResolve: (() => void) | null = null
|
|
50
53
|
|
|
51
54
|
function checkDrained() {
|
|
@@ -56,6 +59,11 @@ export function shutdownLatch(): ShutdownLatch {
|
|
|
56
59
|
}
|
|
57
60
|
|
|
58
61
|
return {
|
|
62
|
+
onShutdown(callback) {
|
|
63
|
+
if (shuttingDown) callback()
|
|
64
|
+
else callbacks.add(callback)
|
|
65
|
+
return () => { callbacks.delete(callback) }
|
|
66
|
+
},
|
|
59
67
|
registerActivity(): ActivityHandle {
|
|
60
68
|
if (shuttingDown) {
|
|
61
69
|
throw new ShutdownInProgressError()
|
|
@@ -76,14 +84,19 @@ export function shutdownLatch(): ShutdownLatch {
|
|
|
76
84
|
|
|
77
85
|
initiateShutdown(): Promise<void> {
|
|
78
86
|
shuttingDown = true
|
|
87
|
+
for (const callback of callbacks) {
|
|
88
|
+
try { callback() } catch (error) { console.error("Messaging shutdown callback failed", error) }
|
|
89
|
+
}
|
|
90
|
+
callbacks.clear()
|
|
79
91
|
|
|
80
92
|
if (activeCount === 0) {
|
|
81
93
|
return Promise.resolve()
|
|
82
94
|
}
|
|
83
95
|
|
|
84
|
-
|
|
96
|
+
drainPromise ??= new Promise((resolve) => {
|
|
85
97
|
drainResolve = resolve
|
|
86
98
|
})
|
|
99
|
+
return drainPromise
|
|
87
100
|
},
|
|
88
101
|
|
|
89
102
|
get shuttingDown() {
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import type { ResilienceConfig } from "./resilience.js"
|
|
2
|
+
|
|
3
|
+
/** Retry failures of the async stream, not just synchronous stream creation. */
|
|
4
|
+
export function streamRecovery(
|
|
5
|
+
reopen: () => void,
|
|
6
|
+
canReconnect: () => boolean,
|
|
7
|
+
config: Partial<ResilienceConfig> = {},
|
|
8
|
+
) {
|
|
9
|
+
const log = (message: string) => {
|
|
10
|
+
try {
|
|
11
|
+
;(config.log ?? console.warn)(message)
|
|
12
|
+
} catch {
|
|
13
|
+
/* Diagnostics cannot prevent recovery. */
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
let timer: ReturnType<typeof setTimeout> | undefined
|
|
17
|
+
let stopped = false
|
|
18
|
+
let attempts = 0
|
|
19
|
+
let openedAt = Date.now()
|
|
20
|
+
|
|
21
|
+
function failed(error: unknown) {
|
|
22
|
+
if (stopped || timer || !canReconnect()) return
|
|
23
|
+
if (config.isRetryable?.(error) === false || attempts >= (config.maxAttempts ?? 30)) {
|
|
24
|
+
log(`Provider stream recovery exhausted: ${String(error)}`)
|
|
25
|
+
return
|
|
26
|
+
}
|
|
27
|
+
// Keep a positive floor even when jitter is zero. An immediately-ended
|
|
28
|
+
// async generator must never create a microtask reconnect loop.
|
|
29
|
+
const cap = Math.min(
|
|
30
|
+
(config.initialDelayMs ?? 100) * (config.multiplier ?? 2) ** attempts++,
|
|
31
|
+
config.maxDelayMs ?? 30000,
|
|
32
|
+
)
|
|
33
|
+
const delay = Math.max(1, cap * (0.5 + Math.random() * 0.5))
|
|
34
|
+
log(`Provider stream failed; reconnecting in ${Math.round(delay)}ms: ${String(error)}`)
|
|
35
|
+
timer = setTimeout(() => {
|
|
36
|
+
timer = undefined
|
|
37
|
+
if (stopped || !canReconnect()) return
|
|
38
|
+
openedAt = Date.now()
|
|
39
|
+
try {
|
|
40
|
+
reopen()
|
|
41
|
+
} catch (error) {
|
|
42
|
+
failed(error)
|
|
43
|
+
}
|
|
44
|
+
}, delay)
|
|
45
|
+
timer.unref?.()
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
return {
|
|
49
|
+
failed,
|
|
50
|
+
received() {
|
|
51
|
+
// Subscription acknowledgements alone do not prove a stable connection.
|
|
52
|
+
if (Date.now() - openedAt >= 30000) attempts = 0
|
|
53
|
+
},
|
|
54
|
+
restart() {
|
|
55
|
+
if (stopped || !canReconnect()) return
|
|
56
|
+
clearTimeout(timer)
|
|
57
|
+
timer = undefined
|
|
58
|
+
attempts = 0
|
|
59
|
+
openedAt = Date.now()
|
|
60
|
+
try {
|
|
61
|
+
reopen()
|
|
62
|
+
} catch (error) {
|
|
63
|
+
failed(error)
|
|
64
|
+
}
|
|
65
|
+
},
|
|
66
|
+
stop() {
|
|
67
|
+
stopped = true
|
|
68
|
+
clearTimeout(timer)
|
|
69
|
+
timer = undefined
|
|
70
|
+
},
|
|
71
|
+
}
|
|
72
|
+
}
|
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
import type { SnapshotStore } from "@kronos-ts/core";
|
|
2
|
-
import type { AxonServerStoreSource } from "./connection.js";
|
|
3
|
-
/**
|
|
4
|
-
* A SnapshotStore over Axon Server's gRPC snapshot service, in one context.
|
|
5
|
-
*
|
|
6
|
-
* Uses the `DcbSnapshotStore` gRPC service to store and retrieve state
|
|
7
|
-
* snapshots. `context` is a per-call header, so this shares the one channel
|
|
8
|
-
* `conn` holds with every other context — see `contextView`. Payloads go
|
|
9
|
-
* through the connection's serializer.
|
|
10
|
-
*/
|
|
11
|
-
export declare function axonServerSnapshotStore(conn: AxonServerStoreSource, context: string): SnapshotStore;
|
|
12
|
-
//# sourceMappingURL=axon-server-snapshot-store.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"axon-server-snapshot-store.d.ts","sourceRoot":"","sources":["../src/axon-server-snapshot-store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAY,MAAM,iBAAiB,CAAA;AAE9D,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,iBAAiB,CAAA;AAgD5D;;;;;;;GAOG;AACH,wBAAgB,uBAAuB,CACrC,IAAI,EAAE,qBAAqB,EAC3B,OAAO,EAAE,MAAM,GACd,aAAa,CAgDf"}
|
|
@@ -1,82 +0,0 @@
|
|
|
1
|
-
import { contextView } from "./context-view.js";
|
|
2
|
-
// ---------------------------------------------------------------------------
|
|
3
|
-
// Conversion — framework Snapshot ↔ proto Snapshot
|
|
4
|
-
// ---------------------------------------------------------------------------
|
|
5
|
-
const encoder = new TextEncoder();
|
|
6
|
-
function createSnapshotConverters(serializer) {
|
|
7
|
-
return {
|
|
8
|
-
snapshotToProto(snapshot) {
|
|
9
|
-
const serialized = serializer.serialize(snapshot.payload, "snapshot", "");
|
|
10
|
-
return {
|
|
11
|
-
name: "",
|
|
12
|
-
version: "",
|
|
13
|
-
payload: serialized.data,
|
|
14
|
-
timestamp: BigInt(snapshot.timestamp),
|
|
15
|
-
metadata: snapshot.metadata,
|
|
16
|
-
};
|
|
17
|
-
},
|
|
18
|
-
snapshotFromProto(proto, position) {
|
|
19
|
-
const payload = proto.payload.length > 0
|
|
20
|
-
? serializer.deserialize({ data: proto.payload, type: "snapshot", revision: "" })
|
|
21
|
-
: {};
|
|
22
|
-
return {
|
|
23
|
-
position,
|
|
24
|
-
payload,
|
|
25
|
-
timestamp: Number(proto.timestamp),
|
|
26
|
-
metadata: proto.metadata ?? {},
|
|
27
|
-
};
|
|
28
|
-
},
|
|
29
|
-
};
|
|
30
|
-
}
|
|
31
|
-
function encodeKey(stateName, id) {
|
|
32
|
-
const idStr = typeof id === "object" && id !== null ? JSON.stringify(id) : String(id);
|
|
33
|
-
return encoder.encode(`${stateName}:${idStr}`);
|
|
34
|
-
}
|
|
35
|
-
// ---------------------------------------------------------------------------
|
|
36
|
-
// Axon Server snapshot store
|
|
37
|
-
// ---------------------------------------------------------------------------
|
|
38
|
-
/**
|
|
39
|
-
* A SnapshotStore over Axon Server's gRPC snapshot service, in one context.
|
|
40
|
-
*
|
|
41
|
-
* Uses the `DcbSnapshotStore` gRPC service to store and retrieve state
|
|
42
|
-
* snapshots. `context` is a per-call header, so this shares the one channel
|
|
43
|
-
* `conn` holds with every other context — see `contextView`. Payloads go
|
|
44
|
-
* through the connection's serializer.
|
|
45
|
-
*/
|
|
46
|
-
export function axonServerSnapshotStore(conn, context) {
|
|
47
|
-
const { connection, serializer, metadata: createAxonMetadata } = contextView(conn, context);
|
|
48
|
-
const { snapshotToProto, snapshotFromProto } = createSnapshotConverters(serializer);
|
|
49
|
-
return {
|
|
50
|
-
async store(stateName, id, snapshot) {
|
|
51
|
-
await connection.snapshotStore.add({
|
|
52
|
-
key: encodeKey(stateName, id),
|
|
53
|
-
sequence: snapshot.position,
|
|
54
|
-
prune: true,
|
|
55
|
-
snapshot: snapshotToProto(snapshot),
|
|
56
|
-
}, { metadata: createAxonMetadata() });
|
|
57
|
-
},
|
|
58
|
-
async load(stateName, id) {
|
|
59
|
-
try {
|
|
60
|
-
const response = await connection.snapshotStore.getLast({ key: encodeKey(stateName, id) }, { metadata: createAxonMetadata() });
|
|
61
|
-
if (!response.snapshot) {
|
|
62
|
-
return undefined;
|
|
63
|
-
}
|
|
64
|
-
return snapshotFromProto(response.snapshot, response.sequence);
|
|
65
|
-
}
|
|
66
|
-
catch (err) {
|
|
67
|
-
// Axon Server throws when no snapshot exists — treat as "not found"
|
|
68
|
-
if (String(err).includes("No snapshot found")) {
|
|
69
|
-
return undefined;
|
|
70
|
-
}
|
|
71
|
-
throw err;
|
|
72
|
-
}
|
|
73
|
-
},
|
|
74
|
-
async deleteSnapshots(stateName, id) {
|
|
75
|
-
await connection.snapshotStore.delete({
|
|
76
|
-
key: encodeKey(stateName, id),
|
|
77
|
-
toSequence: BigInt(Number.MAX_SAFE_INTEGER),
|
|
78
|
-
}, { metadata: createAxonMetadata() });
|
|
79
|
-
},
|
|
80
|
-
};
|
|
81
|
-
}
|
|
82
|
-
//# sourceMappingURL=axon-server-snapshot-store.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"axon-server-snapshot-store.js","sourceRoot":"","sources":["../src/axon-server-snapshot-store.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAA;AAG/C,8EAA8E;AAC9E,mDAAmD;AACnD,8EAA8E;AAE9E,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAA;AAEjC,SAAS,wBAAwB,CAAC,UAAsB;IACtD,OAAO;QACL,eAAe,CAAC,QAAkB;YAChC,MAAM,UAAU,GAAG,UAAU,CAAC,SAAS,CAAC,QAAQ,CAAC,OAAO,EAAE,UAAU,EAAE,EAAE,CAAC,CAAA;YACzE,OAAO;gBACL,IAAI,EAAE,EAAE;gBACR,OAAO,EAAE,EAAE;gBACX,OAAO,EAAE,UAAU,CAAC,IAAI;gBACxB,SAAS,EAAE,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC;gBACrC,QAAQ,EAAE,QAAQ,CAAC,QAAQ;aAC5B,CAAA;QACH,CAAC;QAED,iBAAiB,CAAC,KAAoB,EAAE,QAAgB;YACtD,MAAM,OAAO,GACX,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC;gBACtB,CAAC,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;gBACjF,CAAC,CAAC,EAAE,CAAA;YAER,OAAO;gBACL,QAAQ;gBACR,OAAO;gBACP,SAAS,EAAE,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC;gBAClC,QAAQ,EAAE,KAAK,CAAC,QAAQ,IAAI,EAAE;aAC/B,CAAA;QACH,CAAC;KACF,CAAA;AACH,CAAC;AAED,SAAS,SAAS,CAAC,SAAiB,EAAE,EAAW;IAC/C,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,QAAQ,IAAI,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;IACrF,OAAO,OAAO,CAAC,MAAM,CAAC,GAAG,SAAS,IAAI,KAAK,EAAE,CAAC,CAAA;AAChD,CAAC;AAED,8EAA8E;AAC9E,6BAA6B;AAC7B,8EAA8E;AAE9E;;;;;;;GAOG;AACH,MAAM,UAAU,uBAAuB,CACrC,IAA2B,EAC3B,OAAe;IAEf,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,QAAQ,EAAE,kBAAkB,EAAE,GAAG,WAAW,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;IAC3F,MAAM,EAAE,eAAe,EAAE,iBAAiB,EAAE,GAAG,wBAAwB,CAAC,UAAU,CAAC,CAAA;IAEnF,OAAO;QACL,KAAK,CAAC,KAAK,CAAC,SAAiB,EAAE,EAAW,EAAE,QAAkB;YAC5D,MAAM,UAAU,CAAC,aAAa,CAAC,GAAG,CAChC;gBACE,GAAG,EAAE,SAAS,CAAC,SAAS,EAAE,EAAE,CAAC;gBAC7B,QAAQ,EAAE,QAAQ,CAAC,QAAQ;gBAC3B,KAAK,EAAE,IAAI;gBACX,QAAQ,EAAE,eAAe,CAAC,QAAQ,CAAC;aACpC,EACD,EAAE,QAAQ,EAAE,kBAAkB,EAAE,EAAE,CACnC,CAAA;QACH,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,SAAiB,EAAE,EAAW;YACvC,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,aAAa,CAAC,OAAO,CACrD,EAAE,GAAG,EAAE,SAAS,CAAC,SAAS,EAAE,EAAE,CAAC,EAAE,EACjC,EAAE,QAAQ,EAAE,kBAAkB,EAAE,EAAE,CACnC,CAAA;gBAED,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC;oBACvB,OAAO,SAAS,CAAA;gBAClB,CAAC;gBAED,OAAO,iBAAiB,CAAC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAA;YAChE,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,oEAAoE;gBACpE,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,mBAAmB,CAAC,EAAE,CAAC;oBAC9C,OAAO,SAAS,CAAA;gBAClB,CAAC;gBACD,MAAM,GAAG,CAAA;YACX,CAAC;QACH,CAAC;QAED,KAAK,CAAC,eAAe,CAAC,SAAiB,EAAE,EAAW;YAClD,MAAM,UAAU,CAAC,aAAa,CAAC,MAAM,CACnC;gBACE,GAAG,EAAE,SAAS,CAAC,SAAS,EAAE,EAAE,CAAC;gBAC7B,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,gBAAgB,CAAC;aAC5C,EACD,EAAE,QAAQ,EAAE,kBAAkB,EAAE,EAAE,CACnC,CAAA;QACH,CAAC;KACF,CAAA;AACH,CAAC"}
|