@norskvideo/ctl-sdk 0.1.24 → 0.1.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/product-health-monitor.d.ts +20 -4
- package/product-health-monitor.js +39 -18
package/package.json
CHANGED
|
@@ -8,14 +8,30 @@ export interface ProductHealthState {
|
|
|
8
8
|
restartAttempts: number;
|
|
9
9
|
/** Clock value of the most recent restart attempt; gates backoff. */
|
|
10
10
|
lastRestartAt?: number;
|
|
11
|
+
/** URL the last failing probe dialled. Cleared on a healthy probe. */
|
|
12
|
+
lastProbeUrl?: string;
|
|
13
|
+
/** Why the last probe failed — status line or transport error. Cleared on
|
|
14
|
+
* a healthy probe. */
|
|
15
|
+
lastProbeDetail?: string;
|
|
16
|
+
}
|
|
17
|
+
/** What one liveness probe saw. A bare boolean here cost three CI runs that
|
|
18
|
+
* reported a product restart-looping with no record of what was dialled or
|
|
19
|
+
* what came back, so the URL and the reason travel with the verdict. */
|
|
20
|
+
export interface ProductProbeResult {
|
|
21
|
+
ok: boolean;
|
|
22
|
+
/** The address dialled, or "" when the probe declined to dial at all. */
|
|
23
|
+
url: string;
|
|
24
|
+
/** Present whenever `ok` is false: the HTTP status, the transport error, or
|
|
25
|
+
* the reason no dial was attempted. */
|
|
26
|
+
detail?: string;
|
|
11
27
|
}
|
|
12
28
|
export interface ProductHealthMonitorOptions {
|
|
13
29
|
/** Products to consider. Non-container products are ignored (externally
|
|
14
30
|
* owned), so this can safely be the unfiltered `productService.list`. */
|
|
15
31
|
listProducts: () => ProductRegistration[];
|
|
16
|
-
/** Liveness probe
|
|
17
|
-
*
|
|
18
|
-
probe: (reg: ProductRegistration) => Promise<
|
|
32
|
+
/** Liveness probe. Injected so tests don't hit the network; the daemon
|
|
33
|
+
* wires `probeProductHealth`. */
|
|
34
|
+
probe: (reg: ProductRegistration) => Promise<ProductProbeResult>;
|
|
19
35
|
/** Recovery action for a product over threshold. Typically
|
|
20
36
|
* `productService.restart`. Rejection counts as a failed attempt. */
|
|
21
37
|
restart: (name: string) => Promise<void>;
|
|
@@ -69,4 +85,4 @@ export declare class ProductHealthMonitor {
|
|
|
69
85
|
* (default /healthz) with a short timeout. Any non-2xx, network error, or
|
|
70
86
|
* timeout reads as unhealthy. Container-only — dev products are externally
|
|
71
87
|
* owned and never reach here. */
|
|
72
|
-
export declare function probeProductHealth(reg: ProductRegistration, timeoutMs?: number): Promise<
|
|
88
|
+
export declare function probeProductHealth(reg: ProductRegistration, timeoutMs?: number): Promise<ProductProbeResult>;
|
|
@@ -75,62 +75,83 @@ export class ProductHealthMonitor {
|
|
|
75
75
|
}
|
|
76
76
|
async checkOne(reg) {
|
|
77
77
|
const prev = this.states.get(reg.name) ?? UNKNOWN;
|
|
78
|
-
let
|
|
78
|
+
let result;
|
|
79
79
|
try {
|
|
80
|
-
|
|
80
|
+
result = await this.probe(reg);
|
|
81
81
|
}
|
|
82
|
-
catch {
|
|
83
|
-
|
|
82
|
+
catch (e) {
|
|
83
|
+
result = { ok: false, url: "", detail: `probe threw: ${e instanceof Error ? e.message : String(e)}` };
|
|
84
84
|
}
|
|
85
|
-
if (
|
|
85
|
+
if (result.ok) {
|
|
86
86
|
this.set(reg.name, { status: "healthy", consecutiveFailures: 0, restartAttempts: 0 }, prev);
|
|
87
87
|
return;
|
|
88
88
|
}
|
|
89
89
|
const consecutiveFailures = prev.consecutiveFailures + 1;
|
|
90
|
+
const probed = { lastProbeUrl: result.url, lastProbeDetail: result.detail };
|
|
91
|
+
// One line per unhealthy episode, not per sweep: enough to diagnose a
|
|
92
|
+
// restart loop without a warn every interval for a product known bad.
|
|
93
|
+
if (prev.consecutiveFailures === 0) {
|
|
94
|
+
logger.warn(`Product '${reg.name}' health probe failed: ${result.url || "not dialled"} — ${result.detail ?? "no detail"}`);
|
|
95
|
+
}
|
|
96
|
+
else {
|
|
97
|
+
logger.debug(`Product '${reg.name}' health probe failed (${consecutiveFailures}): ${result.url || "not dialled"} — ${result.detail ?? "no detail"}`);
|
|
98
|
+
}
|
|
90
99
|
const belowThreshold = consecutiveFailures < this.failureThreshold;
|
|
91
100
|
const gaveUp = prev.restartAttempts >= this.maxRestarts;
|
|
92
101
|
const backoffElapsed = prev.lastRestartAt === undefined || this.now() - prev.lastRestartAt >= this.restartBackoffMs;
|
|
93
102
|
if (belowThreshold || gaveUp || !backoffElapsed) {
|
|
94
|
-
this.set(reg.name, { ...prev, status: "unhealthy", consecutiveFailures }, prev);
|
|
103
|
+
this.set(reg.name, { ...prev, ...probed, status: "unhealthy", consecutiveFailures }, prev);
|
|
95
104
|
return;
|
|
96
105
|
}
|
|
97
106
|
// Threshold reached, attempts left, backoff elapsed: attempt recovery.
|
|
98
107
|
const at = this.now();
|
|
99
108
|
const restartAttempts = prev.restartAttempts + 1;
|
|
100
|
-
this.set(reg.name, { status: "restarting", consecutiveFailures: 0, restartAttempts, lastRestartAt: at }, prev);
|
|
109
|
+
this.set(reg.name, { ...probed, status: "restarting", consecutiveFailures: 0, restartAttempts, lastRestartAt: at }, prev);
|
|
101
110
|
try {
|
|
102
111
|
await this.restart(reg.name);
|
|
103
112
|
// Stay "restarting"; the next sweep re-probes to confirm recovery.
|
|
104
113
|
}
|
|
105
114
|
catch (e) {
|
|
106
115
|
logger.warn(`Product '${reg.name}': restart failed — ${e instanceof Error ? e.message : String(e)}`);
|
|
107
|
-
this.set(reg.name, { status: "unhealthy", consecutiveFailures: 0, restartAttempts, lastRestartAt: at }, prev);
|
|
116
|
+
this.set(reg.name, { ...probed, status: "unhealthy", consecutiveFailures: 0, restartAttempts, lastRestartAt: at }, prev);
|
|
108
117
|
}
|
|
109
118
|
}
|
|
110
119
|
set(name, next, prev) {
|
|
111
120
|
this.states.set(name, next);
|
|
112
121
|
if (next.status !== prev.status) {
|
|
113
|
-
|
|
122
|
+
const why = next.status === "healthy" ? "" : detailSuffix(next);
|
|
123
|
+
logger.info(`Product '${name}' health: ${prev.status} -> ${next.status}${why}`);
|
|
114
124
|
this.onChange?.(name, next);
|
|
115
125
|
}
|
|
116
126
|
}
|
|
117
127
|
}
|
|
128
|
+
/** Renders the recorded probe failure onto a transition log line, so the CI
|
|
129
|
+
* log of a restart loop carries the address and the reason. */
|
|
130
|
+
function detailSuffix(state) {
|
|
131
|
+
if (state.lastProbeUrl === undefined && state.lastProbeDetail === undefined)
|
|
132
|
+
return "";
|
|
133
|
+
return ` (${state.lastProbeUrl || "not dialled"} — ${state.lastProbeDetail ?? "no detail"})`;
|
|
134
|
+
}
|
|
118
135
|
/** Default liveness probe: GET the product's manifest-declared health path
|
|
119
136
|
* (default /healthz) with a short timeout. Any non-2xx, network error, or
|
|
120
137
|
* timeout reads as unhealthy. Container-only — dev products are externally
|
|
121
138
|
* owned and never reach here. */
|
|
122
139
|
export async function probeProductHealth(reg, timeoutMs = 3_000) {
|
|
123
|
-
if (reg.spec.kind !== "container"
|
|
124
|
-
return false;
|
|
140
|
+
if (reg.spec.kind !== "container") {
|
|
141
|
+
return { ok: false, url: "", detail: `not a container product (kind '${reg.spec.kind}')` };
|
|
142
|
+
}
|
|
143
|
+
if (reg.port === undefined && reg.reachHost === undefined) {
|
|
144
|
+
return { ok: false, url: "", detail: "registration has no published port and no reachHost" };
|
|
145
|
+
}
|
|
125
146
|
const path = reg.manifest.api?.healthCheckPath ?? "/healthz";
|
|
126
|
-
const
|
|
147
|
+
const url = `${productBaseUrl(reg)}${path.startsWith("/") ? path : `/${path}`}`;
|
|
127
148
|
try {
|
|
128
|
-
const r = await fetch(
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
return r.
|
|
149
|
+
const r = await fetch(url, { signal: AbortSignal.timeout(timeoutMs) });
|
|
150
|
+
if (r.ok)
|
|
151
|
+
return { ok: true, url };
|
|
152
|
+
return { ok: false, url, detail: `HTTP ${r.status} ${r.statusText}`.trim() };
|
|
132
153
|
}
|
|
133
|
-
catch {
|
|
134
|
-
return false;
|
|
154
|
+
catch (e) {
|
|
155
|
+
return { ok: false, url, detail: e instanceof Error ? `${e.name}: ${e.message}` : String(e) };
|
|
135
156
|
}
|
|
136
157
|
}
|