@norskvideo/ctl-sdk 0.1.23 → 0.1.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docker-runner.d.ts +9 -2
- package/docker-runner.js +4 -3
- package/package.json +1 -1
- package/product-health-monitor.d.ts +20 -4
- package/product-health-monitor.js +39 -18
package/docker-runner.d.ts
CHANGED
|
@@ -4,10 +4,17 @@ export declare const CONTAINER_INTERNAL_PORT = 4321;
|
|
|
4
4
|
* `norsk-ctl.role` label (proxy, proxy-oauth2, cpu-monitor, product); the
|
|
5
5
|
* value doubles as the card's component name. */
|
|
6
6
|
export declare const PRODUCT_ROLE_LABEL = "norsk-ctl.role=product";
|
|
7
|
+
/** How to place a product container. The network is the host's call: ctl puts
|
|
8
|
+
* the control plane on the same bridge as nginx and every launched instance,
|
|
9
|
+
* which is what makes it reachable from a daemon that is itself a container
|
|
10
|
+
* (see product-reach.ts). Omitted leaves it on docker's default bridge. */
|
|
11
|
+
export interface ProductRunOpts {
|
|
12
|
+
network?: string;
|
|
13
|
+
}
|
|
7
14
|
/** argv (sans leading "docker") that launches a product container: detached,
|
|
8
15
|
* auto-removed, loopback-published to its internal 4321, and role-labelled so
|
|
9
16
|
* it shows up in the Infrastructure tab. */
|
|
10
|
-
export declare function productRunArgs(image: string, hostPort: number): string[];
|
|
17
|
+
export declare function productRunArgs(image: string, hostPort: number, opts?: ProductRunOpts): string[];
|
|
11
18
|
/** Stable container name for a product, derived from its manifest name, so
|
|
12
19
|
* `docker ps` shows `norsk-product-studio` instead of a random docker alias.
|
|
13
20
|
* Mirrors the singleton naming of norsk-proxy / norsk-ctl-cpu-monitor. */
|
|
@@ -16,7 +23,7 @@ export declare function productContainerName(productName: string): string;
|
|
|
16
23
|
* so there is nothing to refresh; a tag can move under a registration. */
|
|
17
24
|
export declare function isDigestRef(image: string): boolean;
|
|
18
25
|
export declare function dockerPull(image: string): Promise<void>;
|
|
19
|
-
export declare function dockerRun(image: string, hostPort: number): Promise<string>;
|
|
26
|
+
export declare function dockerRun(image: string, hostPort: number, opts?: ProductRunOpts): Promise<string>;
|
|
20
27
|
/** The container's address on its first network (the default bridge for a
|
|
21
28
|
* product container), or undefined when docker cannot say. */
|
|
22
29
|
export declare function dockerAddress(containerId: string): Promise<string | undefined>;
|
package/docker-runner.js
CHANGED
|
@@ -8,13 +8,14 @@ export const PRODUCT_ROLE_LABEL = "norsk-ctl.role=product";
|
|
|
8
8
|
/** argv (sans leading "docker") that launches a product container: detached,
|
|
9
9
|
* auto-removed, loopback-published to its internal 4321, and role-labelled so
|
|
10
10
|
* it shows up in the Infrastructure tab. */
|
|
11
|
-
export function productRunArgs(image, hostPort) {
|
|
11
|
+
export function productRunArgs(image, hostPort, opts = {}) {
|
|
12
12
|
return [
|
|
13
13
|
"run",
|
|
14
14
|
"-d",
|
|
15
15
|
"--rm",
|
|
16
16
|
"--label",
|
|
17
17
|
PRODUCT_ROLE_LABEL,
|
|
18
|
+
...(opts.network ? ["--network", opts.network] : []),
|
|
18
19
|
"-p",
|
|
19
20
|
`127.0.0.1:${hostPort}:${CONTAINER_INTERNAL_PORT}`,
|
|
20
21
|
image,
|
|
@@ -43,8 +44,8 @@ export async function dockerPull(image) {
|
|
|
43
44
|
throw new ProductError("DOCKER_PULL_FAILED", `docker pull failed (exit ${exitCode}): ${stderr || "no stderr"}`);
|
|
44
45
|
}
|
|
45
46
|
}
|
|
46
|
-
export async function dockerRun(image, hostPort) {
|
|
47
|
-
const proc = Bun.spawn(["docker", ...productRunArgs(image, hostPort)], { stdout: "pipe", stderr: "pipe" });
|
|
47
|
+
export async function dockerRun(image, hostPort, opts = {}) {
|
|
48
|
+
const proc = Bun.spawn(["docker", ...productRunArgs(image, hostPort, opts)], { stdout: "pipe", stderr: "pipe" });
|
|
48
49
|
const exitCode = await proc.exited;
|
|
49
50
|
const stdout = (await new Response(proc.stdout).text()).trim();
|
|
50
51
|
const stderr = (await new Response(proc.stderr).text()).trim();
|
package/package.json
CHANGED
|
@@ -8,14 +8,30 @@ export interface ProductHealthState {
|
|
|
8
8
|
restartAttempts: number;
|
|
9
9
|
/** Clock value of the most recent restart attempt; gates backoff. */
|
|
10
10
|
lastRestartAt?: number;
|
|
11
|
+
/** URL the last failing probe dialled. Cleared on a healthy probe. */
|
|
12
|
+
lastProbeUrl?: string;
|
|
13
|
+
/** Why the last probe failed — status line or transport error. Cleared on
|
|
14
|
+
* a healthy probe. */
|
|
15
|
+
lastProbeDetail?: string;
|
|
16
|
+
}
|
|
17
|
+
/** What one liveness probe saw. A bare boolean here cost three CI runs that
|
|
18
|
+
* reported a product restart-looping with no record of what was dialled or
|
|
19
|
+
* what came back, so the URL and the reason travel with the verdict. */
|
|
20
|
+
export interface ProductProbeResult {
|
|
21
|
+
ok: boolean;
|
|
22
|
+
/** The address dialled, or "" when the probe declined to dial at all. */
|
|
23
|
+
url: string;
|
|
24
|
+
/** Present whenever `ok` is false: the HTTP status, the transport error, or
|
|
25
|
+
* the reason no dial was attempted. */
|
|
26
|
+
detail?: string;
|
|
11
27
|
}
|
|
12
28
|
export interface ProductHealthMonitorOptions {
|
|
13
29
|
/** Products to consider. Non-container products are ignored (externally
|
|
14
30
|
* owned), so this can safely be the unfiltered `productService.list`. */
|
|
15
31
|
listProducts: () => ProductRegistration[];
|
|
16
|
-
/** Liveness probe
|
|
17
|
-
*
|
|
18
|
-
probe: (reg: ProductRegistration) => Promise<
|
|
32
|
+
/** Liveness probe. Injected so tests don't hit the network; the daemon
|
|
33
|
+
* wires `probeProductHealth`. */
|
|
34
|
+
probe: (reg: ProductRegistration) => Promise<ProductProbeResult>;
|
|
19
35
|
/** Recovery action for a product over threshold. Typically
|
|
20
36
|
* `productService.restart`. Rejection counts as a failed attempt. */
|
|
21
37
|
restart: (name: string) => Promise<void>;
|
|
@@ -69,4 +85,4 @@ export declare class ProductHealthMonitor {
|
|
|
69
85
|
* (default /healthz) with a short timeout. Any non-2xx, network error, or
|
|
70
86
|
* timeout reads as unhealthy. Container-only — dev products are externally
|
|
71
87
|
* owned and never reach here. */
|
|
72
|
-
export declare function probeProductHealth(reg: ProductRegistration, timeoutMs?: number): Promise<
|
|
88
|
+
export declare function probeProductHealth(reg: ProductRegistration, timeoutMs?: number): Promise<ProductProbeResult>;
|
|
@@ -75,62 +75,83 @@ export class ProductHealthMonitor {
|
|
|
75
75
|
}
|
|
76
76
|
async checkOne(reg) {
|
|
77
77
|
const prev = this.states.get(reg.name) ?? UNKNOWN;
|
|
78
|
-
let
|
|
78
|
+
let result;
|
|
79
79
|
try {
|
|
80
|
-
|
|
80
|
+
result = await this.probe(reg);
|
|
81
81
|
}
|
|
82
|
-
catch {
|
|
83
|
-
|
|
82
|
+
catch (e) {
|
|
83
|
+
result = { ok: false, url: "", detail: `probe threw: ${e instanceof Error ? e.message : String(e)}` };
|
|
84
84
|
}
|
|
85
|
-
if (
|
|
85
|
+
if (result.ok) {
|
|
86
86
|
this.set(reg.name, { status: "healthy", consecutiveFailures: 0, restartAttempts: 0 }, prev);
|
|
87
87
|
return;
|
|
88
88
|
}
|
|
89
89
|
const consecutiveFailures = prev.consecutiveFailures + 1;
|
|
90
|
+
const probed = { lastProbeUrl: result.url, lastProbeDetail: result.detail };
|
|
91
|
+
// One line per unhealthy episode, not per sweep: enough to diagnose a
|
|
92
|
+
// restart loop without a warn every interval for a product known bad.
|
|
93
|
+
if (prev.consecutiveFailures === 0) {
|
|
94
|
+
logger.warn(`Product '${reg.name}' health probe failed: ${result.url || "not dialled"} — ${result.detail ?? "no detail"}`);
|
|
95
|
+
}
|
|
96
|
+
else {
|
|
97
|
+
logger.debug(`Product '${reg.name}' health probe failed (${consecutiveFailures}): ${result.url || "not dialled"} — ${result.detail ?? "no detail"}`);
|
|
98
|
+
}
|
|
90
99
|
const belowThreshold = consecutiveFailures < this.failureThreshold;
|
|
91
100
|
const gaveUp = prev.restartAttempts >= this.maxRestarts;
|
|
92
101
|
const backoffElapsed = prev.lastRestartAt === undefined || this.now() - prev.lastRestartAt >= this.restartBackoffMs;
|
|
93
102
|
if (belowThreshold || gaveUp || !backoffElapsed) {
|
|
94
|
-
this.set(reg.name, { ...prev, status: "unhealthy", consecutiveFailures }, prev);
|
|
103
|
+
this.set(reg.name, { ...prev, ...probed, status: "unhealthy", consecutiveFailures }, prev);
|
|
95
104
|
return;
|
|
96
105
|
}
|
|
97
106
|
// Threshold reached, attempts left, backoff elapsed: attempt recovery.
|
|
98
107
|
const at = this.now();
|
|
99
108
|
const restartAttempts = prev.restartAttempts + 1;
|
|
100
|
-
this.set(reg.name, { status: "restarting", consecutiveFailures: 0, restartAttempts, lastRestartAt: at }, prev);
|
|
109
|
+
this.set(reg.name, { ...probed, status: "restarting", consecutiveFailures: 0, restartAttempts, lastRestartAt: at }, prev);
|
|
101
110
|
try {
|
|
102
111
|
await this.restart(reg.name);
|
|
103
112
|
// Stay "restarting"; the next sweep re-probes to confirm recovery.
|
|
104
113
|
}
|
|
105
114
|
catch (e) {
|
|
106
115
|
logger.warn(`Product '${reg.name}': restart failed — ${e instanceof Error ? e.message : String(e)}`);
|
|
107
|
-
this.set(reg.name, { status: "unhealthy", consecutiveFailures: 0, restartAttempts, lastRestartAt: at }, prev);
|
|
116
|
+
this.set(reg.name, { ...probed, status: "unhealthy", consecutiveFailures: 0, restartAttempts, lastRestartAt: at }, prev);
|
|
108
117
|
}
|
|
109
118
|
}
|
|
110
119
|
set(name, next, prev) {
|
|
111
120
|
this.states.set(name, next);
|
|
112
121
|
if (next.status !== prev.status) {
|
|
113
|
-
|
|
122
|
+
const why = next.status === "healthy" ? "" : detailSuffix(next);
|
|
123
|
+
logger.info(`Product '${name}' health: ${prev.status} -> ${next.status}${why}`);
|
|
114
124
|
this.onChange?.(name, next);
|
|
115
125
|
}
|
|
116
126
|
}
|
|
117
127
|
}
|
|
128
|
+
/** Renders the recorded probe failure onto a transition log line, so the CI
|
|
129
|
+
* log of a restart loop carries the address and the reason. */
|
|
130
|
+
function detailSuffix(state) {
|
|
131
|
+
if (state.lastProbeUrl === undefined && state.lastProbeDetail === undefined)
|
|
132
|
+
return "";
|
|
133
|
+
return ` (${state.lastProbeUrl || "not dialled"} — ${state.lastProbeDetail ?? "no detail"})`;
|
|
134
|
+
}
|
|
118
135
|
/** Default liveness probe: GET the product's manifest-declared health path
|
|
119
136
|
* (default /healthz) with a short timeout. Any non-2xx, network error, or
|
|
120
137
|
* timeout reads as unhealthy. Container-only — dev products are externally
|
|
121
138
|
* owned and never reach here. */
|
|
122
139
|
export async function probeProductHealth(reg, timeoutMs = 3_000) {
|
|
123
|
-
if (reg.spec.kind !== "container"
|
|
124
|
-
return false;
|
|
140
|
+
if (reg.spec.kind !== "container") {
|
|
141
|
+
return { ok: false, url: "", detail: `not a container product (kind '${reg.spec.kind}')` };
|
|
142
|
+
}
|
|
143
|
+
if (reg.port === undefined && reg.reachHost === undefined) {
|
|
144
|
+
return { ok: false, url: "", detail: "registration has no published port and no reachHost" };
|
|
145
|
+
}
|
|
125
146
|
const path = reg.manifest.api?.healthCheckPath ?? "/healthz";
|
|
126
|
-
const
|
|
147
|
+
const url = `${productBaseUrl(reg)}${path.startsWith("/") ? path : `/${path}`}`;
|
|
127
148
|
try {
|
|
128
|
-
const r = await fetch(
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
return r.
|
|
149
|
+
const r = await fetch(url, { signal: AbortSignal.timeout(timeoutMs) });
|
|
150
|
+
if (r.ok)
|
|
151
|
+
return { ok: true, url };
|
|
152
|
+
return { ok: false, url, detail: `HTTP ${r.status} ${r.statusText}`.trim() };
|
|
132
153
|
}
|
|
133
|
-
catch {
|
|
134
|
-
return false;
|
|
154
|
+
catch (e) {
|
|
155
|
+
return { ok: false, url, detail: e instanceof Error ? `${e.name}: ${e.message}` : String(e) };
|
|
135
156
|
}
|
|
136
157
|
}
|