@ultimat3/core 22.4.0 → 22.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/src/config-health.ts +58 -0
- package/src/config.ts +10 -14
- package/src/index.ts +3 -3
- package/src/lifecycle-readiness.ts +21 -0
- package/src/lifecycle.ts +18 -24
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ultimat3/core",
|
|
3
|
-
"version": "22.
|
|
3
|
+
"version": "22.5.0",
|
|
4
4
|
"description": "Ultimate's foundation: errors, context, env, config, clock, ids, logging, telemetry, lifecycle",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -40,6 +40,6 @@
|
|
|
40
40
|
"test": "bun test"
|
|
41
41
|
},
|
|
42
42
|
"dependencies": {
|
|
43
|
-
"@ultimat3/schema": "22.
|
|
43
|
+
"@ultimat3/schema": "22.5.0"
|
|
44
44
|
}
|
|
45
45
|
}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
// Single responsibility: the `health` and `drain` sections of `app.config.ts` — what `/readyz`
|
|
2
|
+
// means and how a SIGTERM'd process leaves the load balancer — and the domain of the readiness
|
|
3
|
+
// mode, so the config validator and `configureLifecycle` refuse the same values.
|
|
4
|
+
|
|
5
|
+
import { UltimateError } from './errors';
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* `'dependencies'` (the default): `/readyz` is 503 while starting, draining, or while ANY registered
|
|
9
|
+
* check fails. `'process'`: 503 only while starting or draining — a failing check is REPORTED in the
|
|
10
|
+
* body (`checks.database: 'failing'`) and the status stays 200.
|
|
11
|
+
*
|
|
12
|
+
* Why the second exists: every replica shares the dependency, so one database blip marks the WHOLE
|
|
13
|
+
* fleet unready at once and the ingress answers "no available server" for every URL — pages that
|
|
14
|
+
* never touch the database included — where the app itself would have served them, or answered its
|
|
15
|
+
* own 503 for the ones that do. `/readyz?deep=1` always answers in `'dependencies'` mode, so
|
|
16
|
+
* monitoring still sees the dependency fail.
|
|
17
|
+
*/
|
|
18
|
+
export type ReadinessMode = 'dependencies' | 'process';
|
|
19
|
+
|
|
20
|
+
export const READINESS_MODES: readonly ReadinessMode[] = ['dependencies', 'process'];
|
|
21
|
+
|
|
22
|
+
/** `app.config.ts`'s `health` section. Read by the production boot into `configureLifecycle`. */
|
|
23
|
+
export interface HealthConfig {
|
|
24
|
+
readonly readiness: ReadinessMode;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* How a SIGTERM'd process leaves the load balancer. Read by `@ultimat3/http`'s `createServer`
|
|
29
|
+
* (`ServerOptions.drain`), which hands it to core's `configureLifecycle`.
|
|
30
|
+
*/
|
|
31
|
+
export interface DrainConfig {
|
|
32
|
+
/**
|
|
33
|
+
* `/readyz` answers 503 for this long before the listener closes, so endpoints stop routing here
|
|
34
|
+
* first. Default 0 in development/test and 5000 everywhere else — a process naming NO environment
|
|
35
|
+
* included. A whole number, 0–60000. The chart's `terminationGracePeriodSeconds` must exceed it
|
|
36
|
+
* plus the drain budget.
|
|
37
|
+
*/
|
|
38
|
+
readonly readinessGraceMs: number;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** Why a value is not a readiness mode, or `undefined` when it is one. */
|
|
42
|
+
export function readinessModeIssue(value: unknown): string | undefined {
|
|
43
|
+
if (READINESS_MODES.some((mode) => mode === value)) return undefined;
|
|
44
|
+
const said = typeof value === 'string' ? `"${value}"` : typeof value;
|
|
45
|
+
return `health.readiness ${said} is not one of ${READINESS_MODES.join(', ')}`;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** `configureLifecycle`'s screen: the mode, or `X_CONFIG_INVALID` naming the key. */
|
|
49
|
+
export function assertReadinessMode(value: unknown): ReadinessMode {
|
|
50
|
+
const issue = readinessModeIssue(value);
|
|
51
|
+
if (issue === undefined) return value as ReadinessMode;
|
|
52
|
+
throw new UltimateError({
|
|
53
|
+
code: 'X_CONFIG_INVALID',
|
|
54
|
+
cause: issue,
|
|
55
|
+
fix: "set health: { readiness: 'process' } (or 'dependencies', the default) in app.config.ts",
|
|
56
|
+
meta: { key: 'health.readiness' },
|
|
57
|
+
});
|
|
58
|
+
}
|
package/src/config.ts
CHANGED
|
@@ -9,6 +9,8 @@ import { CURRENCY_CODE_PATTERN } from '@ultimat3/schema';
|
|
|
9
9
|
import { CACHE_TIERS, type CacheTierName } from './cache-vocabulary';
|
|
10
10
|
import { countIssue } from './config-count';
|
|
11
11
|
import { BASE_FIX, CACHE_TIER_FIX, TIMEZONE_FIX } from './config-fixes';
|
|
12
|
+
import type { DrainConfig, HealthConfig } from './config-health';
|
|
13
|
+
import { readinessModeIssue } from './config-health';
|
|
12
14
|
import { type Input, lastSaid, layered } from './config-merge';
|
|
13
15
|
import type { PwaConfig, PwaOfflineConfig } from './config-pwa';
|
|
14
16
|
import { PWA_FIX, pwaIssues } from './config-pwa';
|
|
@@ -183,20 +185,6 @@ export interface AiConfig {
|
|
|
183
185
|
readonly mcp: McpConfig;
|
|
184
186
|
}
|
|
185
187
|
|
|
186
|
-
/**
|
|
187
|
-
* How a SIGTERM'd process leaves the load balancer. Read by `@ultimat3/http`'s `createServer`
|
|
188
|
-
* (`ServerOptions.drain`), which hands it to core's `configureLifecycle`.
|
|
189
|
-
*/
|
|
190
|
-
export interface DrainConfig {
|
|
191
|
-
/**
|
|
192
|
-
* `/readyz` answers 503 for this long before the listener closes, so endpoints stop routing here
|
|
193
|
-
* first. Default 0 in development/test and 5000 everywhere else — a process naming NO environment
|
|
194
|
-
* included. A whole number, 0–60000. The chart's `terminationGracePeriodSeconds` must exceed it
|
|
195
|
-
* plus the drain budget.
|
|
196
|
-
*/
|
|
197
|
-
readonly readinessGraceMs: number;
|
|
198
|
-
}
|
|
199
|
-
|
|
200
188
|
export interface AppConfig {
|
|
201
189
|
readonly name: string;
|
|
202
190
|
readonly locales: readonly string[];
|
|
@@ -214,6 +202,7 @@ export interface AppConfig {
|
|
|
214
202
|
readonly notify: NotifyConfig;
|
|
215
203
|
readonly ai: AiConfig;
|
|
216
204
|
readonly drain: DrainConfig;
|
|
205
|
+
readonly health: HealthConfig;
|
|
217
206
|
readonly site: SiteConfig;
|
|
218
207
|
readonly seo: SeoConfig;
|
|
219
208
|
}
|
|
@@ -251,6 +240,7 @@ export interface AppConfigInput extends SiteSectionsInput {
|
|
|
251
240
|
readonly notify?: Input<NotifyConfig> | undefined;
|
|
252
241
|
readonly ai?: AiConfigInput | undefined;
|
|
253
242
|
readonly drain?: Input<DrainConfig> | undefined;
|
|
243
|
+
readonly health?: Input<HealthConfig> | undefined;
|
|
254
244
|
}
|
|
255
245
|
|
|
256
246
|
/** An overlay from `config/<concern>.ts`. No `name` — the base owns it. */
|
|
@@ -308,6 +298,7 @@ function defaults(name: string): Omit<AppConfig, 'name' | 'site' | 'seo'> {
|
|
|
308
298
|
ai: { mcp: { expose: true, path: '/mcp' } },
|
|
309
299
|
// Read from the process env when the config is DEFINED — the same env the drain will run in.
|
|
310
300
|
drain: { readinessGraceMs: defaultReadinessGraceMs() },
|
|
301
|
+
health: { readiness: 'dependencies' },
|
|
311
302
|
};
|
|
312
303
|
}
|
|
313
304
|
|
|
@@ -352,6 +343,7 @@ function validate(config: AppConfig): void {
|
|
|
352
343
|
countIssue('jobs.visibilityTimeoutMs', config.jobs.visibilityTimeoutMs, 1),
|
|
353
344
|
countIssue('cache.defaultTtlMs', config.cache.defaultTtlMs, 0),
|
|
354
345
|
readinessGraceIssue(config.drain.readinessGraceMs),
|
|
346
|
+
readinessModeIssue(config.health.readiness),
|
|
355
347
|
];
|
|
356
348
|
for (const issue of counts) if (issue !== undefined) issues.push(issue);
|
|
357
349
|
if (config.jobs.queues.length === 0) issues.push('jobs.queues must list at least one queue');
|
|
@@ -492,6 +484,10 @@ export function defineConfig(
|
|
|
492
484
|
base.drain,
|
|
493
485
|
layers.map((layer) => layer.drain),
|
|
494
486
|
),
|
|
487
|
+
health: layered(
|
|
488
|
+
base.health,
|
|
489
|
+
layers.map((layer) => layer.health),
|
|
490
|
+
),
|
|
495
491
|
...mergeSite(layers),
|
|
496
492
|
};
|
|
497
493
|
|
package/src/index.ts
CHANGED
|
@@ -98,7 +98,6 @@ export type {
|
|
|
98
98
|
AuthConfig,
|
|
99
99
|
CacheConfig,
|
|
100
100
|
DatabaseConfig,
|
|
101
|
-
DrainConfig,
|
|
102
101
|
JobsConfig,
|
|
103
102
|
McpConfig,
|
|
104
103
|
NotifyConfig,
|
|
@@ -109,6 +108,8 @@ export type {
|
|
|
109
108
|
ThemeMode,
|
|
110
109
|
} from './config';
|
|
111
110
|
export { defineConfig, INBOX_RETENTION_KEYS } from './config';
|
|
111
|
+
export type { DrainConfig, HealthConfig, ReadinessMode } from './config-health';
|
|
112
|
+
export { READINESS_MODES } from './config-health';
|
|
112
113
|
export type {
|
|
113
114
|
PwaColors,
|
|
114
115
|
PwaConfig,
|
|
@@ -515,8 +516,6 @@ export type {
|
|
|
515
516
|
LifecycleOptions,
|
|
516
517
|
OnShutdownOptions,
|
|
517
518
|
ProcessSignal,
|
|
518
|
-
ReadinessCheck,
|
|
519
|
-
ReadinessStatus,
|
|
520
519
|
ShutdownHook,
|
|
521
520
|
ShutdownPhase,
|
|
522
521
|
ShutdownReason,
|
|
@@ -548,6 +547,7 @@ export {
|
|
|
548
547
|
READINESS_GRACE_DEFAULT_MS,
|
|
549
548
|
READINESS_GRACE_MAX_MS,
|
|
550
549
|
} from './lifecycle-grace';
|
|
550
|
+
export type { ReadinessCheck, ReadinessStatus } from './lifecycle-readiness';
|
|
551
551
|
export type { SignalHandlerOptions } from './lifecycle-signals';
|
|
552
552
|
export { installSignalHandlers } from './lifecycle-signals';
|
|
553
553
|
export { isSelfOrigin, listeningOrigins, markListening, resetListeners } from './listeners';
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
// Single responsibility: the readiness CHECK — its signature and its two answers. The mode that
|
|
2
|
+
// decides what a failing one does to `/readyz` is `config-health.ts`'s.
|
|
3
|
+
|
|
4
|
+
export type ReadinessStatus = 'ok' | 'failing';
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Synchronous, and that is the design, not a limitation. **Do not widen this to
|
|
8
|
+
* `() => Promise<boolean>`** — the signature is the mechanism.
|
|
9
|
+
*
|
|
10
|
+
* A readiness endpoint that does I/O is a liveness bomb. A probe that awaits a network call takes
|
|
11
|
+
* as long as the dependency does, so a slow database makes the endpoint miss its `timeoutSeconds`,
|
|
12
|
+
* the kubelet reads that as unready, and capacity is pulled from an already-struggling system —
|
|
13
|
+
* the outage the probe existed to prevent, caused by the probe. Worse under a liveness probe
|
|
14
|
+
* sharing the handler: the pod is killed and restarts into the same slow database, cold.
|
|
15
|
+
*
|
|
16
|
+
* So the owner of the dependency keeps a boolean fresh — a pool exposes `isOpen`, a background
|
|
17
|
+
* poller flips a flag on its own schedule with its own timeout — and this reads it. That puts the
|
|
18
|
+
* waiting where a timeout can be tuned, and leaves this path unable to block. A check that throws
|
|
19
|
+
* is `failing`.
|
|
20
|
+
*/
|
|
21
|
+
export type ReadinessCheck = () => boolean;
|
package/src/lifecycle.ts
CHANGED
|
@@ -3,11 +3,14 @@
|
|
|
3
3
|
// and reports the same /healthz + /readyz state.
|
|
4
4
|
|
|
5
5
|
import { type Clock, systemClock } from './clock';
|
|
6
|
+
import type { ReadinessMode } from './config-health';
|
|
7
|
+
import { assertReadinessMode } from './config-health';
|
|
6
8
|
import { UltimateError } from './errors';
|
|
7
9
|
import { finiteCount } from './finite-option';
|
|
8
10
|
import { settleWithin } from './lifecycle-deadline';
|
|
9
11
|
import { lifecycleDrained } from './lifecycle-errors';
|
|
10
12
|
import { defaultReadinessGraceMs, readinessGraceIssue } from './lifecycle-grace';
|
|
13
|
+
import type { ReadinessCheck, ReadinessStatus } from './lifecycle-readiness';
|
|
11
14
|
import { type LogFields, type Logger, logger as rootLogger } from './logger';
|
|
12
15
|
|
|
13
16
|
export type HealthState = 'starting' | 'ready' | 'draining' | 'stopped';
|
|
@@ -64,29 +67,12 @@ export interface LifecycleOptions {
|
|
|
64
67
|
* A whole number from 0 to 60000; 0 is no grace.
|
|
65
68
|
*/
|
|
66
69
|
readonly readinessGraceMs?: number | undefined;
|
|
70
|
+
/** What a failing check does to `/readyz` — see `ReadinessMode`. Default `'dependencies'`. */
|
|
71
|
+
readonly readiness?: ReadinessMode | undefined;
|
|
67
72
|
readonly clock?: Clock | undefined;
|
|
68
73
|
readonly logger?: Logger | undefined;
|
|
69
74
|
}
|
|
70
75
|
|
|
71
|
-
export type ReadinessStatus = 'ok' | 'failing';
|
|
72
|
-
|
|
73
|
-
/**
|
|
74
|
-
* Synchronous, and that is the design, not a limitation. **Do not widen this to
|
|
75
|
-
* `() => Promise<boolean>`** — the signature is the mechanism.
|
|
76
|
-
*
|
|
77
|
-
* A readiness endpoint that does I/O is a liveness bomb. A probe that awaits a network call takes
|
|
78
|
-
* as long as the dependency does, so a slow database makes the endpoint miss its `timeoutSeconds`,
|
|
79
|
-
* the kubelet reads that as unready, and capacity is pulled from an already-struggling system —
|
|
80
|
-
* the outage the probe existed to prevent, caused by the probe. Worse under a liveness probe
|
|
81
|
-
* sharing the handler: the pod is killed and restarts into the same slow database, cold.
|
|
82
|
-
*
|
|
83
|
-
* So the owner of the dependency keeps a boolean fresh — a pool exposes `isOpen`, a background
|
|
84
|
-
* poller flips a flag on its own schedule with its own timeout — and this reads it. That puts the
|
|
85
|
-
* waiting where a timeout can be tuned, and leaves this path unable to block. A check that throws
|
|
86
|
-
* is `failing`.
|
|
87
|
-
*/
|
|
88
|
-
export type ReadinessCheck = () => boolean;
|
|
89
|
-
|
|
90
76
|
export interface HealthReport {
|
|
91
77
|
readonly state: HealthState;
|
|
92
78
|
readonly ready: boolean;
|
|
@@ -123,6 +109,7 @@ const DEFAULT_DEADLINE_MS = 25_000;
|
|
|
123
109
|
let deadlineMs = DEFAULT_DEADLINE_MS;
|
|
124
110
|
/** `undefined` means "the environment's default", read when a drain starts, not at import. */
|
|
125
111
|
let graceMs: number | undefined;
|
|
112
|
+
let readinessMode: ReadinessMode = 'dependencies';
|
|
126
113
|
let clock: Clock = systemClock;
|
|
127
114
|
let log: Logger = rootLogger;
|
|
128
115
|
let state: HealthState = 'starting';
|
|
@@ -155,6 +142,7 @@ export function configureLifecycle(options: LifecycleOptions): void {
|
|
|
155
142
|
}
|
|
156
143
|
graceMs = options.readinessGraceMs;
|
|
157
144
|
}
|
|
145
|
+
if (options.readiness !== undefined) readinessMode = assertReadinessMode(options.readiness);
|
|
158
146
|
if (options.clock !== undefined) {
|
|
159
147
|
clock = options.clock;
|
|
160
148
|
startedAtMono = clock.monotonic();
|
|
@@ -451,13 +439,15 @@ export function drain(signal = 'manual'): Promise<void> {
|
|
|
451
439
|
return drainPromise;
|
|
452
440
|
}
|
|
453
441
|
|
|
454
|
-
export function healthReport(): HealthReport {
|
|
442
|
+
export function healthReport(mode: ReadinessMode = readinessMode): HealthReport {
|
|
455
443
|
const checks = readinessChecks();
|
|
444
|
+
const dependencies = Object.values(checks).every((status) => status === 'ok');
|
|
456
445
|
return {
|
|
457
446
|
state,
|
|
458
447
|
// `ready` is the same predicate `/readyz` answers on, so a body and its status can never
|
|
459
448
|
// disagree — a 200 whose body says `ready: false` is the bug this shares one source to avoid.
|
|
460
|
-
|
|
449
|
+
// `'process'` mode leaves the checks OUT of it, and still reports every one of them below.
|
|
450
|
+
ready: state === 'ready' && (mode === 'process' || dependencies),
|
|
461
451
|
uptimeMs: Math.round(clock.monotonic() - startedAtMono),
|
|
462
452
|
inflight,
|
|
463
453
|
buildId: process.env['BUILD_ID'] ?? 'dev',
|
|
@@ -477,9 +467,12 @@ export function healthzPayload(): HealthPayload {
|
|
|
477
467
|
return { ok, status: ok ? 200 : 503, body };
|
|
478
468
|
}
|
|
479
469
|
|
|
480
|
-
/**
|
|
481
|
-
|
|
482
|
-
|
|
470
|
+
/**
|
|
471
|
+
* Readiness: may this instance receive traffic? 503 while starting or draining, and — in the
|
|
472
|
+
* default `'dependencies'` mode, or always with `deep` (`/readyz?deep=1`) — while any check fails.
|
|
473
|
+
*/
|
|
474
|
+
export function readyzPayload(options: { readonly deep?: boolean } = {}): HealthPayload {
|
|
475
|
+
const body = healthReport(options.deep === true ? 'dependencies' : readinessMode);
|
|
483
476
|
return { ok: body.ready, status: body.ready ? 200 : 503, body };
|
|
484
477
|
}
|
|
485
478
|
|
|
@@ -487,6 +480,7 @@ export function readyzPayload(): HealthPayload {
|
|
|
487
480
|
export function resetLifecycle(): void {
|
|
488
481
|
deadlineMs = DEFAULT_DEADLINE_MS;
|
|
489
482
|
graceMs = undefined;
|
|
483
|
+
readinessMode = 'dependencies';
|
|
490
484
|
clock = systemClock;
|
|
491
485
|
log = rootLogger;
|
|
492
486
|
state = 'starting';
|