@ultimat3/core 22.4.0 → 22.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +5 -2
- package/README.md +1 -0
- package/package.json +2 -2
- package/src/canonical-json.ts +4 -0
- package/src/config-health.ts +58 -0
- package/src/config.ts +10 -14
- package/src/cursor.ts +8 -0
- package/src/index.ts +9 -3
- package/src/keyed-fingerprint.ts +69 -0
- package/src/lifecycle-readiness.ts +21 -0
- package/src/lifecycle.ts +18 -24
package/CLAUDE.md
CHANGED
|
@@ -95,9 +95,12 @@ top-level `UltimateError` use in `error-codes.ts`.
|
|
|
95
95
|
derives their retry classification from the same set. It is a `SIDE_EFFECTS_ANCHORS` entry.
|
|
96
96
|
- `timing-safe-equal.ts` is the one constant-time comparison (`@ultimat3/auth`, `@ultimat3/storage`).
|
|
97
97
|
- **`canonical-json.ts`: `canonicalJson` is INJECTIVE and `fingerprint` is SHA-256/16 of it** — the
|
|
98
|
-
hash every sharing key is taken over (`
|
|
99
|
-
`
|
|
98
|
+
hash every in-memory sharing key is taken over (`query`'s `queryHash`, `realtime`'s `qid`).
|
|
99
|
+
`NaN`, `±Infinity` and `-0` are bare tokens; `Date`, `Map` and `Set` are TAGGED. Never a
|
|
100
100
|
fourth copy, never parseable (`@ultimat3/action`'s `stableStringify` is the document form).
|
|
101
|
+
- **A PERSISTED fingerprint is `keyedFingerprint`** (`keyed-fingerprint.ts`: HMAC under a per-purpose
|
|
102
|
+
key derived from the cursor secret, `h1:` versioned) — `action`'s idempotency `requestHash`. An
|
|
103
|
+
unkeyed prefix in a table is an offline oracle for a short account number in the input.
|
|
101
104
|
- **`decimal-order.ts`'s `compareDecimalText` answers `undefined` for a non-decimal**, and only a
|
|
102
105
|
caller that knows the column's kind may ask (`@ultimat3/entity`'s `compareByKind`) — never
|
|
103
106
|
`@ultimat3/query`, whose `OrderKey` has no kind.
|
package/README.md
CHANGED
|
@@ -498,6 +498,7 @@ never a silently wrong page.
|
|
|
498
498
|
|---|---|
|
|
499
499
|
| Signature | truncated HMAC-SHA256, compared in constant time |
|
|
500
500
|
| Secret | `configureCursorSigning()` at boot, else `ULTIMATE_CURSOR_SECRET`. **Read when a cursor is signed, never at import** — an app whose `openSecrets()` sets the variable during boot would otherwise sign every cursor with the dev key. Rotating it invalidates every open cursor |
|
|
501
|
+
| Also keys | `keyedFingerprint(value, purpose)` — `h1:<key id>:<HMAC>` over `canonicalJson`, under a per-purpose key derived from this secret; the fingerprint to PERSIST (`@ultimat3/action`'s idempotency `requestHash`). `compareFingerprint` answers `match` / `mismatch` / `unverifiable` (other key), and still checks a legacy bare `fingerprint()` exactly. Rotating the secret makes in-window stored fingerprints `unverifiable` |
|
|
501
502
|
| Signed, not encrypted | the client already has these rows; what it must not do is *invent* a position |
|
|
502
503
|
| `usesDevCursorSecret()` | true while the shipped dev key is in use |
|
|
503
504
|
| `resetCursorSigning()` | test seam: forget `configureCursorSigning` and fall back to the environment |
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ultimat3/core",
|
|
3
|
-
"version": "22.
|
|
3
|
+
"version": "22.5.1",
|
|
4
4
|
"description": "Ultimate's foundation: errors, context, env, config, clock, ids, logging, telemetry, lifecycle",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -40,6 +40,6 @@
|
|
|
40
40
|
"test": "bun test"
|
|
41
41
|
},
|
|
42
42
|
"dependencies": {
|
|
43
|
-
"@ultimat3/schema": "22.
|
|
43
|
+
"@ultimat3/schema": "22.5.1"
|
|
44
44
|
}
|
|
45
45
|
}
|
package/src/canonical-json.ts
CHANGED
|
@@ -119,6 +119,10 @@ export function canonicalJson(value: unknown): string {
|
|
|
119
119
|
* hands one caller another's rows. FNV-1a/32, which two of the three copies started as, is
|
|
120
120
|
* 4x10^9 values and brute-forceable offline in seconds: an input landing on another read's key was
|
|
121
121
|
* something an attacker could mint rather than something they had to wait for.
|
|
122
|
+
*
|
|
123
|
+
* UNKEYED, so never the thing to PERSIST beside input that may hold a low-entropy secret — a
|
|
124
|
+
* 64-bit prefix of a known hash is an offline oracle for a short account number. A stored
|
|
125
|
+
* fingerprint is `keyedFingerprint` (`keyed-fingerprint.ts`).
|
|
122
126
|
*/
|
|
123
127
|
export function fingerprint(value: unknown): string {
|
|
124
128
|
return new Bun.CryptoHasher('sha256').update(canonicalJson(value)).digest('hex').slice(0, 16);
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
// Single responsibility: the `health` and `drain` sections of `app.config.ts` — what `/readyz`
|
|
2
|
+
// means and how a SIGTERM'd process leaves the load balancer — and the domain of the readiness
|
|
3
|
+
// mode, so the config validator and `configureLifecycle` refuse the same values.
|
|
4
|
+
|
|
5
|
+
import { UltimateError } from './errors';
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* `'dependencies'` (the default): `/readyz` is 503 while starting, draining, or while ANY registered
|
|
9
|
+
* check fails. `'process'`: 503 only while starting or draining — a failing check is REPORTED in the
|
|
10
|
+
* body (`checks.database: 'failing'`) and the status stays 200.
|
|
11
|
+
*
|
|
12
|
+
* Why the second exists: every replica shares the dependency, so one database blip marks the WHOLE
|
|
13
|
+
* fleet unready at once and the ingress answers "no available server" for every URL — pages that
|
|
14
|
+
* never touch the database included — where the app itself would have served them, or answered its
|
|
15
|
+
* own 503 for the ones that do. `/readyz?deep=1` always answers in `'dependencies'` mode, so
|
|
16
|
+
* monitoring still sees the dependency fail.
|
|
17
|
+
*/
|
|
18
|
+
export type ReadinessMode = 'dependencies' | 'process';
|
|
19
|
+
|
|
20
|
+
export const READINESS_MODES: readonly ReadinessMode[] = ['dependencies', 'process'];
|
|
21
|
+
|
|
22
|
+
/** `app.config.ts`'s `health` section. Read by the production boot into `configureLifecycle`. */
|
|
23
|
+
export interface HealthConfig {
|
|
24
|
+
readonly readiness: ReadinessMode;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* How a SIGTERM'd process leaves the load balancer. Read by `@ultimat3/http`'s `createServer`
|
|
29
|
+
* (`ServerOptions.drain`), which hands it to core's `configureLifecycle`.
|
|
30
|
+
*/
|
|
31
|
+
export interface DrainConfig {
|
|
32
|
+
/**
|
|
33
|
+
* `/readyz` answers 503 for this long before the listener closes, so endpoints stop routing here
|
|
34
|
+
* first. Default 0 in development/test and 5000 everywhere else — a process naming NO environment
|
|
35
|
+
* included. A whole number, 0–60000. The chart's `terminationGracePeriodSeconds` must exceed it
|
|
36
|
+
* plus the drain budget.
|
|
37
|
+
*/
|
|
38
|
+
readonly readinessGraceMs: number;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** Why a value is not a readiness mode, or `undefined` when it is one. */
|
|
42
|
+
export function readinessModeIssue(value: unknown): string | undefined {
|
|
43
|
+
if (READINESS_MODES.some((mode) => mode === value)) return undefined;
|
|
44
|
+
const said = typeof value === 'string' ? `"${value}"` : typeof value;
|
|
45
|
+
return `health.readiness ${said} is not one of ${READINESS_MODES.join(', ')}`;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** `configureLifecycle`'s screen: the mode, or `X_CONFIG_INVALID` naming the key. */
|
|
49
|
+
export function assertReadinessMode(value: unknown): ReadinessMode {
|
|
50
|
+
const issue = readinessModeIssue(value);
|
|
51
|
+
if (issue === undefined) return value as ReadinessMode;
|
|
52
|
+
throw new UltimateError({
|
|
53
|
+
code: 'X_CONFIG_INVALID',
|
|
54
|
+
cause: issue,
|
|
55
|
+
fix: "set health: { readiness: 'process' } (or 'dependencies', the default) in app.config.ts",
|
|
56
|
+
meta: { key: 'health.readiness' },
|
|
57
|
+
});
|
|
58
|
+
}
|
package/src/config.ts
CHANGED
|
@@ -9,6 +9,8 @@ import { CURRENCY_CODE_PATTERN } from '@ultimat3/schema';
|
|
|
9
9
|
import { CACHE_TIERS, type CacheTierName } from './cache-vocabulary';
|
|
10
10
|
import { countIssue } from './config-count';
|
|
11
11
|
import { BASE_FIX, CACHE_TIER_FIX, TIMEZONE_FIX } from './config-fixes';
|
|
12
|
+
import type { DrainConfig, HealthConfig } from './config-health';
|
|
13
|
+
import { readinessModeIssue } from './config-health';
|
|
12
14
|
import { type Input, lastSaid, layered } from './config-merge';
|
|
13
15
|
import type { PwaConfig, PwaOfflineConfig } from './config-pwa';
|
|
14
16
|
import { PWA_FIX, pwaIssues } from './config-pwa';
|
|
@@ -183,20 +185,6 @@ export interface AiConfig {
|
|
|
183
185
|
readonly mcp: McpConfig;
|
|
184
186
|
}
|
|
185
187
|
|
|
186
|
-
/**
|
|
187
|
-
* How a SIGTERM'd process leaves the load balancer. Read by `@ultimat3/http`'s `createServer`
|
|
188
|
-
* (`ServerOptions.drain`), which hands it to core's `configureLifecycle`.
|
|
189
|
-
*/
|
|
190
|
-
export interface DrainConfig {
|
|
191
|
-
/**
|
|
192
|
-
* `/readyz` answers 503 for this long before the listener closes, so endpoints stop routing here
|
|
193
|
-
* first. Default 0 in development/test and 5000 everywhere else — a process naming NO environment
|
|
194
|
-
* included. A whole number, 0–60000. The chart's `terminationGracePeriodSeconds` must exceed it
|
|
195
|
-
* plus the drain budget.
|
|
196
|
-
*/
|
|
197
|
-
readonly readinessGraceMs: number;
|
|
198
|
-
}
|
|
199
|
-
|
|
200
188
|
export interface AppConfig {
|
|
201
189
|
readonly name: string;
|
|
202
190
|
readonly locales: readonly string[];
|
|
@@ -214,6 +202,7 @@ export interface AppConfig {
|
|
|
214
202
|
readonly notify: NotifyConfig;
|
|
215
203
|
readonly ai: AiConfig;
|
|
216
204
|
readonly drain: DrainConfig;
|
|
205
|
+
readonly health: HealthConfig;
|
|
217
206
|
readonly site: SiteConfig;
|
|
218
207
|
readonly seo: SeoConfig;
|
|
219
208
|
}
|
|
@@ -251,6 +240,7 @@ export interface AppConfigInput extends SiteSectionsInput {
|
|
|
251
240
|
readonly notify?: Input<NotifyConfig> | undefined;
|
|
252
241
|
readonly ai?: AiConfigInput | undefined;
|
|
253
242
|
readonly drain?: Input<DrainConfig> | undefined;
|
|
243
|
+
readonly health?: Input<HealthConfig> | undefined;
|
|
254
244
|
}
|
|
255
245
|
|
|
256
246
|
/** An overlay from `config/<concern>.ts`. No `name` — the base owns it. */
|
|
@@ -308,6 +298,7 @@ function defaults(name: string): Omit<AppConfig, 'name' | 'site' | 'seo'> {
|
|
|
308
298
|
ai: { mcp: { expose: true, path: '/mcp' } },
|
|
309
299
|
// Read from the process env when the config is DEFINED — the same env the drain will run in.
|
|
310
300
|
drain: { readinessGraceMs: defaultReadinessGraceMs() },
|
|
301
|
+
health: { readiness: 'dependencies' },
|
|
311
302
|
};
|
|
312
303
|
}
|
|
313
304
|
|
|
@@ -352,6 +343,7 @@ function validate(config: AppConfig): void {
|
|
|
352
343
|
countIssue('jobs.visibilityTimeoutMs', config.jobs.visibilityTimeoutMs, 1),
|
|
353
344
|
countIssue('cache.defaultTtlMs', config.cache.defaultTtlMs, 0),
|
|
354
345
|
readinessGraceIssue(config.drain.readinessGraceMs),
|
|
346
|
+
readinessModeIssue(config.health.readiness),
|
|
355
347
|
];
|
|
356
348
|
for (const issue of counts) if (issue !== undefined) issues.push(issue);
|
|
357
349
|
if (config.jobs.queues.length === 0) issues.push('jobs.queues must list at least one queue');
|
|
@@ -492,6 +484,10 @@ export function defineConfig(
|
|
|
492
484
|
base.drain,
|
|
493
485
|
layers.map((layer) => layer.drain),
|
|
494
486
|
),
|
|
487
|
+
health: layered(
|
|
488
|
+
base.health,
|
|
489
|
+
layers.map((layer) => layer.health),
|
|
490
|
+
),
|
|
495
491
|
...mergeSite(layers),
|
|
496
492
|
};
|
|
497
493
|
|
package/src/cursor.ts
CHANGED
|
@@ -52,6 +52,14 @@ function currentSecret(): string {
|
|
|
52
52
|
return configured ?? Bun.env['ULTIMATE_CURSOR_SECRET'] ?? DEV_SECRET;
|
|
53
53
|
}
|
|
54
54
|
|
|
55
|
+
/**
|
|
56
|
+
* The app's signing secret, exactly as cursor signing reads it. Internal to `@ultimat3/core`:
|
|
57
|
+
* `keyed-fingerprint.ts` derives its own key from it, so an app has one secret to set, not two.
|
|
58
|
+
*/
|
|
59
|
+
export function currentSigningSecret(): string {
|
|
60
|
+
return currentSecret();
|
|
61
|
+
}
|
|
62
|
+
|
|
55
63
|
/** Set once at boot from the app secret. Rotating it invalidates every open cursor. */
|
|
56
64
|
export function configureCursorSigning(next: string): void {
|
|
57
65
|
configured = next;
|
package/src/index.ts
CHANGED
|
@@ -98,7 +98,6 @@ export type {
|
|
|
98
98
|
AuthConfig,
|
|
99
99
|
CacheConfig,
|
|
100
100
|
DatabaseConfig,
|
|
101
|
-
DrainConfig,
|
|
102
101
|
JobsConfig,
|
|
103
102
|
McpConfig,
|
|
104
103
|
NotifyConfig,
|
|
@@ -109,6 +108,8 @@ export type {
|
|
|
109
108
|
ThemeMode,
|
|
110
109
|
} from './config';
|
|
111
110
|
export { defineConfig, INBOX_RETENTION_KEYS } from './config';
|
|
111
|
+
export type { DrainConfig, HealthConfig, ReadinessMode } from './config-health';
|
|
112
|
+
export { READINESS_MODES } from './config-health';
|
|
112
113
|
export type {
|
|
113
114
|
PwaColors,
|
|
114
115
|
PwaConfig,
|
|
@@ -508,6 +509,12 @@ export {
|
|
|
508
509
|
} from './intl-cache';
|
|
509
510
|
export { isIsoDateTime } from './iso-date';
|
|
510
511
|
export { isJsonObject } from './json-object';
|
|
512
|
+
export type { FingerprintMatch } from './keyed-fingerprint';
|
|
513
|
+
export {
|
|
514
|
+
compareFingerprint,
|
|
515
|
+
KEYED_FINGERPRINT_VERSION,
|
|
516
|
+
keyedFingerprint,
|
|
517
|
+
} from './keyed-fingerprint';
|
|
511
518
|
export type {
|
|
512
519
|
HealthPayload,
|
|
513
520
|
HealthReport,
|
|
@@ -515,8 +522,6 @@ export type {
|
|
|
515
522
|
LifecycleOptions,
|
|
516
523
|
OnShutdownOptions,
|
|
517
524
|
ProcessSignal,
|
|
518
|
-
ReadinessCheck,
|
|
519
|
-
ReadinessStatus,
|
|
520
525
|
ShutdownHook,
|
|
521
526
|
ShutdownPhase,
|
|
522
527
|
ShutdownReason,
|
|
@@ -548,6 +553,7 @@ export {
|
|
|
548
553
|
READINESS_GRACE_DEFAULT_MS,
|
|
549
554
|
READINESS_GRACE_MAX_MS,
|
|
550
555
|
} from './lifecycle-grace';
|
|
556
|
+
export type { ReadinessCheck, ReadinessStatus } from './lifecycle-readiness';
|
|
551
557
|
export type { SignalHandlerOptions } from './lifecycle-signals';
|
|
552
558
|
export { installSignalHandlers } from './lifecycle-signals';
|
|
553
559
|
export { isSelfOrigin, listeningOrigins, markListening, resetListeners } from './listeners';
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
// Single responsibility: a fingerprint of caller input that is safe to PERSIST. `fingerprint()` is
|
|
2
|
+
// an unkeyed SHA-256 prefix — fine as an in-memory sharing key, but stored beside a row it is an
|
|
3
|
+
// offline oracle: a 6–20-digit account number or an ID number in an input, with the other fields
|
|
4
|
+
// known, is brute-forced from a database read. Keyed with HMAC-SHA-256, the stored value proves
|
|
5
|
+
// nothing to anyone without the app's signing secret.
|
|
6
|
+
|
|
7
|
+
import { canonicalJson, fingerprint } from './canonical-json';
|
|
8
|
+
import { currentSigningSecret } from './cursor';
|
|
9
|
+
import { timingSafeEqual } from './timing-safe-equal';
|
|
10
|
+
|
|
11
|
+
/** The format tag. A stored value without it predates keying (see `compareFingerprint`). */
|
|
12
|
+
export const KEYED_FINGERPRINT_VERSION = 'h1';
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* What a stored fingerprint says about an input:
|
|
16
|
+
* - `match` / `mismatch` — computed under the key in force now (or the legacy unkeyed form);
|
|
17
|
+
* - `unverifiable` — keyed under a secret this process does not hold (a rotation), or a format this
|
|
18
|
+
* build does not know. The input can be neither confirmed nor refused.
|
|
19
|
+
*/
|
|
20
|
+
export type FingerprintMatch = 'match' | 'mismatch' | 'unverifiable';
|
|
21
|
+
|
|
22
|
+
const LEGACY = /^[0-9a-f]{16}$/;
|
|
23
|
+
|
|
24
|
+
/** One key per purpose, derived from the signing secret, so no two uses share a MAC key. */
|
|
25
|
+
function derivedKey(purpose: string): string {
|
|
26
|
+
return new Bun.CryptoHasher('sha256', currentSigningSecret())
|
|
27
|
+
.update(`ultimate:keyed-fingerprint:${purpose}`)
|
|
28
|
+
.digest('hex');
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Names the key a fingerprint was made under, so a rotation reads as `unverifiable` rather than as
|
|
33
|
+
* a mismatch. 32 bits of a hash of a 256-bit derived key: a label, useless for recovering it.
|
|
34
|
+
*/
|
|
35
|
+
function keyId(key: string): string {
|
|
36
|
+
return new Bun.CryptoHasher('sha256').update(key).digest('hex').slice(0, 8);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* `h1:<key id>:<HMAC-SHA-256 over canonical JSON, 128 bits>`. `purpose` separates uses: the same
|
|
41
|
+
* input fingerprints differently for two purposes. Under the shipped development secret this is as
|
|
42
|
+
* public as `fingerprint()` — production boot refuses that secret (`X_CURSOR_SECRET_DEV`).
|
|
43
|
+
*/
|
|
44
|
+
export function keyedFingerprint(value: unknown, purpose: string): string {
|
|
45
|
+
const key = derivedKey(purpose);
|
|
46
|
+
const mac = new Bun.CryptoHasher('sha256', key)
|
|
47
|
+
.update(canonicalJson(value))
|
|
48
|
+
.digest('hex')
|
|
49
|
+
.slice(0, 32);
|
|
50
|
+
return `${KEYED_FINGERPRINT_VERSION}:${keyId(key)}:${mac}`;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Compare a STORED fingerprint with an input. A legacy unkeyed value (16 hex characters, written by
|
|
55
|
+
* a build before keying) is checked against `fingerprint()` computed here and never stored, so rows
|
|
56
|
+
* written across an upgrade keep their exact semantics until they age out.
|
|
57
|
+
*/
|
|
58
|
+
export function compareFingerprint(
|
|
59
|
+
stored: string,
|
|
60
|
+
value: unknown,
|
|
61
|
+
purpose: string,
|
|
62
|
+
): FingerprintMatch {
|
|
63
|
+
if (LEGACY.test(stored))
|
|
64
|
+
return timingSafeEqual(stored, fingerprint(value)) ? 'match' : 'mismatch';
|
|
65
|
+
const current = keyedFingerprint(value, purpose);
|
|
66
|
+
const prefix = current.slice(0, current.lastIndexOf(':') + 1);
|
|
67
|
+
if (!stored.startsWith(prefix)) return 'unverifiable';
|
|
68
|
+
return timingSafeEqual(stored, current) ? 'match' : 'mismatch';
|
|
69
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
// Single responsibility: the readiness CHECK — its signature and its two answers. The mode that
|
|
2
|
+
// decides what a failing one does to `/readyz` is `config-health.ts`'s.
|
|
3
|
+
|
|
4
|
+
export type ReadinessStatus = 'ok' | 'failing';
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Synchronous, and that is the design, not a limitation. **Do not widen this to
|
|
8
|
+
* `() => Promise<boolean>`** — the signature is the mechanism.
|
|
9
|
+
*
|
|
10
|
+
* A readiness endpoint that does I/O is a liveness bomb. A probe that awaits a network call takes
|
|
11
|
+
* as long as the dependency does, so a slow database makes the endpoint miss its `timeoutSeconds`,
|
|
12
|
+
* the kubelet reads that as unready, and capacity is pulled from an already-struggling system —
|
|
13
|
+
* the outage the probe existed to prevent, caused by the probe. Worse under a liveness probe
|
|
14
|
+
* sharing the handler: the pod is killed and restarts into the same slow database, cold.
|
|
15
|
+
*
|
|
16
|
+
* So the owner of the dependency keeps a boolean fresh — a pool exposes `isOpen`, a background
|
|
17
|
+
* poller flips a flag on its own schedule with its own timeout — and this reads it. That puts the
|
|
18
|
+
* waiting where a timeout can be tuned, and leaves this path unable to block. A check that throws
|
|
19
|
+
* is `failing`.
|
|
20
|
+
*/
|
|
21
|
+
export type ReadinessCheck = () => boolean;
|
package/src/lifecycle.ts
CHANGED
|
@@ -3,11 +3,14 @@
|
|
|
3
3
|
// and reports the same /healthz + /readyz state.
|
|
4
4
|
|
|
5
5
|
import { type Clock, systemClock } from './clock';
|
|
6
|
+
import type { ReadinessMode } from './config-health';
|
|
7
|
+
import { assertReadinessMode } from './config-health';
|
|
6
8
|
import { UltimateError } from './errors';
|
|
7
9
|
import { finiteCount } from './finite-option';
|
|
8
10
|
import { settleWithin } from './lifecycle-deadline';
|
|
9
11
|
import { lifecycleDrained } from './lifecycle-errors';
|
|
10
12
|
import { defaultReadinessGraceMs, readinessGraceIssue } from './lifecycle-grace';
|
|
13
|
+
import type { ReadinessCheck, ReadinessStatus } from './lifecycle-readiness';
|
|
11
14
|
import { type LogFields, type Logger, logger as rootLogger } from './logger';
|
|
12
15
|
|
|
13
16
|
export type HealthState = 'starting' | 'ready' | 'draining' | 'stopped';
|
|
@@ -64,29 +67,12 @@ export interface LifecycleOptions {
|
|
|
64
67
|
* A whole number from 0 to 60000; 0 is no grace.
|
|
65
68
|
*/
|
|
66
69
|
readonly readinessGraceMs?: number | undefined;
|
|
70
|
+
/** What a failing check does to `/readyz` — see `ReadinessMode`. Default `'dependencies'`. */
|
|
71
|
+
readonly readiness?: ReadinessMode | undefined;
|
|
67
72
|
readonly clock?: Clock | undefined;
|
|
68
73
|
readonly logger?: Logger | undefined;
|
|
69
74
|
}
|
|
70
75
|
|
|
71
|
-
export type ReadinessStatus = 'ok' | 'failing';
|
|
72
|
-
|
|
73
|
-
/**
|
|
74
|
-
* Synchronous, and that is the design, not a limitation. **Do not widen this to
|
|
75
|
-
* `() => Promise<boolean>`** — the signature is the mechanism.
|
|
76
|
-
*
|
|
77
|
-
* A readiness endpoint that does I/O is a liveness bomb. A probe that awaits a network call takes
|
|
78
|
-
* as long as the dependency does, so a slow database makes the endpoint miss its `timeoutSeconds`,
|
|
79
|
-
* the kubelet reads that as unready, and capacity is pulled from an already-struggling system —
|
|
80
|
-
* the outage the probe existed to prevent, caused by the probe. Worse under a liveness probe
|
|
81
|
-
* sharing the handler: the pod is killed and restarts into the same slow database, cold.
|
|
82
|
-
*
|
|
83
|
-
* So the owner of the dependency keeps a boolean fresh — a pool exposes `isOpen`, a background
|
|
84
|
-
* poller flips a flag on its own schedule with its own timeout — and this reads it. That puts the
|
|
85
|
-
* waiting where a timeout can be tuned, and leaves this path unable to block. A check that throws
|
|
86
|
-
* is `failing`.
|
|
87
|
-
*/
|
|
88
|
-
export type ReadinessCheck = () => boolean;
|
|
89
|
-
|
|
90
76
|
export interface HealthReport {
|
|
91
77
|
readonly state: HealthState;
|
|
92
78
|
readonly ready: boolean;
|
|
@@ -123,6 +109,7 @@ const DEFAULT_DEADLINE_MS = 25_000;
|
|
|
123
109
|
let deadlineMs = DEFAULT_DEADLINE_MS;
|
|
124
110
|
/** `undefined` means "the environment's default", read when a drain starts, not at import. */
|
|
125
111
|
let graceMs: number | undefined;
|
|
112
|
+
let readinessMode: ReadinessMode = 'dependencies';
|
|
126
113
|
let clock: Clock = systemClock;
|
|
127
114
|
let log: Logger = rootLogger;
|
|
128
115
|
let state: HealthState = 'starting';
|
|
@@ -155,6 +142,7 @@ export function configureLifecycle(options: LifecycleOptions): void {
|
|
|
155
142
|
}
|
|
156
143
|
graceMs = options.readinessGraceMs;
|
|
157
144
|
}
|
|
145
|
+
if (options.readiness !== undefined) readinessMode = assertReadinessMode(options.readiness);
|
|
158
146
|
if (options.clock !== undefined) {
|
|
159
147
|
clock = options.clock;
|
|
160
148
|
startedAtMono = clock.monotonic();
|
|
@@ -451,13 +439,15 @@ export function drain(signal = 'manual'): Promise<void> {
|
|
|
451
439
|
return drainPromise;
|
|
452
440
|
}
|
|
453
441
|
|
|
454
|
-
export function healthReport(): HealthReport {
|
|
442
|
+
export function healthReport(mode: ReadinessMode = readinessMode): HealthReport {
|
|
455
443
|
const checks = readinessChecks();
|
|
444
|
+
const dependencies = Object.values(checks).every((status) => status === 'ok');
|
|
456
445
|
return {
|
|
457
446
|
state,
|
|
458
447
|
// `ready` is the same predicate `/readyz` answers on, so a body and its status can never
|
|
459
448
|
// disagree — a 200 whose body says `ready: false` is the bug this shares one source to avoid.
|
|
460
|
-
|
|
449
|
+
// `'process'` mode leaves the checks OUT of it, and still reports every one of them below.
|
|
450
|
+
ready: state === 'ready' && (mode === 'process' || dependencies),
|
|
461
451
|
uptimeMs: Math.round(clock.monotonic() - startedAtMono),
|
|
462
452
|
inflight,
|
|
463
453
|
buildId: process.env['BUILD_ID'] ?? 'dev',
|
|
@@ -477,9 +467,12 @@ export function healthzPayload(): HealthPayload {
|
|
|
477
467
|
return { ok, status: ok ? 200 : 503, body };
|
|
478
468
|
}
|
|
479
469
|
|
|
480
|
-
/**
|
|
481
|
-
|
|
482
|
-
|
|
470
|
+
/**
|
|
471
|
+
* Readiness: may this instance receive traffic? 503 while starting or draining, and — in the
|
|
472
|
+
* default `'dependencies'` mode, or always with `deep` (`/readyz?deep=1`) — while any check fails.
|
|
473
|
+
*/
|
|
474
|
+
export function readyzPayload(options: { readonly deep?: boolean } = {}): HealthPayload {
|
|
475
|
+
const body = healthReport(options.deep === true ? 'dependencies' : readinessMode);
|
|
483
476
|
return { ok: body.ready, status: body.ready ? 200 : 503, body };
|
|
484
477
|
}
|
|
485
478
|
|
|
@@ -487,6 +480,7 @@ export function readyzPayload(): HealthPayload {
|
|
|
487
480
|
export function resetLifecycle(): void {
|
|
488
481
|
deadlineMs = DEFAULT_DEADLINE_MS;
|
|
489
482
|
graceMs = undefined;
|
|
483
|
+
readinessMode = 'dependencies';
|
|
490
484
|
clock = systemClock;
|
|
491
485
|
log = rootLogger;
|
|
492
486
|
state = 'starting';
|