@ultimat3/core 1.2.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +272 -0
- package/README.md +248 -10
- package/package.json +2 -1
- package/src/actor.ts +118 -4
- package/src/app-version.ts +32 -0
- package/src/assert.ts +5 -1
- package/src/config.ts +47 -12
- package/src/context.ts +30 -3
- package/src/cursor.ts +25 -4
- package/src/env-example.ts +2 -1
- package/src/env.ts +14 -3
- package/src/environment.ts +39 -13
- package/src/error-codes.ts +15 -1
- package/src/error-render.ts +249 -0
- package/src/error-reporter-sentry.ts +175 -0
- package/src/error-reporter.ts +212 -0
- package/src/error-retry.ts +112 -0
- package/src/errors.ts +55 -7
- package/src/exports/error-contract.ts +62 -0
- package/src/exports/observability.ts +161 -0
- package/src/exports/secrets.ts +71 -0
- package/src/ids.ts +49 -7
- package/src/impersonate.ts +62 -0
- package/src/index.ts +279 -113
- package/src/intl-cache.ts +43 -0
- package/src/lifecycle-deadline.ts +73 -0
- package/src/lifecycle-errors.ts +33 -0
- package/src/lifecycle.ts +237 -34
- package/src/logger.ts +99 -9
- package/src/mcp-exposure.ts +32 -0
- package/src/metrics.ts +0 -0
- package/src/otlp-metric-exporter.ts +136 -0
- package/src/otlp-span-exporter.ts +170 -0
- package/src/otlp.ts +217 -0
- package/src/read-capped.ts +47 -0
- package/src/runtime-metrics.ts +15 -0
- package/src/safe-url.ts +50 -0
- package/src/sampler.ts +126 -0
- package/src/schema-error-codes.ts +28 -0
- package/src/secrets-errors.ts +143 -0
- package/src/secrets-store.ts +173 -0
- package/src/secrets.ts +292 -0
- package/src/telemetry.ts +43 -11
- package/src/timing-safe-equal.ts +18 -0
- package/src/type-pins.ts +93 -0
- package/src/version.ts +53 -4
package/src/lifecycle.ts
CHANGED
|
@@ -3,7 +3,10 @@
|
|
|
3
3
|
// and reports the same /healthz + /readyz state.
|
|
4
4
|
|
|
5
5
|
import { type Clock, systemClock } from './clock';
|
|
6
|
-
import {
|
|
6
|
+
import { UltimateError } from './errors';
|
|
7
|
+
import { settleWithin } from './lifecycle-deadline';
|
|
8
|
+
import { lifecycleDrained } from './lifecycle-errors';
|
|
9
|
+
import { type LogFields, type Logger, logger as rootLogger } from './logger';
|
|
7
10
|
|
|
8
11
|
export type HealthState = 'starting' | 'ready' | 'draining' | 'stopped';
|
|
9
12
|
|
|
@@ -17,7 +20,13 @@ export type ProcessSignal = 'SIGTERM' | 'SIGINT' | 'SIGHUP' | 'SIGQUIT';
|
|
|
17
20
|
|
|
18
21
|
export interface ShutdownReason {
|
|
19
22
|
readonly signal: string;
|
|
20
|
-
/**
|
|
23
|
+
/**
|
|
24
|
+
* Real monotonic ms (`systemClock`) after which hooks are abandoned — deliberately NOT the
|
|
25
|
+
* injected clock. The budget this bounds is `terminationGracePeriodSeconds`, counted by the
|
|
26
|
+
* kubelet in real seconds, so a frozen clock must be unable to extend it: read off `clock` a
|
|
27
|
+
* test that advanced an hour of fake time handed the drain a 16-minute grace period, while
|
|
28
|
+
* `waitForIdle` went on sleeping on a real `setTimeout`. `clock` still owns `uptimeMs`.
|
|
29
|
+
*/
|
|
21
30
|
readonly deadlineAt: number;
|
|
22
31
|
}
|
|
23
32
|
|
|
@@ -28,17 +37,51 @@ export interface OnShutdownOptions {
|
|
|
28
37
|
}
|
|
29
38
|
|
|
30
39
|
export interface LifecycleOptions {
|
|
40
|
+
/**
|
|
41
|
+
* The whole drain's budget — the in-flight wait AND every hook, in every phase. 25s by default,
|
|
42
|
+
* and **enforced whether or not an app sets it**: `ShutdownReason.deadlineAt` was always computed
|
|
43
|
+
* and handed to every hook, so the deadline was declared by the design and only the enforcement
|
|
44
|
+
* was missing. No hook reads `deadlineAt`, which is why it has to be imposed here.
|
|
45
|
+
*
|
|
46
|
+
* The lever is a LARGER value, not the absence of one: a `worker` holding a 10-minute job wants
|
|
47
|
+
* `configureLifecycle({ deadlineMs: 600_000 })` and a `terminationGracePeriodSeconds` at least as
|
|
48
|
+
* large. Left at 25s it is abandoned and the process exits clean — the row's visibility lease
|
|
49
|
+
* lapses and another worker re-claims it, which is what at-least-once already promises. The
|
|
50
|
+
* alternative is not "the job finishes": it is the same duplicate, delivered by SIGKILL at the
|
|
51
|
+
* kubelet's grace period, with no log line naming what overran.
|
|
52
|
+
*/
|
|
31
53
|
readonly deadlineMs?: number | undefined;
|
|
32
54
|
readonly clock?: Clock | undefined;
|
|
33
55
|
readonly logger?: Logger | undefined;
|
|
34
56
|
}
|
|
35
57
|
|
|
58
|
+
export type ReadinessStatus = 'ok' | 'failing';
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Synchronous, and that is the design, not a limitation. **Do not widen this to
|
|
62
|
+
* `() => Promise<boolean>`** — the signature is the mechanism.
|
|
63
|
+
*
|
|
64
|
+
* A readiness endpoint that does I/O is a liveness bomb. A probe that awaits a network call takes
|
|
65
|
+
* as long as the dependency does, so a slow database makes the endpoint miss its `timeoutSeconds`,
|
|
66
|
+
* the kubelet reads that as unready, and capacity is pulled from an already-struggling system —
|
|
67
|
+
* the outage the probe existed to prevent, caused by the probe. Worse under a liveness probe
|
|
68
|
+
* sharing the handler: the pod is killed and restarts into the same slow database, cold.
|
|
69
|
+
*
|
|
70
|
+
* So the owner of the dependency keeps a boolean fresh — a pool exposes `isOpen`, a background
|
|
71
|
+
* poller flips a flag on its own schedule with its own timeout — and this reads it. That puts the
|
|
72
|
+
* waiting where a timeout can be tuned, and leaves this path unable to block. A check that throws
|
|
73
|
+
* is `failing`.
|
|
74
|
+
*/
|
|
75
|
+
export type ReadinessCheck = () => boolean;
|
|
76
|
+
|
|
36
77
|
export interface HealthReport {
|
|
37
78
|
readonly state: HealthState;
|
|
38
79
|
readonly ready: boolean;
|
|
39
80
|
readonly uptimeMs: number;
|
|
40
81
|
readonly inflight: number;
|
|
41
82
|
readonly buildId: string;
|
|
83
|
+
/** Named, because "alert on check failures BY CHECK NAME" is not writable against a boolean. */
|
|
84
|
+
readonly checks: Readonly<Record<string, ReadinessStatus>>;
|
|
42
85
|
}
|
|
43
86
|
|
|
44
87
|
export interface HealthPayload {
|
|
@@ -65,6 +108,7 @@ let inflight = 0;
|
|
|
65
108
|
let registrations: Registration[] = [];
|
|
66
109
|
let drainPromise: Promise<void> | undefined;
|
|
67
110
|
let idleWaiters: (() => void)[] = [];
|
|
111
|
+
const readiness = new Map<string, ReadinessCheck>();
|
|
68
112
|
|
|
69
113
|
export function configureLifecycle(options: LifecycleOptions): void {
|
|
70
114
|
if (options.deadlineMs !== undefined) deadlineMs = options.deadlineMs;
|
|
@@ -79,14 +123,106 @@ export function lifecycleState(): HealthState {
|
|
|
79
123
|
return state;
|
|
80
124
|
}
|
|
81
125
|
|
|
126
|
+
/**
|
|
127
|
+
* "This process bound its socket." NOT "this process can serve a request" — that is what the
|
|
128
|
+
* readiness checks answer. Before them, `markReady()` was the whole of `/readyz`, so a pod went
|
|
129
|
+
* green the instant it bound and the load balancer sent traffic into a Postgres pool that had not
|
|
130
|
+
* opened a connection yet; `maxUnavailable: 0` does not help when readiness lies.
|
|
131
|
+
*
|
|
132
|
+
* **A drained lifecycle refuses this, and that is the whole of "one process, one lifecycle."**
|
|
133
|
+
* `state` never leaves `stopped` and `drain()` memoizes, so a role that marked ready after a drain
|
|
134
|
+
* used to be told nothing and go on to bind a socket answering 503 to everything, with no drain
|
|
135
|
+
* left to close it. `X_LIFECYCLE_DRAINED` is that mistake named at the moment it is made, rather
|
|
136
|
+
* than a lifecycle that can be restarted: two live lifecycles racing one shutdown is a worse
|
|
137
|
+
* mechanism than a boot that fails.
|
|
138
|
+
*/
|
|
82
139
|
export function markReady(): void {
|
|
140
|
+
if (isDraining()) throw lifecycleDrained(state === 'draining' ? 'draining' : 'stopped');
|
|
83
141
|
if (state === 'starting') state = 'ready';
|
|
84
142
|
}
|
|
85
143
|
|
|
144
|
+
/**
|
|
145
|
+
* Register a named readiness check. Returns its unregister — the same shape as `onShutdown`, and
|
|
146
|
+
* owned by whoever can be started twice, for the same reason.
|
|
147
|
+
*/
|
|
148
|
+
export function registerReadinessCheck(name: string, check: ReadinessCheck): () => void {
|
|
149
|
+
if (readiness.has(name)) {
|
|
150
|
+
throw new UltimateError({
|
|
151
|
+
code: 'X_READINESS_CHECK_DUPLICATE',
|
|
152
|
+
cause: `a readiness check named "${name}" is already registered (have: ${[...readiness.keys()].join(', ')})`,
|
|
153
|
+
fix: `name the second check for what it actually probes, e.g. registerReadinessCheck('${name}-replica', check) — or hold the unregister the first registration returned and call it first`,
|
|
154
|
+
meta: { name },
|
|
155
|
+
});
|
|
156
|
+
}
|
|
157
|
+
readiness.set(name, check);
|
|
158
|
+
return () => {
|
|
159
|
+
if (readiness.get(name) === check) readiness.delete(name);
|
|
160
|
+
};
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/** Test-only: registered checks. A count that climbs across a start/stop cycle is a leak. */
|
|
164
|
+
export function readinessCheckCount(): number {
|
|
165
|
+
return readiness.size;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
/**
|
|
169
|
+
* Every line this file emits, and the only way it emits one. `log` is an injection seam
|
|
170
|
+
* (`configureLifecycle({ logger })`), so an app's `Logger` decides whether a log call can throw —
|
|
171
|
+
* and a throw here does not lose a line, it replaces the event. Inside `drain()` it rejected
|
|
172
|
+
* `drainPromise`: `state` never reached 'stopped', the memo re-rejected for every later caller,
|
|
173
|
+
* and on Bun the unhandled rejection ended the process the drain was trying to end cleanly.
|
|
174
|
+
* Inside `readinessChecks()` it replaced the probe's answer with a throw.
|
|
175
|
+
*
|
|
176
|
+
* A lifecycle that cannot report is still a lifecycle: the line falls back to core's own
|
|
177
|
+
* `rootLogger`, which is total by construction (`logger.ts`), and failing that is dropped.
|
|
178
|
+
*/
|
|
179
|
+
function report(level: 'info' | 'warn' | 'error', message: string, fields: LogFields): void {
|
|
180
|
+
try {
|
|
181
|
+
log[level](message, fields);
|
|
182
|
+
return;
|
|
183
|
+
} catch {
|
|
184
|
+
// Fall through — the injected sink is gone, and the fallback below is the last one there is.
|
|
185
|
+
}
|
|
186
|
+
if (log === rootLogger) return;
|
|
187
|
+
try {
|
|
188
|
+
rootLogger[level](message, fields);
|
|
189
|
+
} catch {
|
|
190
|
+
// Both sinks are gone. Dropping the line is the only remaining option that still ends the
|
|
191
|
+
// process, which is the outcome every caller of this file depends on.
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
/** Every check, run now, by name. A check that throws is `failing` — never an unhandled error. */
|
|
196
|
+
export function readinessChecks(): Readonly<Record<string, ReadinessStatus>> {
|
|
197
|
+
const results: Record<string, ReadinessStatus> = {};
|
|
198
|
+
for (const [name, check] of readiness) {
|
|
199
|
+
try {
|
|
200
|
+
results[name] = check() ? 'ok' : 'failing';
|
|
201
|
+
} catch (thrown) {
|
|
202
|
+
results[name] = 'failing';
|
|
203
|
+
report('warn', 'readiness check threw', { check: name, error: thrown });
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
return results;
|
|
207
|
+
}
|
|
208
|
+
|
|
86
209
|
export function inflightCount(): number {
|
|
87
210
|
return inflight;
|
|
88
211
|
}
|
|
89
212
|
|
|
213
|
+
/** Test-only: drains still waiting on in-flight work. A count stuck above zero is a leak. */
|
|
214
|
+
export function idleWaiterCount(): number {
|
|
215
|
+
return idleWaiters.length;
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
/**
|
|
219
|
+
* Test-only: hooks still registered. A count that climbs across a start/stop cycle is a leak —
|
|
220
|
+
* the registration retains its closure, and the next drain runs it against a torn-down resource.
|
|
221
|
+
*/
|
|
222
|
+
export function shutdownHookCount(): number {
|
|
223
|
+
return registrations.length;
|
|
224
|
+
}
|
|
225
|
+
|
|
90
226
|
/** Register a drain hook. Returns an unregister function. */
|
|
91
227
|
export function onShutdown(
|
|
92
228
|
name: string,
|
|
@@ -127,23 +263,66 @@ export function isDraining(): boolean {
|
|
|
127
263
|
function waitForIdle(timeoutMs: number): Promise<boolean> {
|
|
128
264
|
if (inflight === 0) return Promise.resolve(true);
|
|
129
265
|
return new Promise<boolean>((resolve) => {
|
|
130
|
-
const
|
|
131
|
-
idleWaiters.push(() => {
|
|
266
|
+
const waiter = (): void => {
|
|
132
267
|
clearTimeout(timer);
|
|
133
268
|
resolve(true);
|
|
134
|
-
}
|
|
269
|
+
};
|
|
270
|
+
// A drain that times out must not leave its waiter in the queue forever — the next
|
|
271
|
+
// `beginWork()` to reach zero would still hold and invoke it, a dangling closure over a
|
|
272
|
+
// promise nothing is awaiting anymore.
|
|
273
|
+
const timer = setTimeout(() => {
|
|
274
|
+
idleWaiters = idleWaiters.filter((candidate) => candidate !== waiter);
|
|
275
|
+
resolve(false);
|
|
276
|
+
}, timeoutMs);
|
|
277
|
+
idleWaiters.push(waiter);
|
|
135
278
|
});
|
|
136
279
|
}
|
|
137
280
|
|
|
281
|
+
/**
|
|
282
|
+
* The budget every drain is bounded by — `DEFAULT_DEADLINE_MS` until an app raises it. There is no
|
|
283
|
+
* unbounded state: `ShutdownReason.deadlineAt` was always computed and handed to every hook, so the
|
|
284
|
+
* deadline was declared by the design all along and only the enforcement was missing.
|
|
285
|
+
*
|
|
286
|
+
* The ONE place the budget is decided, and exported so a test can pin it: 25s is far above any
|
|
287
|
+
* drain a test can wait out, so the default needs a probe and not only a stopwatch.
|
|
288
|
+
*/
|
|
289
|
+
export function drainDeadlineMs(): number {
|
|
290
|
+
return deadlineMs;
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
/**
|
|
294
|
+
* What is left of that budget. Read per hook, not per phase: the deadline bounds the WHOLE drain,
|
|
295
|
+
* so a hook that spent it leaves nothing for the ones behind it — which is what
|
|
296
|
+
* `terminationGracePeriodSeconds` means, and what makes the SUM of the phases bounded rather than
|
|
297
|
+
* each one of them separately. Returns `number`, never `number | undefined`: "no budget" is not a
|
|
298
|
+
* state this file has, and the type is what keeps it from becoming one again.
|
|
299
|
+
*/
|
|
300
|
+
function remainingBudget(reason: ShutdownReason): number {
|
|
301
|
+
return Math.max(0, reason.deadlineAt - systemClock.monotonic());
|
|
302
|
+
}
|
|
303
|
+
|
|
138
304
|
async function runPhase(phase: ShutdownPhase, reason: ShutdownReason): Promise<void> {
|
|
139
305
|
for (const registration of registrations.filter((entry) => entry.phase === phase)) {
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
306
|
+
const outcome = await settleWithin(() => registration.hook(reason), remainingBudget(reason));
|
|
307
|
+
if (outcome.kind === 'failed') {
|
|
308
|
+
report('error', 'shutdown hook failed', {
|
|
309
|
+
hook: registration.name,
|
|
310
|
+
phase,
|
|
311
|
+
error: outcome.error,
|
|
312
|
+
});
|
|
313
|
+
continue;
|
|
314
|
+
}
|
|
315
|
+
if (outcome.kind === 'abandoned') {
|
|
316
|
+
// Abandoned, not merely logged. A deadline that waited anyway would leave the kubelet to
|
|
317
|
+
// SIGKILL this process — the every-deploy duplicate that draining exists to prevent — so
|
|
318
|
+
// the drain moves on and the hook is left running with nobody reading it. The cost of that
|
|
319
|
+
// choice is real and named in the cause: a write it had in flight may be half done.
|
|
320
|
+
report('warn', 'X_SHUTDOWN_TIMEOUT', {
|
|
321
|
+
code: 'X_SHUTDOWN_TIMEOUT',
|
|
322
|
+
cause: `the "${registration.name}" shutdown hook (phase: ${phase}) was still running at the ${deadlineMs}ms drain deadline and has been ABANDONED — the process exits without it, so anything it had in flight may be incomplete`,
|
|
323
|
+
fix: `raise the budget past the work this hook does — configureLifecycle({ deadlineMs: 600_000 }) for a 10-minute job — and set terminationGracePeriodSeconds to at least as many seconds, or make the "${registration.name}" hook return once it has stopped accepting work rather than once it has finished`,
|
|
144
324
|
hook: registration.name,
|
|
145
325
|
phase,
|
|
146
|
-
error: thrown,
|
|
147
326
|
});
|
|
148
327
|
}
|
|
149
328
|
}
|
|
@@ -153,26 +332,37 @@ async function runPhase(phase: ShutdownPhase, reason: ShutdownReason): Promise<v
|
|
|
153
332
|
export function drain(signal = 'manual'): Promise<void> {
|
|
154
333
|
if (drainPromise !== undefined) return drainPromise;
|
|
155
334
|
state = 'draining';
|
|
156
|
-
const reason: ShutdownReason = { signal, deadlineAt:
|
|
335
|
+
const reason: ShutdownReason = { signal, deadlineAt: systemClock.monotonic() + deadlineMs };
|
|
157
336
|
|
|
158
337
|
drainPromise = (async () => {
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
338
|
+
try {
|
|
339
|
+
report('info', 'draining', { signal, deadlineMs, inflight });
|
|
340
|
+
await runPhase('accept', reason);
|
|
341
|
+
|
|
342
|
+
// Real monotonic, like `deadlineAt` itself: `waitForIdle` sleeps on a real `setTimeout`, and
|
|
343
|
+
// a budget read off an injected clock is a number that timer will never honour.
|
|
344
|
+
const remaining = Math.max(0, reason.deadlineAt - systemClock.monotonic());
|
|
345
|
+
const idle = await waitForIdle(remaining);
|
|
346
|
+
if (!idle) {
|
|
347
|
+
report('warn', 'X_SHUTDOWN_TIMEOUT', {
|
|
348
|
+
code: 'X_SHUTDOWN_TIMEOUT',
|
|
349
|
+
cause: `${inflight} in-flight operations still running after ${deadlineMs}ms`,
|
|
350
|
+
fix: 'raise the budget past the slowest handler — configureLifecycle({ deadlineMs: 600_000 }) for a 10-minute one — and set terminationGracePeriodSeconds to at least as many seconds, or shorten the handler',
|
|
351
|
+
});
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
await runPhase('inflight', reason);
|
|
355
|
+
await runPhase('close', reason);
|
|
356
|
+
} catch (thrown) {
|
|
357
|
+
// Nothing above should reach here — every hook is caught by `settleWithin` and every line
|
|
358
|
+
// goes through `report`. If something does, the drain still ENDS: a rejected `drainPromise`
|
|
359
|
+
// is a memo that re-rejects for every later caller and an unhandled rejection that kills the
|
|
360
|
+
// process mid-drain, which is strictly worse than a drain that finished badly and said so.
|
|
361
|
+
report('error', 'drain failed', { signal, error: thrown });
|
|
362
|
+
} finally {
|
|
363
|
+
state = 'stopped';
|
|
170
364
|
}
|
|
171
|
-
|
|
172
|
-
await runPhase('inflight', reason);
|
|
173
|
-
await runPhase('close', reason);
|
|
174
|
-
state = 'stopped';
|
|
175
|
-
log.info('stopped', { signal });
|
|
365
|
+
report('info', 'stopped', { signal });
|
|
176
366
|
})();
|
|
177
367
|
|
|
178
368
|
return drainPromise;
|
|
@@ -191,9 +381,14 @@ export function installSignalHandlers(options?: SignalHandlerOptions): () => voi
|
|
|
191
381
|
|
|
192
382
|
for (const signal of signals) {
|
|
193
383
|
const handler = (): void => {
|
|
194
|
-
|
|
384
|
+
// Attached on BOTH settle paths, for the reason `settleWithin` gives: an unhandled rejection
|
|
385
|
+
// ends the process before the drain does, and the exit is what the kubelet is waiting for.
|
|
386
|
+
// `drain()` cannot reject today — that is the `try/finally` above, not luck — and this is
|
|
387
|
+
// the one line that keeps it true when someone changes the body.
|
|
388
|
+
const done = (): void => {
|
|
195
389
|
if (options?.exit === true) process.exit(0);
|
|
196
|
-
}
|
|
390
|
+
};
|
|
391
|
+
void drain(signal).then(done, done);
|
|
197
392
|
};
|
|
198
393
|
handlers.set(signal, handler);
|
|
199
394
|
process.on(signal, handler);
|
|
@@ -205,27 +400,34 @@ export function installSignalHandlers(options?: SignalHandlerOptions): () => voi
|
|
|
205
400
|
}
|
|
206
401
|
|
|
207
402
|
export function healthReport(): HealthReport {
|
|
403
|
+
const checks = readinessChecks();
|
|
208
404
|
return {
|
|
209
405
|
state,
|
|
210
|
-
ready
|
|
406
|
+
// `ready` is the same predicate `/readyz` answers on, so a body and its status can never
|
|
407
|
+
// disagree — a 200 whose body says `ready: false` is the bug this shares one source to avoid.
|
|
408
|
+
ready: state === 'ready' && Object.values(checks).every((status) => status === 'ok'),
|
|
211
409
|
uptimeMs: Math.round(clock.monotonic() - startedAtMono),
|
|
212
410
|
inflight,
|
|
213
411
|
buildId: process.env['BUILD_ID'] ?? 'dev',
|
|
412
|
+
checks,
|
|
214
413
|
};
|
|
215
414
|
}
|
|
216
415
|
|
|
217
|
-
/**
|
|
416
|
+
/**
|
|
417
|
+
* Liveness: the process exists and is not wedged. Stays 200 while draining, and deliberately
|
|
418
|
+
* ignores the checks — a database outage that failed liveness everywhere would restart the whole
|
|
419
|
+
* fleet into the same outage, with cold caches and no connections.
|
|
420
|
+
*/
|
|
218
421
|
export function healthzPayload(): HealthPayload {
|
|
219
422
|
const body = healthReport();
|
|
220
423
|
const ok = state !== 'stopped';
|
|
221
424
|
return { ok, status: ok ? 200 : 503, body };
|
|
222
425
|
}
|
|
223
426
|
|
|
224
|
-
/** Readiness: may this instance receive traffic? 503 while starting or
|
|
427
|
+
/** Readiness: may this instance receive traffic? 503 while starting, draining or any check fails. */
|
|
225
428
|
export function readyzPayload(): HealthPayload {
|
|
226
429
|
const body = healthReport();
|
|
227
|
-
|
|
228
|
-
return { ok, status: ok ? 200 : 503, body };
|
|
430
|
+
return { ok: body.ready, status: body.ready ? 200 : 503, body };
|
|
229
431
|
}
|
|
230
432
|
|
|
231
433
|
/** Test-only: forget all hooks and return to `starting`. */
|
|
@@ -239,4 +441,5 @@ export function resetLifecycle(): void {
|
|
|
239
441
|
registrations = [];
|
|
240
442
|
drainPromise = undefined;
|
|
241
443
|
idleWaiters = [];
|
|
444
|
+
readiness.clear();
|
|
242
445
|
}
|
package/src/logger.ts
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
// default because the primary reader is an agent tailing `x logs --json`.
|
|
3
3
|
|
|
4
4
|
import { type Clock, systemClock } from './clock';
|
|
5
|
+
import { renderCauseValue } from './error-render';
|
|
5
6
|
import { isUltimateError } from './errors';
|
|
6
7
|
import { isSecret, REDACTED } from './secret';
|
|
7
8
|
|
|
@@ -48,15 +49,34 @@ export interface LoggerOptions {
|
|
|
48
49
|
readonly writer?: ((line: string, level: LogLevel) => void) | undefined;
|
|
49
50
|
}
|
|
50
51
|
|
|
52
|
+
/**
|
|
53
|
+
* LOWERCASE, always: `isRedactedKey` lowercases its lookup, so `apiKey`/`accessToken`/
|
|
54
|
+
* `refreshToken` sat here for three releases matching nothing — and those are the exact field
|
|
55
|
+
* names on `@ultimat3/auth`'s `OAuthTokens`. Matching is exact-key and never substring, so a
|
|
56
|
+
* spelling that is not in this set is not redacted: both the camel and the snake wire spelling of
|
|
57
|
+
* each credential is listed. Add through `redactKeys()` (which lowercases) rather than here.
|
|
58
|
+
*/
|
|
51
59
|
const redactedKeys = new Set<string>([
|
|
52
60
|
'password',
|
|
53
61
|
'token',
|
|
54
62
|
'secret',
|
|
55
63
|
'authorization',
|
|
56
64
|
'cookie',
|
|
57
|
-
'
|
|
58
|
-
'
|
|
59
|
-
'
|
|
65
|
+
'set-cookie',
|
|
66
|
+
'apikey',
|
|
67
|
+
'api_key',
|
|
68
|
+
'accesstoken',
|
|
69
|
+
'access_token',
|
|
70
|
+
'refreshtoken',
|
|
71
|
+
'refresh_token',
|
|
72
|
+
'idtoken',
|
|
73
|
+
'id_token',
|
|
74
|
+
'sessiontoken',
|
|
75
|
+
'session_token',
|
|
76
|
+
'clientsecret',
|
|
77
|
+
'client_secret',
|
|
78
|
+
'privatekey',
|
|
79
|
+
'private_key',
|
|
60
80
|
]);
|
|
61
81
|
|
|
62
82
|
/** Mark keys as secret everywhere. `defineEnv()` calls this for every `secret: true` var. */
|
|
@@ -83,31 +103,101 @@ function defaultWriter(line: string, level: LogLevel): void {
|
|
|
83
103
|
stream.write(`${line}\n`);
|
|
84
104
|
}
|
|
85
105
|
|
|
106
|
+
/**
|
|
107
|
+
* TOTAL, on purpose. `lifecycle.ts` logs the value a shutdown hook threw and the value a
|
|
108
|
+
* readiness check threw — both caught, both arbitrary — so a renderer that throws here escapes
|
|
109
|
+
* `runPhase`'s catch, rejects the drain promise, and `installSignalHandlers` never reaches
|
|
110
|
+
* `process.exit(0)`: SIGTERM hangs, and `/readyz` dies with the check it was reporting on. A log
|
|
111
|
+
* line must never replace the event it describes.
|
|
112
|
+
*
|
|
113
|
+
* Degradation is per KEY, the same shape `renderMetaRecord` uses: one hostile getter must not
|
|
114
|
+
* cost a reader the fields beside it.
|
|
115
|
+
*/
|
|
86
116
|
function serialiseValue(value: unknown, depth: number): unknown {
|
|
117
|
+
try {
|
|
118
|
+
return serialise(value, depth);
|
|
119
|
+
} catch {
|
|
120
|
+
// `instanceof`, `Object.keys` and `toJSON` are all property reads on a value the framework
|
|
121
|
+
// did not build; `renderCauseValue` is the one renderer that cannot itself throw.
|
|
122
|
+
return renderCauseValue(value);
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
function serialise(value: unknown, depth: number): unknown {
|
|
127
|
+
// `JSON.stringify` raises a `TypeError` on a bigint, so the whole line died for one field.
|
|
128
|
+
if (typeof value === 'bigint') return renderCauseValue(value);
|
|
87
129
|
if (value === null || typeof value !== 'object') return value;
|
|
88
130
|
// Before every other branch: a `Secret` is redacted by VALUE, so it stays redacted under a key
|
|
89
131
|
// nobody listed — `{ dsn: secret(url) }` is the leak key-name redaction cannot see.
|
|
90
132
|
if (isSecret(value)) return REDACTED;
|
|
91
|
-
|
|
133
|
+
// `toISOString()` THROWS on an invalid Date, and an invalid Date is exactly the value worth
|
|
134
|
+
// logging when a schedule went wrong.
|
|
135
|
+
if (value instanceof Date) {
|
|
136
|
+
return Number.isNaN(value.getTime()) ? 'an invalid Date' : value.toISOString();
|
|
137
|
+
}
|
|
92
138
|
if (isUltimateError(value)) return value.toJSON();
|
|
93
139
|
if (value instanceof Error) return { name: value.name, message: value.message };
|
|
94
140
|
if (depth >= 6) return '[depth-limit]';
|
|
95
141
|
if (Array.isArray(value)) return value.map((item) => serialiseValue(item, depth + 1));
|
|
142
|
+
const source = value as Record<string, unknown>;
|
|
96
143
|
const out: Record<string, unknown> = {};
|
|
97
|
-
for (const
|
|
98
|
-
out[key] = isRedactedKey(key) ? REDACTED :
|
|
144
|
+
for (const key of Object.keys(source)) {
|
|
145
|
+
out[key] = isRedactedKey(key) ? REDACTED : entryValue(source, key, depth);
|
|
99
146
|
}
|
|
100
147
|
return out;
|
|
101
148
|
}
|
|
102
149
|
|
|
150
|
+
/** One field. The `try` covers the property READ — `serialiseValue` above is already total. */
|
|
151
|
+
function entryValue(source: Record<string, unknown>, key: string, depth: number): unknown {
|
|
152
|
+
try {
|
|
153
|
+
return serialiseValue(source[key], depth + 1);
|
|
154
|
+
} catch {
|
|
155
|
+
return 'a value that cannot be read';
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
|
|
103
159
|
function redactFields(fields: LogFields): Record<string, unknown> {
|
|
104
160
|
const out: Record<string, unknown> = {};
|
|
105
|
-
|
|
106
|
-
|
|
161
|
+
const source = fields as Record<string, unknown>;
|
|
162
|
+
// `Object.keys` before the values, so the read of each value is its own guarded step: a field
|
|
163
|
+
// record is the caller's object, and enumerating it eagerly threw on the first hostile getter.
|
|
164
|
+
for (const key of ownKeys(source)) {
|
|
165
|
+
out[key] = isRedactedKey(key) ? REDACTED : entryValue(source, key, 0);
|
|
107
166
|
}
|
|
108
167
|
return out;
|
|
109
168
|
}
|
|
110
169
|
|
|
170
|
+
function ownKeys(source: Record<string, unknown>): readonly string[] {
|
|
171
|
+
try {
|
|
172
|
+
return Object.keys(source);
|
|
173
|
+
} catch {
|
|
174
|
+
return [];
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/**
|
|
179
|
+
* The last guard. The walk above already degraded every hostile field, so reaching the fallback
|
|
180
|
+
* means the assembled line itself refused to serialise — and the answer to that is still a line,
|
|
181
|
+
* not a throw propagating out of `log.error` into whatever `catch` block called it.
|
|
182
|
+
*/
|
|
183
|
+
function renderLine(
|
|
184
|
+
line: Readonly<Record<string, unknown>>,
|
|
185
|
+
level: LogLevel,
|
|
186
|
+
message: string,
|
|
187
|
+
ts: unknown,
|
|
188
|
+
): string {
|
|
189
|
+
try {
|
|
190
|
+
return JSON.stringify(line);
|
|
191
|
+
} catch {
|
|
192
|
+
return JSON.stringify({
|
|
193
|
+
ts: typeof ts === 'string' ? ts : '',
|
|
194
|
+
level,
|
|
195
|
+
msg: message,
|
|
196
|
+
logFields: 'a log line that cannot be serialised',
|
|
197
|
+
});
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
|
|
111
201
|
function envLevel(): LogLevel {
|
|
112
202
|
const raw = process.env['LOG_LEVEL'];
|
|
113
203
|
return raw !== undefined && (LOG_LEVELS as readonly string[]).includes(raw)
|
|
@@ -132,7 +222,7 @@ export function createLogger(options?: LoggerOptions): Logger {
|
|
|
132
222
|
...redactFields(contextFields() ?? {}),
|
|
133
223
|
...redactFields(fields ?? {}),
|
|
134
224
|
};
|
|
135
|
-
writer(
|
|
225
|
+
writer(renderLine(line, lineLevel, message, line.ts), lineLevel);
|
|
136
226
|
}
|
|
137
227
|
|
|
138
228
|
return {
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
// The one answer to "did this primitive opt into being an MCP tool?" — a literal `expose: true`.
|
|
2
|
+
// Core owns it because its readers span tiers 3-5 — `action`, `query`, `mcp`, `ai`, `manifest` —
|
|
3
|
+
// and this is the only tier all of them reach, the same reason `timing-safe-equal.ts` lives here.
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* The `mcp` block, read structurally. Each package keeps its own richer declaration —
|
|
7
|
+
* `ActionMcp` carries `visibleTo`, `@ultimat3/mcp`'s `McpExposure` carries `name` — and hands it
|
|
8
|
+
* here; restating the one field they share binds this to none of them.
|
|
9
|
+
*/
|
|
10
|
+
export interface McpExposureDeclaration {
|
|
11
|
+
readonly expose?: boolean | undefined;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Opt-in, never opt-out: silence exposes nothing. An absent block, an omitted `expose` and a
|
|
16
|
+
* literal `false` are one answer, because a tool the author never asked for is a capability
|
|
17
|
+
* handed to every agent that can reach the surface — and writing an action is not a request to
|
|
18
|
+
* hand one out.
|
|
19
|
+
*
|
|
20
|
+
* Six readers decided it three ways until 2026-08: `=== true` where a tool is actually built,
|
|
21
|
+
* `!== false` in the OpenAPI hint and `?? true` in the manifest fact. So an action with no `mcp`
|
|
22
|
+
* block was published as a tool by the contract and refused by every surface that could have
|
|
23
|
+
* called one.
|
|
24
|
+
*
|
|
25
|
+
* The one deliberate exception is `@ultimat3/admin`'s OWN catalog, whose every tool is already
|
|
26
|
+
* gated on an admin permission and whose CRUD tools carry no `mcp` block at all; there
|
|
27
|
+
* `expose: false` withdraws a tool. That surface says so in `mcp-tools.ts` and in
|
|
28
|
+
* `wiki/Admin-Dashboard.md`. Nothing else may grow a second default.
|
|
29
|
+
*/
|
|
30
|
+
export function isMcpExposed(declared: McpExposureDeclaration | undefined): boolean {
|
|
31
|
+
return declared?.expose === true;
|
|
32
|
+
}
|
package/src/metrics.ts
CHANGED
|
Binary file
|