@ultimat3/core 1.1.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +252 -0
- package/README.md +217 -10
- package/package.json +2 -1
- package/src/actor.ts +118 -4
- package/src/app-version.ts +32 -0
- package/src/assert.ts +5 -1
- package/src/config.ts +47 -12
- package/src/context.ts +30 -3
- package/src/cursor.ts +25 -4
- package/src/env-example.ts +2 -1
- package/src/env.ts +14 -3
- package/src/environment.ts +39 -13
- package/src/error-codes.ts +13 -0
- package/src/error-render.ts +249 -0
- package/src/error-reporter-sentry.ts +175 -0
- package/src/error-reporter.ts +212 -0
- package/src/error-retry.ts +100 -0
- package/src/errors.ts +55 -7
- package/src/exports/error-contract.ts +61 -0
- package/src/exports/observability.ts +161 -0
- package/src/exports/secrets.ts +71 -0
- package/src/ids.ts +49 -7
- package/src/impersonate.ts +62 -0
- package/src/index.ts +277 -113
- package/src/lifecycle-deadline.ts +73 -0
- package/src/lifecycle-errors.ts +33 -0
- package/src/lifecycle.ts +178 -16
- package/src/logger.ts +99 -9
- package/src/mcp-exposure.ts +32 -0
- package/src/metrics.ts +0 -0
- package/src/otlp-metric-exporter.ts +136 -0
- package/src/otlp-span-exporter.ts +170 -0
- package/src/otlp.ts +217 -0
- package/src/read-capped.ts +47 -0
- package/src/runtime-metrics.ts +15 -0
- package/src/safe-url.ts +50 -0
- package/src/sampler.ts +126 -0
- package/src/schema-error-codes.ts +28 -0
- package/src/secrets-errors.ts +143 -0
- package/src/secrets-store.ts +173 -0
- package/src/secrets.ts +292 -0
- package/src/telemetry.ts +43 -11
- package/src/timing-safe-equal.ts +18 -0
- package/src/type-pins.ts +93 -0
- package/src/version.ts +53 -4
package/src/lifecycle.ts
CHANGED
|
@@ -3,6 +3,9 @@
|
|
|
3
3
|
// and reports the same /healthz + /readyz state.
|
|
4
4
|
|
|
5
5
|
import { type Clock, systemClock } from './clock';
|
|
6
|
+
import { UltimateError } from './errors';
|
|
7
|
+
import { settleWithin } from './lifecycle-deadline';
|
|
8
|
+
import { lifecycleDrained } from './lifecycle-errors';
|
|
6
9
|
import { type Logger, logger as rootLogger } from './logger';
|
|
7
10
|
|
|
8
11
|
export type HealthState = 'starting' | 'ready' | 'draining' | 'stopped';
|
|
@@ -17,7 +20,13 @@ export type ProcessSignal = 'SIGTERM' | 'SIGINT' | 'SIGHUP' | 'SIGQUIT';
|
|
|
17
20
|
|
|
18
21
|
export interface ShutdownReason {
|
|
19
22
|
readonly signal: string;
|
|
20
|
-
/**
|
|
23
|
+
/**
|
|
24
|
+
* Real monotonic ms (`systemClock`) after which hooks are abandoned — deliberately NOT the
|
|
25
|
+
* injected clock. The budget this bounds is `terminationGracePeriodSeconds`, counted by the
|
|
26
|
+
* kubelet in real seconds, so a frozen clock must be unable to extend it: read off `clock` a
|
|
27
|
+
* test that advanced an hour of fake time handed the drain a 16-minute grace period, while
|
|
28
|
+
* `waitForIdle` went on sleeping on a real `setTimeout`. `clock` still owns `uptimeMs`.
|
|
29
|
+
*/
|
|
21
30
|
readonly deadlineAt: number;
|
|
22
31
|
}
|
|
23
32
|
|
|
@@ -28,17 +37,51 @@ export interface OnShutdownOptions {
|
|
|
28
37
|
}
|
|
29
38
|
|
|
30
39
|
export interface LifecycleOptions {
|
|
40
|
+
/**
|
|
41
|
+
* The whole drain's budget — the in-flight wait AND every hook, in every phase. 25s by default,
|
|
42
|
+
* and **enforced whether or not an app sets it**: `ShutdownReason.deadlineAt` was always computed
|
|
43
|
+
* and handed to every hook, so the deadline was declared by the design and only the enforcement
|
|
44
|
+
* was missing. No hook reads `deadlineAt`, which is why it has to be imposed here.
|
|
45
|
+
*
|
|
46
|
+
* The lever is a LARGER value, not the absence of one: a `worker` holding a 10-minute job wants
|
|
47
|
+
* `configureLifecycle({ deadlineMs: 600_000 })` and a `terminationGracePeriodSeconds` at least as
|
|
48
|
+
* large. Left at 25s it is abandoned and the process exits clean — the row's visibility lease
|
|
49
|
+
* lapses and another worker re-claims it, which is what at-least-once already promises. The
|
|
50
|
+
* alternative is not "the job finishes": it is the same duplicate, delivered by SIGKILL at the
|
|
51
|
+
* kubelet's grace period, with no log line naming what overran.
|
|
52
|
+
*/
|
|
31
53
|
readonly deadlineMs?: number | undefined;
|
|
32
54
|
readonly clock?: Clock | undefined;
|
|
33
55
|
readonly logger?: Logger | undefined;
|
|
34
56
|
}
|
|
35
57
|
|
|
58
|
+
export type ReadinessStatus = 'ok' | 'failing';
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Synchronous, and that is the design, not a limitation. **Do not widen this to
|
|
62
|
+
* `() => Promise<boolean>`** — the signature is the mechanism.
|
|
63
|
+
*
|
|
64
|
+
* A readiness endpoint that does I/O is a liveness bomb. A probe that awaits a network call takes
|
|
65
|
+
* as long as the dependency does, so a slow database makes the endpoint miss its `timeoutSeconds`,
|
|
66
|
+
* the kubelet reads that as unready, and capacity is pulled from an already-struggling system —
|
|
67
|
+
* the outage the probe existed to prevent, caused by the probe. Worse under a liveness probe
|
|
68
|
+
* sharing the handler: the pod is killed and restarts into the same slow database, cold.
|
|
69
|
+
*
|
|
70
|
+
* So the owner of the dependency keeps a boolean fresh — a pool exposes `isOpen`, a background
|
|
71
|
+
* poller flips a flag on its own schedule with its own timeout — and this reads it. That puts the
|
|
72
|
+
* waiting where a timeout can be tuned, and leaves this path unable to block. A check that throws
|
|
73
|
+
* is `failing`.
|
|
74
|
+
*/
|
|
75
|
+
export type ReadinessCheck = () => boolean;
|
|
76
|
+
|
|
36
77
|
export interface HealthReport {
|
|
37
78
|
readonly state: HealthState;
|
|
38
79
|
readonly ready: boolean;
|
|
39
80
|
readonly uptimeMs: number;
|
|
40
81
|
readonly inflight: number;
|
|
41
82
|
readonly buildId: string;
|
|
83
|
+
/** Named, because "alert on check failures BY CHECK NAME" is not writable against a boolean. */
|
|
84
|
+
readonly checks: Readonly<Record<string, ReadinessStatus>>;
|
|
42
85
|
}
|
|
43
86
|
|
|
44
87
|
export interface HealthPayload {
|
|
@@ -65,6 +108,7 @@ let inflight = 0;
|
|
|
65
108
|
let registrations: Registration[] = [];
|
|
66
109
|
let drainPromise: Promise<void> | undefined;
|
|
67
110
|
let idleWaiters: (() => void)[] = [];
|
|
111
|
+
const readiness = new Map<string, ReadinessCheck>();
|
|
68
112
|
|
|
69
113
|
export function configureLifecycle(options: LifecycleOptions): void {
|
|
70
114
|
if (options.deadlineMs !== undefined) deadlineMs = options.deadlineMs;
|
|
@@ -79,14 +123,79 @@ export function lifecycleState(): HealthState {
|
|
|
79
123
|
return state;
|
|
80
124
|
}
|
|
81
125
|
|
|
126
|
+
/**
|
|
127
|
+
* "This process bound its socket." NOT "this process can serve a request" — that is what the
|
|
128
|
+
* readiness checks answer. Before them, `markReady()` was the whole of `/readyz`, so a pod went
|
|
129
|
+
* green the instant it bound and the load balancer sent traffic into a Postgres pool that had not
|
|
130
|
+
* opened a connection yet; `maxUnavailable: 0` does not help when readiness lies.
|
|
131
|
+
*
|
|
132
|
+
* **A drained lifecycle refuses this, and that is the whole of "one process, one lifecycle."**
|
|
133
|
+
* `state` never leaves `stopped` and `drain()` memoizes, so a role that marked ready after a drain
|
|
134
|
+
* used to be told nothing and go on to bind a socket answering 503 to everything, with no drain
|
|
135
|
+
* left to close it. `X_LIFECYCLE_DRAINED` is that mistake named at the moment it is made, rather
|
|
136
|
+
* than a lifecycle that can be restarted: two live lifecycles racing one shutdown is a worse
|
|
137
|
+
* mechanism than a boot that fails.
|
|
138
|
+
*/
|
|
82
139
|
export function markReady(): void {
|
|
140
|
+
if (isDraining()) throw lifecycleDrained(state === 'draining' ? 'draining' : 'stopped');
|
|
83
141
|
if (state === 'starting') state = 'ready';
|
|
84
142
|
}
|
|
85
143
|
|
|
144
|
+
/**
|
|
145
|
+
* Register a named readiness check. Returns its unregister — the same shape as `onShutdown`, and
|
|
146
|
+
* owned by whoever can be started twice, for the same reason.
|
|
147
|
+
*/
|
|
148
|
+
export function registerReadinessCheck(name: string, check: ReadinessCheck): () => void {
|
|
149
|
+
if (readiness.has(name)) {
|
|
150
|
+
throw new UltimateError({
|
|
151
|
+
code: 'X_READINESS_CHECK_DUPLICATE',
|
|
152
|
+
cause: `a readiness check named "${name}" is already registered (have: ${[...readiness.keys()].join(', ')})`,
|
|
153
|
+
fix: `name the second check for what it actually probes, e.g. registerReadinessCheck('${name}-replica', check) — or hold the unregister the first registration returned and call it first`,
|
|
154
|
+
meta: { name },
|
|
155
|
+
});
|
|
156
|
+
}
|
|
157
|
+
readiness.set(name, check);
|
|
158
|
+
return () => {
|
|
159
|
+
if (readiness.get(name) === check) readiness.delete(name);
|
|
160
|
+
};
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/** Test-only: registered checks. A count that climbs across a start/stop cycle is a leak. */
|
|
164
|
+
export function readinessCheckCount(): number {
|
|
165
|
+
return readiness.size;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
/** Every check, run now, by name. A check that throws is `failing` — never an unhandled error. */
|
|
169
|
+
export function readinessChecks(): Readonly<Record<string, ReadinessStatus>> {
|
|
170
|
+
const results: Record<string, ReadinessStatus> = {};
|
|
171
|
+
for (const [name, check] of readiness) {
|
|
172
|
+
try {
|
|
173
|
+
results[name] = check() ? 'ok' : 'failing';
|
|
174
|
+
} catch (thrown) {
|
|
175
|
+
results[name] = 'failing';
|
|
176
|
+
log.warn('readiness check threw', { check: name, error: thrown });
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
return results;
|
|
180
|
+
}
|
|
181
|
+
|
|
86
182
|
export function inflightCount(): number {
|
|
87
183
|
return inflight;
|
|
88
184
|
}
|
|
89
185
|
|
|
186
|
+
/** Test-only: drains still waiting on in-flight work. A count stuck above zero is a leak. */
|
|
187
|
+
export function idleWaiterCount(): number {
|
|
188
|
+
return idleWaiters.length;
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
/**
|
|
192
|
+
* Test-only: hooks still registered. A count that climbs across a start/stop cycle is a leak —
|
|
193
|
+
* the registration retains its closure, and the next drain runs it against a torn-down resource.
|
|
194
|
+
*/
|
|
195
|
+
export function shutdownHookCount(): number {
|
|
196
|
+
return registrations.length;
|
|
197
|
+
}
|
|
198
|
+
|
|
90
199
|
/** Register a drain hook. Returns an unregister function. */
|
|
91
200
|
export function onShutdown(
|
|
92
201
|
name: string,
|
|
@@ -127,23 +236,66 @@ export function isDraining(): boolean {
|
|
|
127
236
|
function waitForIdle(timeoutMs: number): Promise<boolean> {
|
|
128
237
|
if (inflight === 0) return Promise.resolve(true);
|
|
129
238
|
return new Promise<boolean>((resolve) => {
|
|
130
|
-
const
|
|
131
|
-
idleWaiters.push(() => {
|
|
239
|
+
const waiter = (): void => {
|
|
132
240
|
clearTimeout(timer);
|
|
133
241
|
resolve(true);
|
|
134
|
-
}
|
|
242
|
+
};
|
|
243
|
+
// A drain that times out must not leave its waiter in the queue forever — the next
|
|
244
|
+
// `beginWork()` to reach zero would still hold and invoke it, a dangling closure over a
|
|
245
|
+
// promise nothing is awaiting anymore.
|
|
246
|
+
const timer = setTimeout(() => {
|
|
247
|
+
idleWaiters = idleWaiters.filter((candidate) => candidate !== waiter);
|
|
248
|
+
resolve(false);
|
|
249
|
+
}, timeoutMs);
|
|
250
|
+
idleWaiters.push(waiter);
|
|
135
251
|
});
|
|
136
252
|
}
|
|
137
253
|
|
|
254
|
+
/**
|
|
255
|
+
* The budget every drain is bounded by — `DEFAULT_DEADLINE_MS` until an app raises it. There is no
|
|
256
|
+
* unbounded state: `ShutdownReason.deadlineAt` was always computed and handed to every hook, so the
|
|
257
|
+
* deadline was declared by the design all along and only the enforcement was missing.
|
|
258
|
+
*
|
|
259
|
+
* The ONE place the budget is decided, and exported so a test can pin it: 25s is far above any
|
|
260
|
+
* drain a test can wait out, so the default needs a probe and not only a stopwatch.
|
|
261
|
+
*/
|
|
262
|
+
export function drainDeadlineMs(): number {
|
|
263
|
+
return deadlineMs;
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
/**
|
|
267
|
+
* What is left of that budget. Read per hook, not per phase: the deadline bounds the WHOLE drain,
|
|
268
|
+
* so a hook that spent it leaves nothing for the ones behind it — which is what
|
|
269
|
+
* `terminationGracePeriodSeconds` means, and what makes the SUM of the phases bounded rather than
|
|
270
|
+
* each one of them separately. Returns `number`, never `number | undefined`: "no budget" is not a
|
|
271
|
+
* state this file has, and the type is what keeps it from becoming one again.
|
|
272
|
+
*/
|
|
273
|
+
function remainingBudget(reason: ShutdownReason): number {
|
|
274
|
+
return Math.max(0, reason.deadlineAt - systemClock.monotonic());
|
|
275
|
+
}
|
|
276
|
+
|
|
138
277
|
async function runPhase(phase: ShutdownPhase, reason: ShutdownReason): Promise<void> {
|
|
139
278
|
for (const registration of registrations.filter((entry) => entry.phase === phase)) {
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
} catch (thrown) {
|
|
279
|
+
const outcome = await settleWithin(() => registration.hook(reason), remainingBudget(reason));
|
|
280
|
+
if (outcome.kind === 'failed') {
|
|
143
281
|
log.error('shutdown hook failed', {
|
|
144
282
|
hook: registration.name,
|
|
145
283
|
phase,
|
|
146
|
-
error:
|
|
284
|
+
error: outcome.error,
|
|
285
|
+
});
|
|
286
|
+
continue;
|
|
287
|
+
}
|
|
288
|
+
if (outcome.kind === 'abandoned') {
|
|
289
|
+
// Abandoned, not merely logged. A deadline that waited anyway would leave the kubelet to
|
|
290
|
+
// SIGKILL this process — the every-deploy duplicate that draining exists to prevent — so
|
|
291
|
+
// the drain moves on and the hook is left running with nobody reading it. The cost of that
|
|
292
|
+
// choice is real and named in the cause: a write it had in flight may be half done.
|
|
293
|
+
log.warn('X_SHUTDOWN_TIMEOUT', {
|
|
294
|
+
code: 'X_SHUTDOWN_TIMEOUT',
|
|
295
|
+
cause: `the "${registration.name}" shutdown hook (phase: ${phase}) was still running at the ${deadlineMs}ms drain deadline and has been ABANDONED — the process exits without it, so anything it had in flight may be incomplete`,
|
|
296
|
+
fix: `raise the budget past the work this hook does — configureLifecycle({ deadlineMs: 600_000 }) for a 10-minute job — and set terminationGracePeriodSeconds to at least as many seconds, or make the "${registration.name}" hook return once it has stopped accepting work rather than once it has finished`,
|
|
297
|
+
hook: registration.name,
|
|
298
|
+
phase,
|
|
147
299
|
});
|
|
148
300
|
}
|
|
149
301
|
}
|
|
@@ -153,19 +305,21 @@ async function runPhase(phase: ShutdownPhase, reason: ShutdownReason): Promise<v
|
|
|
153
305
|
export function drain(signal = 'manual'): Promise<void> {
|
|
154
306
|
if (drainPromise !== undefined) return drainPromise;
|
|
155
307
|
state = 'draining';
|
|
156
|
-
const reason: ShutdownReason = { signal, deadlineAt:
|
|
308
|
+
const reason: ShutdownReason = { signal, deadlineAt: systemClock.monotonic() + deadlineMs };
|
|
157
309
|
|
|
158
310
|
drainPromise = (async () => {
|
|
159
311
|
log.info('draining', { signal, deadlineMs, inflight });
|
|
160
312
|
await runPhase('accept', reason);
|
|
161
313
|
|
|
162
|
-
|
|
314
|
+
// Real monotonic, like `deadlineAt` itself: `waitForIdle` sleeps on a real `setTimeout`, and a
|
|
315
|
+
// budget read off an injected clock is a number that timer will never honour.
|
|
316
|
+
const remaining = Math.max(0, reason.deadlineAt - systemClock.monotonic());
|
|
163
317
|
const idle = await waitForIdle(remaining);
|
|
164
318
|
if (!idle) {
|
|
165
319
|
log.warn('X_SHUTDOWN_TIMEOUT', {
|
|
166
320
|
code: 'X_SHUTDOWN_TIMEOUT',
|
|
167
321
|
cause: `${inflight} in-flight operations still running after ${deadlineMs}ms`,
|
|
168
|
-
fix: 'raise configureLifecycle({ deadlineMs }) or shorten the
|
|
322
|
+
fix: 'raise the budget past the slowest handler — configureLifecycle({ deadlineMs: 600_000 }) for a 10-minute one — and set terminationGracePeriodSeconds to at least as many seconds, or shorten the handler',
|
|
169
323
|
});
|
|
170
324
|
}
|
|
171
325
|
|
|
@@ -205,27 +359,34 @@ export function installSignalHandlers(options?: SignalHandlerOptions): () => voi
|
|
|
205
359
|
}
|
|
206
360
|
|
|
207
361
|
export function healthReport(): HealthReport {
|
|
362
|
+
const checks = readinessChecks();
|
|
208
363
|
return {
|
|
209
364
|
state,
|
|
210
|
-
ready
|
|
365
|
+
// `ready` is the same predicate `/readyz` answers on, so a body and its status can never
|
|
366
|
+
// disagree — a 200 whose body says `ready: false` is the bug this shares one source to avoid.
|
|
367
|
+
ready: state === 'ready' && Object.values(checks).every((status) => status === 'ok'),
|
|
211
368
|
uptimeMs: Math.round(clock.monotonic() - startedAtMono),
|
|
212
369
|
inflight,
|
|
213
370
|
buildId: process.env['BUILD_ID'] ?? 'dev',
|
|
371
|
+
checks,
|
|
214
372
|
};
|
|
215
373
|
}
|
|
216
374
|
|
|
217
|
-
/**
|
|
375
|
+
/**
|
|
376
|
+
* Liveness: the process exists and is not wedged. Stays 200 while draining, and deliberately
|
|
377
|
+
* ignores the checks — a database outage that failed liveness everywhere would restart the whole
|
|
378
|
+
* fleet into the same outage, with cold caches and no connections.
|
|
379
|
+
*/
|
|
218
380
|
export function healthzPayload(): HealthPayload {
|
|
219
381
|
const body = healthReport();
|
|
220
382
|
const ok = state !== 'stopped';
|
|
221
383
|
return { ok, status: ok ? 200 : 503, body };
|
|
222
384
|
}
|
|
223
385
|
|
|
224
|
-
/** Readiness: may this instance receive traffic? 503 while starting or
|
|
386
|
+
/** Readiness: may this instance receive traffic? 503 while starting, draining or any check fails. */
|
|
225
387
|
export function readyzPayload(): HealthPayload {
|
|
226
388
|
const body = healthReport();
|
|
227
|
-
|
|
228
|
-
return { ok, status: ok ? 200 : 503, body };
|
|
389
|
+
return { ok: body.ready, status: body.ready ? 200 : 503, body };
|
|
229
390
|
}
|
|
230
391
|
|
|
231
392
|
/** Test-only: forget all hooks and return to `starting`. */
|
|
@@ -239,4 +400,5 @@ export function resetLifecycle(): void {
|
|
|
239
400
|
registrations = [];
|
|
240
401
|
drainPromise = undefined;
|
|
241
402
|
idleWaiters = [];
|
|
403
|
+
readiness.clear();
|
|
242
404
|
}
|
package/src/logger.ts
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
// default because the primary reader is an agent tailing `x logs --json`.
|
|
3
3
|
|
|
4
4
|
import { type Clock, systemClock } from './clock';
|
|
5
|
+
import { renderCauseValue } from './error-render';
|
|
5
6
|
import { isUltimateError } from './errors';
|
|
6
7
|
import { isSecret, REDACTED } from './secret';
|
|
7
8
|
|
|
@@ -48,15 +49,34 @@ export interface LoggerOptions {
|
|
|
48
49
|
readonly writer?: ((line: string, level: LogLevel) => void) | undefined;
|
|
49
50
|
}
|
|
50
51
|
|
|
52
|
+
/**
|
|
53
|
+
* LOWERCASE, always: `isRedactedKey` lowercases its lookup, so `apiKey`/`accessToken`/
|
|
54
|
+
* `refreshToken` sat here for three releases matching nothing — and those are the exact field
|
|
55
|
+
* names on `@ultimat3/auth`'s `OAuthTokens`. Matching is exact-key and never substring, so a
|
|
56
|
+
* spelling that is not in this set is not redacted: both the camel and the snake wire spelling of
|
|
57
|
+
* each credential is listed. Add through `redactKeys()` (which lowercases) rather than here.
|
|
58
|
+
*/
|
|
51
59
|
const redactedKeys = new Set<string>([
|
|
52
60
|
'password',
|
|
53
61
|
'token',
|
|
54
62
|
'secret',
|
|
55
63
|
'authorization',
|
|
56
64
|
'cookie',
|
|
57
|
-
'
|
|
58
|
-
'
|
|
59
|
-
'
|
|
65
|
+
'set-cookie',
|
|
66
|
+
'apikey',
|
|
67
|
+
'api_key',
|
|
68
|
+
'accesstoken',
|
|
69
|
+
'access_token',
|
|
70
|
+
'refreshtoken',
|
|
71
|
+
'refresh_token',
|
|
72
|
+
'idtoken',
|
|
73
|
+
'id_token',
|
|
74
|
+
'sessiontoken',
|
|
75
|
+
'session_token',
|
|
76
|
+
'clientsecret',
|
|
77
|
+
'client_secret',
|
|
78
|
+
'privatekey',
|
|
79
|
+
'private_key',
|
|
60
80
|
]);
|
|
61
81
|
|
|
62
82
|
/** Mark keys as secret everywhere. `defineEnv()` calls this for every `secret: true` var. */
|
|
@@ -83,31 +103,101 @@ function defaultWriter(line: string, level: LogLevel): void {
|
|
|
83
103
|
stream.write(`${line}\n`);
|
|
84
104
|
}
|
|
85
105
|
|
|
106
|
+
/**
|
|
107
|
+
* TOTAL, on purpose. `lifecycle.ts` logs the value a shutdown hook threw and the value a
|
|
108
|
+
* readiness check threw — both caught, both arbitrary — so a renderer that throws here escapes
|
|
109
|
+
* `runPhase`'s catch, rejects the drain promise, and `installSignalHandlers` never reaches
|
|
110
|
+
* `process.exit(0)`: SIGTERM hangs, and `/readyz` dies with the check it was reporting on. A log
|
|
111
|
+
* line must never replace the event it describes.
|
|
112
|
+
*
|
|
113
|
+
* Degradation is per KEY, the same shape `renderMetaRecord` uses: one hostile getter must not
|
|
114
|
+
* cost a reader the fields beside it.
|
|
115
|
+
*/
|
|
86
116
|
function serialiseValue(value: unknown, depth: number): unknown {
|
|
117
|
+
try {
|
|
118
|
+
return serialise(value, depth);
|
|
119
|
+
} catch {
|
|
120
|
+
// `instanceof`, `Object.keys` and `toJSON` are all property reads on a value the framework
|
|
121
|
+
// did not build; `renderCauseValue` is the one renderer that cannot itself throw.
|
|
122
|
+
return renderCauseValue(value);
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
function serialise(value: unknown, depth: number): unknown {
|
|
127
|
+
// `JSON.stringify` raises a `TypeError` on a bigint, so the whole line died for one field.
|
|
128
|
+
if (typeof value === 'bigint') return renderCauseValue(value);
|
|
87
129
|
if (value === null || typeof value !== 'object') return value;
|
|
88
130
|
// Before every other branch: a `Secret` is redacted by VALUE, so it stays redacted under a key
|
|
89
131
|
// nobody listed — `{ dsn: secret(url) }` is the leak key-name redaction cannot see.
|
|
90
132
|
if (isSecret(value)) return REDACTED;
|
|
91
|
-
|
|
133
|
+
// `toISOString()` THROWS on an invalid Date, and an invalid Date is exactly the value worth
|
|
134
|
+
// logging when a schedule went wrong.
|
|
135
|
+
if (value instanceof Date) {
|
|
136
|
+
return Number.isNaN(value.getTime()) ? 'an invalid Date' : value.toISOString();
|
|
137
|
+
}
|
|
92
138
|
if (isUltimateError(value)) return value.toJSON();
|
|
93
139
|
if (value instanceof Error) return { name: value.name, message: value.message };
|
|
94
140
|
if (depth >= 6) return '[depth-limit]';
|
|
95
141
|
if (Array.isArray(value)) return value.map((item) => serialiseValue(item, depth + 1));
|
|
142
|
+
const source = value as Record<string, unknown>;
|
|
96
143
|
const out: Record<string, unknown> = {};
|
|
97
|
-
for (const
|
|
98
|
-
out[key] = isRedactedKey(key) ? REDACTED :
|
|
144
|
+
for (const key of Object.keys(source)) {
|
|
145
|
+
out[key] = isRedactedKey(key) ? REDACTED : entryValue(source, key, depth);
|
|
99
146
|
}
|
|
100
147
|
return out;
|
|
101
148
|
}
|
|
102
149
|
|
|
150
|
+
/** One field. The `try` covers the property READ — `serialiseValue` above is already total. */
|
|
151
|
+
function entryValue(source: Record<string, unknown>, key: string, depth: number): unknown {
|
|
152
|
+
try {
|
|
153
|
+
return serialiseValue(source[key], depth + 1);
|
|
154
|
+
} catch {
|
|
155
|
+
return 'a value that cannot be read';
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
|
|
103
159
|
function redactFields(fields: LogFields): Record<string, unknown> {
|
|
104
160
|
const out: Record<string, unknown> = {};
|
|
105
|
-
|
|
106
|
-
|
|
161
|
+
const source = fields as Record<string, unknown>;
|
|
162
|
+
// `Object.keys` before the values, so the read of each value is its own guarded step: a field
|
|
163
|
+
// record is the caller's object, and enumerating it eagerly threw on the first hostile getter.
|
|
164
|
+
for (const key of ownKeys(source)) {
|
|
165
|
+
out[key] = isRedactedKey(key) ? REDACTED : entryValue(source, key, 0);
|
|
107
166
|
}
|
|
108
167
|
return out;
|
|
109
168
|
}
|
|
110
169
|
|
|
170
|
+
function ownKeys(source: Record<string, unknown>): readonly string[] {
|
|
171
|
+
try {
|
|
172
|
+
return Object.keys(source);
|
|
173
|
+
} catch {
|
|
174
|
+
return [];
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/**
|
|
179
|
+
* The last guard. The walk above already degraded every hostile field, so reaching the fallback
|
|
180
|
+
* means the assembled line itself refused to serialise — and the answer to that is still a line,
|
|
181
|
+
* not a throw propagating out of `log.error` into whatever `catch` block called it.
|
|
182
|
+
*/
|
|
183
|
+
function renderLine(
|
|
184
|
+
line: Readonly<Record<string, unknown>>,
|
|
185
|
+
level: LogLevel,
|
|
186
|
+
message: string,
|
|
187
|
+
ts: unknown,
|
|
188
|
+
): string {
|
|
189
|
+
try {
|
|
190
|
+
return JSON.stringify(line);
|
|
191
|
+
} catch {
|
|
192
|
+
return JSON.stringify({
|
|
193
|
+
ts: typeof ts === 'string' ? ts : '',
|
|
194
|
+
level,
|
|
195
|
+
msg: message,
|
|
196
|
+
logFields: 'a log line that cannot be serialised',
|
|
197
|
+
});
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
|
|
111
201
|
function envLevel(): LogLevel {
|
|
112
202
|
const raw = process.env['LOG_LEVEL'];
|
|
113
203
|
return raw !== undefined && (LOG_LEVELS as readonly string[]).includes(raw)
|
|
@@ -132,7 +222,7 @@ export function createLogger(options?: LoggerOptions): Logger {
|
|
|
132
222
|
...redactFields(contextFields() ?? {}),
|
|
133
223
|
...redactFields(fields ?? {}),
|
|
134
224
|
};
|
|
135
|
-
writer(
|
|
225
|
+
writer(renderLine(line, lineLevel, message, line.ts), lineLevel);
|
|
136
226
|
}
|
|
137
227
|
|
|
138
228
|
return {
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
// The one answer to "did this primitive opt into being an MCP tool?" — a literal `expose: true`.
|
|
2
|
+
// Core owns it because its readers span tiers 3-5 — `action`, `query`, `mcp`, `ai`, `manifest` —
|
|
3
|
+
// and this is the only tier all of them reach, the same reason `timing-safe-equal.ts` lives here.
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* The `mcp` block, read structurally. Each package keeps its own richer declaration —
|
|
7
|
+
* `ActionMcp` carries `visibleTo`, `@ultimat3/mcp`'s `McpExposure` carries `name` — and hands it
|
|
8
|
+
* here; restating the one field they share binds this to none of them.
|
|
9
|
+
*/
|
|
10
|
+
export interface McpExposureDeclaration {
|
|
11
|
+
readonly expose?: boolean | undefined;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Opt-in, never opt-out: silence exposes nothing. An absent block, an omitted `expose` and a
|
|
16
|
+
* literal `false` are one answer, because a tool the author never asked for is a capability
|
|
17
|
+
* handed to every agent that can reach the surface — and writing an action is not a request to
|
|
18
|
+
* hand one out.
|
|
19
|
+
*
|
|
20
|
+
* Six readers decided it three ways until 2026-08: `=== true` where a tool is actually built,
|
|
21
|
+
* `!== false` in the OpenAPI hint and `?? true` in the manifest fact. So an action with no `mcp`
|
|
22
|
+
* block was published as a tool by the contract and refused by every surface that could have
|
|
23
|
+
* called one.
|
|
24
|
+
*
|
|
25
|
+
* The one deliberate exception is `@ultimat3/admin`'s OWN catalog, whose every tool is already
|
|
26
|
+
* gated on an admin permission and whose CRUD tools carry no `mcp` block at all; there
|
|
27
|
+
* `expose: false` withdraws a tool. That surface says so in `mcp-tools.ts` and in
|
|
28
|
+
* `wiki/Admin-Dashboard.md`. Nothing else may grow a second default.
|
|
29
|
+
*/
|
|
30
|
+
export function isMcpExposed(declared: McpExposureDeclaration | undefined): boolean {
|
|
31
|
+
return declared?.expose === true;
|
|
32
|
+
}
|
package/src/metrics.ts
CHANGED
|
Binary file
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
// Single responsibility: a `MetricExporter` that POSTs OTLP/HTTP JSON to a collector. No batching
|
|
2
|
+
// — `collectMetrics()` already produces one whole snapshot per tick, so a tick is a request.
|
|
3
|
+
|
|
4
|
+
import type {
|
|
5
|
+
HistogramPoint,
|
|
6
|
+
MetricCollection,
|
|
7
|
+
MetricExporter,
|
|
8
|
+
MetricPoint,
|
|
9
|
+
ReadableMetric,
|
|
10
|
+
} from './metrics';
|
|
11
|
+
import {
|
|
12
|
+
OTLP_SCOPE,
|
|
13
|
+
otlpAttributes,
|
|
14
|
+
otlpEndpoint,
|
|
15
|
+
otlpHeaders,
|
|
16
|
+
otlpResource,
|
|
17
|
+
postOtlp,
|
|
18
|
+
unixNano,
|
|
19
|
+
} from './otlp';
|
|
20
|
+
|
|
21
|
+
/** `AGGREGATION_TEMPORALITY_CUMULATIVE`. The only temporality `metrics.ts` produces. */
|
|
22
|
+
const CUMULATIVE = 2;
|
|
23
|
+
|
|
24
|
+
function isHistogramPoint(point: MetricPoint): point is HistogramPoint {
|
|
25
|
+
return 'buckets' in point;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function numberPoint(point: MetricPoint, at: string, startedAt: string): unknown {
|
|
29
|
+
return {
|
|
30
|
+
attributes: otlpAttributes(point.attributes),
|
|
31
|
+
startTimeUnixNano: startedAt,
|
|
32
|
+
timeUnixNano: at,
|
|
33
|
+
asDouble: point.value,
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
function histogramDataPoint(point: HistogramPoint, at: string, startedAt: string): unknown {
|
|
38
|
+
return {
|
|
39
|
+
attributes: otlpAttributes(point.attributes),
|
|
40
|
+
startTimeUnixNano: startedAt,
|
|
41
|
+
timeUnixNano: at,
|
|
42
|
+
count: String(point.count),
|
|
43
|
+
sum: point.value,
|
|
44
|
+
// 64-bit counts, so the JSON encoding spells them as strings — the same rule `intValue` follows.
|
|
45
|
+
bucketCounts: point.buckets.map((count) => String(count)),
|
|
46
|
+
explicitBounds: [...point.bounds],
|
|
47
|
+
...(point.count === 0 ? {} : { min: point.min, max: point.max }),
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function metricJson(metric: ReadableMetric, at: string, startedAt: string): unknown {
|
|
52
|
+
const { name, unit, description, kind } = metric.descriptor;
|
|
53
|
+
const head = { name, unit, description };
|
|
54
|
+
if (kind === 'histogram') {
|
|
55
|
+
const dataPoints = metric.points
|
|
56
|
+
.filter(isHistogramPoint)
|
|
57
|
+
.map((point) => histogramDataPoint(point, at, startedAt));
|
|
58
|
+
return { ...head, histogram: { dataPoints, aggregationTemporality: CUMULATIVE } };
|
|
59
|
+
}
|
|
60
|
+
const dataPoints = metric.points.map((point) => numberPoint(point, at, startedAt));
|
|
61
|
+
if (kind === 'gauge') return { ...head, gauge: { dataPoints } };
|
|
62
|
+
return {
|
|
63
|
+
...head,
|
|
64
|
+
sum: { dataPoints, aggregationTemporality: CUMULATIVE, isMonotonic: true },
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Pure, so the wire format is a unit test. `startedAt` is the process start: cumulative points
|
|
70
|
+
* carry the instant the sum began, and a reader that sees it move knows the process restarted —
|
|
71
|
+
* which is the whole reason OTLP carries it separately from the observation time.
|
|
72
|
+
*/
|
|
73
|
+
export function otlpMetricsRequest(collection: MetricCollection, startedAtMs: number): unknown {
|
|
74
|
+
const at = unixNano(collection.at);
|
|
75
|
+
const startedAt = unixNano(startedAtMs);
|
|
76
|
+
return {
|
|
77
|
+
resourceMetrics: [
|
|
78
|
+
{
|
|
79
|
+
resource: otlpResource(collection.resource),
|
|
80
|
+
scopeMetrics: [
|
|
81
|
+
{
|
|
82
|
+
scope: OTLP_SCOPE,
|
|
83
|
+
metrics: collection.metrics.map((metric) => metricJson(metric, at, startedAt)),
|
|
84
|
+
},
|
|
85
|
+
],
|
|
86
|
+
},
|
|
87
|
+
],
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
export interface OtlpMetricExporterOptions {
|
|
92
|
+
/** Overrides `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT` / `OTEL_EXPORTER_OTLP_ENDPOINT`. */
|
|
93
|
+
readonly endpoint?: string | undefined;
|
|
94
|
+
/** Merged over `OTEL_EXPORTER_OTLP_HEADERS`. */
|
|
95
|
+
readonly headers?: Readonly<Record<string, string>> | undefined;
|
|
96
|
+
/** Default 10000ms. */
|
|
97
|
+
readonly timeoutMs?: number | undefined;
|
|
98
|
+
/** Injected by tests; the preload seals the real one. */
|
|
99
|
+
readonly fetch?: typeof globalThis.fetch | undefined;
|
|
100
|
+
/**
|
|
101
|
+
* Epoch ms this process's counters started at. Defaults to the first export, which is close
|
|
102
|
+
* enough for a rate and exact for a restart detector; pass the boot time when you have it.
|
|
103
|
+
*/
|
|
104
|
+
readonly startedAtMs?: number | undefined;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
export interface OtlpMetricExporter extends MetricExporter {
|
|
108
|
+
/** Resolves once the last POST settles — `exportMetrics()` itself is fire-and-forget. */
|
|
109
|
+
flush(): Promise<void>;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* Throws `X_OTLP_ENDPOINT_INVALID` at construction when nothing configured an endpoint. Ask
|
|
114
|
+
* `tryOtlpEndpoint('metrics')` first when the exporter is optional.
|
|
115
|
+
*/
|
|
116
|
+
export function otlpMetricExporter(options: OtlpMetricExporterOptions = {}): OtlpMetricExporter {
|
|
117
|
+
const url = otlpEndpoint('metrics', options.endpoint);
|
|
118
|
+
const headers = otlpHeaders(options.headers);
|
|
119
|
+
const timeoutMs = options.timeoutMs ?? 10_000;
|
|
120
|
+
const send = options.fetch ?? globalThis.fetch;
|
|
121
|
+
let startedAtMs = options.startedAtMs;
|
|
122
|
+
let inflight: Promise<void> = Promise.resolve();
|
|
123
|
+
|
|
124
|
+
return {
|
|
125
|
+
export(collection: MetricCollection): void {
|
|
126
|
+
startedAtMs ??= collection.at;
|
|
127
|
+
const body = JSON.stringify(otlpMetricsRequest(collection, startedAtMs));
|
|
128
|
+
// Chained, so a slow collector cannot make two snapshots arrive out of order and turn a
|
|
129
|
+
// cumulative counter into an apparent reset.
|
|
130
|
+
inflight = inflight.then(() => postOtlp({ url, headers, body, timeoutMs, fetch: send }));
|
|
131
|
+
},
|
|
132
|
+
flush(): Promise<void> {
|
|
133
|
+
return inflight;
|
|
134
|
+
},
|
|
135
|
+
};
|
|
136
|
+
}
|