@ultimat3/core 2.0.0 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +46 -0
- package/README.md +39 -0
- package/package.json +1 -1
- package/src/actor.ts +27 -1
- package/src/canonical-json.ts +102 -0
- package/src/config.ts +33 -10
- package/src/decimal-order.ts +76 -0
- package/src/env-example.ts +9 -1
- package/src/error-codes.ts +2 -1
- package/src/error-retry.ts +40 -14
- package/src/exports/error-contract.ts +1 -0
- package/src/image/errors.ts +1 -1
- package/src/index.ts +10 -12
- package/src/intl-cache.ts +43 -0
- package/src/lifecycle.ts +65 -24
- package/src/logger.ts +21 -2
- package/src/metrics-types.ts +102 -0
- package/src/metrics.ts +107 -92
- package/src/otlp-metric-exporter.ts +26 -3
- package/src/otlp-span-exporter.ts +28 -3
- package/src/type-pins.ts +10 -3
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
// One bounded cache, on one canonical key, for every `Intl` formatter the framework builds.
|
|
2
|
+
// A locale and a zone both arrive from a request header, so an unbounded `Map` keyed on the
|
|
3
|
+
// caller's spelling is memory the client chooses — 31 MB and 55.1 MB, measured `As of 2026-08` and
|
|
4
|
+
// written up in the README. The bound and the canonical key are two halves of ONE rule.
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Above the full canonical IANA set (445 zones as of tzdata 2025) so a correct app never evicts,
|
|
8
|
+
* and small enough that the worst case is a few megabytes rather than a leak. A miss costs one
|
|
9
|
+
* `Intl` construction, never a wrong answer — which is what makes a bound safe here at all.
|
|
10
|
+
*/
|
|
11
|
+
export const MAX_CACHED_FORMATTERS = 512;
|
|
12
|
+
|
|
13
|
+
/** FIFO — a `Map` iterates in insertion order, so the first key inserted is the first evicted. */
|
|
14
|
+
export function cachedFormatter<T>(cache: Map<string, T>, key: string, build: () => T): T {
|
|
15
|
+
// Membership decides, never truthiness: `T` is the caller's, so a stored `undefined` is a hit.
|
|
16
|
+
// The cast is sound because `has` just proved the key is present, which `get`'s signature cannot.
|
|
17
|
+
if (cache.has(key)) return cache.get(key) as T;
|
|
18
|
+
const formatter = build();
|
|
19
|
+
if (cache.size >= MAX_CACHED_FORMATTERS) {
|
|
20
|
+
const oldest = cache.keys().next().value;
|
|
21
|
+
if (oldest !== undefined) cache.delete(oldest);
|
|
22
|
+
}
|
|
23
|
+
cache.set(key, formatter);
|
|
24
|
+
return formatter;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* The canonical BCP 47 spelling, or `undefined` when the tag is not structurally valid at all
|
|
29
|
+
* (`en_US`, `''`, `not a locale`). Well-formed but unknown to ICU (`zz`) is a locale — `Intl`
|
|
30
|
+
* falls back for it, and refusing here would be stricter than the formatters this feeds.
|
|
31
|
+
*
|
|
32
|
+
* Deliberately **not** memoised: this is string work, and a `Map` keyed on a header value is the
|
|
33
|
+
* unbounded cache the bound above exists to prevent.
|
|
34
|
+
*/
|
|
35
|
+
export function canonicalLocale(locale: string): string | undefined {
|
|
36
|
+
try {
|
|
37
|
+
// `getCanonicalLocales` runs the same IsStructurallyValidLanguageTag check that
|
|
38
|
+
// `supportedLocalesOf` throws on, and unlike it, hands back the canonical spelling.
|
|
39
|
+
return Intl.getCanonicalLocales(locale)[0];
|
|
40
|
+
} catch {
|
|
41
|
+
return undefined;
|
|
42
|
+
}
|
|
43
|
+
}
|
package/src/lifecycle.ts
CHANGED
|
@@ -6,7 +6,7 @@ import { type Clock, systemClock } from './clock';
|
|
|
6
6
|
import { UltimateError } from './errors';
|
|
7
7
|
import { settleWithin } from './lifecycle-deadline';
|
|
8
8
|
import { lifecycleDrained } from './lifecycle-errors';
|
|
9
|
-
import { type Logger, logger as rootLogger } from './logger';
|
|
9
|
+
import { type LogFields, type Logger, logger as rootLogger } from './logger';
|
|
10
10
|
|
|
11
11
|
export type HealthState = 'starting' | 'ready' | 'draining' | 'stopped';
|
|
12
12
|
|
|
@@ -165,6 +165,33 @@ export function readinessCheckCount(): number {
|
|
|
165
165
|
return readiness.size;
|
|
166
166
|
}
|
|
167
167
|
|
|
168
|
+
/**
|
|
169
|
+
* Every line this file emits, and the only way it emits one. `log` is an injection seam
|
|
170
|
+
* (`configureLifecycle({ logger })`), so an app's `Logger` decides whether a log call can throw —
|
|
171
|
+
* and a throw here does not lose a line, it replaces the event. Inside `drain()` it rejected
|
|
172
|
+
* `drainPromise`: `state` never reached 'stopped', the memo re-rejected for every later caller,
|
|
173
|
+
* and on Bun the unhandled rejection ended the process the drain was trying to end cleanly.
|
|
174
|
+
* Inside `readinessChecks()` it replaced the probe's answer with a throw.
|
|
175
|
+
*
|
|
176
|
+
* A lifecycle that cannot report is still a lifecycle: the line falls back to core's own
|
|
177
|
+
* `rootLogger`, which is total by construction (`logger.ts`), and failing that is dropped.
|
|
178
|
+
*/
|
|
179
|
+
function report(level: 'info' | 'warn' | 'error', message: string, fields: LogFields): void {
|
|
180
|
+
try {
|
|
181
|
+
log[level](message, fields);
|
|
182
|
+
return;
|
|
183
|
+
} catch {
|
|
184
|
+
// Fall through — the injected sink is gone, and the fallback below is the last one there is.
|
|
185
|
+
}
|
|
186
|
+
if (log === rootLogger) return;
|
|
187
|
+
try {
|
|
188
|
+
rootLogger[level](message, fields);
|
|
189
|
+
} catch {
|
|
190
|
+
// Both sinks are gone. Dropping the line is the only remaining option that still ends the
|
|
191
|
+
// process, which is the outcome every caller of this file depends on.
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
|
|
168
195
|
/** Every check, run now, by name. A check that throws is `failing` — never an unhandled error. */
|
|
169
196
|
export function readinessChecks(): Readonly<Record<string, ReadinessStatus>> {
|
|
170
197
|
const results: Record<string, ReadinessStatus> = {};
|
|
@@ -173,7 +200,7 @@ export function readinessChecks(): Readonly<Record<string, ReadinessStatus>> {
|
|
|
173
200
|
results[name] = check() ? 'ok' : 'failing';
|
|
174
201
|
} catch (thrown) {
|
|
175
202
|
results[name] = 'failing';
|
|
176
|
-
|
|
203
|
+
report('warn', 'readiness check threw', { check: name, error: thrown });
|
|
177
204
|
}
|
|
178
205
|
}
|
|
179
206
|
return results;
|
|
@@ -278,7 +305,7 @@ async function runPhase(phase: ShutdownPhase, reason: ShutdownReason): Promise<v
|
|
|
278
305
|
for (const registration of registrations.filter((entry) => entry.phase === phase)) {
|
|
279
306
|
const outcome = await settleWithin(() => registration.hook(reason), remainingBudget(reason));
|
|
280
307
|
if (outcome.kind === 'failed') {
|
|
281
|
-
|
|
308
|
+
report('error', 'shutdown hook failed', {
|
|
282
309
|
hook: registration.name,
|
|
283
310
|
phase,
|
|
284
311
|
error: outcome.error,
|
|
@@ -290,7 +317,7 @@ async function runPhase(phase: ShutdownPhase, reason: ShutdownReason): Promise<v
|
|
|
290
317
|
// SIGKILL this process — the every-deploy duplicate that draining exists to prevent — so
|
|
291
318
|
// the drain moves on and the hook is left running with nobody reading it. The cost of that
|
|
292
319
|
// choice is real and named in the cause: a write it had in flight may be half done.
|
|
293
|
-
|
|
320
|
+
report('warn', 'X_SHUTDOWN_TIMEOUT', {
|
|
294
321
|
code: 'X_SHUTDOWN_TIMEOUT',
|
|
295
322
|
cause: `the "${registration.name}" shutdown hook (phase: ${phase}) was still running at the ${deadlineMs}ms drain deadline and has been ABANDONED — the process exits without it, so anything it had in flight may be incomplete`,
|
|
296
323
|
fix: `raise the budget past the work this hook does — configureLifecycle({ deadlineMs: 600_000 }) for a 10-minute job — and set terminationGracePeriodSeconds to at least as many seconds, or make the "${registration.name}" hook return once it has stopped accepting work rather than once it has finished`,
|
|
@@ -308,25 +335,34 @@ export function drain(signal = 'manual'): Promise<void> {
|
|
|
308
335
|
const reason: ShutdownReason = { signal, deadlineAt: systemClock.monotonic() + deadlineMs };
|
|
309
336
|
|
|
310
337
|
drainPromise = (async () => {
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
338
|
+
try {
|
|
339
|
+
report('info', 'draining', { signal, deadlineMs, inflight });
|
|
340
|
+
await runPhase('accept', reason);
|
|
341
|
+
|
|
342
|
+
// Real monotonic, like `deadlineAt` itself: `waitForIdle` sleeps on a real `setTimeout`, and
|
|
343
|
+
// a budget read off an injected clock is a number that timer will never honour.
|
|
344
|
+
const remaining = Math.max(0, reason.deadlineAt - systemClock.monotonic());
|
|
345
|
+
const idle = await waitForIdle(remaining);
|
|
346
|
+
if (!idle) {
|
|
347
|
+
report('warn', 'X_SHUTDOWN_TIMEOUT', {
|
|
348
|
+
code: 'X_SHUTDOWN_TIMEOUT',
|
|
349
|
+
cause: `${inflight} in-flight operations still running after ${deadlineMs}ms`,
|
|
350
|
+
fix: 'raise the budget past the slowest handler — configureLifecycle({ deadlineMs: 600_000 }) for a 10-minute one — and set terminationGracePeriodSeconds to at least as many seconds, or shorten the handler',
|
|
351
|
+
});
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
await runPhase('inflight', reason);
|
|
355
|
+
await runPhase('close', reason);
|
|
356
|
+
} catch (thrown) {
|
|
357
|
+
// Nothing above should reach here — every hook is caught by `settleWithin` and every line
|
|
358
|
+
// goes through `report`. If something does, the drain still ENDS: a rejected `drainPromise`
|
|
359
|
+
// is a memo that re-rejects for every later caller and an unhandled rejection that kills the
|
|
360
|
+
// process mid-drain, which is strictly worse than a drain that finished badly and said so.
|
|
361
|
+
report('error', 'drain failed', { signal, error: thrown });
|
|
362
|
+
} finally {
|
|
363
|
+
state = 'stopped';
|
|
324
364
|
}
|
|
325
|
-
|
|
326
|
-
await runPhase('inflight', reason);
|
|
327
|
-
await runPhase('close', reason);
|
|
328
|
-
state = 'stopped';
|
|
329
|
-
log.info('stopped', { signal });
|
|
365
|
+
report('info', 'stopped', { signal });
|
|
330
366
|
})();
|
|
331
367
|
|
|
332
368
|
return drainPromise;
|
|
@@ -345,9 +381,14 @@ export function installSignalHandlers(options?: SignalHandlerOptions): () => voi
|
|
|
345
381
|
|
|
346
382
|
for (const signal of signals) {
|
|
347
383
|
const handler = (): void => {
|
|
348
|
-
|
|
384
|
+
// Attached on BOTH settle paths, for the reason `settleWithin` gives: an unhandled rejection
|
|
385
|
+
// ends the process before the drain does, and the exit is what the kubelet is waiting for.
|
|
386
|
+
// `drain()` cannot reject today — that is the `try/finally` above, not luck — and this is
|
|
387
|
+
// the one line that keeps it true when someone changes the body.
|
|
388
|
+
const done = (): void => {
|
|
349
389
|
if (options?.exit === true) process.exit(0);
|
|
350
|
-
}
|
|
390
|
+
};
|
|
391
|
+
void drain(signal).then(done, done);
|
|
351
392
|
};
|
|
352
393
|
handlers.set(signal, handler);
|
|
353
394
|
process.on(signal, handler);
|
package/src/logger.ts
CHANGED
|
@@ -10,6 +10,9 @@ import { isSecret, REDACTED } from './secret';
|
|
|
10
10
|
// it without importing the logger, and two constants spelled the same is one rename from a leak.
|
|
11
11
|
export { REDACTED } from './secret';
|
|
12
12
|
|
|
13
|
+
/** What a `Date` this file cannot render says instead — the line survives, the value is named. */
|
|
14
|
+
const INVALID_DATE = 'an invalid Date';
|
|
15
|
+
|
|
13
16
|
export const LOG_LEVELS = ['trace', 'debug', 'info', 'warn', 'error', 'fatal', 'silent'] as const;
|
|
14
17
|
|
|
15
18
|
export type LogLevel = (typeof LOG_LEVELS)[number];
|
|
@@ -133,7 +136,7 @@ function serialise(value: unknown, depth: number): unknown {
|
|
|
133
136
|
// `toISOString()` THROWS on an invalid Date, and an invalid Date is exactly the value worth
|
|
134
137
|
// logging when a schedule went wrong.
|
|
135
138
|
if (value instanceof Date) {
|
|
136
|
-
return Number.isNaN(value.getTime()) ?
|
|
139
|
+
return Number.isNaN(value.getTime()) ? INVALID_DATE : value.toISOString();
|
|
137
140
|
}
|
|
138
141
|
if (isUltimateError(value)) return value.toJSON();
|
|
139
142
|
if (value instanceof Error) return { name: value.name, message: value.message };
|
|
@@ -198,6 +201,22 @@ function renderLine(
|
|
|
198
201
|
}
|
|
199
202
|
}
|
|
200
203
|
|
|
204
|
+
/**
|
|
205
|
+
* The one value in a line that is not the caller's, and it was the one read left unguarded:
|
|
206
|
+
* `toISOString()` raises `RangeError` on an invalid `Date`, and a `Clock` is injected — a frozen
|
|
207
|
+
* clock set from a bad string, or a clock whose `now()` throws, took the whole line with it. The
|
|
208
|
+
* same marker `serialise` gives an invalid `Date` in a FIELD, so one vocabulary covers both.
|
|
209
|
+
*/
|
|
210
|
+
function timestamp(clock: Clock): string {
|
|
211
|
+
try {
|
|
212
|
+
const at = clock.now();
|
|
213
|
+
if (at instanceof Date && !Number.isNaN(at.getTime())) return at.toISOString();
|
|
214
|
+
} catch {
|
|
215
|
+
// A clock that fights being read is exactly the moment a line is worth keeping.
|
|
216
|
+
}
|
|
217
|
+
return INVALID_DATE;
|
|
218
|
+
}
|
|
219
|
+
|
|
201
220
|
function envLevel(): LogLevel {
|
|
202
221
|
const raw = process.env['LOG_LEVEL'];
|
|
203
222
|
return raw !== undefined && (LOG_LEVELS as readonly string[]).includes(raw)
|
|
@@ -215,7 +234,7 @@ export function createLogger(options?: LoggerOptions): Logger {
|
|
|
215
234
|
function emit(lineLevel: LogLevel, message: string, fields?: LogFields): void {
|
|
216
235
|
if (LEVEL_WEIGHT[lineLevel] < threshold) return;
|
|
217
236
|
const line = {
|
|
218
|
-
ts: clock
|
|
237
|
+
ts: timestamp(clock),
|
|
219
238
|
level: lineLevel,
|
|
220
239
|
msg: message,
|
|
221
240
|
...redactFields(bound),
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
// Single responsibility: the OpenTelemetry-shaped metric DATA MODEL — what a point, a
|
|
2
|
+
// descriptor, a collection and an instrument's options are. No registry and no state: `metrics.ts`
|
|
3
|
+
// owns those, and a reader (`metrics-text.ts`, an exporter) needs the shapes without them.
|
|
4
|
+
|
|
5
|
+
import type { SpanResource } from './telemetry';
|
|
6
|
+
|
|
7
|
+
export type MetricKind = 'counter' | 'gauge' | 'histogram';
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Narrower than a span attribute on purpose: metric attributes become time-series labels, every
|
|
11
|
+
* distinct combination is a stored series, and an array label has no meaning in any exposition
|
|
12
|
+
* format. Keep the cardinality low — a user id here is an outage, and `maxSeries` is the ceiling
|
|
13
|
+
* that makes "keep it low" a mechanism instead of this sentence.
|
|
14
|
+
*/
|
|
15
|
+
export type MetricAttributeValue = string | number | boolean;
|
|
16
|
+
|
|
17
|
+
export interface MetricAttributes {
|
|
18
|
+
readonly [key: string]: MetricAttributeValue;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export interface MetricDescriptor {
|
|
22
|
+
readonly name: string;
|
|
23
|
+
readonly kind: MetricKind;
|
|
24
|
+
/** UCUM, as OTel spells it: `1`, `s`, `By`, `{request}`. */
|
|
25
|
+
readonly unit: string;
|
|
26
|
+
readonly description: string;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export interface MetricPoint {
|
|
30
|
+
readonly attributes: MetricAttributes;
|
|
31
|
+
/** Counter: cumulative sum since process start. Gauge: last value. Histogram: sum. */
|
|
32
|
+
readonly value: number;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export interface HistogramPoint extends MetricPoint {
|
|
36
|
+
readonly count: number;
|
|
37
|
+
readonly min: number;
|
|
38
|
+
readonly max: number;
|
|
39
|
+
/** Explicit upper bounds; `buckets` is one longer, the last being the `+Inf` overflow. */
|
|
40
|
+
readonly bounds: readonly number[];
|
|
41
|
+
readonly buckets: readonly number[];
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export interface ReadableMetric {
|
|
45
|
+
readonly descriptor: MetricDescriptor;
|
|
46
|
+
readonly points: readonly MetricPoint[];
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export interface MetricCollection {
|
|
50
|
+
/** Epoch milliseconds, from the configured clock. */
|
|
51
|
+
readonly at: number;
|
|
52
|
+
readonly resource: SpanResource;
|
|
53
|
+
readonly metrics: readonly ReadableMetric[];
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** The driver seam. OTLP, Prometheus remote-write or a vendor SDK all arrive as one of these. */
|
|
57
|
+
export interface MetricExporter {
|
|
58
|
+
export(collection: MetricCollection): void;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export interface Counter {
|
|
62
|
+
add(value?: number, attributes?: MetricAttributes): void;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export interface Gauge {
|
|
66
|
+
/** Set the current value. */
|
|
67
|
+
record(value: number, attributes?: MetricAttributes): void;
|
|
68
|
+
/** Move the current value — `+1` on connect, `-1` on disconnect. */
|
|
69
|
+
add(delta: number, attributes?: MetricAttributes): void;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
export interface Histogram {
|
|
73
|
+
record(value: number, attributes?: MetricAttributes): void;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
export interface InstrumentOptions {
|
|
77
|
+
readonly unit?: string | undefined;
|
|
78
|
+
readonly description?: string | undefined;
|
|
79
|
+
/**
|
|
80
|
+
* Distinct label sets this instrument may store. Past it every new set folds into one overflow
|
|
81
|
+
* series. Defaults to `DEFAULT_MAX_SERIES`; the first declaration of a name wins.
|
|
82
|
+
*/
|
|
83
|
+
readonly maxSeries?: number | undefined;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
export interface GaugeOptions extends InstrumentOptions {
|
|
87
|
+
/**
|
|
88
|
+
* Async instrument: read at collection time instead of being pushed. Never stale.
|
|
89
|
+
* Stated twice for one name with two different callbacks is `X_METRIC_NAME_INVALID`, not a
|
|
90
|
+
* silent win for the first — see `assertSameDeclaration`.
|
|
91
|
+
*/
|
|
92
|
+
readonly observe?: (() => number) | undefined;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
export interface HistogramOptions extends InstrumentOptions {
|
|
96
|
+
/**
|
|
97
|
+
* Explicit bucket boundaries, ascending. Defaults to the OTel latency-in-seconds set.
|
|
98
|
+
* Stated twice for one name with two different sets is `X_METRIC_NAME_INVALID`, not a silent
|
|
99
|
+
* win for the first — see `assertSameDeclaration`.
|
|
100
|
+
*/
|
|
101
|
+
readonly bounds?: readonly number[] | undefined;
|
|
102
|
+
}
|
package/src/metrics.ts
CHANGED
|
@@ -3,9 +3,44 @@
|
|
|
3
3
|
// always on, a no-op exporter by default, and the wire format supplied by a driver, never here.
|
|
4
4
|
|
|
5
5
|
import { type Clock, systemClock } from './clock';
|
|
6
|
+
import { renderThrowable } from './error-render';
|
|
6
7
|
import { type CodedErrorInit, UltimateError } from './errors';
|
|
7
8
|
import { logger } from './logger';
|
|
8
|
-
import
|
|
9
|
+
import type {
|
|
10
|
+
Counter,
|
|
11
|
+
Gauge,
|
|
12
|
+
GaugeOptions,
|
|
13
|
+
Histogram,
|
|
14
|
+
HistogramOptions,
|
|
15
|
+
InstrumentOptions,
|
|
16
|
+
MetricAttributes,
|
|
17
|
+
MetricCollection,
|
|
18
|
+
MetricDescriptor,
|
|
19
|
+
MetricExporter,
|
|
20
|
+
MetricKind,
|
|
21
|
+
MetricPoint,
|
|
22
|
+
} from './metrics-types';
|
|
23
|
+
import { serviceResource } from './telemetry';
|
|
24
|
+
|
|
25
|
+
// The data model is a module of its own; the public surface is unchanged, so nothing that imports
|
|
26
|
+
// a metric type from here has to learn a second path.
|
|
27
|
+
export type {
|
|
28
|
+
Counter,
|
|
29
|
+
Gauge,
|
|
30
|
+
GaugeOptions,
|
|
31
|
+
Histogram,
|
|
32
|
+
HistogramOptions,
|
|
33
|
+
HistogramPoint,
|
|
34
|
+
InstrumentOptions,
|
|
35
|
+
MetricAttributes,
|
|
36
|
+
MetricAttributeValue,
|
|
37
|
+
MetricCollection,
|
|
38
|
+
MetricDescriptor,
|
|
39
|
+
MetricExporter,
|
|
40
|
+
MetricKind,
|
|
41
|
+
MetricPoint,
|
|
42
|
+
ReadableMetric,
|
|
43
|
+
} from './metrics-types';
|
|
9
44
|
|
|
10
45
|
export class MetricNameInvalidError extends UltimateError {
|
|
11
46
|
static readonly code = 'X_METRIC_NAME_INVALID';
|
|
@@ -31,95 +66,6 @@ export class MetricCardinalityError extends UltimateError {
|
|
|
31
66
|
}
|
|
32
67
|
}
|
|
33
68
|
|
|
34
|
-
export type MetricKind = 'counter' | 'gauge' | 'histogram';
|
|
35
|
-
|
|
36
|
-
/**
|
|
37
|
-
* Narrower than a span attribute on purpose: metric attributes become time-series labels, every
|
|
38
|
-
* distinct combination is a stored series, and an array label has no meaning in any exposition
|
|
39
|
-
* format. Keep the cardinality low — a user id here is an outage, and `maxSeries` is the ceiling
|
|
40
|
-
* that makes "keep it low" a mechanism instead of this sentence.
|
|
41
|
-
*/
|
|
42
|
-
export type MetricAttributeValue = string | number | boolean;
|
|
43
|
-
|
|
44
|
-
export interface MetricAttributes {
|
|
45
|
-
readonly [key: string]: MetricAttributeValue;
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
export interface MetricDescriptor {
|
|
49
|
-
readonly name: string;
|
|
50
|
-
readonly kind: MetricKind;
|
|
51
|
-
/** UCUM, as OTel spells it: `1`, `s`, `By`, `{request}`. */
|
|
52
|
-
readonly unit: string;
|
|
53
|
-
readonly description: string;
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
export interface MetricPoint {
|
|
57
|
-
readonly attributes: MetricAttributes;
|
|
58
|
-
/** Counter: cumulative sum since process start. Gauge: last value. Histogram: sum. */
|
|
59
|
-
readonly value: number;
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
export interface HistogramPoint extends MetricPoint {
|
|
63
|
-
readonly count: number;
|
|
64
|
-
readonly min: number;
|
|
65
|
-
readonly max: number;
|
|
66
|
-
/** Explicit upper bounds; `buckets` is one longer, the last being the `+Inf` overflow. */
|
|
67
|
-
readonly bounds: readonly number[];
|
|
68
|
-
readonly buckets: readonly number[];
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
export interface ReadableMetric {
|
|
72
|
-
readonly descriptor: MetricDescriptor;
|
|
73
|
-
readonly points: readonly MetricPoint[];
|
|
74
|
-
}
|
|
75
|
-
|
|
76
|
-
export interface MetricCollection {
|
|
77
|
-
/** Epoch milliseconds, from the configured clock. */
|
|
78
|
-
readonly at: number;
|
|
79
|
-
readonly resource: SpanResource;
|
|
80
|
-
readonly metrics: readonly ReadableMetric[];
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
/** The driver seam. OTLP, Prometheus remote-write or a vendor SDK all arrive as one of these. */
|
|
84
|
-
export interface MetricExporter {
|
|
85
|
-
export(collection: MetricCollection): void;
|
|
86
|
-
}
|
|
87
|
-
|
|
88
|
-
export interface Counter {
|
|
89
|
-
add(value?: number, attributes?: MetricAttributes): void;
|
|
90
|
-
}
|
|
91
|
-
|
|
92
|
-
export interface Gauge {
|
|
93
|
-
/** Set the current value. */
|
|
94
|
-
record(value: number, attributes?: MetricAttributes): void;
|
|
95
|
-
/** Move the current value — `+1` on connect, `-1` on disconnect. */
|
|
96
|
-
add(delta: number, attributes?: MetricAttributes): void;
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
export interface Histogram {
|
|
100
|
-
record(value: number, attributes?: MetricAttributes): void;
|
|
101
|
-
}
|
|
102
|
-
|
|
103
|
-
export interface InstrumentOptions {
|
|
104
|
-
readonly unit?: string | undefined;
|
|
105
|
-
readonly description?: string | undefined;
|
|
106
|
-
/**
|
|
107
|
-
* Distinct label sets this instrument may store. Past it every new set folds into one overflow
|
|
108
|
-
* series. Defaults to `DEFAULT_MAX_SERIES`; the first declaration of a name wins.
|
|
109
|
-
*/
|
|
110
|
-
readonly maxSeries?: number | undefined;
|
|
111
|
-
}
|
|
112
|
-
|
|
113
|
-
export interface GaugeOptions extends InstrumentOptions {
|
|
114
|
-
/** Async instrument: read at collection time instead of being pushed. Never stale. */
|
|
115
|
-
readonly observe?: (() => number) | undefined;
|
|
116
|
-
}
|
|
117
|
-
|
|
118
|
-
export interface HistogramOptions extends InstrumentOptions {
|
|
119
|
-
/** Explicit bucket boundaries, ascending. Defaults to the OTel latency-in-seconds set. */
|
|
120
|
-
readonly bounds?: readonly number[] | undefined;
|
|
121
|
-
}
|
|
122
|
-
|
|
123
69
|
/**
|
|
124
70
|
* The per-instrument series ceiling. 2000 is roomy for a bounded label set — every route pattern
|
|
125
71
|
* times every status class times every method — and small enough that the process notices an
|
|
@@ -196,6 +142,8 @@ interface Instrument {
|
|
|
196
142
|
readonly maxSeries: number;
|
|
197
143
|
/** Reported once. A cardinality blow-up is one bug, not one log line per call. */
|
|
198
144
|
overflowed: boolean;
|
|
145
|
+
/** Reported once, for the same reason: a scrape every 15s must not become a log every 15s. */
|
|
146
|
+
observeFailed: boolean;
|
|
199
147
|
}
|
|
200
148
|
|
|
201
149
|
const instruments = new Map<string, Instrument>();
|
|
@@ -222,6 +170,7 @@ export function resetMetrics(): void {
|
|
|
222
170
|
for (const instrument of instruments.values()) {
|
|
223
171
|
instrument.series.clear();
|
|
224
172
|
instrument.overflowed = false;
|
|
173
|
+
instrument.observeFailed = false;
|
|
225
174
|
}
|
|
226
175
|
}
|
|
227
176
|
|
|
@@ -263,6 +212,7 @@ function declare(name: string, kind: MetricKind, options: GaugeOptions & Histogr
|
|
|
263
212
|
meta: { name, declared: existing.descriptor.kind, requested: kind },
|
|
264
213
|
});
|
|
265
214
|
}
|
|
215
|
+
assertSameDeclaration(name, existing, options);
|
|
266
216
|
return existing;
|
|
267
217
|
}
|
|
268
218
|
const maxSeries = options.maxSeries ?? DEFAULT_MAX_SERIES;
|
|
@@ -285,11 +235,45 @@ function declare(name: string, kind: MetricKind, options: GaugeOptions & Histogr
|
|
|
285
235
|
observe: options.observe,
|
|
286
236
|
maxSeries,
|
|
287
237
|
overflowed: false,
|
|
238
|
+
observeFailed: false,
|
|
288
239
|
};
|
|
289
240
|
instruments.set(name, instrument);
|
|
290
241
|
return instrument;
|
|
291
242
|
}
|
|
292
243
|
|
|
244
|
+
/**
|
|
245
|
+
* A second declaration that STATES a different shape is refused. The first declaration wins, so a
|
|
246
|
+
* second `histogram(name, { bounds })` recorded into buckets another module chose and a second
|
|
247
|
+
* `gauge(name, { observe })` was collected through the first module's observer — silently, in both
|
|
248
|
+
* cases, which is the whole failure. An OMITTED option is not a conflict: `gauge(name)` is how a
|
|
249
|
+
* module takes a handle on an instrument someone else declared, and `maxSeries` keeps its shipped
|
|
250
|
+
* first-declaration-wins rule because it decides a ceiling rather than what gets recorded.
|
|
251
|
+
*/
|
|
252
|
+
function assertSameDeclaration(
|
|
253
|
+
name: string,
|
|
254
|
+
existing: Instrument,
|
|
255
|
+
options: GaugeOptions & HistogramOptions,
|
|
256
|
+
): void {
|
|
257
|
+
const { bounds, observe } = options;
|
|
258
|
+
if (bounds !== undefined && !sameBounds(existing.bounds, bounds)) {
|
|
259
|
+
throw new MetricNameInvalidError({
|
|
260
|
+
cause: `"${name}" is already declared with bounds [${existing.bounds.join(', ')}] and is redeclared with [${bounds.join(', ')}]; the first declaration wins, so the second set would never be used`,
|
|
261
|
+
fix: `declare "${name}" once and export the handle — import it where you record — or give the second instrument its own name`,
|
|
262
|
+
meta: { name, declared: existing.bounds.join(','), requested: bounds.join(',') },
|
|
263
|
+
});
|
|
264
|
+
}
|
|
265
|
+
if (observe !== undefined && observe !== existing.observe) {
|
|
266
|
+
throw new MetricNameInvalidError({
|
|
267
|
+
cause: `"${name}" is already declared with an observe() callback and is redeclared with a different one; the first declaration wins, so the second callback would never be read`,
|
|
268
|
+
fix: `declare "${name}" once and export the handle — import it where you read — or give the second gauge its own name`,
|
|
269
|
+
meta: { name },
|
|
270
|
+
});
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
const sameBounds = (left: readonly number[], right: readonly number[]): boolean =>
|
|
275
|
+
left.length === right.length && left.every((bound, index) => bound === right[index]);
|
|
276
|
+
|
|
293
277
|
/**
|
|
294
278
|
* Reported through the logger rather than thrown: the call site is `orderCounter.add(1, …)` deep
|
|
295
279
|
* inside a request, and killing that request would turn a metrics bug into a user-visible outage
|
|
@@ -308,6 +292,26 @@ function reportOverflow(instrument: Instrument): void {
|
|
|
308
292
|
logger.error(error.format(), { code: error.code, metric: name });
|
|
309
293
|
}
|
|
310
294
|
|
|
295
|
+
/**
|
|
296
|
+
* Reported through the logger for `reportOverflow`'s reason and once for the same one — a scrape
|
|
297
|
+
* runs on a timer, so a permanently broken observer would otherwise write a log line every
|
|
298
|
+
* interval forever. A recurrence after the first is therefore silent by design; the missing series
|
|
299
|
+
* is the signal that outlives the line.
|
|
300
|
+
*/
|
|
301
|
+
function reportObserveFailure(instrument: Instrument, thrown: unknown): void {
|
|
302
|
+
if (instrument.observeFailed) return;
|
|
303
|
+
instrument.observeFailed = true;
|
|
304
|
+
const { name, kind } = instrument.descriptor;
|
|
305
|
+
const error = new MetricValueInvalidError({
|
|
306
|
+
// `renderThrowable`, never `${thrown}`: the value is whatever the app's callback threw, and a
|
|
307
|
+
// `.message` read on it is the one that throws where there is nothing left to answer with.
|
|
308
|
+
cause: `the observe() callback of ${name} did not produce a value: ${renderThrowable(thrown)}; this instrument contributes no point until it does`,
|
|
309
|
+
fix: `make the observe() callback of ${name} total — return a finite number when the resource it reads is gone, e.g. ${kind}('${name}', { observe: () => pool?.size ?? 0 })`,
|
|
310
|
+
meta: { metric: name },
|
|
311
|
+
});
|
|
312
|
+
logger.error(error.format(), { code: error.code, metric: name });
|
|
313
|
+
}
|
|
314
|
+
|
|
311
315
|
function createSeries(instrument: Instrument, key: string, attributes: MetricAttributes): Series {
|
|
312
316
|
const created: Series = {
|
|
313
317
|
attributes,
|
|
@@ -391,8 +395,19 @@ export function histogram(name: string, options?: HistogramOptions): Histogram {
|
|
|
391
395
|
}
|
|
392
396
|
|
|
393
397
|
function pointsOf(instrument: Instrument): readonly MetricPoint[] {
|
|
394
|
-
|
|
395
|
-
|
|
398
|
+
const observe = instrument.observe;
|
|
399
|
+
if (observe !== undefined) {
|
|
400
|
+
try {
|
|
401
|
+
return [{ attributes: {}, value: finite(instrument.descriptor.name, observe()) }];
|
|
402
|
+
} catch (thrown) {
|
|
403
|
+
// The callback is the app's, run at SCRAPE time with no call site to blame: `() => pool.size`
|
|
404
|
+
// after a drain throws, and an unguarded read here took every other instrument down with it
|
|
405
|
+
// — /metrics 500s, `http_requests_total` goes invisible, and `startMetricExport`'s timer
|
|
406
|
+
// callback raises where nothing can catch it. One hostile observer costs its own point only,
|
|
407
|
+
// the same degradation `readinessChecks()` and the logger's per-key walk already make.
|
|
408
|
+
reportObserveFailure(instrument, thrown);
|
|
409
|
+
return [];
|
|
410
|
+
}
|
|
396
411
|
}
|
|
397
412
|
return [...instrument.series.values()].map((series) =>
|
|
398
413
|
instrument.descriptor.kind === 'histogram'
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
// Single responsibility: a `MetricExporter` that POSTs OTLP/HTTP JSON to a collector. No batching
|
|
2
2
|
// — `collectMetrics()` already produces one whole snapshot per tick, so a tick is a request.
|
|
3
3
|
|
|
4
|
+
import { renderThrowable } from './error-render';
|
|
5
|
+
import { logger } from './logger';
|
|
4
6
|
import type {
|
|
5
7
|
HistogramPoint,
|
|
6
8
|
MetricCollection,
|
|
@@ -124,10 +126,31 @@ export function otlpMetricExporter(options: OtlpMetricExporterOptions = {}): Otl
|
|
|
124
126
|
return {
|
|
125
127
|
export(collection: MetricCollection): void {
|
|
126
128
|
startedAtMs ??= collection.at;
|
|
127
|
-
|
|
129
|
+
let body: string;
|
|
130
|
+
try {
|
|
131
|
+
// `export` is called from a timer, not awaited by anyone, so this throw had nowhere to go
|
|
132
|
+
// but into the metric loop that called it: `MetricAttributeValue` is a compile-time claim,
|
|
133
|
+
// and an attribute the app spelled as an object or a bigint reaches `otlpAttributes` as a
|
|
134
|
+
// TypeError. Dropped with a line, the same degradation `postOtlp` already applies to a
|
|
135
|
+
// collector that is down — telemetry is best-effort and must never end the process.
|
|
136
|
+
body = JSON.stringify(otlpMetricsRequest(collection, startedAtMs));
|
|
137
|
+
} catch (failure) {
|
|
138
|
+
logger.warn('otlp metric snapshot dropped', {
|
|
139
|
+
url,
|
|
140
|
+
metrics: collection.metrics.length,
|
|
141
|
+
error: renderThrowable(failure),
|
|
142
|
+
});
|
|
143
|
+
return;
|
|
144
|
+
}
|
|
128
145
|
// Chained, so a slow collector cannot make two snapshots arrive out of order and turn a
|
|
129
|
-
// cumulative counter into an apparent reset.
|
|
130
|
-
|
|
146
|
+
// cumulative counter into an apparent reset. Chained on a SETTLED shadow, for the reason
|
|
147
|
+
// `otlp-span-exporter.ts` spells out: a chain that carries a rejection forward stops calling
|
|
148
|
+
// `postOtlp` for the life of the process, in silence.
|
|
149
|
+
const settled = inflight.then(
|
|
150
|
+
() => undefined,
|
|
151
|
+
() => undefined,
|
|
152
|
+
);
|
|
153
|
+
inflight = settled.then(() => postOtlp({ url, headers, body, timeoutMs, fetch: send }));
|
|
131
154
|
},
|
|
132
155
|
flush(): Promise<void> {
|
|
133
156
|
return inflight;
|