@gr8ful/spf 0.17.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/skill/references/config.md +6 -3
- package/assets/skill/references/observability.md +115 -1
- package/dist/cli/index.js +6 -0
- package/dist/core/agent_cc.d.ts +11 -0
- package/dist/core/agent_cc.js +39 -1
- package/dist/core/agent_flue.js +26 -0
- package/dist/core/agent_opencode.d.ts +105 -3
- package/dist/core/agent_opencode.js +169 -17
- package/dist/core/agents.d.ts +6 -0
- package/dist/core/agents.js +22 -0
- package/dist/core/data_types.d.ts +60 -0
- package/dist/core/data_types.js +69 -0
- package/dist/core/otel.d.ts +182 -48
- package/dist/core/otel.js +373 -158
- package/dist/core/otel_metrics.d.ts +127 -0
- package/dist/core/otel_metrics.js +221 -0
- package/dist/core/otel_propagation.d.ts +159 -0
- package/dist/core/otel_propagation.js +225 -0
- package/package.json +14 -1
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* OpenTelemetry metrics export — a PROCESS-scoped sibling to `otel.ts`'s
|
|
3
|
+
* per-run span exporter, built on the real `@opentelemetry/sdk-metrics` +
|
|
4
|
+
* `@opentelemetry/exporter-metrics-otlp-http` (v1's header explained why
|
|
5
|
+
* metrics were deliberately cut from the hand-rolled encoder: "the OTLP
|
|
6
|
+
* metrics data model — temporality, monotonicity, cumulative-vs-delta — is
|
|
7
|
+
* exactly where a hand-rolled encoder produces numbers a backend silently
|
|
8
|
+
* misreads". That risk is why this module exists only now that a REAL SDK
|
|
9
|
+
* owns the encoding, never before).
|
|
10
|
+
*
|
|
11
|
+
* PROCESS-scoped, not per-run — the one significant lifecycle difference from
|
|
12
|
+
* `otel.ts`. `spf watch`'s daemon loop runs many sessions in one process, one
|
|
13
|
+
* per claimed issue; a MeterProvider created per-run would repeat the same
|
|
14
|
+
* unbounded-growth mistake `otel.ts`'s RUN-SCOPED CLEANUP note (#26) already
|
|
15
|
+
* fixed once for spans. There is exactly one `MeterProvider` for the life of
|
|
16
|
+
* the `spf` process, created lazily on first use and torn down once, from
|
|
17
|
+
* `src/cli/index.ts`'s existing `finally` block, alongside `otel.flushAll()`.
|
|
18
|
+
*
|
|
19
|
+
* SAME ACTIVATION GATE AS TRACES: `observability.otel.endpoint` must be set —
|
|
20
|
+
* no ambient `OTEL_EXPORTER_OTLP_*` env var can turn this on (see `otel.ts`'s
|
|
21
|
+
* EXPLICIT CONFIG ONLY). `observability.otel.metrics: false` additionally
|
|
22
|
+
* opts OUT of metrics while leaving trace export on; there is no way to have
|
|
23
|
+
* metrics without traces, since the gate is the trace endpoint's presence.
|
|
24
|
+
*
|
|
25
|
+
* NEVER THROWS. Every public method here follows the same discipline
|
|
26
|
+
* `otel.ts` documents at its own top: a metrics failure must never surface as
|
|
27
|
+
* a caller's exception, so every instrument call is wrapped and any error is
|
|
28
|
+
* logged at most once and then swallowed.
|
|
29
|
+
*
|
|
30
|
+
* INSTRUMENTS:
|
|
31
|
+
* spf.tokens (Counter, unit "token") attrs: kind (input|output|
|
|
32
|
+
* cache_read|cache_write), agent, model
|
|
33
|
+
* spf.cost_usd (Counter, unit "USD") attrs: agent, model
|
|
34
|
+
* spf.phase.duration(Histogram, unit "s") attrs: kind, owner, status
|
|
35
|
+
* spf.gate.result (Counter) attrs: gate, result (pass|fail)
|
|
36
|
+
* spf.agent.calls (Counter) attrs: agent, model, coding_agent
|
|
37
|
+
* spf.otel.dropped_spans / spf.otel.dropped_span_events (Counter) — the
|
|
38
|
+
* v1 "stamp it on the final flush's resource attribute" hack has no home in
|
|
39
|
+
* the real SDK (a `Resource` is immutable per exporter instance); these
|
|
40
|
+
* replace it, recorded once per exporter's lifetime, same as the warn log.
|
|
41
|
+
*
|
|
42
|
+
* WIRE TRANSPORT mirrors `otel.ts`'s trace exporter: JSON-encoded OTLP/HTTP
|
|
43
|
+
* (`OTLPMetricExporter`'s default), `keepAlive: false` so a lingering
|
|
44
|
+
* keep-alive socket can never be the reason `spf` fails to exit, aimed at
|
|
45
|
+
* `resolveMetricsUrl(otel.endpoint)` — the same collector host, `/v1/metrics`
|
|
46
|
+
* instead of `/v1/traces` (or the given path verbatim, for a collector
|
|
47
|
+
* fronted by a router that doesn't use the standard suffixes at all).
|
|
48
|
+
*/
|
|
49
|
+
import { type PushMetricExporter } from "@opentelemetry/sdk-metrics";
|
|
50
|
+
import { type SFConfig } from "./data_types.ts";
|
|
51
|
+
/**
|
|
52
|
+
* A bare origin gets `/v1/metrics` appended; an endpoint that already ends in
|
|
53
|
+
* `/v1/traces` (the documented shape for `observability.otel.endpoint`) has
|
|
54
|
+
* that suffix swapped for `/v1/metrics` — same collector host, the metrics
|
|
55
|
+
* signal's own path. Anything else (a router/proxy path that doesn't use the
|
|
56
|
+
* standard suffixes) is left exactly as given, matching `otel.ts`'s
|
|
57
|
+
* `resolveTracesUrl`'s "the operator's URL is not ours to rewrite" rule.
|
|
58
|
+
*/
|
|
59
|
+
export declare function resolveMetricsUrl(endpoint: string): string;
|
|
60
|
+
export interface OtelMetricsInit {
|
|
61
|
+
endpoint: string;
|
|
62
|
+
headers?: Record<string, string>;
|
|
63
|
+
serviceName: string;
|
|
64
|
+
/** Injectable for tests; defaults to stderr. */
|
|
65
|
+
log?: (message: string) => void;
|
|
66
|
+
/** Injectable for tests — swap in an `InMemoryMetricExporter`/a short interval without touching the network. */
|
|
67
|
+
exporter?: PushMetricExporter;
|
|
68
|
+
exportIntervalMillis?: number;
|
|
69
|
+
}
|
|
70
|
+
export declare class OtelMetrics {
|
|
71
|
+
private readonly provider;
|
|
72
|
+
private readonly log;
|
|
73
|
+
private loggedFailure;
|
|
74
|
+
private readonly tokens;
|
|
75
|
+
private readonly cost;
|
|
76
|
+
private readonly phaseDuration;
|
|
77
|
+
private readonly gateResult;
|
|
78
|
+
private readonly agentCalls;
|
|
79
|
+
private readonly droppedSpans;
|
|
80
|
+
private readonly droppedSpanEvents;
|
|
81
|
+
constructor(init: OtelMetricsInit);
|
|
82
|
+
recordTokens(kind: "input" | "output" | "cache_read" | "cache_write", value: number, attrs: {
|
|
83
|
+
agent: string;
|
|
84
|
+
model: string;
|
|
85
|
+
}): void;
|
|
86
|
+
recordCost(value: number, attrs: {
|
|
87
|
+
agent: string;
|
|
88
|
+
model: string;
|
|
89
|
+
}): void;
|
|
90
|
+
recordPhaseDuration(seconds: number, attrs: {
|
|
91
|
+
kind: string;
|
|
92
|
+
owner: string;
|
|
93
|
+
status: string;
|
|
94
|
+
}): void;
|
|
95
|
+
recordGateResult(gate: string, passed: boolean): void;
|
|
96
|
+
recordAgentCall(attrs: {
|
|
97
|
+
agent: string;
|
|
98
|
+
model: string;
|
|
99
|
+
codingAgent: string;
|
|
100
|
+
}): void;
|
|
101
|
+
recordDroppedSpans(n: number): void;
|
|
102
|
+
recordDroppedSpanEvents(n: number): void;
|
|
103
|
+
/** Force an out-of-band export. Tests use this instead of waiting on `exportIntervalMillis`. Never throws. */
|
|
104
|
+
forceFlush(): Promise<void>;
|
|
105
|
+
/** Never throws. */
|
|
106
|
+
shutdown(budgetMs?: number): Promise<void>;
|
|
107
|
+
private guard;
|
|
108
|
+
private logFailureOnce;
|
|
109
|
+
}
|
|
110
|
+
/**
|
|
111
|
+
* Resolve (once per process) the shared metrics handle from
|
|
112
|
+
* `cfg.observability.otel`, or `null` when metrics are off — absent
|
|
113
|
+
* `endpoint` (same gate as traces) or `metrics: false`. Every call after the
|
|
114
|
+
* first, for the life of the process, returns the SAME instance regardless
|
|
115
|
+
* of what `cfg` says (this is the "exactly once, never per-issue" contract
|
|
116
|
+
* `spf watch`'s daemon loop needs — see the module header) — call
|
|
117
|
+
* `resetOtelMetricsForTest()` between test cases that need a fresh one.
|
|
118
|
+
*/
|
|
119
|
+
export declare function resolveOtelMetrics(cfg: SFConfig, log?: (message: string) => void): OtelMetrics | null;
|
|
120
|
+
/**
|
|
121
|
+
* Shut down the shared MeterProvider, if one was ever created. A no-op when
|
|
122
|
+
* metrics were never configured. Called once, from `src/cli/index.ts`'s
|
|
123
|
+
* `finally`, alongside `otel.flushAll()`. Never throws.
|
|
124
|
+
*/
|
|
125
|
+
export declare function shutdownOtelMetrics(budgetMs?: number): Promise<void>;
|
|
126
|
+
/** Tests only: forget the shared instance so cases cannot leak into each other. */
|
|
127
|
+
export declare function resetOtelMetricsForTest(): void;
|
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* OpenTelemetry metrics export — a PROCESS-scoped sibling to `otel.ts`'s
|
|
3
|
+
* per-run span exporter, built on the real `@opentelemetry/sdk-metrics` +
|
|
4
|
+
* `@opentelemetry/exporter-metrics-otlp-http` (v1's header explained why
|
|
5
|
+
* metrics were deliberately cut from the hand-rolled encoder: "the OTLP
|
|
6
|
+
* metrics data model — temporality, monotonicity, cumulative-vs-delta — is
|
|
7
|
+
* exactly where a hand-rolled encoder produces numbers a backend silently
|
|
8
|
+
* misreads". That risk is why this module exists only now that a REAL SDK
|
|
9
|
+
* owns the encoding, never before).
|
|
10
|
+
*
|
|
11
|
+
* PROCESS-scoped, not per-run — the one significant lifecycle difference from
|
|
12
|
+
* `otel.ts`. `spf watch`'s daemon loop runs many sessions in one process, one
|
|
13
|
+
* per claimed issue; a MeterProvider created per-run would repeat the same
|
|
14
|
+
* unbounded-growth mistake `otel.ts`'s RUN-SCOPED CLEANUP note (#26) already
|
|
15
|
+
* fixed once for spans. There is exactly one `MeterProvider` for the life of
|
|
16
|
+
* the `spf` process, created lazily on first use and torn down once, from
|
|
17
|
+
* `src/cli/index.ts`'s existing `finally` block, alongside `otel.flushAll()`.
|
|
18
|
+
*
|
|
19
|
+
* SAME ACTIVATION GATE AS TRACES: `observability.otel.endpoint` must be set —
|
|
20
|
+
* no ambient `OTEL_EXPORTER_OTLP_*` env var can turn this on (see `otel.ts`'s
|
|
21
|
+
* EXPLICIT CONFIG ONLY). `observability.otel.metrics: false` additionally
|
|
22
|
+
* opts OUT of metrics while leaving trace export on; there is no way to have
|
|
23
|
+
* metrics without traces, since the gate is the trace endpoint's presence.
|
|
24
|
+
*
|
|
25
|
+
* NEVER THROWS. Every public method here follows the same discipline
|
|
26
|
+
* `otel.ts` documents at its own top: a metrics failure must never surface as
|
|
27
|
+
* a caller's exception, so every instrument call is wrapped and any error is
|
|
28
|
+
* logged at most once and then swallowed.
|
|
29
|
+
*
|
|
30
|
+
* INSTRUMENTS:
|
|
31
|
+
* spf.tokens (Counter, unit "token") attrs: kind (input|output|
|
|
32
|
+
* cache_read|cache_write), agent, model
|
|
33
|
+
* spf.cost_usd (Counter, unit "USD") attrs: agent, model
|
|
34
|
+
* spf.phase.duration(Histogram, unit "s") attrs: kind, owner, status
|
|
35
|
+
* spf.gate.result (Counter) attrs: gate, result (pass|fail)
|
|
36
|
+
* spf.agent.calls (Counter) attrs: agent, model, coding_agent
|
|
37
|
+
* spf.otel.dropped_spans / spf.otel.dropped_span_events (Counter) — the
|
|
38
|
+
* v1 "stamp it on the final flush's resource attribute" hack has no home in
|
|
39
|
+
* the real SDK (a `Resource` is immutable per exporter instance); these
|
|
40
|
+
* replace it, recorded once per exporter's lifetime, same as the warn log.
|
|
41
|
+
*
|
|
42
|
+
* WIRE TRANSPORT mirrors `otel.ts`'s trace exporter: JSON-encoded OTLP/HTTP
|
|
43
|
+
* (`OTLPMetricExporter`'s default), `keepAlive: false` so a lingering
|
|
44
|
+
* keep-alive socket can never be the reason `spf` fails to exit, aimed at
|
|
45
|
+
* `resolveMetricsUrl(otel.endpoint)` — the same collector host, `/v1/metrics`
|
|
46
|
+
* instead of `/v1/traces` (or the given path verbatim, for a collector
|
|
47
|
+
* fronted by a router that doesn't use the standard suffixes at all).
|
|
48
|
+
*/
|
|
49
|
+
import { OTLPMetricExporter } from "@opentelemetry/exporter-metrics-otlp-http";
|
|
50
|
+
import { resourceFromAttributes } from "@opentelemetry/resources";
|
|
51
|
+
import { MeterProvider, PeriodicExportingMetricReader } from "@opentelemetry/sdk-metrics";
|
|
52
|
+
import { applyOtelEnvSupplement } from "./data_types.js";
|
|
53
|
+
/** Never emits on its own — `forceFlush()`/tests drive collection explicitly, so a process never waits `exportIntervalMillis` for its final data to go out. */
|
|
54
|
+
const EXPORT_INTERVAL_MS = 60_000;
|
|
55
|
+
const EXPORT_TIMEOUT_MS = 2_000;
|
|
56
|
+
/**
|
|
57
|
+
* A bare origin gets `/v1/metrics` appended; an endpoint that already ends in
|
|
58
|
+
* `/v1/traces` (the documented shape for `observability.otel.endpoint`) has
|
|
59
|
+
* that suffix swapped for `/v1/metrics` — same collector host, the metrics
|
|
60
|
+
* signal's own path. Anything else (a router/proxy path that doesn't use the
|
|
61
|
+
* standard suffixes) is left exactly as given, matching `otel.ts`'s
|
|
62
|
+
* `resolveTracesUrl`'s "the operator's URL is not ours to rewrite" rule.
|
|
63
|
+
*/
|
|
64
|
+
export function resolveMetricsUrl(endpoint) {
|
|
65
|
+
let url;
|
|
66
|
+
try {
|
|
67
|
+
url = new URL(endpoint);
|
|
68
|
+
}
|
|
69
|
+
catch {
|
|
70
|
+
return endpoint;
|
|
71
|
+
}
|
|
72
|
+
if (url.pathname === "" || url.pathname === "/") {
|
|
73
|
+
url.pathname = "/v1/metrics";
|
|
74
|
+
return url.toString();
|
|
75
|
+
}
|
|
76
|
+
if (url.pathname.endsWith("/v1/traces")) {
|
|
77
|
+
url.pathname = url.pathname.slice(0, -"/v1/traces".length) + "/v1/metrics";
|
|
78
|
+
return url.toString();
|
|
79
|
+
}
|
|
80
|
+
return url.toString();
|
|
81
|
+
}
|
|
82
|
+
export class OtelMetrics {
|
|
83
|
+
provider;
|
|
84
|
+
log;
|
|
85
|
+
loggedFailure = false;
|
|
86
|
+
tokens;
|
|
87
|
+
cost;
|
|
88
|
+
phaseDuration;
|
|
89
|
+
gateResult;
|
|
90
|
+
agentCalls;
|
|
91
|
+
droppedSpans;
|
|
92
|
+
droppedSpanEvents;
|
|
93
|
+
constructor(init) {
|
|
94
|
+
this.log = init.log ?? ((m) => console.error(m));
|
|
95
|
+
const exporter = init.exporter ??
|
|
96
|
+
new OTLPMetricExporter({
|
|
97
|
+
url: resolveMetricsUrl(init.endpoint),
|
|
98
|
+
headers: init.headers,
|
|
99
|
+
timeoutMillis: EXPORT_TIMEOUT_MS,
|
|
100
|
+
keepAlive: false,
|
|
101
|
+
});
|
|
102
|
+
const reader = new PeriodicExportingMetricReader({
|
|
103
|
+
exporter,
|
|
104
|
+
exportIntervalMillis: init.exportIntervalMillis ?? EXPORT_INTERVAL_MS,
|
|
105
|
+
exportTimeoutMillis: EXPORT_TIMEOUT_MS,
|
|
106
|
+
});
|
|
107
|
+
this.provider = new MeterProvider({
|
|
108
|
+
resource: resourceFromAttributes({ "service.name": init.serviceName }),
|
|
109
|
+
readers: [reader],
|
|
110
|
+
});
|
|
111
|
+
const meter = this.provider.getMeter("spf", "1");
|
|
112
|
+
this.tokens = meter.createCounter("spf.tokens", { unit: "token", description: "Tokens spent per agent call, by kind" });
|
|
113
|
+
this.cost = meter.createCounter("spf.cost_usd", { unit: "USD", description: "Dollar cost spent per agent call" });
|
|
114
|
+
this.phaseDuration = meter.createHistogram("spf.phase.duration", { unit: "s", description: "Phase wall-clock duration" });
|
|
115
|
+
this.gateResult = meter.createCounter("spf.gate.result", { description: "Gate verdicts, pass or fail" });
|
|
116
|
+
this.agentCalls = meter.createCounter("spf.agent.calls", { description: "Agent calls completed" });
|
|
117
|
+
this.droppedSpans = meter.createCounter("spf.otel.dropped_spans", { description: "Spans dropped by the bounded queue (see otel.ts)" });
|
|
118
|
+
this.droppedSpanEvents = meter.createCounter("spf.otel.dropped_span_events", { description: "Span events dropped by the per-phase cap" });
|
|
119
|
+
}
|
|
120
|
+
recordTokens(kind, value, attrs) {
|
|
121
|
+
this.guard(() => this.tokens.add(value, { kind, agent: attrs.agent, model: attrs.model }));
|
|
122
|
+
}
|
|
123
|
+
recordCost(value, attrs) {
|
|
124
|
+
this.guard(() => this.cost.add(value, { agent: attrs.agent, model: attrs.model }));
|
|
125
|
+
}
|
|
126
|
+
recordPhaseDuration(seconds, attrs) {
|
|
127
|
+
this.guard(() => this.phaseDuration.record(seconds, { kind: attrs.kind, owner: attrs.owner, status: attrs.status }));
|
|
128
|
+
}
|
|
129
|
+
recordGateResult(gate, passed) {
|
|
130
|
+
this.guard(() => this.gateResult.add(1, { gate, result: passed ? "pass" : "fail" }));
|
|
131
|
+
}
|
|
132
|
+
recordAgentCall(attrs) {
|
|
133
|
+
this.guard(() => this.agentCalls.add(1, { agent: attrs.agent, model: attrs.model, coding_agent: attrs.codingAgent }));
|
|
134
|
+
}
|
|
135
|
+
recordDroppedSpans(n) {
|
|
136
|
+
this.guard(() => this.droppedSpans.add(n));
|
|
137
|
+
}
|
|
138
|
+
recordDroppedSpanEvents(n) {
|
|
139
|
+
this.guard(() => this.droppedSpanEvents.add(n));
|
|
140
|
+
}
|
|
141
|
+
/** Force an out-of-band export. Tests use this instead of waiting on `exportIntervalMillis`. Never throws. */
|
|
142
|
+
async forceFlush() {
|
|
143
|
+
try {
|
|
144
|
+
await this.provider.forceFlush();
|
|
145
|
+
}
|
|
146
|
+
catch (error) {
|
|
147
|
+
this.logFailureOnce(error);
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
/** Never throws. */
|
|
151
|
+
async shutdown(budgetMs = EXPORT_TIMEOUT_MS) {
|
|
152
|
+
let deadline = null;
|
|
153
|
+
const budget = new Promise((resolve) => {
|
|
154
|
+
deadline = setTimeout(resolve, budgetMs);
|
|
155
|
+
deadline.unref?.();
|
|
156
|
+
});
|
|
157
|
+
try {
|
|
158
|
+
await Promise.race([this.provider.shutdown(), budget]);
|
|
159
|
+
}
|
|
160
|
+
catch (error) {
|
|
161
|
+
this.logFailureOnce(error);
|
|
162
|
+
}
|
|
163
|
+
finally {
|
|
164
|
+
if (deadline)
|
|
165
|
+
clearTimeout(deadline);
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
guard(fn) {
|
|
169
|
+
try {
|
|
170
|
+
fn();
|
|
171
|
+
}
|
|
172
|
+
catch (error) {
|
|
173
|
+
this.logFailureOnce(error);
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
logFailureOnce(error) {
|
|
177
|
+
if (this.loggedFailure)
|
|
178
|
+
return;
|
|
179
|
+
this.loggedFailure = true;
|
|
180
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
181
|
+
this.log(`spf: otel metrics failed (${message}) — metrics for this process are incomplete; runs are unaffected`);
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
// ── module-level lifecycle: ONE MeterProvider for the life of the process ──
|
|
185
|
+
let SHARED; // undefined = not yet resolved; null = resolved-and-off
|
|
186
|
+
/**
|
|
187
|
+
* Resolve (once per process) the shared metrics handle from
|
|
188
|
+
* `cfg.observability.otel`, or `null` when metrics are off — absent
|
|
189
|
+
* `endpoint` (same gate as traces) or `metrics: false`. Every call after the
|
|
190
|
+
* first, for the life of the process, returns the SAME instance regardless
|
|
191
|
+
* of what `cfg` says (this is the "exactly once, never per-issue" contract
|
|
192
|
+
* `spf watch`'s daemon loop needs — see the module header) — call
|
|
193
|
+
* `resetOtelMetricsForTest()` between test cases that need a fresh one.
|
|
194
|
+
*/
|
|
195
|
+
export function resolveOtelMetrics(cfg, log) {
|
|
196
|
+
if (SHARED !== undefined)
|
|
197
|
+
return SHARED;
|
|
198
|
+
const rawOtel = cfg.observability.otel;
|
|
199
|
+
if (!rawOtel || !rawOtel.endpoint || rawOtel.metrics === false) {
|
|
200
|
+
SHARED = null;
|
|
201
|
+
return null;
|
|
202
|
+
}
|
|
203
|
+
const otel = applyOtelEnvSupplement(rawOtel);
|
|
204
|
+
SHARED = new OtelMetrics({ endpoint: otel.endpoint, headers: otel.headers, serviceName: otel.service_name || "spf", log });
|
|
205
|
+
return SHARED;
|
|
206
|
+
}
|
|
207
|
+
/**
|
|
208
|
+
* Shut down the shared MeterProvider, if one was ever created. A no-op when
|
|
209
|
+
* metrics were never configured. Called once, from `src/cli/index.ts`'s
|
|
210
|
+
* `finally`, alongside `otel.flushAll()`. Never throws.
|
|
211
|
+
*/
|
|
212
|
+
export async function shutdownOtelMetrics(budgetMs) {
|
|
213
|
+
const shared = SHARED;
|
|
214
|
+
SHARED = undefined;
|
|
215
|
+
if (shared)
|
|
216
|
+
await shared.shutdown(budgetMs);
|
|
217
|
+
}
|
|
218
|
+
/** Tests only: forget the shared instance so cases cannot leak into each other. */
|
|
219
|
+
export function resetOtelMetricsForTest() {
|
|
220
|
+
SHARED = undefined;
|
|
221
|
+
}
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Outbound trace-context propagation for `coding_agent: flue` — the
|
|
3
|
+
* "best-effort" half of the two propagation paths this repo's OTel spike
|
|
4
|
+
* documented (`claude_code` gets `agent_cc.ts`'s single `spawn()` choke
|
|
5
|
+
* point instead; see that module).
|
|
6
|
+
*
|
|
7
|
+
* `@flue/opentelemetry`'s own docs (fetched at
|
|
8
|
+
* https://flueframework.com/docs/ecosystem/tooling/opentelemetry/, cited in
|
|
9
|
+
* this repo's phase0 spike) are explicit that `dispatch()` "does not
|
|
10
|
+
* propagate trace context currently" and that "custom header propagation to
|
|
11
|
+
* model providers is not documented" — so the literal ask ("propagate via
|
|
12
|
+
* @flue/opentelemetry") is not fully satisfiable by that package alone. What
|
|
13
|
+
* IS real and verifiable: `@opentelemetry/instrumentation-http` and
|
|
14
|
+
* `-undici` create a real client span (with a real, non-noop SpanContext)
|
|
15
|
+
* around every outbound `http`/`https`/`fetch`(undici) call made from this
|
|
16
|
+
* process, and the OTel API's global propagator is what those
|
|
17
|
+
* instrumentations use to inject `traceparent` (and, here, `x-request-id`)
|
|
18
|
+
* into that call's headers — REGARDLESS of which provider SDK issued it.
|
|
19
|
+
* This reaches every provider whose Node SDK issues requests through
|
|
20
|
+
* Node's own `http`/`https` modules or `undici` (verified: `fetch()`,
|
|
21
|
+
* `https.request()`). It does NOT reach a provider transport that bypasses
|
|
22
|
+
* both (unverified in this repo for the Anthropic/Google/Mistral SDKs'
|
|
23
|
+
* internal transports specifically — flagged, not assumed, per the spike).
|
|
24
|
+
*
|
|
25
|
+
* WHY A REAL GLOBAL TracerProvider IS REQUIRED, NOT OPTIONAL: the
|
|
26
|
+
* W3CTraceContextPropagator's `inject()` silently skips writing a
|
|
27
|
+
* `traceparent` header when the active SpanContext is INVALID
|
|
28
|
+
* (`isSpanContextValid()` false) — which is exactly what every span is when
|
|
29
|
+
* no global TracerProvider has ever been registered (the API's default is a
|
|
30
|
+
* no-op tracer). So this module registers a real (if minimal)
|
|
31
|
+
* `BasicTracerProvider` — with its own `BatchSpanProcessor` ->
|
|
32
|
+
* `OTLPTraceExporter` aimed at the SAME collector `observability.otel.
|
|
33
|
+
* endpoint` names — alongside the propagator and the two instrumentations.
|
|
34
|
+
* This also happens to be exactly what `@flue/opentelemetry`'s own docs ask
|
|
35
|
+
* for ("Configure the SDK first, then register one instrumentation
|
|
36
|
+
* instance") — Flue's own spans (`invoke_agent`, `chat <model>`,
|
|
37
|
+
* `execute_tool`) now have somewhere real to go, which they did not before.
|
|
38
|
+
*
|
|
39
|
+
* FLUE SPANS JOIN SPF's DETERMINISTIC TRACE (v3; issue #80). The naive
|
|
40
|
+
* approach — extracting SPF's agent-call traceparent into the active
|
|
41
|
+
* context around `dispatch()` — was tried and REJECTED by design review:
|
|
42
|
+
* flue's node runtime executes submissions in ONE process-lifetime claim
|
|
43
|
+
* loop (`builtin-providers ... claimLoop()`, started by the first
|
|
44
|
+
* dispatch's `finally`), whose async context is captured once at loop
|
|
45
|
+
* creation. A dispatch-time context wrap therefore joins only the FIRST
|
|
46
|
+
* flue agent in a process and silently MIS-ATTRIBUTES every later agent's
|
|
47
|
+
* spans into the first agent's trace — worse than a separate trace.
|
|
48
|
+
*
|
|
49
|
+
* The mechanism below instead uses the instrumentation's own
|
|
50
|
+
* `resolveRootContext(event, ctx)` option (typed in
|
|
51
|
+
* `@flue/opentelemetry`'s public d.mts; verified in its dist: consulted
|
|
52
|
+
* per span exactly when a span has neither an explicit parent nor an
|
|
53
|
+
* active-context SpanContext — i.e. per span, per submission, no matter
|
|
54
|
+
* what context the claim loop was captured in). SPF keeps a small
|
|
55
|
+
* instance-id -> traceparent map (`registerFlueSessionTrace`, populated by
|
|
56
|
+
* `agent_flue.ts`'s `run()` around each agent call), and the resolver
|
|
57
|
+
* matches on `ctx.id` — flue's documented "stable agent instance id during
|
|
58
|
+
* agent processing", which is the id SPF mints and hands to
|
|
59
|
+
* `init(SfAgent, { id })`. Extraction goes through the globally
|
|
60
|
+
* registered propagator against ROOT_CONTEXT, so no leaked loop context
|
|
61
|
+
* can stick. Consequences, all intended:
|
|
62
|
+
* - Flue's spans inherit SPF's sha256 trace id, parented under the
|
|
63
|
+
* right agent-call span PER SESSION — correct under multiple agents
|
|
64
|
+
* per process, concurrent agents, and claim-loop restarts alike. Span
|
|
65
|
+
* ids are SDK-random; only the trace id is shared.
|
|
66
|
+
* - The http/undici client spans' injected `traceparent` carries the
|
|
67
|
+
* deterministic id too, so Switchyard/vLLM hops land as descendants of
|
|
68
|
+
* SPF's trace — parity with `claude_code`'s `ANTHROPIC_CUSTOM_HEADERS`
|
|
69
|
+
* path. `x-request-id` stays the this-span id for request-keyed
|
|
70
|
+
* correlation, unchanged.
|
|
71
|
+
* - Unmapped sessions (never registered, restarted process with a
|
|
72
|
+
* durable backlog, post-`unregister` straggler bookkeeping spans)
|
|
73
|
+
* resolve to an unparented root — flue's spans root a separate SDK
|
|
74
|
+
* trace exactly as v1 did, correlatable by `x-request-id`/`spf.adw_id`/
|
|
75
|
+
* time window. Degraded join, never an error and never MIS-attributed.
|
|
76
|
+
* - flue's internal `executionContext.traceCarrier` (typed but not on
|
|
77
|
+
* the public `AgentDispatchRequest` surface) stays unused — noted here
|
|
78
|
+
* as flue's own escape hatch, not something SPF reaches into.
|
|
79
|
+
*
|
|
80
|
+
* REGISTRATION TIMING. `installFluePropagation()` is called from
|
|
81
|
+
* `agent_flue.ts`'s `run()`, before `ensureRuntime()`/dispatch — i.e. before
|
|
82
|
+
* the actual outbound call, which is the only ordering that matters for
|
|
83
|
+
* `instrumentation-undici` (subscribes to `undici`'s own `diagnostics_channel`
|
|
84
|
+
* events; any registration before the request fires is picked up regardless
|
|
85
|
+
* of when `undici`/`fetch` was first imported) and, in practice, for
|
|
86
|
+
* `instrumentation-http` too (Node's `http`/`https` modules are shared,
|
|
87
|
+
* monkey-patchable singletons; a later `require("http")` elsewhere in the
|
|
88
|
+
* process still resolves to the SAME, already-patched module object). This
|
|
89
|
+
* is a real, load-bearing difference from a truly cold, "before ANY other
|
|
90
|
+
* import" registration (which would require moving this into `cli/bin.ts`,
|
|
91
|
+
* ahead of that file's own deliberately-static-import-free module graph) —
|
|
92
|
+
* documented as the honest scope of what's verified, not claimed as more.
|
|
93
|
+
*
|
|
94
|
+
* NO-OP WHEN UNCONFIGURED. Gated on the exact same `observability.otel.
|
|
95
|
+
* endpoint` presence check as `otel.ts` and `otel_metrics.ts` — no ambient
|
|
96
|
+
* `OTEL_*` env var activates any of this on its own. Idempotent: registers
|
|
97
|
+
* exactly once per process, on whichever call (across however many `flue`
|
|
98
|
+
* agent dispatches this process makes) happens to arrive first.
|
|
99
|
+
*/
|
|
100
|
+
import { type Context, type TextMapPropagator, type TextMapSetter } from "@opentelemetry/api";
|
|
101
|
+
/** The one custom propagation field this repo adds beyond the standard W3C `traceparent`: the current span's own id, for a collector/log pipeline that correlates by request rather than by trace. */
|
|
102
|
+
export declare const X_REQUEST_ID_HEADER = "x-request-id";
|
|
103
|
+
/**
|
|
104
|
+
* Injects `x-request-id` from whatever span is active at the point of the
|
|
105
|
+
* outbound call — the same id that would appear as that span's `spanId` on
|
|
106
|
+
* the (separate — see the module header) Flue-side trace. One-directional:
|
|
107
|
+
* `extract()` is a pass-through, since nothing on the INBOUND side of an
|
|
108
|
+
* outbound provider call needs to read this back.
|
|
109
|
+
*/
|
|
110
|
+
export declare class XRequestIdPropagator implements TextMapPropagator {
|
|
111
|
+
inject(ctx: Context, carrier: unknown, setter: TextMapSetter): void;
|
|
112
|
+
extract(ctx: Context): Context;
|
|
113
|
+
fields(): string[];
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* Registers `traceparent` (SPF's deterministic agent-call span, as a W3C
|
|
117
|
+
* carrier string) as the trace root for flue spans belonging to `sessionId`
|
|
118
|
+
* — flue's instance id, minted by SPF and handed to `init(SfAgent, { id })`.
|
|
119
|
+
* Overwrites a prior registration for the same id (a same-phase retry is
|
|
120
|
+
* the same logical call; the current call wins).
|
|
121
|
+
*/
|
|
122
|
+
export declare function registerFlueSessionTrace(sessionId: string, traceparent: string): void;
|
|
123
|
+
/**
|
|
124
|
+
* Idempotent — called from `run()`'s `finally`. Post-settlement bookkeeping
|
|
125
|
+
* spans flue mints after this point simply resolve to an unparented root
|
|
126
|
+
* (separate trace), which is preferable to leaking a registration whose id
|
|
127
|
+
* a REUSED session id could collide with on a later phase.
|
|
128
|
+
*/
|
|
129
|
+
export declare function unregisterFlueSessionTrace(sessionId: string): void;
|
|
130
|
+
/**
|
|
131
|
+
* The `resolveRootContext` implementation handed to
|
|
132
|
+
* `createOpenTelemetryInstrumentation` — consulted per root-span creation
|
|
133
|
+
* (see the module header). Matches on `ctx.id` (flue's documented stable
|
|
134
|
+
* agent instance id during processing) and returns SPF's agent-call span
|
|
135
|
+
* as an extracted REMOTE parent, pulled from ROOT_CONTEXT so no ambient
|
|
136
|
+
* claim-loop context can leak in. Returns `undefined` (flue mints an
|
|
137
|
+
* unparented root span of its own) for an unmapped session id, a malformed
|
|
138
|
+
* traceparent, an absent ctx — and, with no global propagator installed,
|
|
139
|
+
* for everything. Exported for tests.
|
|
140
|
+
*/
|
|
141
|
+
export declare function resolveFlueRootContext(_event: unknown, ctx: {
|
|
142
|
+
id?: string;
|
|
143
|
+
} | undefined): Context | undefined;
|
|
144
|
+
export interface FluePropagationConfig {
|
|
145
|
+
endpoint: string;
|
|
146
|
+
headers?: Record<string, string>;
|
|
147
|
+
service_name: string;
|
|
148
|
+
}
|
|
149
|
+
/**
|
|
150
|
+
* Idempotent: the first call in this process wins; every later call
|
|
151
|
+
* (another `flue` agent dispatch, possibly with a different `cfg`) is a
|
|
152
|
+
* silent no-op, matching the "process-scoped, created exactly once" rule
|
|
153
|
+
* `otel_metrics.ts` documents for the same reason (`spf watch`'s daemon
|
|
154
|
+
* loop). Never throws — a failure to install best-effort propagation must
|
|
155
|
+
* never fail an agent dispatch.
|
|
156
|
+
*/
|
|
157
|
+
export declare function installFluePropagation(cfg: FluePropagationConfig | undefined | null, log?: (message: string) => void): void;
|
|
158
|
+
/** Tests only: forget global installation state. Does NOT undo `setGlobalTracerProvider`/`registerInstrumentations` (the OTel API has no supported "un-register" — tests that need isolation run in a fresh process). */
|
|
159
|
+
export declare function resetFluePropagationForTest(): void;
|