@gr8ful/spf 0.17.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/skill/references/config.md +6 -3
- package/assets/skill/references/observability.md +115 -1
- package/dist/cli/index.js +6 -0
- package/dist/core/agent_cc.d.ts +11 -0
- package/dist/core/agent_cc.js +39 -1
- package/dist/core/agent_flue.js +26 -0
- package/dist/core/agent_opencode.d.ts +105 -3
- package/dist/core/agent_opencode.js +169 -17
- package/dist/core/agents.d.ts +6 -0
- package/dist/core/agents.js +22 -0
- package/dist/core/data_types.d.ts +60 -0
- package/dist/core/data_types.js +69 -0
- package/dist/core/otel.d.ts +182 -48
- package/dist/core/otel.js +373 -158
- package/dist/core/otel_metrics.d.ts +127 -0
- package/dist/core/otel_metrics.js +221 -0
- package/dist/core/otel_propagation.d.ts +159 -0
- package/dist/core/otel_propagation.js +225 -0
- package/package.json +14 -1
package/dist/core/agents.d.ts
CHANGED
|
@@ -146,6 +146,12 @@ interface RunForAgents {
|
|
|
146
146
|
envelopeRow: (phase: Phase, agent: string, outputType: string, payloadJson: string, valid: boolean, attempt: number) => Promise<void>;
|
|
147
147
|
gateRow: (phase: Phase, gate: string, report: GateReport, attempt: number) => Promise<void>;
|
|
148
148
|
agentSessionRow: (adwId: string, agent: AgentConfig, sessionId: string, contextTokens?: number, contextWindow?: number) => Promise<void>;
|
|
149
|
+
otel?: {
|
|
150
|
+
agentCallTraceContext: (phaseId: string, agentName: string) => {
|
|
151
|
+
traceparent: string;
|
|
152
|
+
spanId: string;
|
|
153
|
+
} | null;
|
|
154
|
+
} | null;
|
|
149
155
|
};
|
|
150
156
|
console: {
|
|
151
157
|
agentStarted: (name: string, model: string, sessionId: string) => Promise<void>;
|
package/dist/core/agents.js
CHANGED
|
@@ -808,6 +808,19 @@ export async function execute(run, phase, call) {
|
|
|
808
808
|
// WHOLE run rather than on the first call of each phase. See
|
|
809
809
|
// `assertRunBudget` for why it is checked before, not after.
|
|
810
810
|
assertRunBudget(run);
|
|
811
|
+
// Outbound OTel propagation (SPF's otel-sdk extension): the CURRENTLY
|
|
812
|
+
// OPEN agent-call span's own trace context, when one exists — `null`
|
|
813
|
+
// whenever `observability.otel` is unconfigured for this run (the
|
|
814
|
+
// common case, and byte-identical to before this field existed) or the
|
|
815
|
+
// exporter has no agent call open (shouldn't happen here — agent_start
|
|
816
|
+
// fires before `send()` is ever reached — but a `null` is a silent
|
|
817
|
+
// no-op either way, never an error). `otelBlock` threads the SAME
|
|
818
|
+
// endpoint/headers/service_name through so `agent_flue.ts` can install
|
|
819
|
+
// its own (separate, process-scoped) http/undici propagation without
|
|
820
|
+
// needing the whole `SFConfig` — see `data_types.ts`'s `AgentRequest.
|
|
821
|
+
// otel` doc comment.
|
|
822
|
+
const otelBlock = run.cfg.observability.otel;
|
|
823
|
+
const otelCtx = otelBlock ? (run.tracer.otel?.agentCallTraceContext(phase.phase_id, agent.name) ?? null) : null;
|
|
811
824
|
const request = {
|
|
812
825
|
prompt: promptText,
|
|
813
826
|
system_prompt: systemText,
|
|
@@ -822,6 +835,15 @@ export async function execute(run, phase, call) {
|
|
|
822
835
|
flue_db_path: path.join(run.data_dir, "flue.db"),
|
|
823
836
|
env: agentEnv(agent),
|
|
824
837
|
sandbox: spec,
|
|
838
|
+
otel: otelCtx && otelBlock
|
|
839
|
+
? {
|
|
840
|
+
traceparent: otelCtx.traceparent,
|
|
841
|
+
x_request_id: otelCtx.spanId,
|
|
842
|
+
endpoint: otelBlock.endpoint,
|
|
843
|
+
headers: otelBlock.headers,
|
|
844
|
+
service_name: otelBlock.service_name,
|
|
845
|
+
}
|
|
846
|
+
: undefined,
|
|
825
847
|
};
|
|
826
848
|
const forward = eventForwarder(run, phase, agent.name, agent.coding_agent);
|
|
827
849
|
// Best-effort, fire-and-forget: these fire from a plain process-lifecycle
|
|
@@ -685,6 +685,7 @@ export declare const AgentConfigSchema: v.ObjectSchema<{
|
|
|
685
685
|
readonly writes: v.OptionalSchema<v.NullableSchema<v.ArraySchema<v.StringSchema<undefined>, undefined>, undefined>, undefined>;
|
|
686
686
|
readonly env_allowlist: v.OptionalSchema<v.NullableSchema<v.ArraySchema<v.StringSchema<undefined>, undefined>, undefined>, undefined>;
|
|
687
687
|
readonly sandbox: v.OptionalSchema<v.NullableSchema<v.PicklistSchema<["local", "opensandbox", "cloudflare"], undefined>, undefined>, undefined>;
|
|
688
|
+
readonly lora_adapter: v.OptionalSchema<v.StringSchema<undefined>, undefined>;
|
|
688
689
|
}, undefined>;
|
|
689
690
|
export type AgentConfig = v.InferOutput<typeof AgentConfigSchema>;
|
|
690
691
|
export declare const ConfigDefaultsSchema: v.ObjectSchema<{
|
|
@@ -756,13 +757,45 @@ export type ConfigDefaults = v.InferOutput<typeof ConfigDefaultsSchema>;
|
|
|
756
757
|
* (`https://collector:4318/v1/traces`) or a bare origin (`/v1/traces` is
|
|
757
758
|
* appended — see `resolveTracesUrl`). `headers` is where a collector's auth
|
|
758
759
|
* token goes; its VALUES are treated as secrets and never logged.
|
|
760
|
+
*
|
|
761
|
+
* `metrics` (default `true`) additionally gates `otel_metrics.ts`'s
|
|
762
|
+
* process-scoped meter — set `false` to keep trace export on while opting
|
|
763
|
+
* out of the metrics pipeline entirely. It has no effect on activation
|
|
764
|
+
* either way: `endpoint`'s presence is still the sole switch for BOTH
|
|
765
|
+
* signals, this field only narrows what a configured block sends.
|
|
766
|
+
*
|
|
767
|
+
* `allow_env` (default `false`) is a narrow, opt-in exception to EXPLICIT
|
|
768
|
+
* CONFIG ONLY (see `core/otel.ts`'s header): when `true` AND this block is
|
|
769
|
+
* ALREADY active (`endpoint` set here, in the file), the standard
|
|
770
|
+
* `OTEL_EXPORTER_OTLP_ENDPOINT`/`OTEL_EXPORTER_OTLP_HEADERS` env vars may
|
|
771
|
+
* SUPPLEMENT it — e.g. a CI-injected collector token added to `headers`
|
|
772
|
+
* without checking it into the repo. It can NEVER activate export from
|
|
773
|
+
* nothing (there is no `endpoint` to fall back to when this block is absent
|
|
774
|
+
* at all — that case is unaffected by this flag either way) and a
|
|
775
|
+
* config-declared value always wins over the env on conflict. Defaults to
|
|
776
|
+
* `false` because this is still a deliberate loosening of an adversarially-
|
|
777
|
+
* reviewed, test-pinned invariant — an operator opts in per-repo.
|
|
759
778
|
*/
|
|
760
779
|
export declare const OTelConfigSchema: v.ObjectSchema<{
|
|
761
780
|
readonly endpoint: v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.UrlAction<string, undefined>]>;
|
|
762
781
|
readonly headers: v.OptionalSchema<v.RecordSchema<v.StringSchema<undefined>, v.StringSchema<undefined>, undefined>, undefined>;
|
|
763
782
|
readonly service_name: v.OptionalSchema<v.StringSchema<undefined>, "spf">;
|
|
783
|
+
readonly metrics: v.OptionalSchema<v.BooleanSchema<undefined>, true>;
|
|
784
|
+
readonly allow_env: v.OptionalSchema<v.BooleanSchema<undefined>, false>;
|
|
764
785
|
}, undefined>;
|
|
765
786
|
export type OTelConfig = v.InferOutput<typeof OTelConfigSchema>;
|
|
787
|
+
/**
|
|
788
|
+
* `allow_env`'s narrow env-supplement path (see `OTelConfigSchema`'s doc
|
|
789
|
+
* comment above): called ONLY when `cfg.observability.otel` already exists
|
|
790
|
+
* with a real `endpoint` — this function never activates anything on its
|
|
791
|
+
* own and is never called when the block is absent. `OTEL_EXPORTER_OTLP_
|
|
792
|
+
* HEADERS` follows the OTel spec's env-var shape (comma-separated
|
|
793
|
+
* `key=value` pairs, URL-decoded); config-declared headers win over the env
|
|
794
|
+
* on a key collision. `OTEL_EXPORTER_OTLP_ENDPOINT`, when set, overrides
|
|
795
|
+
* `endpoint` itself — the one field this can change, since a CI runner
|
|
796
|
+
* commonly injects a different collector per job/environment.
|
|
797
|
+
*/
|
|
798
|
+
export declare function applyOtelEnvSupplement(otel: OTelConfig, env?: NodeJS.ProcessEnv): OTelConfig;
|
|
766
799
|
/**
|
|
767
800
|
* MIGRATION NOTE (BT-issue #66, 3 PRs): `observability.db` used to be ONLY a
|
|
768
801
|
* bare string (a local sqlite path, defaulting to ".spf/data/spf.db"). This
|
|
@@ -868,6 +901,8 @@ export declare const ObservabilityConfigSchema: v.ObjectSchema<{
|
|
|
868
901
|
readonly endpoint: v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.UrlAction<string, undefined>]>;
|
|
869
902
|
readonly headers: v.OptionalSchema<v.RecordSchema<v.StringSchema<undefined>, v.StringSchema<undefined>, undefined>, undefined>;
|
|
870
903
|
readonly service_name: v.OptionalSchema<v.StringSchema<undefined>, "spf">;
|
|
904
|
+
readonly metrics: v.OptionalSchema<v.BooleanSchema<undefined>, true>;
|
|
905
|
+
readonly allow_env: v.OptionalSchema<v.BooleanSchema<undefined>, false>;
|
|
871
906
|
}, undefined>, undefined>;
|
|
872
907
|
}, undefined>;
|
|
873
908
|
export type ObservabilityConfig = v.InferOutput<typeof ObservabilityConfigSchema>;
|
|
@@ -1525,6 +1560,8 @@ export declare const SFConfigSchema: v.ObjectSchema<{
|
|
|
1525
1560
|
readonly endpoint: v.SchemaWithPipe<readonly [v.StringSchema<undefined>, v.UrlAction<string, undefined>]>;
|
|
1526
1561
|
readonly headers: v.OptionalSchema<v.RecordSchema<v.StringSchema<undefined>, v.StringSchema<undefined>, undefined>, undefined>;
|
|
1527
1562
|
readonly service_name: v.OptionalSchema<v.StringSchema<undefined>, "spf">;
|
|
1563
|
+
readonly metrics: v.OptionalSchema<v.BooleanSchema<undefined>, true>;
|
|
1564
|
+
readonly allow_env: v.OptionalSchema<v.BooleanSchema<undefined>, false>;
|
|
1528
1565
|
}, undefined>, undefined>;
|
|
1529
1566
|
}, undefined>, () => {
|
|
1530
1567
|
db: string | {
|
|
@@ -1543,6 +1580,8 @@ export declare const SFConfigSchema: v.ObjectSchema<{
|
|
|
1543
1580
|
[x: string]: string;
|
|
1544
1581
|
} | undefined;
|
|
1545
1582
|
service_name: string;
|
|
1583
|
+
metrics: boolean;
|
|
1584
|
+
allow_env: boolean;
|
|
1546
1585
|
} | undefined;
|
|
1547
1586
|
}>;
|
|
1548
1587
|
readonly agents: v.OptionalSchema<v.ArraySchema<v.ObjectSchema<{
|
|
@@ -1561,6 +1600,7 @@ export declare const SFConfigSchema: v.ObjectSchema<{
|
|
|
1561
1600
|
readonly writes: v.OptionalSchema<v.NullableSchema<v.ArraySchema<v.StringSchema<undefined>, undefined>, undefined>, undefined>;
|
|
1562
1601
|
readonly env_allowlist: v.OptionalSchema<v.NullableSchema<v.ArraySchema<v.StringSchema<undefined>, undefined>, undefined>, undefined>;
|
|
1563
1602
|
readonly sandbox: v.OptionalSchema<v.NullableSchema<v.PicklistSchema<["local", "opensandbox", "cloudflare"], undefined>, undefined>, undefined>;
|
|
1603
|
+
readonly lora_adapter: v.OptionalSchema<v.StringSchema<undefined>, undefined>;
|
|
1564
1604
|
}, undefined>, undefined>, () => never[]>;
|
|
1565
1605
|
readonly quality: v.OptionalSchema<v.ObjectSchema<{
|
|
1566
1606
|
readonly checks: v.OptionalSchema<v.ArraySchema<v.ObjectSchema<{
|
|
@@ -1974,6 +2014,26 @@ export interface AgentRequest {
|
|
|
1974
2014
|
env?: Record<string, string>;
|
|
1975
2015
|
/** Absent (the default) => local(), byte-identical to before this field existed. See sandbox.ts. */
|
|
1976
2016
|
sandbox?: SandboxSpec;
|
|
2017
|
+
/**
|
|
2018
|
+
* Outbound OTel trace-context propagation — set by `agents.ts`'s `send()`
|
|
2019
|
+
* from `otel.ts`'s `OtelExporter.agentCallTraceContext()` ONLY when
|
|
2020
|
+
* `observability.otel` is configured for this run; absent otherwise, and
|
|
2021
|
+
* every backend that ignores it (`opencode` today) is byte-identical to
|
|
2022
|
+
* before this field existed. `traceparent`/`x_request_id` are this call's
|
|
2023
|
+
* own span context (`agent_cc.ts`'s single `spawn()` choke point turns
|
|
2024
|
+
* them into `TRACEPARENT`/`ANTHROPIC_CUSTOM_HEADERS`); `endpoint`/
|
|
2025
|
+
* `headers`/`service_name` are the SAME `observability.otel` block,
|
|
2026
|
+
* carried through so `agent_flue.ts` can install its own (separate,
|
|
2027
|
+
* process-scoped — see `otel_propagation.ts`) global http/undici
|
|
2028
|
+
* propagation without needing the full `SFConfig`.
|
|
2029
|
+
*/
|
|
2030
|
+
otel?: {
|
|
2031
|
+
traceparent: string;
|
|
2032
|
+
x_request_id: string;
|
|
2033
|
+
endpoint: string;
|
|
2034
|
+
headers?: Record<string, string>;
|
|
2035
|
+
service_name: string;
|
|
2036
|
+
};
|
|
1977
2037
|
}
|
|
1978
2038
|
/**
|
|
1979
2039
|
* Tokens and the dollars they cost, per component, summed over a call.
|
package/dist/core/data_types.js
CHANGED
|
@@ -508,6 +508,13 @@ export const AgentConfigSchema = v.object({
|
|
|
508
508
|
// "local" -> force local for this agent
|
|
509
509
|
// null -> same as unset (the writes/env_allowlist spelling)
|
|
510
510
|
sandbox: v.optional(v.nullable(SandboxBackendSchema)),
|
|
511
|
+
// Explicit override for `otel.ts`'s `spf.lora_adapter` span attribute —
|
|
512
|
+
// the unambiguous source of truth when set. Unset (the common case) falls
|
|
513
|
+
// back to parsing `model` itself; see `loraAdapterFor()`'s own doc comment
|
|
514
|
+
// in `core/otel.ts` for the two zero-config conventions it recognizes.
|
|
515
|
+
// Purely descriptive: it does not change routing, dispatch, or which
|
|
516
|
+
// model actually serves the call — only what an OTel backend sees.
|
|
517
|
+
lora_adapter: v.optional(v.string()),
|
|
511
518
|
});
|
|
512
519
|
export const ConfigDefaultsSchema = v.object({
|
|
513
520
|
coding_agent: v.optional(v.picklist(["flue", "claude_code", "opencode"]), "flue"),
|
|
@@ -581,12 +588,74 @@ export const ConfigDefaultsSchema = v.object({
|
|
|
581
588
|
* (`https://collector:4318/v1/traces`) or a bare origin (`/v1/traces` is
|
|
582
589
|
* appended — see `resolveTracesUrl`). `headers` is where a collector's auth
|
|
583
590
|
* token goes; its VALUES are treated as secrets and never logged.
|
|
591
|
+
*
|
|
592
|
+
* `metrics` (default `true`) additionally gates `otel_metrics.ts`'s
|
|
593
|
+
* process-scoped meter — set `false` to keep trace export on while opting
|
|
594
|
+
* out of the metrics pipeline entirely. It has no effect on activation
|
|
595
|
+
* either way: `endpoint`'s presence is still the sole switch for BOTH
|
|
596
|
+
* signals, this field only narrows what a configured block sends.
|
|
597
|
+
*
|
|
598
|
+
* `allow_env` (default `false`) is a narrow, opt-in exception to EXPLICIT
|
|
599
|
+
* CONFIG ONLY (see `core/otel.ts`'s header): when `true` AND this block is
|
|
600
|
+
* ALREADY active (`endpoint` set here, in the file), the standard
|
|
601
|
+
* `OTEL_EXPORTER_OTLP_ENDPOINT`/`OTEL_EXPORTER_OTLP_HEADERS` env vars may
|
|
602
|
+
* SUPPLEMENT it — e.g. a CI-injected collector token added to `headers`
|
|
603
|
+
* without checking it into the repo. It can NEVER activate export from
|
|
604
|
+
* nothing (there is no `endpoint` to fall back to when this block is absent
|
|
605
|
+
* at all — that case is unaffected by this flag either way) and a
|
|
606
|
+
* config-declared value always wins over the env on conflict. Defaults to
|
|
607
|
+
* `false` because this is still a deliberate loosening of an adversarially-
|
|
608
|
+
* reviewed, test-pinned invariant — an operator opts in per-repo.
|
|
584
609
|
*/
|
|
585
610
|
export const OTelConfigSchema = v.object({
|
|
586
611
|
endpoint: v.pipe(v.string(), v.url()),
|
|
587
612
|
headers: v.optional(v.record(v.string(), v.string()), undefined),
|
|
588
613
|
service_name: v.optional(v.string(), "spf"),
|
|
614
|
+
metrics: v.optional(v.boolean(), true),
|
|
615
|
+
allow_env: v.optional(v.boolean(), false),
|
|
589
616
|
});
|
|
617
|
+
/**
|
|
618
|
+
* `allow_env`'s narrow env-supplement path (see `OTelConfigSchema`'s doc
|
|
619
|
+
* comment above): called ONLY when `cfg.observability.otel` already exists
|
|
620
|
+
* with a real `endpoint` — this function never activates anything on its
|
|
621
|
+
* own and is never called when the block is absent. `OTEL_EXPORTER_OTLP_
|
|
622
|
+
* HEADERS` follows the OTel spec's env-var shape (comma-separated
|
|
623
|
+
* `key=value` pairs, URL-decoded); config-declared headers win over the env
|
|
624
|
+
* on a key collision. `OTEL_EXPORTER_OTLP_ENDPOINT`, when set, overrides
|
|
625
|
+
* `endpoint` itself — the one field this can change, since a CI runner
|
|
626
|
+
* commonly injects a different collector per job/environment.
|
|
627
|
+
*/
|
|
628
|
+
export function applyOtelEnvSupplement(otel, env = process.env) {
|
|
629
|
+
if (!otel.allow_env)
|
|
630
|
+
return otel;
|
|
631
|
+
const envEndpoint = env["OTEL_EXPORTER_OTLP_ENDPOINT"];
|
|
632
|
+
const envHeadersRaw = env["OTEL_EXPORTER_OTLP_HEADERS"];
|
|
633
|
+
const envHeaders = {};
|
|
634
|
+
if (envHeadersRaw) {
|
|
635
|
+
for (const pair of envHeadersRaw.split(",")) {
|
|
636
|
+
const eq = pair.indexOf("=");
|
|
637
|
+
if (eq <= 0)
|
|
638
|
+
continue;
|
|
639
|
+
const key = pair.slice(0, eq).trim();
|
|
640
|
+
const value = pair.slice(eq + 1).trim();
|
|
641
|
+
if (key) {
|
|
642
|
+
try {
|
|
643
|
+
envHeaders[key] = decodeURIComponent(value);
|
|
644
|
+
}
|
|
645
|
+
catch {
|
|
646
|
+
envHeaders[key] = value;
|
|
647
|
+
}
|
|
648
|
+
}
|
|
649
|
+
}
|
|
650
|
+
}
|
|
651
|
+
if (!envEndpoint && Object.keys(envHeaders).length === 0)
|
|
652
|
+
return otel;
|
|
653
|
+
return {
|
|
654
|
+
...otel,
|
|
655
|
+
endpoint: envEndpoint || otel.endpoint,
|
|
656
|
+
headers: { ...envHeaders, ...(otel.headers ?? {}) }, // config wins on conflict
|
|
657
|
+
};
|
|
658
|
+
}
|
|
590
659
|
/**
|
|
591
660
|
* MIGRATION NOTE (BT-issue #66, 3 PRs): `observability.db` used to be ONLY a
|
|
592
661
|
* bare string (a local sqlite path, defaulting to ".spf/data/spf.db"). This
|
package/dist/core/otel.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* OpenTelemetry span export (
|
|
2
|
+
* OpenTelemetry span export (v2): a config-gated, lossy, fire-and-forget
|
|
3
3
|
* PROJECTION of the trace SQLite already holds. Read this header before
|
|
4
4
|
* changing anything here — every paragraph is a constraint that survived an
|
|
5
5
|
* adversarial review, not a preference.
|
|
@@ -12,35 +12,84 @@
|
|
|
12
12
|
* ever throw into a caller, block a caller, or be awaited by a caller other
|
|
13
13
|
* than the two shutdown paths named under LIFECYCLE below.
|
|
14
14
|
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
15
|
+
* v2 CHANGE (SDK ENCODER SWAP). v1 hand-rolled the entire OTLP/HTTP-JSON wire
|
|
16
|
+
* format with its own `fetch()` call. v2 keeps every invariant below —
|
|
17
|
+
* public API, the deterministic sha256 id scheme, the attribute allowlist,
|
|
18
|
+
* the bounded queue, the per-run lifecycle — byte-for-byte, and replaces
|
|
19
|
+
* ONLY the encoder: spans are now plain objects that structurally satisfy
|
|
20
|
+
* `@opentelemetry/sdk-trace`'s `ReadableSpan` interface (that package's own
|
|
21
|
+
* concrete `Span`/`SpanImpl` class is NOT part of its public API surface —
|
|
22
|
+
* only the type is exported — so a duck-typed object is not a workaround,
|
|
23
|
+
* it is the intended integration point), handed to a real
|
|
24
|
+
* `@opentelemetry/exporter-trace-otlp-http` `OTLPTraceExporter` instance.
|
|
25
|
+
* `IdGenerator.generateSpanId()` takes no arguments and cannot be handed our
|
|
26
|
+
* sha256 ids any other way — this is why the SDK is used AROUND our own ids
|
|
27
|
+
* rather than asked to generate them.
|
|
28
|
+
*
|
|
29
|
+
* VERIFIED WIRE-SHAPE DIFFERENCES from the old hand-rolled encoder (proven
|
|
30
|
+
* against a real in-process OTLP/HTTP receiver in `src/test/otel.test.ts`,
|
|
31
|
+
* not assumed from docs — this was v1's #1 documented open risk):
|
|
32
|
+
* - `intValue` is a JSON NUMBER (`{"intValue":1234}`), not a numeric
|
|
33
|
+
* STRING. The real JSON serializer's `toAnyValue()` (`@opentelemetry/
|
|
34
|
+
* otlp-transformer`) picks `intValue` whenever `Number.isInteger(value)`
|
|
35
|
+
* and never stringifies it — proto3 JSON's "int64 as string" rule is a
|
|
36
|
+
* PROTOBUF-JSON convention this exporter's plain-JSON path does not
|
|
37
|
+
* follow. A whole-number COST (e.g. exactly `$2`) is therefore
|
|
38
|
+
* indistinguishable on the wire from an integer attribute — a real,
|
|
39
|
+
* accepted limitation of `number`-typed OTel attributes, not a bug
|
|
40
|
+
* introduced here.
|
|
41
|
+
* - `startTimeUnixNano`/`endTimeUnixNano`/event `timeUnixNano` ARE
|
|
42
|
+
* STRINGS (`encodeAsString` — nanoseconds via `BigInt`, so no
|
|
43
|
+
* precision loss past 2^53), matching v1's own precision-driven choice.
|
|
44
|
+
* - trace/span ids are lowercase hex STRINGS (the JSON encoder's
|
|
45
|
+
* `encodeSpanContext` is `identity` — our own hex ids pass straight
|
|
46
|
+
* through), matching v1 exactly.
|
|
47
|
+
* - a ROOT span's `parentSpanId` is OMITTED from the wire object entirely
|
|
48
|
+
* (no key at all) rather than v1's explicit `""` — both spellings mean
|
|
49
|
+
* "no parent" per the OTLP proto3-JSON mapping (proto3 JSON drops
|
|
50
|
+
* zero-value/unset fields by default); `src/test/otel.test.ts` asserts
|
|
51
|
+
* `undefined`, not `""`, for a root span now.
|
|
52
|
+
* - extra fields the real exporter adds that v1 never had (`flags`,
|
|
53
|
+
* `traceState`, `droppedAttributesCount`, `droppedEventsCount`,
|
|
54
|
+
* `droppedLinksCount`, `links: []`) are additive and harmless — nothing
|
|
55
|
+
* downstream reads a fixed field LIST, only named fields.
|
|
56
|
+
*
|
|
57
|
+
* SPANS ONLY (from THIS module's own per-run exporter). No `resourceLogs`.
|
|
58
|
+
* Metrics are now real (see `otel_metrics.ts`) but live on their own
|
|
59
|
+
* PROCESS-scoped pipeline with their own real `@opentelemetry/sdk-metrics`
|
|
60
|
+
* temporality/aggregation handling — never hand-rolled, and never mixed into
|
|
61
|
+
* this module's `resourceSpans` payload.
|
|
20
62
|
*
|
|
21
63
|
* EXPLICIT CONFIG ONLY. Activation requires `observability.otel.endpoint` in
|
|
22
64
|
* the config file. This module NEVER reads `OTEL_EXPORTER_OTLP_ENDPOINT` or
|
|
23
|
-
* any other ambient exporter variable: an unrelated
|
|
24
|
-
* from a CI image or a coworker's dotfiles must not
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
*
|
|
65
|
+
* any other ambient exporter variable AS AN ACTIVATION SWITCH: an unrelated
|
|
66
|
+
* shell variable inherited from a CI image or a coworker's dotfiles must not
|
|
67
|
+
* be able to turn a repo's telemetry egress on. `observability.otel.
|
|
68
|
+
* allow_env` (see `data_types.ts`'s `OTelConfigSchema`) is the one narrow,
|
|
69
|
+
* opt-in exception: when `true` AND the block is ALREADY active (`endpoint`
|
|
70
|
+
* set), `OTEL_EXPORTER_OTLP_ENDPOINT`/`OTEL_EXPORTER_OTLP_HEADERS` may
|
|
71
|
+
* SUPPLEMENT it (an env-injected token in CI, say) — never activate it from
|
|
72
|
+
* nothing, and config-declared values always win over the env on conflict.
|
|
73
|
+
* (`SPF_CLAUDE_CMD` is not a precedent for the opposite: that variable is
|
|
74
|
+
* SPF-namespaced and only redirects a LOCAL subprocess — it moves no data
|
|
75
|
+
* off the machine.)
|
|
28
76
|
*
|
|
29
77
|
* ATTRIBUTE ALLOWLIST — exfiltration is the top risk here, because
|
|
30
78
|
* `EventRecord.payload` carries the repository's own source code (tool args,
|
|
31
79
|
* result snippets, diffs, prompts, envelope contents, the operator's request
|
|
32
80
|
* text). The allowlist, in full: phase name/kind/owner/status/seq/attempt,
|
|
33
|
-
* chain name, adw_id, agent name/model/coding_agent, gate name +
|
|
34
|
-
* violation COUNT, token counts (UsageBreakdown fields) + costs,
|
|
35
|
-
* (implied by span start/end), and event TYPE. Everything else is
|
|
36
|
-
* construction, not by filtering:
|
|
81
|
+
* chain name, adw_id, agent name/model/coding_agent/lora_adapter, gate name +
|
|
82
|
+
* passed + violation COUNT, token counts (UsageBreakdown fields) + costs,
|
|
83
|
+
* durations (implied by span start/end), and event TYPE. Everything else is
|
|
84
|
+
* excluded by construction, not by filtering:
|
|
37
85
|
* - This module reads `EventRecord.payload` for FINITE NUMBERS ONLY (see
|
|
38
86
|
* `numOrNull`) and only under known UsageBreakdown/cost keys. A string can
|
|
39
87
|
* never reach an attribute through the payload path. Do not add a
|
|
40
|
-
*
|
|
88
|
+
* string read from `payload` — that single line is the whole
|
|
41
89
|
* exfiltration bug.
|
|
42
|
-
* - Agent model/coding_agent come from the typed `AgentConfig`
|
|
43
|
-
* `recordAgentSession` (config data), NOT from the
|
|
90
|
+
* - Agent model/coding_agent/lora_adapter come from the typed `AgentConfig`
|
|
91
|
+
* handed to `recordAgentSession` (config data), NOT from the
|
|
92
|
+
* `agent_start` payload.
|
|
44
93
|
* - Tool spans are named from `record.name`'s prefix up to the first ":"
|
|
45
94
|
* (see `toolSpanName`). The full `record.name` is a HUMAN LABEL built from
|
|
46
95
|
* real tool arguments (`agent_flue.ts`'s `labelFor` -> "bash: cat
|
|
@@ -71,7 +120,9 @@
|
|
|
71
120
|
* `agent:<phase_id>:<agent>:<n>` for an agent call, `tool:<phase_id>:<event_id>`
|
|
72
121
|
* for a tool call). Determinism means a re-export of the same run lands on the
|
|
73
122
|
* same ids instead of duplicating the trace, and a child span can name its
|
|
74
|
-
* parent's id without waiting for the parent to be emitted.
|
|
123
|
+
* parent's id without waiting for the parent to be emitted. UNCHANGED in v2:
|
|
124
|
+
* the SDK is used to ENCODE spans we already fully control, never to
|
|
125
|
+
* generate their ids.
|
|
75
126
|
* `EventRecord.parent_id` is structurally ALWAYS EMPTY today (SPF's phases are
|
|
76
127
|
* flat siblings; nothing writes nesting), so there is no recorded hierarchy to
|
|
77
128
|
* mine — the parenting above is reconstructed from phase_id + agent-call
|
|
@@ -79,9 +130,9 @@
|
|
|
79
130
|
*
|
|
80
131
|
* PHASE SPANS ARE EMITTED AT PHASE END ONLY. A hung or killed phase is
|
|
81
132
|
* therefore INVISIBLE to the backend (its buffered span events die with it),
|
|
82
|
-
* while SQLite still shows it as `running`. Deliberate v1 trade:
|
|
83
|
-
* span at phase start would require mutating an already-sent
|
|
84
|
-
* has no notion of.
|
|
133
|
+
* while SQLite still shows it as `running`. Deliberate v1 trade, unchanged:
|
|
134
|
+
* streaming a span at phase start would require mutating an already-sent
|
|
135
|
+
* span, which OTLP has no notion of.
|
|
85
136
|
*
|
|
86
137
|
* INBOUND TRACEPARENT. When a valid W3C `traceparent` is present in the
|
|
87
138
|
* environment, its trace-id becomes this run's trace-id and the run's root
|
|
@@ -92,6 +143,37 @@
|
|
|
92
143
|
* silently (see `parseTraceparent`) — a malformed variable must degrade to
|
|
93
144
|
* "own root", never to an error.
|
|
94
145
|
*
|
|
146
|
+
* OUTBOUND PROPAGATION (new in v2). `agentCallTraceContext()` hands back the
|
|
147
|
+
* CURRENTLY OPEN agent call's own trace context (same traceId, same sha256
|
|
148
|
+
* span id already computed by `openAgentCall`) so a caller can propagate it
|
|
149
|
+
* onward — `agents.ts`'s `send()` reads it into `AgentRequest.otel`, which
|
|
150
|
+
* `agent_cc.ts`'s single `spawn()` choke point turns into `TRACEPARENT` +
|
|
151
|
+
* `ANTHROPIC_CUSTOM_HEADERS` env vars for the `claude` CLI subprocess (see
|
|
152
|
+
* that module's own header for the verified env var format). This is a
|
|
153
|
+
* READ of state this exporter already tracks for its own id scheme — it
|
|
154
|
+
* does not change what gets exported, and it is `null` (a silent no-op)
|
|
155
|
+
* whenever no agent call is currently open.
|
|
156
|
+
*
|
|
157
|
+
* LORA ADAPTER ATTRIBUTE. `loraAdapterFor()` resolves `spf.lora_adapter` —
|
|
158
|
+
* see its own doc comment for the two zero-config conventions plus the
|
|
159
|
+
* explicit `AgentConfig.lora_adapter` override, checked in that order. The
|
|
160
|
+
* attribute is omitted entirely when nothing resolves — never a blind copy
|
|
161
|
+
* of a non-LoRA model id.
|
|
162
|
+
*
|
|
163
|
+
* METRICS FAN-OUT (new in v2). `recordPhase`/`recordGate`/`closeAgentCall`
|
|
164
|
+
* additionally fan out to an OPTIONAL, PROCESS-scoped `OtelMetrics` handle
|
|
165
|
+
* (see `otel_metrics.ts`) — `spf.phase.duration`, `spf.gate.result`,
|
|
166
|
+
* `spf.tokens`, `spf.cost_usd`, `spf.agent.calls`. `resolveOtelExporter`
|
|
167
|
+
* resolves it once via `otel_metrics.resolveOtelMetrics(cfg)` and holds the
|
|
168
|
+
* reference; every OTHER call site (`tracer.ts`, `agents.ts`) is BYTE-
|
|
169
|
+
* IDENTICAL to before metrics existed. The v1 "dropped spans" resource
|
|
170
|
+
* attribute hack is GONE (a `Resource` is immutable per-exporter-instance in
|
|
171
|
+
* the real SDK — there is no home for a value that changes after
|
|
172
|
+
* construction) — dropped-span/dropped-event counts now ride as real
|
|
173
|
+
* Counters on the metrics pipeline instead, recorded once (same "once, on
|
|
174
|
+
* the final flush" cadence the warn log already used), with the warn log
|
|
175
|
+
* itself UNCHANGED as the fallback when metrics are off.
|
|
176
|
+
*
|
|
95
177
|
* LIFECYCLE (copied from `notify/notifier.ts`'s discipline, with one
|
|
96
178
|
* addition `notify` doesn't need — see RUN-SCOPED CLEANUP below). A
|
|
97
179
|
* module-level LIVE registry holds every exporter this process created;
|
|
@@ -133,22 +215,28 @@
|
|
|
133
215
|
* Spans go into a BOUNDED queue (`MAX_QUEUED_SPANS`, drop-OLDEST) and leave in
|
|
134
216
|
* batches (`BATCH_SPANS`, or `FLUSH_INTERVAL_MS`, whichever comes first) via an
|
|
135
217
|
* UNREF'D timer that can never hold the process open. Dropped spans are
|
|
136
|
-
* counted
|
|
137
|
-
*
|
|
138
|
-
*
|
|
139
|
-
*
|
|
140
|
-
*
|
|
218
|
+
* counted and reported once as a warn line (and once as a metric, when one is
|
|
219
|
+
* configured — see METRICS FAN-OUT above). The size trigger schedules a timer
|
|
220
|
+
* rather than flushing inline, which also means a synchronous burst of
|
|
221
|
+
* thousands of events exercises the bound (see the queue test) instead of
|
|
222
|
+
* interleaving sends.
|
|
141
223
|
*
|
|
142
|
-
* WIRE
|
|
143
|
-
*
|
|
144
|
-
* `
|
|
145
|
-
*
|
|
146
|
-
* `
|
|
147
|
-
*
|
|
148
|
-
*
|
|
149
|
-
*
|
|
224
|
+
* WIRE TRANSPORT is `@opentelemetry/exporter-trace-otlp-http`'s real
|
|
225
|
+
* `OTLPTraceExporter`, JSON-encoded (its default) against the resolved
|
|
226
|
+
* `/v1/traces` URL — see the VERIFIED WIRE-SHAPE DIFFERENCES note above for
|
|
227
|
+
* exactly how its bytes differ from v1's hand-rolled ones.
|
|
228
|
+
* `keepAlive: false` is passed explicitly: the real Node HTTP agent defaults
|
|
229
|
+
* `keepAlive: true`, which would hold an open socket past this exporter's own
|
|
230
|
+
* bounded `drain()` — the same "must never be the reason a `spf` process
|
|
231
|
+
* lingers" requirement the unref'd flush timer already exists for.
|
|
232
|
+
* `timeoutMillis: SEND_TIMEOUT_MS` bounds the exporter's own internal
|
|
233
|
+
* retrying transport (up to 5 attempts, capped by this same deadline across
|
|
234
|
+
* all of them — verified against `@opentelemetry/otlp-exporter-base`'s
|
|
235
|
+
* `RetryingTransport` source) to the same budget the old hand-rolled
|
|
236
|
+
* `AbortController` enforced.
|
|
150
237
|
*/
|
|
151
|
-
import type
|
|
238
|
+
import { type AgentConfig, type EventRecord, type GateReport, type OTelConfig, type Phase, type SFConfig } from "./data_types.ts";
|
|
239
|
+
import { type OtelMetrics } from "./otel_metrics.ts";
|
|
152
240
|
/** trace-id = first 32 hex of sha256(adw_id). Bespoke convention — see the header. */
|
|
153
241
|
export declare function traceIdFor(adwId: string): string;
|
|
154
242
|
/** span-id = first 16 hex of sha256(key), where key is a phase_id or a synthetic child key. */
|
|
@@ -186,12 +274,7 @@ export declare function resolveTracesUrl(endpoint: string): string;
|
|
|
186
274
|
* both are places a credential is routinely smuggled into a URL.
|
|
187
275
|
*/
|
|
188
276
|
export declare function endpointLabel(endpoint: string): string;
|
|
189
|
-
/**
|
|
190
|
-
* ISO-8601 -> uint64 nanoseconds AS A STRING (see WIRE FORMAT). Unparseable,
|
|
191
|
-
* missing, or pre-epoch input falls back to `fallbackMs`, because a span with
|
|
192
|
-
* a nonsense timestamp is rejected wholesale by most backends while a span
|
|
193
|
-
* with an approximate one is still useful.
|
|
194
|
-
*/
|
|
277
|
+
/** ISO-8601 -> uint64 nanoseconds AS A STRING. Unparseable, missing, or pre-epoch input falls back to `fallbackMs`. Kept as its own public, string-returning helper — pinned by tests since before the SDK swap. */
|
|
195
278
|
export declare function nanosFromIso(iso: string | null | undefined, fallbackMs?: number): string;
|
|
196
279
|
/**
|
|
197
280
|
* The safe half of a `tool_call` event's name. `record.name` for a tool call is
|
|
@@ -208,6 +291,28 @@ export declare function toolSpanName(eventName: string | undefined | null): stri
|
|
|
208
291
|
* `observability.otel` ever reaches a log.
|
|
209
292
|
*/
|
|
210
293
|
export declare function redact(message: string, secrets: Array<string | undefined>): string;
|
|
294
|
+
/**
|
|
295
|
+
* `spf.lora_adapter` — the served LoRA adapter name for this agent's model,
|
|
296
|
+
* when one can be determined. Checked in order, first match wins:
|
|
297
|
+
* 1. `agent.lora_adapter` (explicit config) — set once per agent, the
|
|
298
|
+
* unambiguous source of truth an operator can always fall back to.
|
|
299
|
+
* 2. `provider/base:adapter` — an explicit adapter suffix after the LAST
|
|
300
|
+
* ":" in the model id (e.g. `vllm/nemotron-base:my-lora` -> `my-lora`).
|
|
301
|
+
* 3. `provider/adapter-name` — no ":" in the model id, but the id itself
|
|
302
|
+
* contains "-lora-" (case-insensitive) — this org's own vLLM/Switchyard
|
|
303
|
+
* served-model naming convention (`k8s/manifests/switchyard/
|
|
304
|
+
* configmap-routes.yaml`: "id MUST equal the name= half of the matching
|
|
305
|
+
* --lora-modules entry", e.g. `nemotron-lora-placeholder`) — the WHOLE
|
|
306
|
+
* model id (minus the `provider/` prefix) IS the adapter name in this
|
|
307
|
+
* convention, since vLLM resolves LoRA adapters by served-model name,
|
|
308
|
+
* not by a base-model-plus-suffix split.
|
|
309
|
+
* No match on any of the three -> `null`, and the attribute is omitted
|
|
310
|
+
* entirely — never a blind copy of a non-LoRA model id.
|
|
311
|
+
*/
|
|
312
|
+
export declare function loraAdapterFor(agent: {
|
|
313
|
+
model: string;
|
|
314
|
+
lora_adapter?: string | null;
|
|
315
|
+
}): string | null;
|
|
211
316
|
export interface OtelExporterInit {
|
|
212
317
|
/** The validated `observability.otel` block. Its presence IS the activation switch. */
|
|
213
318
|
cfg: OTelConfig;
|
|
@@ -218,6 +323,8 @@ export interface OtelExporterInit {
|
|
|
218
323
|
log?: (message: string) => void;
|
|
219
324
|
/** Injectable for tests; defaults to `process.env`. Only ever read for `traceparent`. */
|
|
220
325
|
env?: NodeJS.ProcessEnv;
|
|
326
|
+
/** The process-scoped metrics handle (see `otel_metrics.ts`), or `null` when metrics are off/unconfigured. Injectable for tests. */
|
|
327
|
+
metrics?: OtelMetrics | null;
|
|
221
328
|
}
|
|
222
329
|
export declare class OtelExporter {
|
|
223
330
|
private readonly cfg;
|
|
@@ -226,10 +333,14 @@ export declare class OtelExporter {
|
|
|
226
333
|
private readonly serviceName;
|
|
227
334
|
private readonly url;
|
|
228
335
|
private readonly log;
|
|
336
|
+
private readonly metrics;
|
|
229
337
|
private readonly traceId;
|
|
230
338
|
/** "" unless an inbound traceparent parented this run — see INBOUND TRACEPARENT. */
|
|
231
339
|
private readonly rootParentSpanId;
|
|
232
340
|
private readonly rootSpanId;
|
|
341
|
+
private readonly resource;
|
|
342
|
+
private readonly scope;
|
|
343
|
+
private readonly spanExporter;
|
|
233
344
|
private queue;
|
|
234
345
|
private dropped;
|
|
235
346
|
private droppedEvents;
|
|
@@ -248,7 +359,7 @@ export declare class OtelExporter {
|
|
|
248
359
|
* nothing will ever drain.
|
|
249
360
|
*/
|
|
250
361
|
private emittedPhases;
|
|
251
|
-
/** `<phase_id
|
|
362
|
+
/** `<phase_id> <agent>` -> the open agent call, for closing it and parenting tool spans. */
|
|
252
363
|
private openAgents;
|
|
253
364
|
/** How many times an agent has been called in a phase, so a retry gets its own span id. */
|
|
254
365
|
private agentCalls;
|
|
@@ -308,6 +419,17 @@ export declare class OtelExporter {
|
|
|
308
419
|
* the guard is why that is harmless.
|
|
309
420
|
*/
|
|
310
421
|
recordSessionFinish(ok: boolean): void;
|
|
422
|
+
/**
|
|
423
|
+
* The currently-open agent call's trace context — `null` when otel has no
|
|
424
|
+
* agent call open for this phase+agent pair right now (agent_start hasn't
|
|
425
|
+
* fired, or already closed). `agents.ts`'s `send()` reads this into
|
|
426
|
+
* `AgentRequest.otel`; a `null` here just means that field stays unset, a
|
|
427
|
+
* plain no-op for every backend that doesn't propagate it.
|
|
428
|
+
*/
|
|
429
|
+
agentCallTraceContext(phaseId: string, agentName: string): {
|
|
430
|
+
traceparent: string;
|
|
431
|
+
spanId: string;
|
|
432
|
+
} | null;
|
|
311
433
|
/** Queued spans and spans/events dropped so far. For tests and diagnostics. */
|
|
312
434
|
stats(): {
|
|
313
435
|
queued: number;
|
|
@@ -316,9 +438,11 @@ export declare class OtelExporter {
|
|
|
316
438
|
};
|
|
317
439
|
/**
|
|
318
440
|
* The exact JSON body the next flush would POST, without sending or
|
|
319
|
-
* draining
|
|
320
|
-
*
|
|
321
|
-
*
|
|
441
|
+
* draining — via the SAME `JsonTraceSerializer` the real exporter uses
|
|
442
|
+
* internally, so this is not a second, possibly-diverging encoding path.
|
|
443
|
+
* This is the seam `src/test/otel.test.ts` uses to prove the allowlist
|
|
444
|
+
* holds — the assertion is on the literal bytes, so any future attribute
|
|
445
|
+
* that leaks a payload fails a test rather than a review.
|
|
322
446
|
*/
|
|
323
447
|
pendingJson(): string;
|
|
324
448
|
private enqueue;
|
|
@@ -327,7 +451,11 @@ export declare class OtelExporter {
|
|
|
327
451
|
/**
|
|
328
452
|
* Send whatever is queued. Never throws, never rejects: a failed export is a
|
|
329
453
|
* single redacted log line and a swallowed error, because the alternative is
|
|
330
|
-
* an observability feature that can fail a run.
|
|
454
|
+
* an observability feature that can fail a run. `OTLPTraceExporter.export()`
|
|
455
|
+
* itself already never throws and always calls its callback exactly once
|
|
456
|
+
* (verified against `@opentelemetry/otlp-exporter-base`'s
|
|
457
|
+
* `OTLPExportDelegate.export()` source) — the try/catch here is belt-and-
|
|
458
|
+
* braces for that contract, not a load-bearing guard.
|
|
331
459
|
*/
|
|
332
460
|
flush(isFinal?: boolean): Promise<void>;
|
|
333
461
|
/**
|
|
@@ -336,6 +464,7 @@ export declare class OtelExporter {
|
|
|
336
464
|
* `session.ts`'s signal handler (with a tighter budget there).
|
|
337
465
|
*/
|
|
338
466
|
drain(budgetMs?: number): Promise<void>;
|
|
467
|
+
private makeSpan;
|
|
339
468
|
private emitRootSpan;
|
|
340
469
|
private agentKey;
|
|
341
470
|
private openAgentCall;
|
|
@@ -351,7 +480,6 @@ export declare class OtelExporter {
|
|
|
351
480
|
private emitToolSpan;
|
|
352
481
|
private bufferSpanEvent;
|
|
353
482
|
private takeBufferedEvents;
|
|
354
|
-
private payloadFor;
|
|
355
483
|
private logFailureOnce;
|
|
356
484
|
}
|
|
357
485
|
/**
|
|
@@ -361,6 +489,12 @@ export declare class OtelExporter {
|
|
|
361
489
|
* the default for every repo that has not configured an endpoint, and no
|
|
362
490
|
* environment variable can change that (see EXPLICIT CONFIG ONLY).
|
|
363
491
|
*
|
|
492
|
+
* Also resolves (once per process — see `otel_metrics.ts`'s own singleton
|
|
493
|
+
* guard) the shared, PROCESS-scoped `OtelMetrics` handle and holds a
|
|
494
|
+
* reference on the exporter, so `recordPhase`/`recordGate`/`closeAgentCall`
|
|
495
|
+
* can fan out to it without any OTHER call site (`tracer.ts`, `agents.ts`)
|
|
496
|
+
* needing to know metrics exist at all.
|
|
497
|
+
*
|
|
364
498
|
* Registered under `opts.adwId` — the RESOLVED id (`session.ensure`'s own
|
|
365
499
|
* `id`, never a caller's possibly-null `ctx.adw_id`) — which is exactly the
|
|
366
500
|
* key `releaseOtelExporter` below looks it up by. A second registration
|