@introspection-sdk/introspection-node 0.4.1 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +1 -1
- package/.turbo/turbo-check-types.log +4 -0
- package/.turbo/turbo-lint.log +5 -0
- package/dist/ai-sdk-integration.d.ts.map +1 -1
- package/dist/ai-sdk-integration.js +2 -2
- package/dist/claude-hooks.d.ts.map +1 -1
- package/dist/claude-hooks.js +3 -3
- package/dist/client.d.ts +42 -132
- package/dist/client.d.ts.map +1 -1
- package/dist/client.js +66 -365
- package/dist/client.js.map +1 -0
- package/dist/converters/claude.d.ts +5 -1
- package/dist/converters/claude.d.ts.map +1 -1
- package/dist/converters/claude.js +7 -0
- package/dist/converters/claude.js.map +1 -0
- package/dist/converters/gemini.d.ts +90 -0
- package/dist/converters/gemini.d.ts.map +1 -0
- package/dist/converters/gemini.js +171 -0
- package/dist/converters/gemini.js.map +1 -0
- package/dist/converters/index.d.ts +1 -1
- package/dist/converters/index.d.ts.map +1 -1
- package/dist/converters/index.js +2 -1
- package/dist/converters/index.js.map +1 -0
- package/dist/converters/openai.js +1 -0
- package/dist/converters/openai.js.map +1 -0
- package/dist/converters/openclaw.d.ts +16 -0
- package/dist/converters/openclaw.d.ts.map +1 -0
- package/dist/converters/openclaw.js +49 -0
- package/dist/converters/openinference.js +1 -0
- package/dist/converters/openinference.js.map +1 -0
- package/dist/converters/vercel.d.ts.map +1 -1
- package/dist/converters/vercel.js +31 -3
- package/dist/converters/vercel.js.map +1 -0
- package/dist/http.d.ts +36 -0
- package/dist/http.d.ts.map +1 -0
- package/dist/http.js +145 -0
- package/dist/http.js.map +1 -0
- package/dist/index.d.ts +24 -27
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +22 -32
- package/dist/index.js.map +1 -0
- package/dist/langchain-handler.d.ts.map +1 -1
- package/dist/langchain-handler.js +2 -2
- package/dist/mastra-exporter.d.ts.map +1 -1
- package/dist/mastra-exporter.js +2 -2
- package/dist/otel/anthropic.d.ts +62 -0
- package/dist/otel/anthropic.d.ts.map +1 -0
- package/dist/otel/anthropic.js +425 -0
- package/dist/otel/anthropic.js.map +1 -0
- package/dist/otel/claude-hooks.d.ts +346 -0
- package/dist/otel/claude-hooks.d.ts.map +1 -0
- package/dist/otel/claude-hooks.js +645 -0
- package/dist/otel/claude-hooks.js.map +1 -0
- package/dist/otel/claude-wrapper.d.ts +93 -0
- package/dist/otel/claude-wrapper.d.ts.map +1 -0
- package/dist/otel/claude-wrapper.js +155 -0
- package/dist/otel/claude-wrapper.js.map +1 -0
- package/dist/otel/gemini.d.ts +48 -0
- package/dist/otel/gemini.d.ts.map +1 -0
- package/dist/otel/gemini.js +297 -0
- package/dist/otel/gemini.js.map +1 -0
- package/dist/otel/index.d.ts +27 -0
- package/dist/otel/index.d.ts.map +1 -0
- package/dist/otel/index.js +33 -0
- package/dist/otel/index.js.map +1 -0
- package/dist/otel/langchain-handler.d.ts +112 -0
- package/dist/otel/langchain-handler.d.ts.map +1 -0
- package/dist/otel/langchain-handler.js +555 -0
- package/dist/otel/langchain-handler.js.map +1 -0
- package/dist/otel/logs.d.ts +82 -0
- package/dist/otel/logs.d.ts.map +1 -0
- package/dist/otel/logs.js +271 -0
- package/dist/otel/logs.js.map +1 -0
- package/dist/otel/mastra-exporter.d.ts +87 -0
- package/dist/otel/mastra-exporter.d.ts.map +1 -0
- package/dist/otel/mastra-exporter.js +525 -0
- package/dist/otel/mastra-exporter.js.map +1 -0
- package/dist/otel/pi.d.ts +59 -0
- package/dist/otel/pi.d.ts.map +1 -0
- package/dist/otel/pi.js +67 -0
- package/dist/otel/pi.js.map +1 -0
- package/dist/otel/setup.d.ts +23 -0
- package/dist/otel/setup.d.ts.map +1 -0
- package/dist/otel/setup.js +127 -0
- package/dist/otel/setup.js.map +1 -0
- package/dist/otel/span-processor.d.ts +62 -0
- package/dist/otel/span-processor.d.ts.map +1 -0
- package/dist/otel/span-processor.js +319 -0
- package/dist/otel/span-processor.js.map +1 -0
- package/dist/otel/tracing-processor.d.ts +139 -0
- package/dist/otel/tracing-processor.d.ts.map +1 -0
- package/dist/otel/tracing-processor.js +418 -0
- package/dist/otel/tracing-processor.js.map +1 -0
- package/dist/resources/experiments.d.ts +31 -0
- package/dist/resources/experiments.d.ts.map +1 -0
- package/dist/resources/experiments.js +133 -0
- package/dist/resources/experiments.js.map +1 -0
- package/dist/resources/recipes.d.ts +22 -0
- package/dist/resources/recipes.d.ts.map +1 -0
- package/dist/resources/recipes.js +61 -0
- package/dist/resources/recipes.js.map +1 -0
- package/dist/resources/runtimes.d.ts +76 -0
- package/dist/resources/runtimes.d.ts.map +1 -0
- package/dist/resources/runtimes.js +179 -0
- package/dist/resources/runtimes.js.map +1 -0
- package/dist/runner-resources/files.d.ts +35 -0
- package/dist/runner-resources/files.d.ts.map +1 -0
- package/dist/runner-resources/files.js +131 -0
- package/dist/runner-resources/files.js.map +1 -0
- package/dist/runner-resources/tasks.d.ts +39 -0
- package/dist/runner-resources/tasks.d.ts.map +1 -0
- package/dist/runner-resources/tasks.js +129 -0
- package/dist/runner-resources/tasks.js.map +1 -0
- package/dist/runner.d.ts +80 -0
- package/dist/runner.d.ts.map +1 -0
- package/dist/runner.js +159 -0
- package/dist/runner.js.map +1 -0
- package/dist/span-processor.d.ts.map +1 -1
- package/dist/span-processor.js +3 -3
- package/dist/streaming.d.ts +8 -0
- package/dist/streaming.d.ts.map +1 -0
- package/dist/streaming.js +67 -0
- package/dist/streaming.js.map +1 -0
- package/dist/tracing-processor.d.ts.map +1 -1
- package/dist/tracing-processor.js +3 -3
- package/dist/types/genai.d.ts +165 -0
- package/dist/types/genai.d.ts.map +1 -0
- package/dist/types/genai.js +82 -0
- package/dist/types.js +1 -0
- package/dist/types.js.map +1 -0
- package/dist/utils.d.ts +1 -0
- package/dist/utils.d.ts.map +1 -1
- package/dist/utils.js +14 -0
- package/dist/utils.js.map +1 -0
- package/dist/version.js +1 -0
- package/dist/version.js.map +1 -0
- package/package.json +83 -25
- package/src/client.ts +76 -471
- package/src/converters/claude.ts +11 -1
- package/src/converters/gemini.ts +243 -0
- package/src/converters/index.ts +13 -7
- package/src/converters/vercel.ts +41 -3
- package/src/http.ts +163 -0
- package/src/index.ts +110 -75
- package/src/{anthropic.ts → otel/anthropic.ts} +95 -8
- package/src/{claude-hooks.ts → otel/claude-hooks.ts} +125 -24
- package/src/{claude-wrapper.ts → otel/claude-wrapper.ts} +1 -1
- package/src/otel/gemini.ts +397 -0
- package/src/otel/index.ts +80 -0
- package/src/{langchain-handler.ts → otel/langchain-handler.ts} +56 -7
- package/src/otel/logs.ts +394 -0
- package/src/{mastra-exporter.ts → otel/mastra-exporter.ts} +86 -23
- package/src/otel/pi.ts +88 -0
- package/src/otel/setup.ts +174 -0
- package/src/{span-processor.ts → otel/span-processor.ts} +17 -10
- package/src/{tracing-processor.ts → otel/tracing-processor.ts} +100 -23
- package/src/resources/experiments.ts +170 -0
- package/src/resources/recipes.ts +73 -0
- package/src/resources/runtimes.ts +231 -0
- package/src/runner-resources/files.ts +163 -0
- package/src/runner-resources/tasks.ts +156 -0
- package/src/runner.ts +194 -0
- package/src/streaming.ts +57 -0
- package/src/utils.ts +15 -0
- package/src/ai-sdk-integration.ts +0 -689
- package/src/converters/ai-sdk.ts +0 -323
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* One-call tracing bootstrap for Introspection.
|
|
3
|
+
*
|
|
4
|
+
* Wires up the three things every host process needs for baggage-based
|
|
5
|
+
* agent / conversation propagation to work:
|
|
6
|
+
*
|
|
7
|
+
* 1. `AsyncLocalStorageContextManager` — without this, `context.with()` is a
|
|
8
|
+
* no-op (the default `NOOP_CONTEXT_MANAGER` silently drops the context),
|
|
9
|
+
* which makes `IntrospectionClient.withAgent()`, `.withConversation()`,
|
|
10
|
+
* etc. dead code.
|
|
11
|
+
* 2. `W3CBaggagePropagator` — so baggage is carried across HTTP boundaries
|
|
12
|
+
* via the `baggage` header (alongside W3C trace-context).
|
|
13
|
+
* 3. A registered `NodeTracerProvider` with an `IntrospectionSpanProcessor`
|
|
14
|
+
* on it — so spans are exported to Introspection and baggage values
|
|
15
|
+
* (conversation / agent IDs) are merged into span attributes at
|
|
16
|
+
* `onEnd()` time.
|
|
17
|
+
*
|
|
18
|
+
* Call this once at process start (before any instrumentors run). It is
|
|
19
|
+
* idempotent across re-invocations — the cached provider is returned and
|
|
20
|
+
* the global registrations are not touched a second time.
|
|
21
|
+
*
|
|
22
|
+
* **Loud-by-default registration.** OTel's `setGlobalContextManager` /
|
|
23
|
+
* `setGlobalPropagator` silently refuse to replace an existing
|
|
24
|
+
* registration. If anything (another lib, a stale call, a test fixture)
|
|
25
|
+
* has already installed a context manager that does NOT propagate W3C
|
|
26
|
+
* baggage, our `withAgent` / `withConversation` calls become silent
|
|
27
|
+
* no-ops and users get spans with no identity attached — exactly the
|
|
28
|
+
* footgun this SDK exists to prevent.
|
|
29
|
+
*
|
|
30
|
+
* `setupTracing` defends against that:
|
|
31
|
+
* - If no manager / propagator is registered: install ours.
|
|
32
|
+
* - If one is registered: WARN (default) or THROW (strict mode) so the
|
|
33
|
+
* host author sees it, rather than discovering missing baggage in
|
|
34
|
+
* production telemetry weeks later.
|
|
35
|
+
*
|
|
36
|
+
* Pass `{ onConflict: "throw" }` (or set `INTROSPECTION_STRICT_TRACING=1`
|
|
37
|
+
* in the env) for the strict variant. Pass `{ onConflict: "replace" }`
|
|
38
|
+
* to force replacement when you know the prior registration is wrong
|
|
39
|
+
* (e.g. test setup, or migrating from another tracer).
|
|
40
|
+
*
|
|
41
|
+
* @example
|
|
42
|
+
* ```ts
|
|
43
|
+
* import { setupTracing, IntrospectionClient } from "@introspection-sdk/introspection-node";
|
|
44
|
+
*
|
|
45
|
+
* setupTracing();
|
|
46
|
+
*
|
|
47
|
+
* const introspect = new IntrospectionClient();
|
|
48
|
+
*
|
|
49
|
+
* await introspect.withAgent("researcher", "researcher-1", () =>
|
|
50
|
+
* introspect.withConversation(conversationId, undefined, () =>
|
|
51
|
+
* client.messages.create({ ... }),
|
|
52
|
+
* ),
|
|
53
|
+
* );
|
|
54
|
+
* ```
|
|
55
|
+
*/
|
|
56
|
+
import { context, propagation } from "@opentelemetry/api";
|
|
57
|
+
import { AsyncLocalStorageContextManager } from "@opentelemetry/context-async-hooks";
|
|
58
|
+
import {
|
|
59
|
+
CompositePropagator,
|
|
60
|
+
W3CBaggagePropagator,
|
|
61
|
+
W3CTraceContextPropagator,
|
|
62
|
+
} from "@opentelemetry/core";
|
|
63
|
+
import { NodeTracerProvider } from "@opentelemetry/sdk-trace-node";
|
|
64
|
+
|
|
65
|
+
import { logger } from "../utils.js";
|
|
66
|
+
import {
|
|
67
|
+
IntrospectionSpanProcessor,
|
|
68
|
+
type IntrospectionSpanProcessorOptions,
|
|
69
|
+
} from "./span-processor.js";
|
|
70
|
+
|
|
71
|
+
let registeredProvider: NodeTracerProvider | undefined;
|
|
72
|
+
|
|
73
|
+
export type ConflictBehavior = "warn" | "throw" | "replace";
|
|
74
|
+
|
|
75
|
+
export interface SetupTracingOptions extends IntrospectionSpanProcessorOptions {
|
|
76
|
+
/**
|
|
77
|
+
* What to do if a context manager or propagator is already registered.
|
|
78
|
+
*
|
|
79
|
+
* - `"warn"` (default) — log a warning and continue with the existing
|
|
80
|
+
* registration. Spans still flow but baggage may not propagate as
|
|
81
|
+
* expected. Set `INTROSPECTION_STRICT_TRACING=1` to upgrade to throw.
|
|
82
|
+
* - `"throw"` — throw an error. Use in environments where a silent
|
|
83
|
+
* baggage drop is unacceptable (production telemetry pipelines).
|
|
84
|
+
* - `"replace"` — force-register over the existing manager / propagator.
|
|
85
|
+
* Useful in tests or when migrating from another tracer; can break
|
|
86
|
+
* other libraries that relied on the previous registration.
|
|
87
|
+
*/
|
|
88
|
+
onConflict?: ConflictBehavior;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
function defaultConflictBehavior(): ConflictBehavior {
|
|
92
|
+
return process.env.INTROSPECTION_STRICT_TRACING ? "throw" : "warn";
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
function registerContextManager(behavior: ConflictBehavior): void {
|
|
96
|
+
const ok = context.setGlobalContextManager(
|
|
97
|
+
new AsyncLocalStorageContextManager().enable(),
|
|
98
|
+
);
|
|
99
|
+
if (ok) return;
|
|
100
|
+
// OTel silently refuses replacement — we have to disable+retry to force.
|
|
101
|
+
const msg =
|
|
102
|
+
"setupTracing: another OTel context manager was already registered. " +
|
|
103
|
+
"withAgent / withConversation baggage scopes will not propagate unless " +
|
|
104
|
+
"that manager is an AsyncLocalStorageContextManager honoring W3C baggage.";
|
|
105
|
+
if (behavior === "throw") {
|
|
106
|
+
throw new Error(`[introspection] ${msg}`);
|
|
107
|
+
}
|
|
108
|
+
if (behavior === "replace") {
|
|
109
|
+
context.disable();
|
|
110
|
+
if (
|
|
111
|
+
!context.setGlobalContextManager(
|
|
112
|
+
new AsyncLocalStorageContextManager().enable(),
|
|
113
|
+
)
|
|
114
|
+
) {
|
|
115
|
+
throw new Error(
|
|
116
|
+
"[introspection] setupTracing: failed to replace context manager " +
|
|
117
|
+
"even after disable(); the OTel global registry is in a bad state.",
|
|
118
|
+
);
|
|
119
|
+
}
|
|
120
|
+
return;
|
|
121
|
+
}
|
|
122
|
+
logger.warn(msg);
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
function registerPropagator(behavior: ConflictBehavior): void {
|
|
126
|
+
const propagator = new CompositePropagator({
|
|
127
|
+
propagators: [new W3CTraceContextPropagator(), new W3CBaggagePropagator()],
|
|
128
|
+
});
|
|
129
|
+
const ok = propagation.setGlobalPropagator(propagator);
|
|
130
|
+
if (ok) return;
|
|
131
|
+
const msg =
|
|
132
|
+
"setupTracing: another OTel propagator was already registered. " +
|
|
133
|
+
"Cross-process baggage propagation depends on the existing propagator " +
|
|
134
|
+
"carrying the W3C `baggage` header.";
|
|
135
|
+
if (behavior === "throw") {
|
|
136
|
+
throw new Error(`[introspection] ${msg}`);
|
|
137
|
+
}
|
|
138
|
+
if (behavior === "replace") {
|
|
139
|
+
propagation.disable();
|
|
140
|
+
if (!propagation.setGlobalPropagator(propagator)) {
|
|
141
|
+
throw new Error(
|
|
142
|
+
"[introspection] setupTracing: failed to replace propagator even " +
|
|
143
|
+
"after disable(); the OTel global registry is in a bad state.",
|
|
144
|
+
);
|
|
145
|
+
}
|
|
146
|
+
return;
|
|
147
|
+
}
|
|
148
|
+
logger.warn(msg);
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/**
|
|
152
|
+
* Initialise Introspection tracing for a Node.js process.
|
|
153
|
+
*/
|
|
154
|
+
export function setupTracing(
|
|
155
|
+
options?: SetupTracingOptions,
|
|
156
|
+
): NodeTracerProvider {
|
|
157
|
+
if (registeredProvider) {
|
|
158
|
+
return registeredProvider;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
const behavior: ConflictBehavior =
|
|
162
|
+
options?.onConflict ?? defaultConflictBehavior();
|
|
163
|
+
|
|
164
|
+
registerContextManager(behavior);
|
|
165
|
+
registerPropagator(behavior);
|
|
166
|
+
|
|
167
|
+
const provider = new NodeTracerProvider({
|
|
168
|
+
spanProcessors: [new IntrospectionSpanProcessor(options)],
|
|
169
|
+
});
|
|
170
|
+
provider.register();
|
|
171
|
+
|
|
172
|
+
registeredProvider = provider;
|
|
173
|
+
return provider;
|
|
174
|
+
}
|
|
@@ -16,14 +16,14 @@ import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions";
|
|
|
16
16
|
import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-proto";
|
|
17
17
|
import { ExportResult, ExportResultCode } from "@opentelemetry/core";
|
|
18
18
|
import { randomUUID } from "crypto";
|
|
19
|
-
import type { AdvancedOptions } from "
|
|
20
|
-
import { logger } from "
|
|
19
|
+
import type { AdvancedOptions } from "../types.js";
|
|
20
|
+
import { logger, withOtlpHttpsProxy } from "../utils.js";
|
|
21
21
|
import {
|
|
22
22
|
isOpenInferenceSpan,
|
|
23
23
|
replaceOpenInferenceWithGenAI,
|
|
24
24
|
isVercelAISpan,
|
|
25
25
|
convertVercelAIToGenAI,
|
|
26
|
-
} from "
|
|
26
|
+
} from "../converters/index.js";
|
|
27
27
|
|
|
28
28
|
/**
|
|
29
29
|
* Extended span type that includes both v1 and v2 instrumentation properties.
|
|
@@ -157,7 +157,7 @@ export class IntrospectionSpanProcessor implements SpanProcessor {
|
|
|
157
157
|
const advanced = options.advanced || {};
|
|
158
158
|
const baseUrl =
|
|
159
159
|
advanced.baseUrl ||
|
|
160
|
-
process.env.
|
|
160
|
+
process.env.INTROSPECTION_BASE_OTEL_URL ||
|
|
161
161
|
"https://otel.introspection.dev";
|
|
162
162
|
|
|
163
163
|
const headers: Record<string, string> = {
|
|
@@ -180,10 +180,12 @@ export class IntrospectionSpanProcessor implements SpanProcessor {
|
|
|
180
180
|
// Use custom spanExporter if provided (for testing), otherwise create OTLP exporter
|
|
181
181
|
const baseExporter: SpanExporter =
|
|
182
182
|
advanced.spanExporter ||
|
|
183
|
-
new OTLPTraceExporter(
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
183
|
+
new OTLPTraceExporter(
|
|
184
|
+
withOtlpHttpsProxy({
|
|
185
|
+
url: endpoint,
|
|
186
|
+
headers,
|
|
187
|
+
}),
|
|
188
|
+
);
|
|
187
189
|
|
|
188
190
|
// Wrap exporter to add v1/v2 compatibility and basic validation
|
|
189
191
|
const exporter: SpanExporter = {
|
|
@@ -379,11 +381,16 @@ export class IntrospectionSpanProcessor implements SpanProcessor {
|
|
|
379
381
|
attrs["gen_ai.operation.name"] = "chat";
|
|
380
382
|
}
|
|
381
383
|
|
|
382
|
-
// Agent name from baggage
|
|
384
|
+
// Agent name / id from baggage (baggage wins so per-call agent identity
|
|
385
|
+
// overrides any default stamped at instrumentor-construction time).
|
|
383
386
|
const baggageAgentName = baggage?.getEntry("gen_ai.agent.name")?.value;
|
|
384
|
-
if (baggageAgentName
|
|
387
|
+
if (baggageAgentName) {
|
|
385
388
|
attrs["gen_ai.agent.name"] = baggageAgentName;
|
|
386
389
|
}
|
|
390
|
+
const baggageAgentId = baggage?.getEntry("gen_ai.agent.id")?.value;
|
|
391
|
+
if (baggageAgentId) {
|
|
392
|
+
attrs["gen_ai.agent.id"] = baggageAgentId;
|
|
393
|
+
}
|
|
387
394
|
|
|
388
395
|
// Override resource with service name if provided
|
|
389
396
|
let resource: Resource | undefined;
|
|
@@ -27,13 +27,13 @@ import {
|
|
|
27
27
|
convertResponsesOutputsToSemconv,
|
|
28
28
|
convertResponsesToolsToSemconv,
|
|
29
29
|
convertResponsesInstructionsToSemconv,
|
|
30
|
-
} from "
|
|
30
|
+
} from "../converters/openai.js";
|
|
31
31
|
import type {
|
|
32
32
|
Response as OpenAIResponse,
|
|
33
33
|
ResponseInputItem,
|
|
34
|
-
} from "
|
|
35
|
-
import { logger } from "
|
|
36
|
-
import { VERSION } from "
|
|
34
|
+
} from "../converters/openai.js";
|
|
35
|
+
import { logger, withOtlpHttpsProxy } from "../utils.js";
|
|
36
|
+
import { VERSION } from "../version.js";
|
|
37
37
|
|
|
38
38
|
// Type imports from @openai/agents
|
|
39
39
|
import type {
|
|
@@ -79,7 +79,7 @@ export interface TracingProcessorAdvancedOptions {
|
|
|
79
79
|
export interface IntrospectionTracingProcessorOptions {
|
|
80
80
|
/** Authentication token (env: INTROSPECTION_TOKEN) */
|
|
81
81
|
token?: string;
|
|
82
|
-
/** Base URL for the API (env:
|
|
82
|
+
/** Base URL for the API (env: INTROSPECTION_BASE_OTEL_URL, default: "https://otel.introspection.dev") */
|
|
83
83
|
baseUrl?: string;
|
|
84
84
|
/** Service name for telemetry (env: INTROSPECTION_SERVICE_NAME) */
|
|
85
85
|
serviceName?: string;
|
|
@@ -131,7 +131,7 @@ export class IntrospectionTracingProcessor implements TracingProcessor {
|
|
|
131
131
|
|
|
132
132
|
const baseUrl =
|
|
133
133
|
options.baseUrl ||
|
|
134
|
-
process.env.
|
|
134
|
+
process.env.INTROSPECTION_BASE_OTEL_URL ||
|
|
135
135
|
"https://otel.introspection.dev";
|
|
136
136
|
|
|
137
137
|
const headers: Record<string, string> = {
|
|
@@ -150,10 +150,12 @@ export class IntrospectionTracingProcessor implements TracingProcessor {
|
|
|
150
150
|
);
|
|
151
151
|
|
|
152
152
|
// Setup OTLP exporter
|
|
153
|
-
const spanExporter = new OTLPTraceExporter(
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
153
|
+
const spanExporter = new OTLPTraceExporter(
|
|
154
|
+
withOtlpHttpsProxy({
|
|
155
|
+
url: endpoint,
|
|
156
|
+
headers,
|
|
157
|
+
}),
|
|
158
|
+
);
|
|
157
159
|
|
|
158
160
|
// Use SimpleSpanProcessor for maxExportBatchSize=1 (sequential export),
|
|
159
161
|
// otherwise BatchSpanProcessor with configurable batch size.
|
|
@@ -218,7 +220,10 @@ export class IntrospectionTracingProcessor implements TracingProcessor {
|
|
|
218
220
|
async onTraceStart(trace: Trace): Promise<void> {
|
|
219
221
|
logger.debug(`onTraceStart called: ${trace?.name} (${trace?.traceId})`);
|
|
220
222
|
if (trace) {
|
|
221
|
-
|
|
223
|
+
// Use groupId as conversationId when provided so callers can link
|
|
224
|
+
// multiple traces (e.g. multi-turn orchestrator runs) into one conversation.
|
|
225
|
+
const conversationId =
|
|
226
|
+
trace.groupId ?? `intro_conv_${randomUUID().replace(/-/g, "")}`;
|
|
222
227
|
this._conversationIds.set(trace.traceId, conversationId);
|
|
223
228
|
const otelSpan = this._tracer.startSpan(trace.name);
|
|
224
229
|
this._spans.set(trace.traceId, otelSpan);
|
|
@@ -284,14 +289,10 @@ export class IntrospectionTracingProcessor implements TracingProcessor {
|
|
|
284
289
|
* @param span - The OpenAI Agents SDK span.
|
|
285
290
|
*/
|
|
286
291
|
async onSpanEnd(span: Span<SpanData>): Promise<void> {
|
|
287
|
-
logger.debug(
|
|
288
|
-
`onSpanEnd called: ${span?.spanId} (type: ${(span?.spanData as SpanData)?.type})`,
|
|
289
|
-
);
|
|
290
292
|
if (!span) return;
|
|
291
293
|
|
|
292
294
|
const otelSpan = this._spans.get(span.spanId);
|
|
293
295
|
if (!otelSpan) {
|
|
294
|
-
logger.debug(`No OTel span found for spanId: ${span.spanId}`);
|
|
295
296
|
return;
|
|
296
297
|
}
|
|
297
298
|
|
|
@@ -446,6 +447,14 @@ export class IntrospectionTracingProcessor implements TracingProcessor {
|
|
|
446
447
|
usage.output_tokens,
|
|
447
448
|
);
|
|
448
449
|
}
|
|
450
|
+
// OpenAI returns cached token count in input_tokens_details.cached_tokens
|
|
451
|
+
const cachedTokens = usage.input_tokens_details?.cached_tokens;
|
|
452
|
+
if (cachedTokens) {
|
|
453
|
+
otelSpan.setAttribute(
|
|
454
|
+
"gen_ai.usage.cache_read.input_tokens",
|
|
455
|
+
cachedTokens,
|
|
456
|
+
);
|
|
457
|
+
}
|
|
449
458
|
}
|
|
450
459
|
|
|
451
460
|
// Model info
|
|
@@ -472,6 +481,9 @@ export class IntrospectionTracingProcessor implements TracingProcessor {
|
|
|
472
481
|
|
|
473
482
|
/**
|
|
474
483
|
* Extract attributes from generation spans.
|
|
484
|
+
* Handles both structured GenerationUsageData (spanData.usage) and raw
|
|
485
|
+
* Chat Completions API responses stored in spanData.output[0] by
|
|
486
|
+
* OpenAIChatCompletionsModel.
|
|
475
487
|
*/
|
|
476
488
|
private _processGenerationSpan(
|
|
477
489
|
otelSpan: OtelSpan,
|
|
@@ -485,15 +497,78 @@ export class IntrospectionTracingProcessor implements TracingProcessor {
|
|
|
485
497
|
otelSpan.setAttribute("gen_ai.request.model", spanData.model);
|
|
486
498
|
}
|
|
487
499
|
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
500
|
+
// GenerationUsageData uses snake_case: input_tokens / output_tokens.
|
|
501
|
+
if (spanData.usage?.input_tokens) {
|
|
502
|
+
otelSpan.setAttribute(
|
|
503
|
+
"gen_ai.usage.input_tokens",
|
|
504
|
+
spanData.usage.input_tokens,
|
|
505
|
+
);
|
|
506
|
+
}
|
|
507
|
+
if (spanData.usage?.output_tokens) {
|
|
508
|
+
otelSpan.setAttribute(
|
|
509
|
+
"gen_ai.usage.output_tokens",
|
|
510
|
+
spanData.usage.output_tokens,
|
|
511
|
+
);
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
// OpenAIChatCompletionsModel stores the raw ChatCompletion response object
|
|
515
|
+
// in spanData.output[0] (it never sets spanData.usage). Extract usage,
|
|
516
|
+
// response ID, response model, and the actual assistant message from it.
|
|
517
|
+
if (!spanData.usage) {
|
|
518
|
+
const rawResponse = spanData.output?.[0] as
|
|
519
|
+
| Record<string, unknown>
|
|
520
|
+
| undefined;
|
|
521
|
+
|
|
522
|
+
// Response ID — required by the backend processor to register this span
|
|
523
|
+
// as a conversation item (build_request returns None without it).
|
|
524
|
+
const responseId = rawResponse?.id;
|
|
525
|
+
if (typeof responseId === "string" && responseId) {
|
|
526
|
+
otelSpan.setAttribute("gen_ai.response.id", responseId);
|
|
492
527
|
}
|
|
493
|
-
|
|
528
|
+
|
|
529
|
+
// Resolved model name (e.g. "gpt-4o-mini-2024-07-18" vs "gpt-4o-mini").
|
|
530
|
+
const responseModel = rawResponse?.model;
|
|
531
|
+
if (typeof responseModel === "string" && responseModel) {
|
|
532
|
+
otelSpan.setAttribute("gen_ai.response.model", responseModel);
|
|
533
|
+
}
|
|
534
|
+
|
|
535
|
+
const rawUsage = rawResponse?.usage as
|
|
536
|
+
| Record<string, unknown>
|
|
537
|
+
| undefined;
|
|
538
|
+
if (rawUsage) {
|
|
539
|
+
const promptTokens = rawUsage.prompt_tokens;
|
|
540
|
+
const completionTokens = rawUsage.completion_tokens;
|
|
541
|
+
if (typeof promptTokens === "number") {
|
|
542
|
+
otelSpan.setAttribute("gen_ai.usage.input_tokens", promptTokens);
|
|
543
|
+
}
|
|
544
|
+
if (typeof completionTokens === "number") {
|
|
545
|
+
otelSpan.setAttribute("gen_ai.usage.output_tokens", completionTokens);
|
|
546
|
+
}
|
|
547
|
+
const promptDetails = rawUsage.prompt_tokens_details as
|
|
548
|
+
| Record<string, unknown>
|
|
549
|
+
| undefined;
|
|
550
|
+
const cachedTokens = promptDetails?.cached_tokens;
|
|
551
|
+
if (typeof cachedTokens === "number" && cachedTokens > 0) {
|
|
552
|
+
otelSpan.setAttribute(
|
|
553
|
+
"gen_ai.usage.cache_read.input_tokens",
|
|
554
|
+
cachedTokens,
|
|
555
|
+
);
|
|
556
|
+
}
|
|
557
|
+
}
|
|
558
|
+
|
|
559
|
+
// Extract the actual assistant message from choices[0].message so that
|
|
560
|
+
// gen_ai.output.messages is a proper [{role, content}] array rather than
|
|
561
|
+
// the full ChatCompletion envelope (which the backend cannot parse).
|
|
562
|
+
const choices = rawResponse?.choices as
|
|
563
|
+
| Array<Record<string, unknown>>
|
|
564
|
+
| undefined;
|
|
565
|
+
const assistantMessage = choices?.[0]?.message as
|
|
566
|
+
| Record<string, unknown>
|
|
567
|
+
| undefined;
|
|
568
|
+
if (assistantMessage) {
|
|
494
569
|
otelSpan.setAttribute(
|
|
495
|
-
"gen_ai.
|
|
496
|
-
|
|
570
|
+
"gen_ai.output.messages",
|
|
571
|
+
JSON.stringify([assistantMessage]),
|
|
497
572
|
);
|
|
498
573
|
}
|
|
499
574
|
}
|
|
@@ -505,7 +580,9 @@ export class IntrospectionTracingProcessor implements TracingProcessor {
|
|
|
505
580
|
);
|
|
506
581
|
}
|
|
507
582
|
|
|
508
|
-
|
|
583
|
+
// For non-Chat-Completions generation spans that do have usage set,
|
|
584
|
+
// still export output messages in the standard format if present.
|
|
585
|
+
if (spanData.usage && spanData.output) {
|
|
509
586
|
otelSpan.setAttribute(
|
|
510
587
|
"gen_ai.output.messages",
|
|
511
588
|
JSON.stringify(spanData.output),
|
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
Experiment,
|
|
3
|
+
ExperimentEndParams,
|
|
4
|
+
ExperimentCreate,
|
|
5
|
+
ExperimentListParams,
|
|
6
|
+
ExperimentUpdate,
|
|
7
|
+
Paginated,
|
|
8
|
+
RunRequest,
|
|
9
|
+
RunnerSpec,
|
|
10
|
+
Uuid,
|
|
11
|
+
} from "@introspection-sdk/types";
|
|
12
|
+
import type { HttpClient } from "../http.js";
|
|
13
|
+
import type { IntrospectionClient } from "../client.js";
|
|
14
|
+
import { Runner, type RunnerSource } from "../runner.js";
|
|
15
|
+
|
|
16
|
+
interface ExperimentRunRequestBody {
|
|
17
|
+
identity?: {
|
|
18
|
+
user_id?: string;
|
|
19
|
+
anonymous_id?: string;
|
|
20
|
+
conversation_id?: string;
|
|
21
|
+
};
|
|
22
|
+
ttl_seconds?: number;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function toRunBody(opts?: RunRequest): ExperimentRunRequestBody | undefined {
|
|
26
|
+
if (!opts) return undefined;
|
|
27
|
+
const out: ExperimentRunRequestBody = {};
|
|
28
|
+
if (opts.identity) out.identity = opts.identity;
|
|
29
|
+
if (opts.ttl_seconds !== undefined) out.ttl_seconds = opts.ttl_seconds;
|
|
30
|
+
return out;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export class ExperimentsApi {
|
|
34
|
+
constructor(
|
|
35
|
+
private readonly http: HttpClient,
|
|
36
|
+
private readonly client: IntrospectionClient,
|
|
37
|
+
) {}
|
|
38
|
+
|
|
39
|
+
list(params: ExperimentListParams): Promise<Paginated<Experiment>> {
|
|
40
|
+
return this.http.request<Paginated<Experiment>>({
|
|
41
|
+
method: "GET",
|
|
42
|
+
path: "/v1/experiments",
|
|
43
|
+
query: params as unknown as Record<string, unknown>,
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
async *listAll(params: ExperimentListParams): AsyncIterable<Experiment> {
|
|
48
|
+
let next: string | undefined = params.next;
|
|
49
|
+
do {
|
|
50
|
+
const page = await this.list({ ...params, next });
|
|
51
|
+
for (const r of page.records) yield r;
|
|
52
|
+
next = page.next ?? undefined;
|
|
53
|
+
} while (next);
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
get(id: Uuid): Promise<Experiment> {
|
|
57
|
+
return this.http.request<Experiment>({
|
|
58
|
+
method: "GET",
|
|
59
|
+
path: `/v1/experiments/${encodeURIComponent(id)}`,
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
create(input: ExperimentCreate): Promise<Experiment> {
|
|
64
|
+
return this.http.request<Experiment>({
|
|
65
|
+
method: "POST",
|
|
66
|
+
path: "/v1/experiments",
|
|
67
|
+
body: input,
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
update(id: Uuid, input: ExperimentUpdate): Promise<Experiment> {
|
|
72
|
+
return this.http.request<Experiment>({
|
|
73
|
+
method: "PATCH",
|
|
74
|
+
path: `/v1/experiments/${encodeURIComponent(id)}`,
|
|
75
|
+
body: input,
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
delete(id: Uuid): Promise<void> {
|
|
80
|
+
return this.http.request<void>({
|
|
81
|
+
method: "DELETE",
|
|
82
|
+
path: `/v1/experiments/${encodeURIComponent(id)}`,
|
|
83
|
+
expect: "empty",
|
|
84
|
+
});
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
startById(id: Uuid): Promise<Experiment> {
|
|
88
|
+
return this.http.request<Experiment>({
|
|
89
|
+
method: "POST",
|
|
90
|
+
path: `/v1/experiments/${encodeURIComponent(id)}/start`,
|
|
91
|
+
body: {},
|
|
92
|
+
});
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
endById(id: Uuid, params?: ExperimentEndParams): Promise<Experiment> {
|
|
96
|
+
return this.http.request<Experiment>({
|
|
97
|
+
method: "POST",
|
|
98
|
+
path: `/v1/experiments/${encodeURIComponent(id)}/end`,
|
|
99
|
+
body: params ?? {},
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
cancelById(id: Uuid): Promise<Experiment> {
|
|
104
|
+
return this.http.request<Experiment>({
|
|
105
|
+
method: "POST",
|
|
106
|
+
path: `/v1/experiments/${encodeURIComponent(id)}/cancel`,
|
|
107
|
+
body: {},
|
|
108
|
+
});
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
async runById(id: Uuid, opts?: RunRequest): Promise<Runner> {
|
|
112
|
+
const spec = await this.http.request<RunnerSpec>({
|
|
113
|
+
method: "POST",
|
|
114
|
+
path: `/v1/experiments/${encodeURIComponent(id)}/run`,
|
|
115
|
+
body: toRunBody(opts) ?? {},
|
|
116
|
+
});
|
|
117
|
+
const source: RunnerSource = {
|
|
118
|
+
kind: "experiment",
|
|
119
|
+
id,
|
|
120
|
+
options: opts,
|
|
121
|
+
};
|
|
122
|
+
return new Runner(this.client, source, spec);
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
export class ExperimentHandle {
|
|
127
|
+
constructor(
|
|
128
|
+
private readonly api: ExperimentsApi,
|
|
129
|
+
private readonly id: Uuid,
|
|
130
|
+
) {}
|
|
131
|
+
|
|
132
|
+
run(opts?: RunRequest): Promise<Runner> {
|
|
133
|
+
return this.api.runById(this.id, opts);
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
start(): Promise<Experiment> {
|
|
137
|
+
return this.api.startById(this.id);
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
end(opts?: ExperimentEndParams): Promise<Experiment> {
|
|
141
|
+
return this.api.endById(this.id, opts);
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
cancel(): Promise<Experiment> {
|
|
145
|
+
return this.api.cancelById(this.id);
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
export type ExperimentHandleFactory = (id: Uuid) => ExperimentHandle;
|
|
150
|
+
|
|
151
|
+
export function attachExperiments(
|
|
152
|
+
client: IntrospectionClient,
|
|
153
|
+
http: HttpClient,
|
|
154
|
+
): ExperimentsApi & ExperimentHandleFactory {
|
|
155
|
+
const api = new ExperimentsApi(http, client);
|
|
156
|
+
const factory: ExperimentHandleFactory = (id: Uuid) =>
|
|
157
|
+
new ExperimentHandle(api, id);
|
|
158
|
+
const hybrid = factory as ExperimentsApi & ExperimentHandleFactory;
|
|
159
|
+
hybrid.list = api.list.bind(api);
|
|
160
|
+
hybrid.listAll = api.listAll.bind(api);
|
|
161
|
+
hybrid.get = api.get.bind(api);
|
|
162
|
+
hybrid.create = api.create.bind(api);
|
|
163
|
+
hybrid.update = api.update.bind(api);
|
|
164
|
+
hybrid.delete = api.delete.bind(api);
|
|
165
|
+
hybrid.startById = api.startById.bind(api);
|
|
166
|
+
hybrid.endById = api.endById.bind(api);
|
|
167
|
+
hybrid.cancelById = api.cancelById.bind(api);
|
|
168
|
+
hybrid.runById = api.runById.bind(api);
|
|
169
|
+
return hybrid;
|
|
170
|
+
}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
Paginated,
|
|
3
|
+
Recipe,
|
|
4
|
+
RecipeCreate,
|
|
5
|
+
RecipeListParams,
|
|
6
|
+
RecipeUpdate,
|
|
7
|
+
Uuid,
|
|
8
|
+
} from "@introspection-sdk/types";
|
|
9
|
+
import type { HttpClient } from "../http.js";
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Programmatic CRUD for `/v1/recipes` on the CP.
|
|
13
|
+
*
|
|
14
|
+
* Recipes are immutable build artefacts (repository + git ref + commit
|
|
15
|
+
* sha + optional sub-path). They are referenced by runtimes via
|
|
16
|
+
* `runtime.recipe_id` and by experiment arms via `arm.recipe_id`.
|
|
17
|
+
*/
|
|
18
|
+
export class RecipesApi {
|
|
19
|
+
constructor(private readonly http: HttpClient) {}
|
|
20
|
+
|
|
21
|
+
list(params: RecipeListParams): Promise<Paginated<Recipe>> {
|
|
22
|
+
return this.http.request<Paginated<Recipe>>({
|
|
23
|
+
method: "GET",
|
|
24
|
+
path: "/v1/recipes",
|
|
25
|
+
query: params as unknown as Record<string, unknown>,
|
|
26
|
+
});
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** Async-iterate every page that matches `params`. */
|
|
30
|
+
async *listAll(params: RecipeListParams): AsyncIterable<Recipe> {
|
|
31
|
+
let next: string | undefined = params.next;
|
|
32
|
+
do {
|
|
33
|
+
const page = await this.list({ ...params, next });
|
|
34
|
+
for (const r of page.records) yield r;
|
|
35
|
+
next = page.next ?? undefined;
|
|
36
|
+
} while (next);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
get(id: Uuid): Promise<Recipe> {
|
|
40
|
+
return this.http.request<Recipe>({
|
|
41
|
+
method: "GET",
|
|
42
|
+
path: `/v1/recipes/${encodeURIComponent(id)}`,
|
|
43
|
+
});
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
create(input: RecipeCreate): Promise<Recipe> {
|
|
47
|
+
return this.http.request<Recipe>({
|
|
48
|
+
method: "POST",
|
|
49
|
+
path: "/v1/recipes",
|
|
50
|
+
body: input,
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
update(id: Uuid, input: RecipeUpdate): Promise<Recipe> {
|
|
55
|
+
return this.http.request<Recipe>({
|
|
56
|
+
method: "PATCH",
|
|
57
|
+
path: `/v1/recipes/${encodeURIComponent(id)}`,
|
|
58
|
+
body: input,
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
delete(id: Uuid): Promise<void> {
|
|
63
|
+
return this.http.request<void>({
|
|
64
|
+
method: "DELETE",
|
|
65
|
+
path: `/v1/recipes/${encodeURIComponent(id)}`,
|
|
66
|
+
expect: "empty",
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export function attachRecipes(http: HttpClient): RecipesApi {
|
|
72
|
+
return new RecipesApi(http);
|
|
73
|
+
}
|