@introspection-sdk/introspection-node 0.4.2 → 0.4.3

package/src/otel/setup.ts

@@ -0,0 +1,174 @@
+/**
+ * One-call tracing bootstrap for Introspection.
+ *
+ * Wires up the three things every host process needs for baggage-based
+ * agent / conversation propagation to work:
+ *
+ * 1. `AsyncLocalStorageContextManager` — without this, `context.with()` is a
+ *    no-op (the default `NOOP_CONTEXT_MANAGER` silently drops the context),
+ *    which makes `IntrospectionClient.withAgent()`, `.withConversation()`,
+ *    etc. dead code.
+ * 2. `W3CBaggagePropagator` — so baggage is carried across HTTP boundaries
+ *    via the `baggage` header (alongside W3C trace-context).
+ * 3. A registered `NodeTracerProvider` with an `IntrospectionSpanProcessor`
+ *    on it — so spans are exported to Introspection and baggage values
+ *    (conversation / agent IDs) are merged into span attributes at
+ *    `onEnd()` time.
+ *
+ * Call this once at process start (before any instrumentors run). It is
+ * idempotent across re-invocations — the cached provider is returned and
+ * the global registrations are not touched a second time.
+ *
+ * **Loud-by-default registration.** OTel's `setGlobalContextManager` /
+ * `setGlobalPropagator` silently refuse to replace an existing
+ * registration. If anything (another lib, a stale call, a test fixture)
+ * has already installed a context manager that does NOT propagate W3C
+ * baggage, our `withAgent` / `withConversation` calls become silent
+ * no-ops and users get spans with no identity attached — exactly the
+ * footgun this SDK exists to prevent.
+ *
+ * `setupTracing` defends against that:
+ *   - If no manager / propagator is registered: install ours.
+ *   - If one is registered: WARN (default) or THROW (strict mode) so the
+ *     host author sees it, rather than discovering missing baggage in
+ *     production telemetry weeks later.
+ *
+ * Pass `{ onConflict: "throw" }` (or set `INTROSPECTION_STRICT_TRACING=1`
+ * in the env) for the strict variant. Pass `{ onConflict: "replace" }`
+ * to force replacement when you know the prior registration is wrong
+ * (e.g. test setup, or migrating from another tracer).
+ *
+ * @example
+ * ```ts
+ * import { setupTracing, IntrospectionClient } from "@introspection-sdk/introspection-node";
+ *
+ * setupTracing();
+ *
+ * const introspect = new IntrospectionClient();
+ *
+ * await introspect.withAgent("researcher", "researcher-1", () =>
+ *   introspect.withConversation(conversationId, undefined, () =>
+ *     client.messages.create({ ... }),
+ *   ),
+ * );
+ * ```
+ */
+import { context, propagation } from "@opentelemetry/api";
+import { AsyncLocalStorageContextManager } from "@opentelemetry/context-async-hooks";
+import {
+  CompositePropagator,
+  W3CBaggagePropagator,
+  W3CTraceContextPropagator,
+} from "@opentelemetry/core";
+import { NodeTracerProvider } from "@opentelemetry/sdk-trace-node";
+
+import { logger } from "../utils.js";
+import {
+  IntrospectionSpanProcessor,
+  type IntrospectionSpanProcessorOptions,
+} from "./span-processor.js";
+
+let registeredProvider: NodeTracerProvider | undefined;
+
+export type ConflictBehavior = "warn" | "throw" | "replace";
+
+export interface SetupTracingOptions extends IntrospectionSpanProcessorOptions {
+  /**
+   * What to do if a context manager or propagator is already registered.
+   *
+   *   - `"warn"`  (default) — log a warning and continue with the existing
+   *     registration. Spans still flow but baggage may not propagate as
+   *     expected. Set `INTROSPECTION_STRICT_TRACING=1` to upgrade to throw.
+   *   - `"throw"` — throw an error. Use in environments where a silent
+   *     baggage drop is unacceptable (production telemetry pipelines).
+   *   - `"replace"` — force-register over the existing manager / propagator.
+   *     Useful in tests or when migrating from another tracer; can break
+   *     other libraries that relied on the previous registration.
+   */
+  onConflict?: ConflictBehavior;
+}
+
+function defaultConflictBehavior(): ConflictBehavior {
+  return process.env.INTROSPECTION_STRICT_TRACING ? "throw" : "warn";
+}
+
+function registerContextManager(behavior: ConflictBehavior): void {
+  const ok = context.setGlobalContextManager(
+    new AsyncLocalStorageContextManager().enable(),
+  );
+  if (ok) return;
+  // OTel silently refuses replacement — we have to disable+retry to force.
+  const msg =
+    "setupTracing: another OTel context manager was already registered. " +
+    "withAgent / withConversation baggage scopes will not propagate unless " +
+    "that manager is an AsyncLocalStorageContextManager honoring W3C baggage.";
+  if (behavior === "throw") {
+    throw new Error(`[introspection] ${msg}`);
+  }
+  if (behavior === "replace") {
+    context.disable();
+    if (
+      !context.setGlobalContextManager(
+        new AsyncLocalStorageContextManager().enable(),
+      )
+    ) {
+      throw new Error(
+        "[introspection] setupTracing: failed to replace context manager " +
+          "even after disable(); the OTel global registry is in a bad state.",
+      );
+    }
+    return;
+  }
+  logger.warn(msg);
+}
+
+function registerPropagator(behavior: ConflictBehavior): void {
+  const propagator = new CompositePropagator({
+    propagators: [new W3CTraceContextPropagator(), new W3CBaggagePropagator()],
+  });
+  const ok = propagation.setGlobalPropagator(propagator);
+  if (ok) return;
+  const msg =
+    "setupTracing: another OTel propagator was already registered. " +
+    "Cross-process baggage propagation depends on the existing propagator " +
+    "carrying the W3C `baggage` header.";
+  if (behavior === "throw") {
+    throw new Error(`[introspection] ${msg}`);
+  }
+  if (behavior === "replace") {
+    propagation.disable();
+    if (!propagation.setGlobalPropagator(propagator)) {
+      throw new Error(
+        "[introspection] setupTracing: failed to replace propagator even " +
+          "after disable(); the OTel global registry is in a bad state.",
+      );
+    }
+    return;
+  }
+  logger.warn(msg);
+}
+
+/**
+ * Initialise Introspection tracing for a Node.js process.
+ */
+export function setupTracing(
+  options?: SetupTracingOptions,
+): NodeTracerProvider {
+  if (registeredProvider) {
+    return registeredProvider;
+  }
+
+  const behavior: ConflictBehavior =
+    options?.onConflict ?? defaultConflictBehavior();
+
+  registerContextManager(behavior);
+  registerPropagator(behavior);
+
+  const provider = new NodeTracerProvider({
+    spanProcessors: [new IntrospectionSpanProcessor(options)],
+  });
+  provider.register();
+
+  registeredProvider = provider;
+  return provider;
+}

package/src/{span-processor.ts → otel/span-processor.ts}

@@ -16,14 +16,14 @@ import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions";
 import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-proto";
 import { ExportResult, ExportResultCode } from "@opentelemetry/core";
 import { randomUUID } from "crypto";
-import type { AdvancedOptions } from "./types.js";
-import { logger, withOtlpHttpsProxy } from "./utils.js";
+import type { AdvancedOptions } from "../types.js";
+import { logger, withOtlpHttpsProxy } from "../utils.js";
 import {
   isOpenInferenceSpan,
   replaceOpenInferenceWithGenAI,
   isVercelAISpan,
   convertVercelAIToGenAI,
-} from "./converters/index.js";
+} from "../converters/index.js";
 
 /**
  * Extended span type that includes both v1 and v2 instrumentation properties.
@@ -157,7 +157,7 @@ export class IntrospectionSpanProcessor implements SpanProcessor {
     const advanced = options.advanced || {};
     const baseUrl =
       advanced.baseUrl ||
-      process.env.INTROSPECTION_BASE_URL ||
+      process.env.INTROSPECTION_BASE_OTEL_URL ||
       "https://otel.introspection.dev";
 
     const headers: Record<string, string> = {
@@ -381,11 +381,16 @@ export class IntrospectionSpanProcessor implements SpanProcessor {
       attrs["gen_ai.operation.name"] = "chat";
     }
 
-    // Agent name from baggage
+    // Agent name / id from baggage (baggage wins so per-call agent identity
+    // overrides any default stamped at instrumentor-construction time).
     const baggageAgentName = baggage?.getEntry("gen_ai.agent.name")?.value;
-    if (baggageAgentName && !attrs["gen_ai.agent.name"]) {
+    if (baggageAgentName) {
       attrs["gen_ai.agent.name"] = baggageAgentName;
     }
+    const baggageAgentId = baggage?.getEntry("gen_ai.agent.id")?.value;
+    if (baggageAgentId) {
+      attrs["gen_ai.agent.id"] = baggageAgentId;
+    }
 
     // Override resource with service name if provided
     let resource: Resource | undefined;

package/src/{tracing-processor.ts → otel/tracing-processor.ts}

@@ -27,13 +27,13 @@ import {
   convertResponsesOutputsToSemconv,
   convertResponsesToolsToSemconv,
   convertResponsesInstructionsToSemconv,
-} from "./converters/openai.js";
+} from "../converters/openai.js";
 import type {
   Response as OpenAIResponse,
   ResponseInputItem,
-} from "./converters/openai.js";
-import { logger, withOtlpHttpsProxy } from "./utils.js";
-import { VERSION } from "./version.js";
+} from "../converters/openai.js";
+import { logger, withOtlpHttpsProxy } from "../utils.js";
+import { VERSION } from "../version.js";
 
 // Type imports from @openai/agents
 import type {
@@ -79,7 +79,7 @@ export interface TracingProcessorAdvancedOptions {
 export interface IntrospectionTracingProcessorOptions {
   /** Authentication token (env: INTROSPECTION_TOKEN) */
   token?: string;
-  /** Base URL for the API (env: INTROSPECTION_BASE_URL, default: "https://otel.introspection.dev") */
+  /** Base URL for the API (env: INTROSPECTION_BASE_OTEL_URL, default: "https://otel.introspection.dev") */
   baseUrl?: string;
   /** Service name for telemetry (env: INTROSPECTION_SERVICE_NAME) */
   serviceName?: string;
@@ -131,7 +131,7 @@ export class IntrospectionTracingProcessor implements TracingProcessor {
 
       const baseUrl =
         options.baseUrl ||
-        process.env.INTROSPECTION_BASE_URL ||
+        process.env.INTROSPECTION_BASE_OTEL_URL ||
         "https://otel.introspection.dev";
 
       const headers: Record<string, string> = {
@@ -220,7 +220,10 @@ export class IntrospectionTracingProcessor implements TracingProcessor {
   async onTraceStart(trace: Trace): Promise<void> {
     logger.debug(`onTraceStart called: ${trace?.name} (${trace?.traceId})`);
     if (trace) {
-      const conversationId = `intro_conv_${randomUUID().replace(/-/g, "")}`;
+      // Use groupId as conversationId when provided so callers can link
+      // multiple traces (e.g. multi-turn orchestrator runs) into one conversation.
+      const conversationId =
+        trace.groupId ?? `intro_conv_${randomUUID().replace(/-/g, "")}`;
       this._conversationIds.set(trace.traceId, conversationId);
       const otelSpan = this._tracer.startSpan(trace.name);
       this._spans.set(trace.traceId, otelSpan);
@@ -286,14 +289,10 @@ export class IntrospectionTracingProcessor implements TracingProcessor {
    * @param span - The OpenAI Agents SDK span.
    */
   async onSpanEnd(span: Span<SpanData>): Promise<void> {
-    logger.debug(
-      `onSpanEnd called: ${span?.spanId} (type: ${(span?.spanData as SpanData)?.type})`,
-    );
     if (!span) return;
 
     const otelSpan = this._spans.get(span.spanId);
     if (!otelSpan) {
-      logger.debug(`No OTel span found for spanId: ${span.spanId}`);
       return;
     }
 
@@ -448,6 +447,14 @@ export class IntrospectionTracingProcessor implements TracingProcessor {
           usage.output_tokens,
         );
       }
+      // OpenAI returns cached token count in input_tokens_details.cached_tokens
+      const cachedTokens = usage.input_tokens_details?.cached_tokens;
+      if (cachedTokens) {
+        otelSpan.setAttribute(
+          "gen_ai.usage.cache_read.input_tokens",
+          cachedTokens,
+        );
+      }
     }
 
     // Model info
@@ -474,6 +481,9 @@ export class IntrospectionTracingProcessor implements TracingProcessor {
 
   /**
    * Extract attributes from generation spans.
+   * Handles both structured GenerationUsageData (spanData.usage) and raw
+   * Chat Completions API responses stored in spanData.output[0] by
+   * OpenAIChatCompletionsModel.
    */
   private _processGenerationSpan(
     otelSpan: OtelSpan,
@@ -487,15 +497,78 @@ export class IntrospectionTracingProcessor implements TracingProcessor {
       otelSpan.setAttribute("gen_ai.request.model", spanData.model);
     }
 
-    if (spanData.usage) {
-      const usage = spanData.usage as Record<string, number>;
-      if (usage.input_tokens) {
-        otelSpan.setAttribute("gen_ai.usage.input_tokens", usage.input_tokens);
+    // GenerationUsageData uses snake_case: input_tokens / output_tokens.
+    if (spanData.usage?.input_tokens) {
+      otelSpan.setAttribute(
+        "gen_ai.usage.input_tokens",
+        spanData.usage.input_tokens,
+      );
+    }
+    if (spanData.usage?.output_tokens) {
+      otelSpan.setAttribute(
+        "gen_ai.usage.output_tokens",
+        spanData.usage.output_tokens,
+      );
+    }
+
+    // OpenAIChatCompletionsModel stores the raw ChatCompletion response object
+    // in spanData.output[0] (it never sets spanData.usage). Extract usage,
+    // response ID, response model, and the actual assistant message from it.
+    if (!spanData.usage) {
+      const rawResponse = spanData.output?.[0] as
+        | Record<string, unknown>
+        | undefined;
+
+      // Response ID — required by the backend processor to register this span
+      // as a conversation item (build_request returns None without it).
+      const responseId = rawResponse?.id;
+      if (typeof responseId === "string" && responseId) {
+        otelSpan.setAttribute("gen_ai.response.id", responseId);
       }
-      if (usage.output_tokens) {
+
+      // Resolved model name (e.g. "gpt-4o-mini-2024-07-18" vs "gpt-4o-mini").
+      const responseModel = rawResponse?.model;
+      if (typeof responseModel === "string" && responseModel) {
+        otelSpan.setAttribute("gen_ai.response.model", responseModel);
+      }
+
+      const rawUsage = rawResponse?.usage as
+        | Record<string, unknown>
+        | undefined;
+      if (rawUsage) {
+        const promptTokens = rawUsage.prompt_tokens;
+        const completionTokens = rawUsage.completion_tokens;
+        if (typeof promptTokens === "number") {
+          otelSpan.setAttribute("gen_ai.usage.input_tokens", promptTokens);
+        }
+        if (typeof completionTokens === "number") {
+          otelSpan.setAttribute("gen_ai.usage.output_tokens", completionTokens);
+        }
+        const promptDetails = rawUsage.prompt_tokens_details as
+          | Record<string, unknown>
+          | undefined;
+        const cachedTokens = promptDetails?.cached_tokens;
+        if (typeof cachedTokens === "number" && cachedTokens > 0) {
+          otelSpan.setAttribute(
+            "gen_ai.usage.cache_read.input_tokens",
+            cachedTokens,
+          );
+        }
+      }
+
+      // Extract the actual assistant message from choices[0].message so that
+      // gen_ai.output.messages is a proper [{role, content}] array rather than
+      // the full ChatCompletion envelope (which the backend cannot parse).
+      const choices = rawResponse?.choices as
+        | Array<Record<string, unknown>>
+        | undefined;
+      const assistantMessage = choices?.[0]?.message as
+        | Record<string, unknown>
+        | undefined;
+      if (assistantMessage) {
         otelSpan.setAttribute(
-          "gen_ai.usage.output_tokens",
-          usage.output_tokens,
+          "gen_ai.output.messages",
+          JSON.stringify([assistantMessage]),
         );
       }
     }
@@ -507,7 +580,9 @@ export class IntrospectionTracingProcessor implements TracingProcessor {
       );
     }
 
-    if (spanData.output) {
+    // For non-Chat-Completions generation spans that do have usage set,
+    // still export output messages in the standard format if present.
+    if (spanData.usage && spanData.output) {
       otelSpan.setAttribute(
         "gen_ai.output.messages",
         JSON.stringify(spanData.output),

package/src/resources/experiments.ts

@@ -0,0 +1,170 @@
+import type {
+  Experiment,
+  ExperimentEndParams,
+  ExperimentCreate,
+  ExperimentListParams,
+  ExperimentUpdate,
+  Paginated,
+  RunRequest,
+  RunnerSpec,
+  Uuid,
+} from "@introspection-sdk/types";
+import type { HttpClient } from "../http.js";
+import type { IntrospectionClient } from "../client.js";
+import { Runner, type RunnerSource } from "../runner.js";
+
+interface ExperimentRunRequestBody {
+  identity?: {
+    user_id?: string;
+    anonymous_id?: string;
+    conversation_id?: string;
+  };
+  ttl_seconds?: number;
+}
+
+function toRunBody(opts?: RunRequest): ExperimentRunRequestBody | undefined {
+  if (!opts) return undefined;
+  const out: ExperimentRunRequestBody = {};
+  if (opts.identity) out.identity = opts.identity;
+  if (opts.ttl_seconds !== undefined) out.ttl_seconds = opts.ttl_seconds;
+  return out;
+}
+
+export class ExperimentsApi {
+  constructor(
+    private readonly http: HttpClient,
+    private readonly client: IntrospectionClient,
+  ) {}
+
+  list(params: ExperimentListParams): Promise<Paginated<Experiment>> {
+    return this.http.request<Paginated<Experiment>>({
+      method: "GET",
+      path: "/v1/experiments",
+      query: params as unknown as Record<string, unknown>,
+    });
+  }
+
+  async *listAll(params: ExperimentListParams): AsyncIterable<Experiment> {
+    let next: string | undefined = params.next;
+    do {
+      const page = await this.list({ ...params, next });
+      for (const r of page.records) yield r;
+      next = page.next ?? undefined;
+    } while (next);
+  }
+
+  get(id: Uuid): Promise<Experiment> {
+    return this.http.request<Experiment>({
+      method: "GET",
+      path: `/v1/experiments/${encodeURIComponent(id)}`,
+    });
+  }
+
+  create(input: ExperimentCreate): Promise<Experiment> {
+    return this.http.request<Experiment>({
+      method: "POST",
+      path: "/v1/experiments",
+      body: input,
+    });
+  }
+
+  update(id: Uuid, input: ExperimentUpdate): Promise<Experiment> {
+    return this.http.request<Experiment>({
+      method: "PATCH",
+      path: `/v1/experiments/${encodeURIComponent(id)}`,
+      body: input,
+    });
+  }
+
+  delete(id: Uuid): Promise<void> {
+    return this.http.request<void>({
+      method: "DELETE",
+      path: `/v1/experiments/${encodeURIComponent(id)}`,
+      expect: "empty",
+    });
+  }
+
+  startById(id: Uuid): Promise<Experiment> {
+    return this.http.request<Experiment>({
+      method: "POST",
+      path: `/v1/experiments/${encodeURIComponent(id)}/start`,
+      body: {},
+    });
+  }
+
+  endById(id: Uuid, params?: ExperimentEndParams): Promise<Experiment> {
+    return this.http.request<Experiment>({
+      method: "POST",
+      path: `/v1/experiments/${encodeURIComponent(id)}/end`,
+      body: params ?? {},
+    });
+  }
+
+  cancelById(id: Uuid): Promise<Experiment> {
+    return this.http.request<Experiment>({
+      method: "POST",
+      path: `/v1/experiments/${encodeURIComponent(id)}/cancel`,
+      body: {},
+    });
+  }
+
+  async runById(id: Uuid, opts?: RunRequest): Promise<Runner> {
+    const spec = await this.http.request<RunnerSpec>({
+      method: "POST",
+      path: `/v1/experiments/${encodeURIComponent(id)}/run`,
+      body: toRunBody(opts) ?? {},
+    });
+    const source: RunnerSource = {
+      kind: "experiment",
+      id,
+      options: opts,
+    };
+    return new Runner(this.client, source, spec);
+  }
+}
+
+export class ExperimentHandle {
+  constructor(
+    private readonly api: ExperimentsApi,
+    private readonly id: Uuid,
+  ) {}
+
+  run(opts?: RunRequest): Promise<Runner> {
+    return this.api.runById(this.id, opts);
+  }
+
+  start(): Promise<Experiment> {
+    return this.api.startById(this.id);
+  }
+
+  end(opts?: ExperimentEndParams): Promise<Experiment> {
+    return this.api.endById(this.id, opts);
+  }
+
+  cancel(): Promise<Experiment> {
+    return this.api.cancelById(this.id);
+  }
+}
+
+export type ExperimentHandleFactory = (id: Uuid) => ExperimentHandle;
+
+export function attachExperiments(
+  client: IntrospectionClient,
+  http: HttpClient,
+): ExperimentsApi & ExperimentHandleFactory {
+  const api = new ExperimentsApi(http, client);
+  const factory: ExperimentHandleFactory = (id: Uuid) =>
+    new ExperimentHandle(api, id);
+  const hybrid = factory as ExperimentsApi & ExperimentHandleFactory;
+  hybrid.list = api.list.bind(api);
+  hybrid.listAll = api.listAll.bind(api);
+  hybrid.get = api.get.bind(api);
+  hybrid.create = api.create.bind(api);
+  hybrid.update = api.update.bind(api);
+  hybrid.delete = api.delete.bind(api);
+  hybrid.startById = api.startById.bind(api);
+  hybrid.endById = api.endById.bind(api);
+  hybrid.cancelById = api.cancelById.bind(api);
+  hybrid.runById = api.runById.bind(api);
+  return hybrid;
+}

package/src/resources/recipes.ts

@@ -0,0 +1,73 @@
+import type {
+  Paginated,
+  Recipe,
+  RecipeCreate,
+  RecipeListParams,
+  RecipeUpdate,
+  Uuid,
+} from "@introspection-sdk/types";
+import type { HttpClient } from "../http.js";
+
+/**
+ * Programmatic CRUD for `/v1/recipes` on the CP.
+ *
+ * Recipes are immutable build artefacts (repository + git ref + commit
+ * sha + optional sub-path). They are referenced by runtimes via
+ * `runtime.recipe_id` and by experiment arms via `arm.recipe_id`.
+ */
+export class RecipesApi {
+  constructor(private readonly http: HttpClient) {}
+
+  list(params: RecipeListParams): Promise<Paginated<Recipe>> {
+    return this.http.request<Paginated<Recipe>>({
+      method: "GET",
+      path: "/v1/recipes",
+      query: params as unknown as Record<string, unknown>,
+    });
+  }
+
+  /** Async-iterate every page that matches `params`. */
+  async *listAll(params: RecipeListParams): AsyncIterable<Recipe> {
+    let next: string | undefined = params.next;
+    do {
+      const page = await this.list({ ...params, next });
+      for (const r of page.records) yield r;
+      next = page.next ?? undefined;
+    } while (next);
+  }
+
+  get(id: Uuid): Promise<Recipe> {
+    return this.http.request<Recipe>({
+      method: "GET",
+      path: `/v1/recipes/${encodeURIComponent(id)}`,
+    });
+  }
+
+  create(input: RecipeCreate): Promise<Recipe> {
+    return this.http.request<Recipe>({
+      method: "POST",
+      path: "/v1/recipes",
+      body: input,
+    });
+  }
+
+  update(id: Uuid, input: RecipeUpdate): Promise<Recipe> {
+    return this.http.request<Recipe>({
+      method: "PATCH",
+      path: `/v1/recipes/${encodeURIComponent(id)}`,
+      body: input,
+    });
+  }
+
+  delete(id: Uuid): Promise<void> {
+    return this.http.request<void>({
+      method: "DELETE",
+      path: `/v1/recipes/${encodeURIComponent(id)}`,
+      expect: "empty",
+    });
+  }
+}
+
+export function attachRecipes(http: HttpClient): RecipesApi {
+  return new RecipesApi(http);
+}