@orq-ai/evaluatorq 1.3.0 → 1.3.2

package/dist/lib/integrations/simulation/tracing.js

@@ -0,0 +1,310 @@
+/**
+ * OpenTelemetry tracing utilities for the agent simulation module.
+ *
+ * Provides span creation helpers that mirror the redteam module's tracing
+ * patterns, adapted for the TypeScript simulation module. All functions
+ * gracefully degrade to no-ops when tracing is not enabled.
+ *
+ * Span hierarchy:
+ *   orq.simulation.pipeline (root)
+ *     ├── orq.simulation.persona_generation
+ *     ├── orq.simulation.scenario_generation
+ *     ├── orq.simulation.run (per datapoint)
+ *     │   ├── orq.simulation.first_message_generation
+ *     │   └── orq.simulation.turn (per turn)
+ *     │       ├── orq.simulation.target_call
+ *     │       ├── orq.simulation.judge_evaluation
+ *     │       └── orq.simulation.user_simulator_call
+ */
+import { getTracer } from "../../tracing/setup.js";
+// ---------------------------------------------------------------------------
+// Internal span: orq.simulation.*
+// ---------------------------------------------------------------------------
+/**
+ * Execute a function within a simulation span (SpanKind.INTERNAL).
+ *
+ * Gracefully returns `fn(undefined)` when tracing is not enabled.
+ * Automatically records errors and sets span status.
+ */
+export async function withSimulationSpan(name, attributes, fn) {
+    const tracer = getTracer();
+    if (!tracer) {
+        return fn(undefined);
+    }
+    let SpanStatusCode;
+    try {
+        ({ SpanStatusCode } = await import("@opentelemetry/api"));
+    }
+    catch {
+        // OTEL not available, run without span
+        return fn(undefined);
+    }
+    const cleanAttrs = {};
+    if (attributes) {
+        for (const [k, v] of Object.entries(attributes)) {
+            if (v !== undefined) {
+                cleanAttrs[k] = v;
+            }
+        }
+    }
+    return tracer.startActiveSpan(name, { attributes: cleanAttrs }, async (span) => {
+        try {
+            const result = await fn(span);
+            span.setStatus({ code: SpanStatusCode.OK });
+            return result;
+        }
+        catch (error) {
+            span.setStatus({
+                code: SpanStatusCode.ERROR,
+                message: error instanceof Error ? error.message : String(error),
+            });
+            span.recordException(error instanceof Error ? error : new Error(String(error)));
+            if (error instanceof Error) {
+                span.setAttribute("error.type", error.constructor.name);
+            }
+            throw error;
+        }
+        finally {
+            span.end();
+        }
+    });
+}
+/**
+ * Execute a function within a GenAI LLM span (SpanKind.CLIENT).
+ *
+ * Follows OTel GenAI semantic conventions for client inference spans.
+ * Span name is derived as `"{operation} {model}"`.
+ */
+export async function withLLMSpan(options, fn) {
+    const tracer = getTracer();
+    if (!tracer) {
+        return fn(undefined);
+    }
+    let SpanKind;
+    let SpanStatusCode;
+    try {
+        ({ SpanKind, SpanStatusCode } = await import("@opentelemetry/api"));
+    }
+    catch {
+        return fn(undefined);
+    }
+    const operation = options.operation ?? "chat";
+    const provider = options.provider ?? deriveProvider(options.model);
+    const spanName = `${operation} ${options.model}`;
+    const attrs = {
+        "gen_ai.operation.name": operation,
+        "gen_ai.system": provider,
+        "gen_ai.provider.name": provider,
+        "gen_ai.request.model": options.model,
+    };
+    if (options.temperature !== undefined) {
+        attrs["gen_ai.request.temperature"] = options.temperature;
+    }
+    if (options.maxTokens !== undefined) {
+        attrs["gen_ai.request.max_tokens"] = options.maxTokens;
+    }
+    if (options.purpose) {
+        attrs["orq.simulation.llm_purpose"] = options.purpose;
+    }
+    return tracer.startActiveSpan(spanName, { kind: SpanKind.CLIENT, attributes: attrs }, async (span) => {
+        try {
+            const result = await fn(span);
+            span.setStatus({ code: SpanStatusCode.OK });
+            return result;
+        }
+        catch (error) {
+            span.setStatus({
+                code: SpanStatusCode.ERROR,
+                message: error instanceof Error ? error.message : String(error),
+            });
+            span.recordException(error instanceof Error ? error : new Error(String(error)));
+            if (error instanceof Error) {
+                span.setAttribute("error.type", error.constructor.name);
+            }
+            throw error;
+        }
+        finally {
+            span.end();
+        }
+    });
+}
+/**
+ * Record token usage attributes on a span.
+ *
+ * Sets both OTel GenAI names and bare attribute keys for platform
+ * compatibility (matches the redteam module's dual-naming convention).
+ */
+export function recordTokenUsage(span, usage) {
+    if (!span)
+        return;
+    const prompt = usage.promptTokens ?? 0;
+    const completion = usage.completionTokens ?? 0;
+    const total = usage.totalTokens ?? prompt + completion;
+    // OTel GenAI semantic convention names
+    span.setAttribute("gen_ai.usage.input_tokens", prompt);
+    span.setAttribute("gen_ai.usage.output_tokens", completion);
+    span.setAttribute("gen_ai.usage.total_tokens", total);
+    if (usage.cacheReadInputTokens !== undefined) {
+        span.setAttribute("gen_ai.usage.cache_read.input_tokens", usage.cacheReadInputTokens);
+    }
+    if (usage.cacheCreationInputTokens !== undefined) {
+        span.setAttribute("gen_ai.usage.cache_creation.input_tokens", usage.cacheCreationInputTokens);
+    }
+    // Aliases for platform compatibility
+    span.setAttribute("gen_ai.usage.prompt_tokens", prompt);
+    span.setAttribute("gen_ai.usage.completion_tokens", completion);
+    span.setAttribute("prompt_tokens", prompt);
+    span.setAttribute("completion_tokens", completion);
+    span.setAttribute("input_tokens", prompt);
+    span.setAttribute("output_tokens", completion);
+    span.setAttribute("total_tokens", total);
+}
+// Max content length per message to avoid oversized spans (matches redteam)
+const MAX_CONTENT_LEN = 2000;
+function truncate(text) {
+    if (text.length <= MAX_CONTENT_LEN)
+        return text;
+    return `${text.slice(0, MAX_CONTENT_LEN)}…`;
+}
+/**
+ * Serialize an array of chat messages to JSON for span attributes.
+ */
+function serializeMessages(messages) {
+    return JSON.stringify(messages.map((m) => ({ role: m.role, content: truncate(m.content) })));
+}
+/**
+ * The OTel GenAI semconv classifies `gen_ai.input.messages` and
+ * `gen_ai.output.messages` as opt-in because they may carry PII. Honor the
+ * spec env var; default to enabled for the platform UI to keep working.
+ */
+function captureMessageContent() {
+    const flag = process.env.OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT;
+    if (flag === undefined)
+        return true;
+    return flag.toLowerCase() === "true" || flag === "1";
+}
+/**
+ * Record LLM input messages on a span.
+ *
+ * Sets both `gen_ai.input.messages` (OTel GenAI convention) and `input`
+ * (platform fallback), matching the redteam module's dual-attribute pattern.
+ * Suppressed when `OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT=false`.
+ */
+export function recordLLMInput(span, messages) {
+    if (!span || messages.length === 0)
+        return;
+    if (!captureMessageContent())
+        return;
+    const serialized = serializeMessages(messages);
+    span.setAttribute("gen_ai.input.messages", serialized);
+    span.setAttribute("input", serialized);
+}
+/**
+ * Record a single LLM output string on a span.
+ *
+ * Sets `gen_ai.output.messages` and `output` (platform fallback). Suppressed
+ * when `OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT=false`.
+ */
+export function recordLLMOutput(span, output) {
+    if (!span || !output)
+        return;
+    if (!captureMessageContent())
+        return;
+    const serialized = serializeMessages([
+        { role: "assistant", content: output },
+    ]);
+    span.setAttribute("gen_ai.output.messages", serialized);
+    span.setAttribute("output", serialized);
+}
+/**
+ * Record LLM response attributes on a span from an OpenAI-compatible response.
+ *
+ * Sets `gen_ai.output.messages` and `output` with the response content,
+ * plus token usage, finish reasons, and response metadata.
+ */
+export function recordLLMResponse(span, response) {
+    if (!span)
+        return;
+    if (response.id) {
+        span.setAttribute("gen_ai.response.id", response.id);
+    }
+    if (response.model) {
+        span.setAttribute("gen_ai.response.model", response.model);
+    }
+    if (response.usage) {
+        recordTokenUsage(span, {
+            promptTokens: response.usage.prompt_tokens,
+            completionTokens: response.usage.completion_tokens,
+            totalTokens: response.usage.total_tokens,
+            cacheReadInputTokens: response.usage.prompt_tokens_details?.cached_tokens,
+        });
+    }
+    // Record output content (dual-attribute pattern). Opt-in per GenAI semconv.
+    if (captureMessageContent()) {
+        const outputMessages = response.choices
+            ?.filter((c) => c.message?.content)
+            .map((c) => ({
+            role: c.message?.role ?? "assistant",
+            content: c.message?.content ?? "",
+        }));
+        if (outputMessages && outputMessages.length > 0) {
+            const serialized = serializeMessages(outputMessages);
+            span.setAttribute("gen_ai.output.messages", serialized);
+            span.setAttribute("output", serialized);
+        }
+    }
+    const finishReasons = response.choices
+        ?.map((c) => c.finish_reason)
+        .filter((r) => Boolean(r));
+    if (finishReasons && finishReasons.length > 0) {
+        span.setAttribute("gen_ai.response.finish_reasons", finishReasons);
+    }
+}
+// ---------------------------------------------------------------------------
+// Attribute helpers
+// ---------------------------------------------------------------------------
+/**
+ * Batch set multiple attributes on a span. Skips undefined values.
+ */
+export function setSpanAttrs(span, attrs) {
+    if (!span)
+        return;
+    for (const [key, value] of Object.entries(attrs)) {
+        if (value !== undefined) {
+            span.setAttribute(key, value);
+        }
+    }
+}
+/**
+ * Get W3C trace context headers (traceparent/tracestate) for the current
+ * active span. Returns an empty object when tracing is not available.
+ *
+ * Used to propagate trace context into outgoing HTTP requests so the
+ * router can create child spans under the current simulation span.
+ */
+export async function getTraceContextHeaders() {
+    try {
+        const { context, propagation } = await import("@opentelemetry/api");
+        const headers = {};
+        propagation.inject(context.active(), headers);
+        return headers;
+    }
+    catch {
+        return {};
+    }
+}
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+// OTel GenAI semconv `gen_ai.system` enum values. The router uses prefixes
+// like "azure/" that don't map 1:1 to the spec — translate the known ones.
+const PROVIDER_ALIASES = {
+    azure: "azure.ai.openai",
+};
+function deriveProvider(model) {
+    if (model.includes("/")) {
+        const prefix = model.split("/")[0];
+        return PROVIDER_ALIASES[prefix] ?? prefix;
+    }
+    return "openai";
+}

package/dist/lib/integrations/simulation/wrap-agent.js

@@ -3,7 +3,7 @@
  *
  * Follows the same pattern as wrapAISdkAgent() and wrapLangChainAgent().
  */
-import { fromOrqDeployment } from "./adapters.js";
+import { fromOrqAgent } from "./adapters.js";
 import { toOpenResponses } from "./convert.js";
 import { simulate } from "./simulation/index.js";
 /**
@@ -51,7 +51,7 @@ export function wrapSimulationAgent(options) {
         // Resolve the target callback
         let resolvedCallback = targetCallback;
         if (!resolvedCallback && agentKey) {
-            resolvedCallback = fromOrqDeployment(agentKey);
+            resolvedCallback = fromOrqAgent(agentKey);
         }
         if (!resolvedCallback) {
             throw new Error("wrapSimulationAgent requires either targetCallback or agentKey");