@orq-ai/evaluatorq 1.3.1 → 1.3.2

package/dist/lib/integrations/simulation/adapters.d.ts

@@ -8,15 +8,14 @@ import type { ChatMessage } from "./types.js";
 /**
  * Creates a simulation `targetCallback` from an Orq deployment key.
  *
- * This is the same bridge that `simulate()` and `wrapSimulationAgent()` use
- * internally when you pass `agentKey`, but exposed as a standalone function
- * so you can compose it freely.
+ * Uses the deployments API (`client.deployments.invoke()`).
+ * For agents, use {@link fromOrqAgent} instead.
  *
  * @example
  * ```typescript
  * import { fromOrqDeployment, simulate } from "@orq-ai/evaluatorq/simulation";
  *
- * const callback = fromOrqDeployment("my-agent-deployment-key");
+ * const callback = fromOrqDeployment("my-deployment-key");
  *
  * const results = await simulate({
  *   evaluationName: "my-sim",
@@ -26,7 +25,31 @@ import type { ChatMessage } from "./types.js";
  * });
  * ```
  */
-export declare function fromOrqDeployment(agentKey: string): (messages: ChatMessage[]) => Promise<string>;
+export declare function fromOrqDeployment(deploymentKey: string): (messages: ChatMessage[]) => Promise<string>;
+/**
+ * Creates a simulation `targetCallback` from an Orq agent key.
+ *
+ * Uses the agents streaming API to get synchronous responses.
+ * Propagates OTel trace context so agent-side LLM spans appear
+ * under the simulation's `target_call` span.
+ *
+ * This is the adapter used internally when you pass `agentKey` to `simulate()`.
+ *
+ * @example
+ * ```typescript
+ * import { fromOrqAgent, simulate } from "@orq-ai/evaluatorq/simulation";
+ *
+ * const callback = fromOrqAgent("my-agent-key");
+ *
+ * const results = await simulate({
+ *   evaluationName: "my-sim",
+ *   targetCallback: callback,
+ *   personas: [...],
+ *   scenarios: [...],
+ * });
+ * ```
+ */
+export declare function fromOrqAgent(agentKey: string): (messages: ChatMessage[]) => Promise<string>;
 /**
  * Creates a simulation `targetCallback` from a plain function that calls
  * an OpenAI-compatible chat completions API.

package/dist/lib/integrations/simulation/adapters.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"adapters.d.ts","sourceRoot":"","sources":["../../../../src/lib/integrations/simulation/adapters.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAE9C;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAgB,iBAAiB,CAC/B,QAAQ,EAAE,MAAM,GACf,CAAC,QAAQ,EAAE,WAAW,EAAE,KAAK,OAAO,CAAC,MAAM,CAAC,CAS9C;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAgB,mBAAmB,CACjC,EAAE,EAAE,CACF,QAAQ,EAAE,KAAK,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC,KAC/C,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,GAC5B,CAAC,QAAQ,EAAE,WAAW,EAAE,KAAK,OAAO,CAAC,MAAM,CAAC,CAI9C"}
+{"version":3,"file":"adapters.d.ts","sourceRoot":"","sources":["../../../../src/lib/integrations/simulation/adapters.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAE9C;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAgB,iBAAiB,CAC/B,aAAa,EAAE,MAAM,GACpB,CAAC,QAAQ,EAAE,WAAW,EAAE,KAAK,OAAO,CAAC,MAAM,CAAC,CAS9C;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAgB,YAAY,CAC1B,QAAQ,EAAE,MAAM,GACf,CAAC,QAAQ,EAAE,WAAW,EAAE,KAAK,OAAO,CAAC,MAAM,CAAC,CAuG9C;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAgB,mBAAmB,CACjC,EAAE,EAAE,CACF,QAAQ,EAAE,KAAK,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC,KAC/C,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,GAC5B,CAAC,QAAQ,EAAE,WAAW,EAAE,KAAK,OAAO,CAAC,MAAM,CAAC,CAI9C"}

package/dist/lib/integrations/simulation/adapters.js

@@ -4,18 +4,18 @@
  * These helpers create `targetCallback` functions from common agent sources,
  * so users don't need to wire the plumbing themselves.
  */
+import { randomUUID } from "node:crypto";
 /**
  * Creates a simulation `targetCallback` from an Orq deployment key.
  *
- * This is the same bridge that `simulate()` and `wrapSimulationAgent()` use
- * internally when you pass `agentKey`, but exposed as a standalone function
- * so you can compose it freely.
+ * Uses the deployments API (`client.deployments.invoke()`).
+ * For agents, use {@link fromOrqAgent} instead.
  *
  * @example
  * ```typescript
  * import { fromOrqDeployment, simulate } from "@orq-ai/evaluatorq/simulation";
  *
- * const callback = fromOrqDeployment("my-agent-deployment-key");
+ * const callback = fromOrqDeployment("my-deployment-key");
  *
  * const results = await simulate({
  *   evaluationName: "my-sim",
@@ -25,13 +25,119 @@
  * });
  * ```
  */
-export function fromOrqDeployment(agentKey) {
+export function fromOrqDeployment(deploymentKey) {
+    if (!deploymentKey.trim()) {
+        throw new Error("deploymentKey must be a non-empty string");
+    }
+    return async (messages) => {
+        const { invoke } = await import("../../deployment-helper.js");
+        return invoke(deploymentKey, { messages });
+    };
+}
+/**
+ * Creates a simulation `targetCallback` from an Orq agent key.
+ *
+ * Uses the agents streaming API to get synchronous responses.
+ * Propagates OTel trace context so agent-side LLM spans appear
+ * under the simulation's `target_call` span.
+ *
+ * This is the adapter used internally when you pass `agentKey` to `simulate()`.
+ *
+ * @example
+ * ```typescript
+ * import { fromOrqAgent, simulate } from "@orq-ai/evaluatorq/simulation";
+ *
+ * const callback = fromOrqAgent("my-agent-key");
+ *
+ * const results = await simulate({
+ *   evaluationName: "my-sim",
+ *   targetCallback: callback,
+ *   personas: [...],
+ *   scenarios: [...],
+ * });
+ * ```
+ */
+export function fromOrqAgent(agentKey) {
     if (!agentKey.trim()) {
         throw new Error("agentKey must be a non-empty string");
     }
+    // Cache client across calls to avoid creating a new one per turn
+    // biome-ignore lint/suspicious/noExplicitAny: cached client type depends on dynamic import
+    let cachedClient = null;
+    // Multi-turn continuity: the agent stream API maintains conversation state
+    // via task_id. On the first turn, we start a new conversation. On subsequent
+    // turns, we pass the task_id from the previous response to continue the
+    // conversation. Use first message content to identify conversations.
+    const conversationTasks = new Map();
+    // Unique per adapter instance — used as map key prefix to prevent cross-simulation
+    // key collisions when two simulations share the same opening message content.
+    const adapterInstanceId = randomUUID();
     return async (messages) => {
-        const { invoke } = await import("../../deployment-helper.js");
-        return invoke(agentKey, { messages });
+        const apiKey = process.env.ORQ_API_KEY;
+        if (!apiKey) {
+            throw new Error("ORQ_API_KEY environment variable must be set to use the agent adapter.");
+        }
+        if (!cachedClient) {
+            const { Orq } = await import("@orq-ai/node");
+            const serverURL = process.env.ORQ_BASE_URL || "https://my.orq.ai";
+            cachedClient = new Orq({ apiKey, serverURL });
+        }
+        const firstUserMessage = messages.find((m) => m.role === "user");
+        if (!firstUserMessage) {
+            throw new Error(`fromOrqAgent: conversation has no user message to send to "${agentKey}".`);
+        }
+        // Check if this is a continuation of an existing conversation
+        // Key prefixed with adapterInstanceId to prevent collisions across simulations
+        // with identical opening messages.
+        const conversationKey = `${adapterInstanceId}:${firstUserMessage.content}`;
+        const existingTaskId = conversationTasks.get(conversationKey);
+        // Send only the latest user message; prior turns are reconstructed
+        // server-side from the task_id.
+        const lastUserMessage = [...messages]
+            .reverse()
+            .find((m) => m.role === "user");
+        const messageText = lastUserMessage?.content ?? "";
+        // Propagate OTel trace context so the agent's server-side LLM spans
+        // are linked as children of the current simulation span
+        let traceHeaders = {};
+        try {
+            const { getTraceContextHeaders } = await import("./tracing.js");
+            traceHeaders = await getTraceContextHeaders();
+        }
+        catch {
+            // Tracing not available — continue without propagation
+        }
+        // Build request with optional taskId for conversation continuation
+        const streamRequest = {
+            message: {
+                role: "user",
+                parts: [{ kind: "text", text: messageText }],
+            },
+            ...(existingTaskId && { taskId: existingTaskId }),
+        };
+        const stream = await cachedClient.agents.stream(streamRequest, agentKey, {
+            headers: traceHeaders,
+        });
+        // Consume stream and extract the final agent message + task ID
+        let lastMessage;
+        let taskId;
+        for await (const event of stream) {
+            const data = event.data;
+            if (data?.type === "event.agents.inactive") {
+                const innerData = data.data;
+                lastMessage = innerData?.lastMessage ?? "";
+                taskId = innerData?.taskId ?? undefined;
+            }
+        }
+        if (!lastMessage) {
+            throw new Error(`Agent stream for "${agentKey}" ended without an event.agents.inactive event. ` +
+                "The agent may have errored out server-side.");
+        }
+        // Store task ID for conversation continuation
+        if (taskId) {
+            conversationTasks.set(conversationKey, taskId);
+        }
+        return lastMessage;
     };
 }
 /**

package/dist/lib/integrations/simulation/agents/base.d.ts

@@ -57,6 +57,7 @@ export declare abstract class BaseAgent {
         maxTokens?: number;
         timeout?: number;
         signal?: AbortSignal;
+        llmPurpose?: string;
     }): Promise<string>;
     /**
      * Get cumulative token usage for this agent.
@@ -85,6 +86,8 @@ export declare abstract class BaseAgent {
         tools?: OpenAI.Chat.Completions.ChatCompletionTool[];
         /** External abort signal — aborts in-flight LLM requests immediately. */
         signal?: AbortSignal;
+        /** Semantic purpose for OTel span (e.g. "judge", "user_simulator"). */
+        llmPurpose?: string;
     }): Promise<LLMResult>;
 }
 //# sourceMappingURL=base.d.ts.map

package/dist/lib/integrations/simulation/agents/base.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"base.d.ts","sourceRoot":"","sources":["../../../../../src/lib/integrations/simulation/agents/base.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAQ3D;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,6BAA6B,EAAE,CAAC;CACtE;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,uDAAuD;IACvD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,wFAAwF;IACxF,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,2DAA2D;IAC3D,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAUD;;;;;;;;;GASG;AACH,8BAAsB,SAAS;IAC7B,SAAS,CAAC,KAAK,EAAE,MAAM,CAAC;IACxB,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC;IACzB,OAAO,CAAC,WAAW,CAAU;IAC7B,OAAO,CAAC,KAAK,CAAa;gBAEd,MAAM,CAAC,EAAE,WAAW;IA2BhC,qCAAqC;IACrC,QAAQ,KAAK,IAAI,IAAI,MAAM,CAAC;IAE5B,oCAAoC;IACpC,QAAQ,KAAK,YAAY,IAAI,MAAM,CAAC;IAMpC;;;;;;;OAOG;IACG,YAAY,CAChB,QAAQ,EAAE,WAAW,EAAE,EACvB,OAAO,CAAC,EAAE;QACR,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,MAAM,CAAC,EAAE,WAAW,CAAC;KACtB,GACA,OAAO,CAAC,MAAM,CAAC;IAiBlB;;OAEG;IACH,QAAQ,IAAI,UAAU;IAItB;;OAEG;IACH,UAAU,IAAI,IAAI;IAIlB;;;;OAIG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAgB5B;;;;;OAKG;cACa,OAAO,CACrB,QAAQ,EAAE,WAAW,EAAE,EACvB,OAAO,CAAC,EAAE;QACR,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,KAAK,CAAC,EAAE,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,kBAAkB,EAAE,CAAC;QACrD,yEAAyE;QACzE,MAAM,CAAC,EAAE,WAAW,CAAC;KACtB,GACA,OAAO,CAAC,SAAS,CAAC;CAsHtB"}
+{"version":3,"file":"base.d.ts","sourceRoot":"","sources":["../../../../../src/lib/integrations/simulation/agents/base.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,MAAM,MAAM,QAAQ,CAAC;AAQ5B,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAQ3D;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,6BAA6B,EAAE,CAAC;CACtE;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,uDAAuD;IACvD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,wFAAwF;IACxF,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,2DAA2D;IAC3D,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAUD;;;;;;;;;GASG;AACH,8BAAsB,SAAS;IAC7B,SAAS,CAAC,KAAK,EAAE,MAAM,CAAC;IACxB,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC;IACzB,OAAO,CAAC,WAAW,CAAU;IAC7B,OAAO,CAAC,KAAK,CAAa;gBAEd,MAAM,CAAC,EAAE,WAAW;IA2BhC,qCAAqC;IACrC,QAAQ,KAAK,IAAI,IAAI,MAAM,CAAC;IAE5B,oCAAoC;IACpC,QAAQ,KAAK,YAAY,IAAI,MAAM,CAAC;IAMpC;;;;;;;OAOG;IACG,YAAY,CAChB,QAAQ,EAAE,WAAW,EAAE,EACvB,OAAO,CAAC,EAAE;QACR,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,MAAM,CAAC,EAAE,WAAW,CAAC;QACrB,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,GACA,OAAO,CAAC,MAAM,CAAC;IAkBlB;;OAEG;IACH,QAAQ,IAAI,UAAU;IAItB;;OAEG;IACH,UAAU,IAAI,IAAI;IAIlB;;;;OAIG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAgB5B;;;;;OAKG;cACa,OAAO,CACrB,QAAQ,EAAE,WAAW,EAAE,EACvB,OAAO,CAAC,EAAE;QACR,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,KAAK,CAAC,EAAE,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,kBAAkB,EAAE,CAAC;QACrD,yEAAyE;QACzE,MAAM,CAAC,EAAE,WAAW,CAAC;QACrB,uEAAuE;QACvE,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,GACA,OAAO,CAAC,SAAS,CAAC;CAyJtB"}

package/dist/lib/integrations/simulation/agents/base.js

@@ -5,6 +5,7 @@
  * including LLM interaction with retry logic.
  */
 import OpenAI from "openai";
+import { getTraceContextHeaders, recordLLMInput, recordLLMResponse, withLLMSpan, } from "../tracing.js";
 // Retry configuration
 const MAX_RETRY_ATTEMPTS = 5;
 const RETRY_MIN_WAIT_MS = 2_000;
@@ -69,6 +70,7 @@ export class BaseAgent {
             maxTokens: options?.maxTokens,
             timeout: options?.timeout,
             signal: options?.signal,
+            llmPurpose: options?.llmPurpose,
         });
         if (!result.content) {
             throw new Error(`${this.name}: LLM call failed -- no content in response`);
@@ -114,95 +116,115 @@ export class BaseAgent {
         const temperature = options?.temperature ?? 0.7;
         const maxTokens = options?.maxTokens ?? 2048;
         const timeoutS = options?.timeout ?? DEFAULT_TIMEOUT_S;
-        const fullMessages = [
-            { role: "system", content: this.systemPrompt },
-            ...messages.map((m) => ({
-                role: m.role,
-                content: m.content,
-            })),
-        ];
-        let lastError;
-        for (let attempt = 1; attempt <= MAX_RETRY_ATTEMPTS; attempt++) {
-            try {
-                // Bail immediately if already cancelled
-                if (options?.signal?.aborted) {
-                    throw new Error("Cancelled");
-                }
-                const controller = new AbortController();
-                const timer = setTimeout(() => controller.abort(), timeoutS * 1000);
-                // Link external signal to this request's controller
-                const onAbort = () => controller.abort();
-                options?.signal?.addEventListener("abort", onAbort, { once: true });
+        return withLLMSpan({
+            model: this.model,
+            temperature,
+            maxTokens,
+            purpose: options?.llmPurpose,
+        }, async (span) => {
+            const fullMessages = [
+                { role: "system", content: this.systemPrompt },
+                ...messages.map((m) => ({
+                    role: m.role,
+                    content: m.content,
+                })),
+            ];
+            // Record input messages on the span for platform UI display
+            recordLLMInput(span, fullMessages.map((m) => ({
+                role: String(m.role),
+                content: typeof m.content === "string" ? m.content : "",
+            })));
+            // Inject W3C trace context so the router links its spans to the
+            // current simulation trace. The active span and trace context don't
+            // change across retries, so compute the headers once.
+            const traceHeaders = await getTraceContextHeaders();
+            let lastError;
+            for (let attempt = 1; attempt <= MAX_RETRY_ATTEMPTS; attempt++) {
                 try {
-                    const params = {
-                        model: this.model,
-                        messages: fullMessages,
-                        temperature,
-                        max_tokens: maxTokens,
-                    };
-                    if (options?.tools && options.tools.length > 0) {
-                        params.tools = options.tools;
-                        params.tool_choice = "auto";
-                    }
-                    const response = await this.client.chat.completions.create(params, {
-                        signal: controller.signal,
-                    });
-                    clearTimeout(timer);
-                    const choice = response.choices[0];
-                    if (!choice) {
-                        throw new Error(`${this.name}: No choices in response`);
+                    // Bail immediately if already cancelled
+                    if (options?.signal?.aborted) {
+                        throw new Error("Cancelled");
                     }
-                    const message = choice.message;
-                    // Accumulate token usage
-                    if (response.usage) {
-                        this.usage.prompt_tokens += response.usage.prompt_tokens;
-                        this.usage.completion_tokens += response.usage.completion_tokens;
-                        this.usage.total_tokens += response.usage.total_tokens;
+                    const controller = new AbortController();
+                    const timer = setTimeout(() => controller.abort(), timeoutS * 1000);
+                    // Link external signal to this request's controller
+                    const onAbort = () => controller.abort();
+                    options?.signal?.addEventListener("abort", onAbort, { once: true });
+                    try {
+                        const params = {
+                            model: this.model,
+                            messages: fullMessages,
+                            temperature,
+                            max_tokens: maxTokens,
+                        };
+                        if (options?.tools && options.tools.length > 0) {
+                            params.tools = options.tools;
+                            params.tool_choice = "auto";
+                        }
+                        const response = await this.client.chat.completions.create(params, {
+                            signal: controller.signal,
+                            headers: traceHeaders,
+                        });
+                        clearTimeout(timer);
+                        const choice = response.choices[0];
+                        if (!choice) {
+                            throw new Error(`${this.name}: No choices in response`);
+                        }
+                        const message = choice.message;
+                        // Record LLM response on the span (token usage, finish reason, etc.)
+                        recordLLMResponse(span, response);
+                        // Accumulate token usage
+                        if (response.usage) {
+                            this.usage.prompt_tokens += response.usage.prompt_tokens;
+                            this.usage.completion_tokens +=
+                                response.usage.completion_tokens;
+                            this.usage.total_tokens += response.usage.total_tokens;
+                        }
+                        const result = {
+                            content: message.content ?? "",
+                        };
+                        if (message.tool_calls && message.tool_calls.length > 0) {
+                            result.tool_calls = message.tool_calls;
+                        }
+                        return result;
                     }
-                    const result = {
-                        content: message.content ?? "",
-                    };
-                    if (message.tool_calls && message.tool_calls.length > 0) {
-                        result.tool_calls = message.tool_calls;
+                    finally {
+                        clearTimeout(timer);
+                        options?.signal?.removeEventListener("abort", onAbort);
                     }
-                    return result;
-                }
-                finally {
-                    clearTimeout(timer);
-                    options?.signal?.removeEventListener("abort", onAbort);
-                }
-            }
-            catch (err) {
-                lastError = err;
-                // Abort errors (from timeout cancellation) should never be retried
-                if (err instanceof Error && err.name === "AbortError") {
-                    throw err;
-                }
-                // Determine if retryable
-                const isApiError = err instanceof OpenAI.APIError;
-                const status = isApiError ? err.status : undefined;
-                const isNetworkError = !isApiError &&
-                    err instanceof Error &&
-                    "code" in err &&
-                    typeof err.code === "string" &&
-                    /^E(CONN|TIMEOUT|NOTFOUND|RESET)/.test(err.code ?? "");
-                // Re-throw immediately for external cancellation
-                if (options?.signal?.aborted)
-                    throw err;
-                if (!isRetryableStatus(status) && !isNetworkError) {
-                    throw err;
                 }
-                if (attempt < MAX_RETRY_ATTEMPTS) {
-                    const baseWait = RETRY_MIN_WAIT_MS * 2 ** (attempt - 1);
-                    const waitMs = Math.min(baseWait, RETRY_MAX_WAIT_MS);
-                    // Add jitter (0-25% of wait time)
-                    const jitter = Math.random() * waitMs * 0.25;
-                    await sleepCancellable(waitMs + jitter, options?.signal);
+                catch (err) {
+                    lastError = err;
+                    // Abort errors (from timeout cancellation) should never be retried
+                    if (err instanceof Error && err.name === "AbortError") {
+                        throw err;
+                    }
+                    // Determine if retryable
+                    const isApiError = err instanceof OpenAI.APIError;
+                    const status = isApiError ? err.status : undefined;
+                    const isNetworkError = !isApiError &&
+                        err instanceof Error &&
+                        "code" in err &&
+                        typeof err.code === "string" &&
+                        /^E(CONN|TIMEOUT|NOTFOUND|RESET)/.test(err.code ?? "");
+                    // Re-throw immediately for external cancellation
+                    if (options?.signal?.aborted)
+                        throw err;
+                    if (!isRetryableStatus(status) && !isNetworkError) {
+                        throw err;
+                    }
+                    if (attempt < MAX_RETRY_ATTEMPTS) {
+                        const baseWait = RETRY_MIN_WAIT_MS * 2 ** (attempt - 1);
+                        const waitMs = Math.min(baseWait, RETRY_MAX_WAIT_MS);
+                        // Add jitter (0-25% of wait time)
+                        const jitter = Math.random() * waitMs * 0.25;
+                        await sleepCancellable(waitMs + jitter, options?.signal);
+                    }
                 }
             }
-        }
-        throw (lastError ??
-            new Error(`${this.name}: Max retries (${MAX_RETRY_ATTEMPTS}) exceeded`));
+            throw (lastError ??
+                new Error(`${this.name}: Max retries (${MAX_RETRY_ATTEMPTS}) exceeded`));
+        });
     }
 }
 // ---------------------------------------------------------------------------

package/dist/lib/integrations/simulation/agents/judge.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"judge.d.ts","sourceRoot":"","sources":["../../../../../src/lib/integrations/simulation/agents/judge.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AAEjC,OAAO,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAEpE,OAAO,KAAK,EAAE,WAAW,EAAa,MAAM,WAAW,CAAC;AACxD,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAiCtC,eAAO,MAAM,WAAW,EAAE,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,kBAAkB,EA2DnE,CAAC;AAMF,eAAO,MAAM,oBAAoB,+wCAwB8B,CAAC;AAMhE,MAAM,WAAW,gBAAiB,SAAQ,WAAW;IACnD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,SAAS,EAAE,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAmBD;;;;;GAKG;AACH,qBAAa,UAAW,SAAQ,SAAS;IACvC,OAAO,CAAC,IAAI,CAAS;IACrB,OAAO,CAAC,QAAQ,CAAc;IAC9B,OAAO,CAAC,WAAW,CAAS;gBAEhB,MAAM,CAAC,EAAE,gBAAgB;IAOrC,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED,IAAI,YAAY,IAAI,MAAM,CASzB;IAED;;;;;OAKG;IACG,QAAQ,CACZ,QAAQ,EAAE,WAAW,EAAE,EACvB,OAAO,CAAC,EAAE;QAAE,MAAM,CAAC,EAAE,WAAW,CAAA;KAAE,GACjC,OAAO,CAAC,QAAQ,CAAC;IAuBpB,OAAO,CAAC,aAAa;IAiGrB;;OAEG;IACH,OAAO,CAAC,MAAM,CAAC,oBAAoB;IAkBnC;;OAEG;IACH,OAAO,CAAC,cAAc;CAuBvB"}
+{"version":3,"file":"judge.d.ts","sourceRoot":"","sources":["../../../../../src/lib/integrations/simulation/agents/judge.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AAEjC,OAAO,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAEpE,OAAO,KAAK,EAAE,WAAW,EAAa,MAAM,WAAW,CAAC;AACxD,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAiCtC,eAAO,MAAM,WAAW,EAAE,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,kBAAkB,EA2DnE,CAAC;AAMF,eAAO,MAAM,oBAAoB,+wCAwB8B,CAAC;AAMhE,MAAM,WAAW,gBAAiB,SAAQ,WAAW;IACnD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,SAAS,EAAE,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAmBD;;;;;GAKG;AACH,qBAAa,UAAW,SAAQ,SAAS;IACvC,OAAO,CAAC,IAAI,CAAS;IACrB,OAAO,CAAC,QAAQ,CAAc;IAC9B,OAAO,CAAC,WAAW,CAAS;gBAEhB,MAAM,CAAC,EAAE,gBAAgB;IAOrC,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED,IAAI,YAAY,IAAI,MAAM,CASzB;IAED;;;;;OAKG;IACG,QAAQ,CACZ,QAAQ,EAAE,WAAW,EAAE,EACvB,OAAO,CAAC,EAAE;QAAE,MAAM,CAAC,EAAE,WAAW,CAAA;KAAE,GACjC,OAAO,CAAC,QAAQ,CAAC;IAwBpB,OAAO,CAAC,aAAa;IAiGrB;;OAEG;IACH,OAAO,CAAC,MAAM,CAAC,oBAAoB;IAkBnC;;OAEG;IACH,OAAO,CAAC,cAAc;CAuBvB"}

package/dist/lib/integrations/simulation/agents/judge.js

@@ -171,6 +171,7 @@ export class JudgeAgent extends BaseAgent {
             temperature: 0.0,
             tools: JUDGE_TOOLS,
             signal: options?.signal,
+            llmPurpose: "judge",
         });
         return this.parseJudgment(result);
     }

package/dist/lib/integrations/simulation/agents/user-simulator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"user-simulator.d.ts","sourceRoot":"","sources":["../../../../../src/lib/integrations/simulation/agents/user-simulator.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC/C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAMtC,eAAO,MAAM,6BAA6B,urBAgBE,CAAC;AAM7C,MAAM,WAAW,wBAAyB,SAAQ,WAAW;IAC3D,4DAA4D;IAC5D,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAMD;;;;;GAKG;AACH,qBAAa,kBAAmB,SAAQ,SAAS;IAC/C,OAAO,CAAC,kBAAkB,CAAgB;gBAE9B,MAAM,CAAC,EAAE,wBAAwB;IAK7C,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED,IAAI,YAAY,IAAI,MAAM,CAKzB;IAED;;;;;OAKG;IACG,oBAAoB,CAAC,QAAQ,CAAC,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;IAWrE;;;;;OAKG;IACH,aAAa,CAAC,cAAc,CAAC,EAAE,MAAM,EAAE,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI;CAWvE"}
+{"version":3,"file":"user-simulator.d.ts","sourceRoot":"","sources":["../../../../../src/lib/integrations/simulation/agents/user-simulator.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC/C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAMtC,eAAO,MAAM,6BAA6B,urBAgBE,CAAC;AAM7C,MAAM,WAAW,wBAAyB,SAAQ,WAAW;IAC3D,4DAA4D;IAC5D,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAMD;;;;;GAKG;AACH,qBAAa,kBAAmB,SAAQ,SAAS;IAC/C,OAAO,CAAC,kBAAkB,CAAgB;gBAE9B,MAAM,CAAC,EAAE,wBAAwB;IAK7C,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED,IAAI,YAAY,IAAI,MAAM,CAKzB;IAED;;;;;OAKG;IACG,oBAAoB,CAAC,QAAQ,CAAC,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;IAcrE;;;;;OAKG;IACH,aAAa,CAAC,cAAc,CAAC,EAAE,MAAM,EAAE,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI;CAWvE"}

package/dist/lib/integrations/simulation/agents/user-simulator.js

@@ -61,7 +61,10 @@ export class UserSimulatorAgent extends BaseAgent {
             role: "user",
             content: "Generate your first message to start the conversation. Remember your goal and persona.",
         });
-        return this.respondAsync(promptMessages, { temperature: 0.8 });
+        return this.respondAsync(promptMessages, {
+            temperature: 0.8,
+            llmPurpose: "first_message",
+        });
     }
     /**
      * Update the persona and scenario context.

package/dist/lib/integrations/simulation/generators/first-message-generator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"first-message-generator.d.ts","sourceRoot":"","sources":["../../../../../src/lib/integrations/simulation/generators/first-message-generator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,OAAO,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAuDrD;;GAEG;AACH,MAAM,WAAW,2BAA2B;IAC1C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;;;;GAKG;AACH,qBAAa,qBAAqB;IAChC,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,MAAM,CAAS;gBAEX,MAAM,CAAC,EAAE,2BAA2B;IAkBhD;;;;;;;;;OASG;IACG,QAAQ,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC;CAgDtE"}
+{"version":3,"file":"first-message-generator.d.ts","sourceRoot":"","sources":["../../../../../src/lib/integrations/simulation/generators/first-message-generator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,MAAM,MAAM,QAAQ,CAAC;AAS5B,OAAO,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAuDrD;;GAEG;AACH,MAAM,WAAW,2BAA2B;IAC1C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;;;;GAKG;AACH,qBAAa,qBAAqB;IAChC,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,MAAM,CAAS;gBAEX,MAAM,CAAC,EAAE,2BAA2B;IAkBhD;;;;;;;;;OASG;IACG,QAAQ,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC;CAyFtE"}

package/dist/lib/integrations/simulation/generators/first-message-generator.js

@@ -4,6 +4,7 @@
  * Generates contextually appropriate first messages based on persona and scenario.
  */
 import OpenAI from "openai";
+import { getTraceContextHeaders, recordLLMInput, recordLLMResponse, withLLMSpan, withSimulationSpan, } from "../tracing.js";
 import { buildPersonaSystemPrompt, buildScenarioUserContext, } from "../utils/prompt-builders.js";
 // Temperature setting for message generation
 const TEMPERATURE_FIRST_MESSAGE = 0.8;
@@ -88,9 +89,14 @@ export class FirstMessageGenerator {
      * @returns Generated first message string
      */
     async generate(persona, scenario) {
-        const personaContext = buildPersonaSystemPrompt(persona);
-        const scenarioContext = buildScenarioUserContext(scenario);
-        const userPrompt = `PERSONA:
+        return withSimulationSpan("orq.simulation.first_message_generation", {
+            "orq.simulation.persona": persona.name,
+            "orq.simulation.scenario": scenario.name,
+            "orq.simulation.model": this.model,
+        }, async (_span) => {
+            const personaContext = buildPersonaSystemPrompt(persona);
+            const scenarioContext = buildScenarioUserContext(scenario);
+            const userPrompt = `PERSONA:
 ${personaContext}
 
 SCENARIO:
@@ -99,33 +105,50 @@ ${scenarioContext}
 Generate the FIRST message this user would send to start the conversation.
 The message should immediately convey their goal and emotional state.
 Keep it natural - this is how they would actually open a conversation.`;
-        try {
-            const response = await this.client.chat.completions.create({
-                model: this.model,
-                messages: [
+            try {
+                const llmMessages = [
                     { role: "system", content: FIRST_MESSAGE_PROMPT },
                     { role: "user", content: userPrompt },
-                ],
-                temperature: TEMPERATURE_FIRST_MESSAGE,
-                max_tokens: 500,
-            });
-            let message = response.choices[0]?.message.content ?? "";
-            message = message.trim().replace(/^["']|["']$/g, "");
-            console.debug(`Generated first message: ${message.substring(0, 100)}...`);
-            return message;
-        }
-        catch (e) {
-            // Re-throw auth errors — a bad API key should fail fast, not silently
-            // produce meaningless results for the entire simulation run.
-            if (e instanceof Error &&
-                "status" in e &&
-                (e.status === 401 ||
-                    e.status === 403)) {
-                throw e;
+                ];
+                const response = await withLLMSpan({
+                    model: this.model,
+                    temperature: TEMPERATURE_FIRST_MESSAGE,
+                    maxTokens: 500,
+                    purpose: "first_message",
+                }, async (llmSpan) => {
+                    recordLLMInput(llmSpan, llmMessages.map((m) => ({
+                        role: m.role,
+                        content: typeof m.content === "string" ? m.content : "",
+                    })));
+                    const traceHeaders = await getTraceContextHeaders();
+                    const res = await this.client.chat.completions.create({
+                        model: this.model,
+                        messages: llmMessages,
+                        temperature: TEMPERATURE_FIRST_MESSAGE,
+                        max_tokens: 500,
+                    }, { headers: traceHeaders });
+                    recordLLMResponse(llmSpan, res);
+                    return res;
+                });
+                let message = response.choices[0]?.message.content ?? "";
+                message = message.trim().replace(/^["']|["']$/g, "");
+                console.debug(`Generated first message: ${message.substring(0, 100)}...`);
+                return message;
             }
-            console.warn(`FirstMessageGenerator: API call failed, using generic fallback. Error: ${e}`);
-            // Fallback to a generic message based on scenario (no persona traits applied)
-            return `Hi, I need help with: ${scenario.goal}`;
-        }
+            catch (e) {
+                // Re-throw auth errors — a bad API key should fail fast, not silently
+                // produce meaningless results for the entire simulation run.
+                if (e instanceof Error &&
+                    "status" in e &&
+                    (e.status === 401 ||
+                        e.status === 403)) {
+                    throw e;
+                }
+                console.warn(`FirstMessageGenerator: API call failed, using generic fallback. Error: ${e}`);
+                // Fallback to a generic message based on scenario (no persona traits applied)
+                _span?.setAttribute("orq.simulation.first_message_fallback", true);
+                return `Hi, I need help with: ${scenario.goal}`;
+            }
+        });
     }
 }

package/dist/lib/integrations/simulation/generators/persona-generator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"persona-generator.d.ts","sourceRoot":"","sources":["../../../../../src/lib/integrations/simulation/generators/persona-generator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,OAAO,KAAK,EAAsB,OAAO,EAAE,MAAM,aAAa,CAAC;AAiE/D;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;;;;GAKG;AACH,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,MAAM,CAAS;gBAEX,MAAM,CAAC,EAAE,sBAAsB;IAkB3C;;OAEG;IACH,OAAO,CAAC,MAAM,CAAC,aAAa;IAsC5B;;OAEG;IACG,QAAQ,CAAC,MAAM,EAAE;QACrB,gBAAgB,EAAE,MAAM,CAAC;QACzB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,kBAAkB,CAAC,EAAE,MAAM,CAAC;KAC7B,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IA0CtB;;;;;OAKG;IACG,oBAAoB,CAAC,MAAM,EAAE;QACjC,gBAAgB,EAAE,MAAM,CAAC;QACzB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,kBAAkB,CAAC,EAAE,MAAM,CAAC;KAC7B,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IAqItB;;OAEG;IACH,OAAO,CAAC,mBAAmB;IA2B3B;;OAEG;IACH,OAAO,CAAC,oBAAoB;CAyB7B"}
+{"version":3,"file":"persona-generator.d.ts","sourceRoot":"","sources":["../../../../../src/lib/integrations/simulation/generators/persona-generator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,MAAM,MAAM,QAAQ,CAAC;AAS5B,OAAO,KAAK,EAAsB,OAAO,EAAE,MAAM,aAAa,CAAC;AAiE/D;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;;;;GAKG;AACH,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,MAAM,CAAS;gBAEX,MAAM,CAAC,EAAE,sBAAsB;IAkB3C;;OAEG;IACH,OAAO,CAAC,MAAM,CAAC,aAAa;IAsC5B;;OAEG;IACG,QAAQ,CAAC,MAAM,EAAE;QACrB,gBAAgB,EAAE,MAAM,CAAC;QACzB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,kBAAkB,CAAC,EAAE,MAAM,CAAC;KAC7B,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IA+EtB;;;;;OAKG;IACG,oBAAoB,CAAC,MAAM,EAAE;QACjC,gBAAgB,EAAE,MAAM,CAAC;QACzB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,kBAAkB,CAAC,EAAE,MAAM,CAAC;KAC7B,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IA2KtB;;OAEG;IACH,OAAO,CAAC,mBAAmB;IA2B3B;;OAEG;IACH,OAAO,CAAC,oBAAoB;CAyB7B"}