@orq-ai/evaluatorq 1.3.1 → 1.3.2

package/dist/lib/integrations/simulation/generators/persona-generator.js

@@ -4,6 +4,7 @@
  * Generates user personas from agent descriptions and optional context.
  */
 import OpenAI from "openai";
+import { getTraceContextHeaders, recordLLMInput, recordLLMResponse, withLLMSpan, withSimulationSpan, } from "../tracing.js";
 import { extractJsonFromResponse } from "../utils/extract-json.js";
 import { delimit } from "../utils/sanitize.js";
 // Temperature settings for different generation modes
@@ -133,9 +134,13 @@ export class PersonaGenerator {
      * Generate personas for agent testing.
      */
     async generate(params) {
-        const { agentDescription, context = "", numPersonas = 5, edgeCasePercentage = 0.2, } = params;
-        const numEdgeCases = Math.floor(numPersonas * edgeCasePercentage);
-        const userPrompt = `Agent Description: ${delimit(agentDescription)}
+        return withSimulationSpan("orq.simulation.persona_generation", {
+            "orq.simulation.num_personas": params.numPersonas ?? 5,
+            "orq.simulation.model": this.model,
+        }, async (_span) => {
+            const { agentDescription, context = "", numPersonas = 5, edgeCasePercentage = 0.2, } = params;
+            const numEdgeCases = Math.floor(numPersonas * edgeCasePercentage);
+            const userPrompt = `Agent Description: ${delimit(agentDescription)}
 
 Additional Context: ${delimit(context || "None provided")}
 
@@ -145,21 +150,37 @@ Generate ${numPersonas} diverse personas for testing this agent.
 - Create realistic backgrounds relevant to the agent's domain
 
 Return ONLY a JSON array, no other text.`;
-        const response = await this.client.chat.completions.create({
-            model: this.model,
-            messages: [
+            const llmMessages = [
                 { role: "system", content: PERSONA_GENERATOR_PROMPT },
                 { role: "user", content: userPrompt },
-            ],
-            temperature: TEMPERATURE_CREATIVE,
-            max_tokens: 4000,
+            ];
+            const response = await withLLMSpan({
+                model: this.model,
+                temperature: TEMPERATURE_CREATIVE,
+                maxTokens: 4000,
+                purpose: "persona_generation",
+            }, async (llmSpan) => {
+                recordLLMInput(llmSpan, llmMessages.map((m) => ({
+                    role: m.role,
+                    content: typeof m.content === "string" ? m.content : "",
+                })));
+                const traceHeaders = await getTraceContextHeaders();
+                const res = await this.client.chat.completions.create({
+                    model: this.model,
+                    messages: llmMessages,
+                    temperature: TEMPERATURE_CREATIVE,
+                    max_tokens: 4000,
+                }, { headers: traceHeaders });
+                recordLLMResponse(llmSpan, res);
+                return res;
+            });
+            const content = response.choices[0]?.message.content ?? "[]";
+            const personas = PersonaGenerator.parsePersonas(content);
+            if (personas.length < numPersonas) {
+                console.warn(`PersonaGenerator: requested ${numPersonas} personas but only ${personas.length} were successfully parsed`);
+            }
+            return personas;
         });
-        const content = response.choices[0]?.message.content ?? "[]";
-        const personas = PersonaGenerator.parsePersonas(content);
-        if (personas.length < numPersonas) {
-            console.warn(`PersonaGenerator: requested ${numPersonas} personas but only ${personas.length} were successfully parsed`);
-        }
-        return personas;
     }
     /**
      * Generate personas with guaranteed trait coverage.
@@ -168,74 +189,79 @@ Return ONLY a JSON array, no other text.`;
      * including extreme values that LLMs tend to avoid.
      */
     async generateWithCoverage(params) {
-        const { agentDescription, context = "", numPersonas = 8, edgeCasePercentage = 0.2, } = params;
-        const styles = [
-            "formal",
-            "casual",
-            "terse",
-            "verbose",
-        ];
-        // Explicit trait combinations covering the FULL range (0.0-1.0)
-        const traitTargets = [
-            {
-                patience: 0.1,
-                assertiveness: 0.1,
-                politeness: 0.1,
-                technical_level: 0.1,
-            },
-            {
-                patience: 0.9,
-                assertiveness: 0.1,
-                politeness: 0.9,
-                technical_level: 0.9,
-            },
-            {
-                patience: 0.1,
-                assertiveness: 0.9,
-                politeness: 0.1,
-                technical_level: 0.5,
-            },
-            {
-                patience: 0.5,
-                assertiveness: 0.9,
-                politeness: 0.9,
-                technical_level: 0.1,
-            },
-            {
-                patience: 0.5,
-                assertiveness: 0.5,
-                politeness: 0.5,
-                technical_level: 0.5,
-            },
-            {
-                patience: 0.3,
-                assertiveness: 0.7,
-                politeness: 0.6,
-                technical_level: 0.3,
-            },
-            {
-                patience: 0.7,
-                assertiveness: 0.3,
-                politeness: 0.8,
-                technical_level: 0.7,
-            },
-            {
-                patience: 0.2,
-                assertiveness: 0.8,
-                politeness: 0.3,
-                technical_level: 0.8,
-            },
-        ];
-        const numEdgeCases = Math.floor(numPersonas * edgeCasePercentage);
-        const coverageInstructions = Array.from({ length: Math.min(numPersonas, 8) }, (_, i) => {
-            const target = traitTargets[i % traitTargets.length];
-            return (`- Persona ${i + 1}: communication_style='${styles[i % styles.length]}', ` +
-                `patience=${target.patience.toFixed(1)}, ` +
-                `assertiveness=${target.assertiveness.toFixed(1)}, ` +
-                `politeness=${target.politeness.toFixed(1)}, ` +
-                `technical_level=${target.technical_level.toFixed(1)}`);
-        }).join("\n");
-        const userPrompt = `Agent Description: ${delimit(agentDescription)}
+        return withSimulationSpan("orq.simulation.persona_generation", {
+            "orq.simulation.num_personas": params.numPersonas ?? 8,
+            "orq.simulation.mode": "coverage",
+            "orq.simulation.model": this.model,
+        }, async (_span) => {
+            const { agentDescription, context = "", numPersonas = 8, edgeCasePercentage = 0.2, } = params;
+            const styles = [
+                "formal",
+                "casual",
+                "terse",
+                "verbose",
+            ];
+            // Explicit trait combinations covering the FULL range (0.0-1.0)
+            const traitTargets = [
+                {
+                    patience: 0.1,
+                    assertiveness: 0.1,
+                    politeness: 0.1,
+                    technical_level: 0.1,
+                },
+                {
+                    patience: 0.9,
+                    assertiveness: 0.1,
+                    politeness: 0.9,
+                    technical_level: 0.9,
+                },
+                {
+                    patience: 0.1,
+                    assertiveness: 0.9,
+                    politeness: 0.1,
+                    technical_level: 0.5,
+                },
+                {
+                    patience: 0.5,
+                    assertiveness: 0.9,
+                    politeness: 0.9,
+                    technical_level: 0.1,
+                },
+                {
+                    patience: 0.5,
+                    assertiveness: 0.5,
+                    politeness: 0.5,
+                    technical_level: 0.5,
+                },
+                {
+                    patience: 0.3,
+                    assertiveness: 0.7,
+                    politeness: 0.6,
+                    technical_level: 0.3,
+                },
+                {
+                    patience: 0.7,
+                    assertiveness: 0.3,
+                    politeness: 0.8,
+                    technical_level: 0.7,
+                },
+                {
+                    patience: 0.2,
+                    assertiveness: 0.8,
+                    politeness: 0.3,
+                    technical_level: 0.8,
+                },
+            ];
+            const numEdgeCases = Math.floor(numPersonas * edgeCasePercentage);
+            const coverageInstructions = Array.from({ length: Math.min(numPersonas, 8) }, (_, i) => {
+                const target = traitTargets[i % traitTargets.length];
+                return (`- Persona ${i + 1}: communication_style='${styles[i % styles.length]}', ` +
+                    `patience=${target.patience.toFixed(1)}, ` +
+                    `assertiveness=${target.assertiveness.toFixed(1)}, ` +
+                    `politeness=${target.politeness.toFixed(1)}, ` +
+                    `technical_level=${target.technical_level.toFixed(1)}`);
+            }).join("\n");
+            const userPrompt = `Agent Description: ${delimit(agentDescription)}
 
 Additional Context: ${delimit(context || "None provided")}
 
@@ -252,28 +278,44 @@ IMPORTANT:
 - Create realistic backgrounds relevant to the agent's domain
 
 Return ONLY a JSON array, no other text.`;
-        const response = await this.client.chat.completions.create({
-            model: this.model,
-            messages: [
+            const covMessages = [
                 { role: "system", content: PERSONA_GENERATOR_PROMPT },
                 { role: "user", content: userPrompt },
-            ],
-            temperature: TEMPERATURE_BALANCED,
-            max_tokens: 4000,
+            ];
+            const response = await withLLMSpan({
+                model: this.model,
+                temperature: TEMPERATURE_BALANCED,
+                maxTokens: 4000,
+                purpose: "persona_generation_coverage",
+            }, async (llmSpan) => {
+                recordLLMInput(llmSpan, covMessages.map((m) => ({
+                    role: m.role,
+                    content: typeof m.content === "string" ? m.content : "",
+                })));
+                const traceHeaders = await getTraceContextHeaders();
+                const res = await this.client.chat.completions.create({
+                    model: this.model,
+                    messages: covMessages,
+                    temperature: TEMPERATURE_BALANCED,
+                    max_tokens: 4000,
+                }, { headers: traceHeaders });
+                recordLLMResponse(llmSpan, res);
+                return res;
+            });
+            const content = response.choices[0]?.message.content ?? "[]";
+            let personas = PersonaGenerator.parsePersonas(content);
+            // Validate coverage and fill gaps if needed
+            personas = this.ensureStyleCoverage(personas, styles);
+            this.logTraitCoverageGaps(personas);
+            // Trim to requested count (coverage adjustments may have kept extras)
+            if (personas.length > numPersonas) {
+                personas = personas.slice(0, numPersonas);
+            }
+            if (personas.length < numPersonas) {
+                console.warn(`PersonaGenerator: requested ${numPersonas} personas (with coverage) but only ${personas.length} were successfully parsed`);
+            }
+            return personas;
         });
-        const content = response.choices[0]?.message.content ?? "[]";
-        let personas = PersonaGenerator.parsePersonas(content);
-        // Validate coverage and fill gaps if needed
-        personas = this.ensureStyleCoverage(personas, styles);
-        this.logTraitCoverageGaps(personas);
-        // Trim to requested count (coverage adjustments may have kept extras)
-        if (personas.length > numPersonas) {
-            personas = personas.slice(0, numPersonas);
-        }
-        if (personas.length < numPersonas) {
-            console.warn(`PersonaGenerator: requested ${numPersonas} personas (with coverage) but only ${personas.length} were successfully parsed`);
-        }
-        return personas;
     }
     /**
      * Ensure all communication styles are covered.

package/dist/lib/integrations/simulation/generators/scenario-generator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"scenario-generator.d.ts","sourceRoot":"","sources":["../../../../../src/lib/integrations/simulation/generators/scenario-generator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,OAAO,KAAK,EAGV,QAAQ,EAET,MAAM,aAAa,CAAC;AAsJrB;;GAEG;AACH,MAAM,WAAW,uBAAuB;IACtC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAwDD;;;;;GAKG;AACH,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,MAAM,CAAS;gBAEX,MAAM,CAAC,EAAE,uBAAuB;IAkB5C;;OAEG;IACG,QAAQ,CAAC,MAAM,EAAE;QACrB,gBAAgB,EAAE,MAAM,CAAC;QACzB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,kBAAkB,CAAC,EAAE,MAAM,CAAC;KAC7B,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;IAuDvB;;OAEG;IACG,oBAAoB,CAAC,MAAM,EAAE;QACjC,gBAAgB,EAAE,MAAM,CAAC;QACzB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,kBAAkB,CAAC,EAAE,MAAM,CAAC;KAC7B,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;IAoFvB;;OAEG;IACH,OAAO,CAAC,qBAAqB;IA6B7B;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAyB9B;;OAEG;IACG,iBAAiB,CAAC,MAAM,EAAE;QAC9B,gBAAgB,EAAE,MAAM,CAAC;QACzB,iBAAiB,CAAC,EAAE,QAAQ,EAAE,CAAC;QAC/B,YAAY,CAAC,EAAE,MAAM,CAAC;KACvB,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;IA4DvB;;OAEG;IACG,yBAAyB,CAAC,MAAM,EAAE;QACtC,gBAAgB,EAAE,MAAM,CAAC;QACzB,YAAY,CAAC,EAAE,MAAM,CAAC;KACvB,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;IAuDvB;;OAEG;IACG,yBAAyB,CAAC,MAAM,EAAE;QACtC,gBAAgB,EAAE,MAAM,CAAC;QACzB,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;QACzC,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;QACtB,YAAY,CAAC,EAAE,MAAM,CAAC;KACvB,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;CA2ExB"}
+{"version":3,"file":"scenario-generator.d.ts","sourceRoot":"","sources":["../../../../../src/lib/integrations/simulation/generators/scenario-generator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,MAAM,MAAM,QAAQ,CAAC;AAS5B,OAAO,KAAK,EAGV,QAAQ,EAET,MAAM,aAAa,CAAC;AAsJrB;;GAEG;AACH,MAAM,WAAW,uBAAuB;IACtC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAwDD;;;;;GAKG;AACH,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,MAAM,CAAS;gBAEX,MAAM,CAAC,EAAE,uBAAuB;IAkB5C;;OAEG;IACG,QAAQ,CAAC,MAAM,EAAE;QACrB,gBAAgB,EAAE,MAAM,CAAC;QACzB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,kBAAkB,CAAC,EAAE,MAAM,CAAC;KAC7B,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;IAyFvB;;OAEG;IACG,oBAAoB,CAAC,MAAM,EAAE;QACjC,gBAAgB,EAAE,MAAM,CAAC;QACzB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,kBAAkB,CAAC,EAAE,MAAM,CAAC;KAC7B,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;IAuHvB;;OAEG;IACH,OAAO,CAAC,qBAAqB;IA6B7B;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAyB9B;;OAEG;IACG,iBAAiB,CAAC,MAAM,EAAE;QAC9B,gBAAgB,EAAE,MAAM,CAAC;QACzB,iBAAiB,CAAC,EAAE,QAAQ,EAAE,CAAC;QAC/B,YAAY,CAAC,EAAE,MAAM,CAAC;KACvB,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;IAmGvB;;OAEG;IACG,yBAAyB,CAAC,MAAM,EAAE;QACtC,gBAAgB,EAAE,MAAM,CAAC;QACzB,YAAY,CAAC,EAAE,MAAM,CAAC;KACvB,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;IA0FvB;;OAEG;IACG,yBAAyB,CAAC,MAAM,EAAE;QACtC,gBAAgB,EAAE,MAAM,CAAC;QACzB,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;QACzC,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;QACtB,YAAY,CAAC,EAAE,MAAM,CAAC;KACvB,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;CA8GxB"}