@orq-ai/evaluatorq 1.2.2 → 1.2.3-rc.1

package/dist/lib/integrations/simulation/generators/first-message-generator.d.ts

@@ -0,0 +1,38 @@
+/**
+ * First message generator using LLM.
+ *
+ * Generates contextually appropriate first messages based on persona and scenario.
+ */
+import OpenAI from "openai";
+import type { Persona, Scenario } from "../types.js";
+/**
+ * Configuration for FirstMessageGenerator.
+ */
+export interface FirstMessageGeneratorConfig {
+    model?: string;
+    client?: OpenAI;
+    apiKey?: string;
+}
+/**
+ * Generates first messages for simulations.
+ *
+ * Creates contextually appropriate opening messages based on
+ * persona characteristics and scenario context.
+ */
+export declare class FirstMessageGenerator {
+    private model;
+    private client;
+    constructor(config?: FirstMessageGeneratorConfig);
+    /**
+     * Generate a first message for a simulation.
+     *
+     * Uses the Persona's toSystemPrompt() and Scenario's toUserContext()
+     * methods to build the context for generation.
+     *
+     * @param persona - User persona
+     * @param scenario - Scenario context
+     * @returns Generated first message string
+     */
+    generate(persona: Persona, scenario: Scenario): Promise<string>;
+}
+//# sourceMappingURL=first-message-generator.d.ts.map

package/dist/lib/integrations/simulation/generators/first-message-generator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"first-message-generator.d.ts","sourceRoot":"","sources":["../../../../../src/lib/integrations/simulation/generators/first-message-generator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,OAAO,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAuDrD;;GAEG;AACH,MAAM,WAAW,2BAA2B;IAC1C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;;;;GAKG;AACH,qBAAa,qBAAqB;IAChC,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,MAAM,CAAS;gBAEX,MAAM,CAAC,EAAE,2BAA2B;IAkBhD;;;;;;;;;OASG;IACG,QAAQ,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC;CAgDtE"}

package/dist/lib/integrations/simulation/generators/first-message-generator.js

@@ -0,0 +1,131 @@
+/**
+ * First message generator using LLM.
+ *
+ * Generates contextually appropriate first messages based on persona and scenario.
+ */
+import OpenAI from "openai";
+import { buildPersonaSystemPrompt, buildScenarioUserContext, } from "../utils/prompt-builders.js";
+// Temperature setting for message generation
+const TEMPERATURE_FIRST_MESSAGE = 0.8;
+const FIRST_MESSAGE_PROMPT = `You are generating the authentic first message a user would type to a support agent.
+
+## Your Task
+Create a realistic opening message that sounds like an ACTUAL customer, not a script.
+
+## Guidelines
+
+### Voice Matching (based on persona traits):
+- **Communication style "terse"**: Short sentences, minimal pleasantries, gets straight to the point
+- **Communication style "verbose"**: Detailed explanations, context, multiple sentences
+- **Communication style "formal"**: Professional language, complete sentences, "Dear", "Sincerely"
+- **Communication style "casual"**: Contractions, slang, emojis if appropriate, friendly tone
+
+- **Low patience (0-0.3)**: Frustrated tone, urgency indicators ("I've been waiting", "This is ridiculous")
+- **High patience (0.7-1.0)**: Calm, understanding, may apologize for bothering
+
+- **Low politeness (0-0.3)**: Direct, potentially demanding, no pleasantries
+- **High politeness (0.7-1.0)**: "Please", "Thank you", "I appreciate your help"
+
+- **Low technical level (0-0.3)**: Simple language, may describe problems in non-technical terms
+- **High technical level (0.7-1.0)**: Technical terminology, specific error codes, detailed descriptions
+
+### Emotional States:
+- **Frustrated**: Caps for emphasis, exclamation marks, expressions of disappointment
+- **Confused**: Questions, uncertainty ("I'm not sure if...", "Am I doing something wrong?")
+- **Urgent**: Time pressure mentioned, immediate action requested
+- **Happy**: Positive tone, compliments, appreciation
+- **Neutral**: Matter-of-fact, balanced
+
+### Message Length:
+- Keep messages 50-200 characters for "terse" style
+- Allow 150-400 characters for "verbose" style
+- Target 80-250 characters for "casual" or "formal"
+
+### DO:
+- Include specific details from the scenario context
+- Sound like a real person typing quickly (minor imperfections are OK)
+- Match the emotional intensity to the starting_emotion
+
+### DON'T:
+- Start with "Dear Support" unless formal style with high politeness
+- Be overly long unless verbose style
+- Use robotic language ("I am writing to inquire about...")
+
+Return ONLY the message text. No quotes, no explanations, no labels.`;
+/**
+ * Generates first messages for simulations.
+ *
+ * Creates contextually appropriate opening messages based on
+ * persona characteristics and scenario context.
+ */
+export class FirstMessageGenerator {
+    model;
+    client;
+    constructor(config) {
+        this.model = config?.model ?? "azure/gpt-4o-mini";
+        if (config?.client) {
+            this.client = config.client;
+        }
+        else {
+            const apiKey = config?.apiKey ?? process.env.ORQ_API_KEY;
+            if (!apiKey) {
+                throw new Error("ORQ_API_KEY environment variable is not set. Set it or pass apiKey/client in config.");
+            }
+            this.client = new OpenAI({
+                baseURL: process.env.ROUTER_BASE_URL || "https://api.orq.ai/v2/router",
+                apiKey,
+            });
+        }
+    }
+    /**
+     * Generate a first message for a simulation.
+     *
+     * Uses the Persona's toSystemPrompt() and Scenario's toUserContext()
+     * methods to build the context for generation.
+     *
+     * @param persona - User persona
+     * @param scenario - Scenario context
+     * @returns Generated first message string
+     */
+    async generate(persona, scenario) {
+        const personaContext = buildPersonaSystemPrompt(persona);
+        const scenarioContext = buildScenarioUserContext(scenario);
+        const userPrompt = `PERSONA:
+${personaContext}
+
+SCENARIO:
+${scenarioContext}
+
+Generate the FIRST message this user would send to start the conversation.
+The message should immediately convey their goal and emotional state.
+Keep it natural - this is how they would actually open a conversation.`;
+        try {
+            const response = await this.client.chat.completions.create({
+                model: this.model,
+                messages: [
+                    { role: "system", content: FIRST_MESSAGE_PROMPT },
+                    { role: "user", content: userPrompt },
+                ],
+                temperature: TEMPERATURE_FIRST_MESSAGE,
+                max_tokens: 500,
+            });
+            let message = response.choices[0]?.message.content ?? "";
+            message = message.trim().replace(/^["']|["']$/g, "");
+            console.debug(`Generated first message: ${message.substring(0, 100)}...`);
+            return message;
+        }
+        catch (e) {
+            // Re-throw auth errors — a bad API key should fail fast, not silently
+            // produce meaningless results for the entire simulation run.
+            if (e instanceof Error &&
+                "status" in e &&
+                (e.status === 401 ||
+                    e.status === 403)) {
+                throw e;
+            }
+            console.warn(`FirstMessageGenerator: API call failed, using generic fallback. Error: ${e}`);
+            // Fallback to a generic message based on scenario (no persona traits applied)
+            return `Hi, I need help with: ${scenario.goal}`;
+        }
+    }
+}

package/dist/lib/integrations/simulation/generators/index.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Generator modules for agent simulation.
+ *
+ * Provides persona, scenario, first message, and datapoint generators
+ * that use LLMs to create diverse test data.
+ */
+export type { DatapointGeneratorConfig } from "./datapoint-generator.js";
+export { DatapointGenerator } from "./datapoint-generator.js";
+export type { FirstMessageGeneratorConfig } from "./first-message-generator.js";
+export { FirstMessageGenerator } from "./first-message-generator.js";
+export type { PersonaGeneratorConfig } from "./persona-generator.js";
+export { PersonaGenerator } from "./persona-generator.js";
+export type { ScenarioGeneratorConfig } from "./scenario-generator.js";
+export { ScenarioGenerator } from "./scenario-generator.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/lib/integrations/simulation/generators/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/lib/integrations/simulation/generators/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,YAAY,EAAE,wBAAwB,EAAE,MAAM,0BAA0B,CAAC;AACzE,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC9D,YAAY,EAAE,2BAA2B,EAAE,MAAM,8BAA8B,CAAC;AAChF,OAAO,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AACrE,YAAY,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAC;AACrE,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,YAAY,EAAE,uBAAuB,EAAE,MAAM,yBAAyB,CAAC;AACvE,OAAO,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC"}

package/dist/lib/integrations/simulation/generators/index.js

@@ -0,0 +1,10 @@
+/**
+ * Generator modules for agent simulation.
+ *
+ * Provides persona, scenario, first message, and datapoint generators
+ * that use LLMs to create diverse test data.
+ */
+export { DatapointGenerator } from "./datapoint-generator.js";
+export { FirstMessageGenerator } from "./first-message-generator.js";
+export { PersonaGenerator } from "./persona-generator.js";
+export { ScenarioGenerator } from "./scenario-generator.js";

package/dist/lib/integrations/simulation/generators/persona-generator.d.ts

@@ -0,0 +1,60 @@
+/**
+ * Persona generator using LLM.
+ *
+ * Generates user personas from agent descriptions and optional context.
+ */
+import OpenAI from "openai";
+import type { Persona } from "../types.js";
+/**
+ * Configuration for PersonaGenerator.
+ */
+export interface PersonaGeneratorConfig {
+    model?: string;
+    client?: OpenAI;
+    apiKey?: string;
+}
+/**
+ * Generates personas from agent descriptions.
+ *
+ * Uses an LLM to create diverse, realistic user personas
+ * based on the agent's purpose and context.
+ */
+export declare class PersonaGenerator {
+    private model;
+    private client;
+    constructor(config?: PersonaGeneratorConfig);
+    /**
+     * Parse LLM response content into Persona objects.
+     */
+    private static parsePersonas;
+    /**
+     * Generate personas for agent testing.
+     */
+    generate(params: {
+        agentDescription: string;
+        context?: string;
+        numPersonas?: number;
+        edgeCasePercentage?: number;
+    }): Promise<Persona[]>;
+    /**
+     * Generate personas with guaranteed trait coverage.
+     *
+     * Ensures all communication styles and trait ranges are represented,
+     * including extreme values that LLMs tend to avoid.
+     */
+    generateWithCoverage(params: {
+        agentDescription: string;
+        context?: string;
+        numPersonas?: number;
+        edgeCasePercentage?: number;
+    }): Promise<Persona[]>;
+    /**
+     * Ensure all communication styles are covered.
+     */
+    private ensureStyleCoverage;
+    /**
+     * Log warnings about missing trait coverage without modifying personas.
+     */
+    private logTraitCoverageGaps;
+}
+//# sourceMappingURL=persona-generator.d.ts.map

package/dist/lib/integrations/simulation/generators/persona-generator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"persona-generator.d.ts","sourceRoot":"","sources":["../../../../../src/lib/integrations/simulation/generators/persona-generator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,OAAO,KAAK,EAAsB,OAAO,EAAE,MAAM,aAAa,CAAC;AAiE/D;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;;;;GAKG;AACH,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,MAAM,CAAS;gBAEX,MAAM,CAAC,EAAE,sBAAsB;IAkB3C;;OAEG;IACH,OAAO,CAAC,MAAM,CAAC,aAAa;IAsC5B;;OAEG;IACG,QAAQ,CAAC,MAAM,EAAE;QACrB,gBAAgB,EAAE,MAAM,CAAC;QACzB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,kBAAkB,CAAC,EAAE,MAAM,CAAC;KAC7B,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IA0CtB;;;;;OAKG;IACG,oBAAoB,CAAC,MAAM,EAAE;QACjC,gBAAgB,EAAE,MAAM,CAAC;QACzB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,kBAAkB,CAAC,EAAE,MAAM,CAAC;KAC7B,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IAqItB;;OAEG;IACH,OAAO,CAAC,mBAAmB;IA2B3B;;OAEG;IACH,OAAO,CAAC,oBAAoB;CAyB7B"}

package/dist/lib/integrations/simulation/generators/persona-generator.js

@@ -0,0 +1,333 @@
+/**
+ * Persona generator using LLM.
+ *
+ * Generates user personas from agent descriptions and optional context.
+ */
+import OpenAI from "openai";
+import { extractJsonFromResponse } from "../utils/extract-json.js";
+import { delimit } from "../utils/sanitize.js";
+// Temperature settings for different generation modes
+const TEMPERATURE_CREATIVE = 0.8;
+const TEMPERATURE_BALANCED = 0.7;
+const VALID_STYLES = new Set(["formal", "casual", "terse", "verbose"]);
+/** Clamp a number to [0, 1]. */
+function clamp01(value) {
+    return Math.max(0, Math.min(1, value));
+}
+const PERSONA_GENERATOR_PROMPT = `You are an expert persona designer for AI agent testing. Create realistic, memorable user personas that feel like real people, not stereotypes.
+
+## Persona Structure
+Each persona must include:
+- **name**: A vivid, specific descriptor (e.g., "Anxious First-Time Buyer", "Retired Engineer Seeking Help")
+- **patience**: 0-1 (0=interrupts constantly, 1=waits indefinitely)
+- **assertiveness**: 0-1 (0=accepts anything, 1=demands specific outcomes)
+- **politeness**: 0-1 (0=rude/demanding, 1=overly polite)
+- **technical_level**: 0-1 (0=never used a computer, 1=software developer)
+- **communication_style**: "formal", "casual", "terse", or "verbose"
+- **background**: DETAILED context (2-3 sentences) explaining WHO they are, WHY they're contacting support, and WHAT their emotional state is
+
+## Quality Guidelines
+
+### DO create personas that are:
+- **Realistic**: Based on actual customer archetypes you'd encounter
+- **Coherent**: Traits that logically fit together (e.g., high technical_level + formal style for an engineer)
+- **Specific**: Unique situations with concrete details (names, specific products, timeframes)
+- **Emotionally grounded**: Clear emotional context that explains their behavior
+
+### DON'T create personas that are:
+- Generic (e.g., "Customer with a problem")
+- Contradictory (e.g., patience=0.1 but described as "patient and understanding")
+- Unrealistic trait combinations (e.g., technical_level=0.9 + communication_style="terse" for a "confused elderly person")
+- All similar - vary trait values across personas, including some with low values and some with high values
+
+## Example HIGH-QUALITY Persona:
+{
+  "name": "Overwhelmed Working Parent",
+  "patience": 0.3,
+  "assertiveness": 0.6,
+  "politeness": 0.7,
+  "technical_level": 0.4,
+  "communication_style": "terse",
+  "background": "Sarah is a working mom with two kids under 5. She ordered a birthday gift (a tablet) for her daughter 2 weeks ago but it hasn't arrived. The party is in 3 days. She's stressed, multitasking while on hold, and needs quick answers without lengthy explanations."
+}
+
+## Example LOW-QUALITY Persona (AVOID):
+{
+  "name": "Angry Customer",
+  "patience": 0.1,
+  "assertiveness": 0.9,
+  "politeness": 0.2,
+  "technical_level": 0.5,
+  "communication_style": "casual",
+  "background": "A customer who is angry about something"  // TOO VAGUE!
+}
+
+Return a JSON array of persona objects.`;
+/**
+ * Generates personas from agent descriptions.
+ *
+ * Uses an LLM to create diverse, realistic user personas
+ * based on the agent's purpose and context.
+ */
+export class PersonaGenerator {
+    model;
+    client;
+    constructor(config) {
+        this.model = config?.model ?? "azure/gpt-4o-mini";
+        if (config?.client) {
+            this.client = config.client;
+        }
+        else {
+            const apiKey = config?.apiKey ?? process.env.ORQ_API_KEY;
+            if (!apiKey) {
+                throw new Error("ORQ_API_KEY environment variable is not set. Set it or pass apiKey/client in config.");
+            }
+            this.client = new OpenAI({
+                baseURL: process.env.ROUTER_BASE_URL || "https://api.orq.ai/v2/router",
+                apiKey,
+            });
+        }
+    }
+    /**
+     * Parse LLM response content into Persona objects.
+     */
+    static parsePersonas(content) {
+        const extracted = extractJsonFromResponse(content);
+        let personaDicts;
+        try {
+            const parsed = JSON.parse(extracted);
+            if (!Array.isArray(parsed)) {
+                console.warn("Failed to parse personas: expected JSON array");
+                return [];
+            }
+            personaDicts = parsed;
+        }
+        catch {
+            console.warn("Failed to parse personas JSON response");
+            return [];
+        }
+        const personas = [];
+        for (const pDict of personaDicts) {
+            try {
+                const p = pDict;
+                const rawStyle = String(p.communication_style ?? "casual");
+                personas.push({
+                    name: String(p.name ?? ""),
+                    patience: clamp01(Number(p.patience ?? 0.5)),
+                    assertiveness: clamp01(Number(p.assertiveness ?? 0.5)),
+                    politeness: clamp01(Number(p.politeness ?? 0.5)),
+                    technical_level: clamp01(Number(p.technical_level ?? 0.5)),
+                    communication_style: VALID_STYLES.has(rawStyle)
+                        ? rawStyle
+                        : "casual",
+                    background: String(p.background ?? ""),
+                });
+            }
+            catch (e) {
+                console.warn(`Failed to parse persona: ${e}`);
+            }
+        }
+        return personas;
+    }
+    /**
+     * Generate personas for agent testing.
+     */
+    async generate(params) {
+        const { agentDescription, context = "", numPersonas = 5, edgeCasePercentage = 0.2, } = params;
+        const numEdgeCases = Math.floor(numPersonas * edgeCasePercentage);
+        const userPrompt = `Agent Description: ${delimit(agentDescription)}
+
+Additional Context: ${delimit(context || "None provided")}
+
+Generate ${numPersonas} diverse personas for testing this agent.
+- Include ${numEdgeCases} edge case/challenging personas
+- Ensure variety in patience, assertiveness, and technical levels
+- Create realistic backgrounds relevant to the agent's domain
+
+Return ONLY a JSON array, no other text.`;
+        const response = await this.client.chat.completions.create({
+            model: this.model,
+            messages: [
+                { role: "system", content: PERSONA_GENERATOR_PROMPT },
+                { role: "user", content: userPrompt },
+            ],
+            temperature: TEMPERATURE_CREATIVE,
+            max_tokens: 4000,
+        });
+        const content = response.choices[0]?.message.content ?? "[]";
+        const personas = PersonaGenerator.parsePersonas(content);
+        if (personas.length < numPersonas) {
+            console.warn(`PersonaGenerator: requested ${numPersonas} personas but only ${personas.length} were successfully parsed`);
+        }
+        return personas;
+    }
+    /**
+     * Generate personas with guaranteed trait coverage.
+     *
+     * Ensures all communication styles and trait ranges are represented,
+     * including extreme values that LLMs tend to avoid.
+     */
+    async generateWithCoverage(params) {
+        const { agentDescription, context = "", numPersonas = 8, edgeCasePercentage = 0.2, } = params;
+        const styles = [
+            "formal",
+            "casual",
+            "terse",
+            "verbose",
+        ];
+        // Explicit trait combinations covering the FULL range (0.0-1.0)
+        const traitTargets = [
+            {
+                patience: 0.1,
+                assertiveness: 0.1,
+                politeness: 0.1,
+                technical_level: 0.1,
+            },
+            {
+                patience: 0.9,
+                assertiveness: 0.1,
+                politeness: 0.9,
+                technical_level: 0.9,
+            },
+            {
+                patience: 0.1,
+                assertiveness: 0.9,
+                politeness: 0.1,
+                technical_level: 0.5,
+            },
+            {
+                patience: 0.5,
+                assertiveness: 0.9,
+                politeness: 0.9,
+                technical_level: 0.1,
+            },
+            {
+                patience: 0.5,
+                assertiveness: 0.5,
+                politeness: 0.5,
+                technical_level: 0.5,
+            },
+            {
+                patience: 0.3,
+                assertiveness: 0.7,
+                politeness: 0.6,
+                technical_level: 0.3,
+            },
+            {
+                patience: 0.7,
+                assertiveness: 0.3,
+                politeness: 0.8,
+                technical_level: 0.7,
+            },
+            {
+                patience: 0.2,
+                assertiveness: 0.8,
+                politeness: 0.3,
+                technical_level: 0.8,
+            },
+        ];
+        const numEdgeCases = Math.floor(numPersonas * edgeCasePercentage);
+        const coverageInstructions = Array.from({ length: Math.min(numPersonas, 8) }, (_, i) => {
+            const target = traitTargets[i % traitTargets.length];
+            return (`- Persona ${i + 1}: communication_style='${styles[i % styles.length]}', ` +
+                `patience=${target.patience.toFixed(1)}, ` +
+                `assertiveness=${target.assertiveness.toFixed(1)}, ` +
+                `politeness=${target.politeness.toFixed(1)}, ` +
+                `technical_level=${target.technical_level.toFixed(1)}`);
+        }).join("\n");
+        const userPrompt = `Agent Description: ${delimit(agentDescription)}
+
+Additional Context: ${delimit(context || "None provided")}
+
+Generate ${numPersonas} personas with EXACT trait values as specified below.
+CRITICAL: Use the EXACT numeric values provided - do NOT adjust them to be more "balanced".
+
+${coverageInstructions}
+
+IMPORTANT:
+- Use the EXACT trait values shown above (e.g., if it says politeness=0.1, use 0.1, not 0.3 or 0.5)
+- Low values (0.1-0.2) represent EXTREME traits - these are intentional, not mistakes
+- Include ${numEdgeCases} edge case/challenging personas
+- Ensure traits span the full 0-1 range across all personas
+- Create realistic backgrounds relevant to the agent's domain
+
+Return ONLY a JSON array, no other text.`;
+        const response = await this.client.chat.completions.create({
+            model: this.model,
+            messages: [
+                { role: "system", content: PERSONA_GENERATOR_PROMPT },
+                { role: "user", content: userPrompt },
+            ],
+            temperature: TEMPERATURE_BALANCED,
+            max_tokens: 4000,
+        });
+        const content = response.choices[0]?.message.content ?? "[]";
+        let personas = PersonaGenerator.parsePersonas(content);
+        // Validate coverage and fill gaps if needed
+        personas = this.ensureStyleCoverage(personas, styles);
+        this.logTraitCoverageGaps(personas);
+        // Trim to requested count (coverage adjustments may have kept extras)
+        if (personas.length > numPersonas) {
+            personas = personas.slice(0, numPersonas);
+        }
+        if (personas.length < numPersonas) {
+            console.warn(`PersonaGenerator: requested ${numPersonas} personas (with coverage) but only ${personas.length} were successfully parsed`);
+        }
+        return personas;
+    }
+    /**
+     * Ensure all communication styles are covered.
+     */
+    ensureStyleCoverage(personas, requiredStyles) {
+        const existingStyles = new Set(personas.map((p) => p.communication_style));
+        const missingStyles = requiredStyles.filter((s) => !existingStyles.has(s));
+        if (missingStyles.length > 0 && personas.length > 0) {
+            for (let i = 0; i < missingStyles.length; i++) {
+                const style = missingStyles[i];
+                if (i < personas.length) {
+                    const p = personas[i];
+                    // Create a new persona with the adjusted style (immutable update)
+                    personas[i] = {
+                        ...p,
+                        communication_style: style,
+                    };
+                    console.debug(`Adjusted persona '${p.name}' to style '${style}' for coverage`);
+                }
+            }
+        }
+        return personas;
+    }
+    /**
+     * Log warnings about missing trait coverage without modifying personas.
+     */
+    logTraitCoverageGaps(personas) {
+        if (personas.length === 0)
+            return;
+        const hasLow = (values) => values.some((v) => v <= 0.2);
+        const hasHigh = (values) => values.some((v) => v >= 0.8);
+        const patienceVals = personas.map((p) => p.patience);
+        const assertiveVals = personas.map((p) => p.assertiveness);
+        const politeVals = personas.map((p) => p.politeness);
+        const techVals = personas.map((p) => p.technical_level);
+        const gaps = [];
+        if (!hasLow(patienceVals))
+            gaps.push("low patience (<0.2)");
+        if (!hasHigh(patienceVals))
+            gaps.push("high patience (>0.8)");
+        if (!hasLow(assertiveVals))
+            gaps.push("low assertiveness (<0.2)");
+        if (!hasHigh(assertiveVals))
+            gaps.push("high assertiveness (>0.8)");
+        if (!hasLow(politeVals))
+            gaps.push("low politeness (<0.2)");
+        if (!hasHigh(politeVals))
+            gaps.push("high politeness (>0.8)");
+        if (!hasLow(techVals))
+            gaps.push("low technical_level (<0.2)");
+        if (!hasHigh(techVals))
+            gaps.push("high technical_level (>0.8)");
+        if (gaps.length > 0) {
+            console.debug(`Trait coverage gaps: ${gaps.join(", ")}`);
+        }
+    }
+}

package/dist/lib/integrations/simulation/generators/scenario-generator.d.ts

@@ -0,0 +1,77 @@
+/**
+ * Scenario generator using LLM.
+ *
+ * Generates test scenarios from agent descriptions and optional context.
+ */
+import OpenAI from "openai";
+import type { Scenario } from "../types.js";
+/**
+ * Configuration for ScenarioGenerator.
+ */
+export interface ScenarioGeneratorConfig {
+    model?: string;
+    client?: OpenAI;
+    apiKey?: string;
+}
+/**
+ * Generates scenarios from agent descriptions.
+ *
+ * Uses an LLM to create diverse test scenarios
+ * based on the agent's purpose and context.
+ */
+export declare class ScenarioGenerator {
+    private model;
+    private client;
+    constructor(config?: ScenarioGeneratorConfig);
+    /**
+     * Generate scenarios for agent testing.
+     */
+    generate(params: {
+        agentDescription: string;
+        context?: string;
+        numScenarios?: number;
+        edgeCasePercentage?: number;
+    }): Promise<Scenario[]>;
+    /**
+     * Generate scenarios with guaranteed emotion and criteria coverage.
+     */
+    generateWithCoverage(params: {
+        agentDescription: string;
+        context?: string;
+        numScenarios?: number;
+        edgeCasePercentage?: number;
+    }): Promise<Scenario[]>;
+    /**
+     * Ensure all starting emotions are covered.
+     */
+    private ensureEmotionCoverage;
+    /**
+     * Ensure at least one must_not_happen criterion exists if none present.
+     */
+    private ensureCriteriaCoverage;
+    /**
+     * Generate edge case scenarios specifically.
+     */
+    generateEdgeCases(params: {
+        agentDescription: string;
+        existingScenarios?: Scenario[];
+        numEdgeCases?: number;
+    }): Promise<Scenario[]>;
+    /**
+     * Generate boundary/out-of-scope test scenarios.
+     */
+    generateBoundaryScenarios(params: {
+        agentDescription: string;
+        numScenarios?: number;
+    }): Promise<Scenario[]>;
+    /**
+     * Generate security test scenarios inspired by OWASP attack patterns.
+     */
+    generateSecurityScenarios(params: {
+        agentDescription: string;
+        seedExamples?: Record<string, unknown>[];
+        categories?: string[];
+        numScenarios?: number;
+    }): Promise<Scenario[]>;
+}
+//# sourceMappingURL=scenario-generator.d.ts.map