@orq-ai/evaluatorq 1.2.2 → 1.2.3-rc.1

package/dist/lib/integrations/simulation/agents/user-simulator.js

@@ -0,0 +1,82 @@
+/**
+ * User simulator agent.
+ *
+ * Simulates user behavior based on a persona and scenario,
+ * generating realistic user messages in conversations.
+ */
+import { BaseAgent } from "./base.js";
+// ---------------------------------------------------------------------------
+// Default user simulator system prompt
+// ---------------------------------------------------------------------------
+export const DEFAULT_USER_SIMULATOR_PROMPT = `You are a user simulator. Your role is to simulate realistic user behavior in a conversation with an AI agent.
+
+You will be given:
+1. A persona describing who you are and how you behave
+2. A scenario describing your goal and context
+
+Your task:
+- Generate realistic user messages based on your persona and scenario
+- Stay in character throughout the conversation
+- Work towards achieving your goal naturally
+- React authentically to the agent's responses
+- Do not break character or acknowledge that you are a simulation
+
+Response format:
+- Respond only with the user's message
+- Do not include any meta-commentary or explanations
+- Keep responses natural and conversational`;
+// ---------------------------------------------------------------------------
+// UserSimulatorAgent
+// ---------------------------------------------------------------------------
+/**
+ * Agent that simulates user behavior.
+ *
+ * Uses a persona and scenario to generate realistic user messages
+ * in a conversation with the agent being tested.
+ */
+export class UserSimulatorAgent extends BaseAgent {
+    customSystemPrompt;
+    constructor(config) {
+        super(config);
+        this.customSystemPrompt = config?.systemPrompt ?? null;
+    }
+    get name() {
+        return "UserSimulator";
+    }
+    get systemPrompt() {
+        if (this.customSystemPrompt) {
+            return `${DEFAULT_USER_SIMULATOR_PROMPT}\n\n---\n\n${this.customSystemPrompt}`;
+        }
+        return DEFAULT_USER_SIMULATOR_PROMPT;
+    }
+    /**
+     * Generate the first message to start a conversation.
+     *
+     * @param messages - Optional context messages
+     * @returns First user message to start the conversation
+     */
+    async generateFirstMessage(messages) {
+        const promptMessages = [...(messages ?? [])];
+        promptMessages.push({
+            role: "user",
+            content: "Generate your first message to start the conversation. Remember your goal and persona.",
+        });
+        return this.respondAsync(promptMessages, { temperature: 0.8 });
+    }
+    /**
+     * Update the persona and scenario context.
+     *
+     * @param personaContext  - Persona-specific context
+     * @param scenarioContext - Scenario-specific context
+     */
+    updateContext(personaContext, scenarioContext) {
+        let combined = "";
+        if (personaContext) {
+            combined += `PERSONA:\n${personaContext}\n\n`;
+        }
+        if (scenarioContext) {
+            combined += `SCENARIO:\n${scenarioContext}`;
+        }
+        this.customSystemPrompt = combined.trim() || null;
+    }
+}

package/dist/lib/integrations/simulation/convert.d.ts

@@ -0,0 +1,22 @@
+/**
+ * Conversion functions from SimulationResult to OpenResponses format.
+ */
+import type { ResponseResource } from "../openresponses/index.js";
+import type { SimulationResult } from "./types.js";
+/**
+ * Converts a SimulationResult to OpenResponses format.
+ *
+ * Mapping:
+ * - messages with role "user"      → input[] as Message with input_text content
+ * - messages with role "assistant"  → output[] as Message with output_text content
+ * - messages with role "system"     → input[] as Message with input_text content
+ * - token_usage                     → Usage
+ * - terminated_by                   → status ("judge" → "completed", others → "incomplete")
+ * - goal_achieved, rules_broken, criteria_results, turn_metrics → metadata
+ *
+ * @param result - The simulation result to convert
+ * @param model - Optional model name to include in the response
+ * @returns A ResponseResource in OpenResponses format
+ */
+export declare function toOpenResponses(result: SimulationResult, model?: string): ResponseResource;
+//# sourceMappingURL=convert.d.ts.map

package/dist/lib/integrations/simulation/convert.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"convert.d.ts","sourceRoot":"","sources":["../../../../src/lib/integrations/simulation/convert.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,KAAK,EAGV,gBAAgB,EAEjB,MAAM,2BAA2B,CAAC;AACnC,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAEnD;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,eAAe,CAC7B,MAAM,EAAE,gBAAgB,EACxB,KAAK,SAAe,GACnB,gBAAgB,CA8GlB"}

package/dist/lib/integrations/simulation/convert.js

@@ -0,0 +1,124 @@
+/**
+ * Conversion functions from SimulationResult to OpenResponses format.
+ */
+import { generateItemId } from "../common/utils.js";
+/**
+ * Converts a SimulationResult to OpenResponses format.
+ *
+ * Mapping:
+ * - messages with role "user"      → input[] as Message with input_text content
+ * - messages with role "assistant"  → output[] as Message with output_text content
+ * - messages with role "system"     → input[] as Message with input_text content
+ * - token_usage                     → Usage
+ * - terminated_by                   → status ("judge" → "completed", others → "incomplete")
+ * - goal_achieved, rules_broken, criteria_results, turn_metrics → metadata
+ *
+ * @param result - The simulation result to convert
+ * @param model - Optional model name to include in the response
+ * @returns A ResponseResource in OpenResponses format
+ */
+export function toOpenResponses(result, model = "simulation") {
+    const now = Math.floor(Date.now() / 1000);
+    const inputItems = [];
+    const outputItems = [];
+    for (const msg of result.messages) {
+        if (msg.role === "user" || msg.role === "system") {
+            const inputMessage = {
+                type: "message",
+                id: generateItemId("msg"),
+                role: msg.role,
+                status: "completed",
+                content: [{ type: "input_text", text: msg.content }],
+            };
+            inputItems.push(inputMessage);
+        }
+        else if (msg.role === "assistant") {
+            const outputMessage = {
+                type: "message",
+                id: generateItemId("msg"),
+                role: "assistant",
+                status: "completed",
+                content: [
+                    {
+                        type: "output_text",
+                        text: msg.content,
+                        annotations: [],
+                        logprobs: [],
+                    },
+                ],
+            };
+            outputItems.push(outputMessage);
+        }
+    }
+    // Map terminated_by to status
+    const status = result.terminated_by === "judge"
+        ? "completed"
+        : result.terminated_by === "error"
+            ? "failed"
+            : "incomplete";
+    const incompleteDetails = status === "incomplete"
+        ? { reason: `${result.terminated_by}: ${result.reason}` }
+        : null;
+    // Build usage from token_usage
+    let usageData = null;
+    if (result.token_usage.total_tokens > 0) {
+        usageData = {
+            input_tokens: result.token_usage.prompt_tokens,
+            input_tokens_details: { cached_tokens: 0 },
+            output_tokens: result.token_usage.completion_tokens,
+            output_tokens_details: { reasoning_tokens: 0 },
+            total_tokens: result.token_usage.total_tokens,
+        };
+    }
+    return {
+        id: generateItemId("resp"),
+        object: "response",
+        created_at: now,
+        completed_at: status === "completed" ? now : null,
+        status,
+        incomplete_details: incompleteDetails,
+        model,
+        previous_response_id: null,
+        instructions: null,
+        input: inputItems,
+        output: outputItems,
+        error: result.terminated_by === "error" ? { message: result.reason } : null,
+        tools: [],
+        tool_choice: "auto",
+        truncation: "disabled",
+        parallel_tool_calls: false,
+        text: {
+            format: { type: "text" },
+        },
+        top_p: 1,
+        presence_penalty: 0,
+        frequency_penalty: 0,
+        top_logprobs: 0,
+        temperature: 1,
+        reasoning: null,
+        user: null,
+        usage: usageData,
+        max_output_tokens: null,
+        max_tool_calls: null,
+        store: false,
+        background: false,
+        service_tier: "default",
+        metadata: {
+            framework: "simulation",
+            goal_achieved: result.goal_achieved,
+            goal_completion_score: result.goal_completion_score,
+            terminated_by: result.terminated_by,
+            reason: result.reason,
+            turn_count: result.turn_count,
+            rules_broken: result.rules_broken,
+            ...(result.criteria_results && {
+                criteria_results: result.criteria_results,
+            }),
+            ...(result.turn_metrics.length > 0 && {
+                turn_metrics: result.turn_metrics,
+            }),
+        },
+        safety_identifier: null,
+        prompt_cache_key: null,
+    };
+}

package/dist/lib/integrations/simulation/evaluators/index.d.ts

@@ -0,0 +1,50 @@
+/**
+ * Built-in evaluators for agent simulation.
+ *
+ * These evaluators assess simulation results using a scorer pattern
+ * compatible with evaluatorq integration.
+ */
+import type { SimulationResult } from "../types.js";
+export type SimulationScorer = (result: SimulationResult) => number;
+/**
+ * Evaluate if the simulation goal was achieved.
+ * Returns 1 if achieved, 0 otherwise.
+ */
+export declare const goalAchievedScorer: SimulationScorer;
+/**
+ * Evaluate how many criteria were met.
+ * Returns a value between 0 and 1 based on the ratio of met criteria.
+ *
+ * Uses the criteria_results from metadata if available; otherwise returns 1.0.
+ */
+export declare const criteriaMetScorer: SimulationScorer;
+/**
+ * Evaluate conversation efficiency (fewer turns = better).
+ * Returns a value between 0 and 1.
+ */
+export declare const turnEfficiencyScorer: SimulationScorer;
+/**
+ * Evaluate overall conversation quality.
+ *
+ * Composite score based on:
+ * - Goal achievement (40%)
+ * - Criteria met (30%)
+ * - Turn efficiency (30%)
+ */
+export declare const conversationQualityScorer: SimulationScorer;
+export declare const SIMULATION_EVALUATORS: Record<string, SimulationScorer>;
+/**
+ * Get a built-in simulation evaluator by name.
+ *
+ * @param name - Evaluator name (goal_achieved, criteria_met, etc.)
+ * @returns The scorer function
+ * @throws Error if evaluator not found
+ */
+export declare function getEvaluator(name: string): SimulationScorer;
+/**
+ * Get all built-in simulation evaluators.
+ *
+ * @returns Record of evaluator name to scorer function
+ */
+export declare function getAllEvaluators(): Record<string, SimulationScorer>;
+//# sourceMappingURL=index.d.ts.map

package/dist/lib/integrations/simulation/evaluators/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/lib/integrations/simulation/evaluators/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAMpD,MAAM,MAAM,gBAAgB,GAAG,CAAC,MAAM,EAAE,gBAAgB,KAAK,MAAM,CAAC;AAMpE;;;GAGG;AACH,eAAO,MAAM,kBAAkB,EAAE,gBAEhC,CAAC;AAEF;;;;;GAKG;AACH,eAAO,MAAM,iBAAiB,EAAE,gBAY/B,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,oBAAoB,EAAE,gBAqBlC,CAAC;AAEF;;;;;;;GAOG;AACH,eAAO,MAAM,yBAAyB,EAAE,gBAOvC,CAAC;AAMF,eAAO,MAAM,qBAAqB,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAKlE,CAAC;AAEF;;;;;;GAMG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,gBAAgB,CAO3D;AAED;;;;GAIG;AACH,wBAAgB,gBAAgB,IAAI,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAEnE"}

package/dist/lib/integrations/simulation/evaluators/index.js

@@ -0,0 +1,100 @@
+/**
+ * Built-in evaluators for agent simulation.
+ *
+ * These evaluators assess simulation results using a scorer pattern
+ * compatible with evaluatorq integration.
+ */
+// ---------------------------------------------------------------------------
+// Individual scorers
+// ---------------------------------------------------------------------------
+/**
+ * Evaluate if the simulation goal was achieved.
+ * Returns 1 if achieved, 0 otherwise.
+ */
+export const goalAchievedScorer = (result) => {
+    return result.goal_achieved ? 1 : 0;
+};
+/**
+ * Evaluate how many criteria were met.
+ * Returns a value between 0 and 1 based on the ratio of met criteria.
+ *
+ * Uses the criteria_results from metadata if available; otherwise returns 1.0.
+ */
+export const criteriaMetScorer = (result) => {
+    const criteriaResults = result.criteria_results ?? {};
+    const keys = Object.keys(criteriaResults);
+    if (keys.length === 0) {
+        return 1.0;
+    }
+    const met = Object.values(criteriaResults).filter((v) => v).length;
+    const total = keys.length;
+    return total > 0 ? met / total : 1.0;
+};
+/**
+ * Evaluate conversation efficiency (fewer turns = better).
+ * Returns a value between 0 and 1.
+ */
+export const turnEfficiencyScorer = (result) => {
+    const totalTurns = result.turn_count;
+    const goalAchieved = result.goal_achieved;
+    if (!goalAchieved) {
+        return 0.0;
+    }
+    if (totalTurns <= 2) {
+        return 1.0;
+    }
+    if (totalTurns <= 4) {
+        return 0.9;
+    }
+    if (totalTurns <= 6) {
+        return 0.7;
+    }
+    return Math.max(0.3, 1.0 - (totalTurns - 6) * 0.1);
+};
+/**
+ * Evaluate overall conversation quality.
+ *
+ * Composite score based on:
+ * - Goal achievement (40%)
+ * - Criteria met (30%)
+ * - Turn efficiency (30%)
+ */
+export const conversationQualityScorer = (result) => {
+    const goalScore = goalAchievedScorer(result);
+    const criteriaScore = criteriaMetScorer(result);
+    const efficiencyScore = turnEfficiencyScorer(result);
+    const score = goalScore * 0.4 + criteriaScore * 0.3 + efficiencyScore * 0.3;
+    return Math.round(score * 100) / 100;
+};
+// ---------------------------------------------------------------------------
+// Registry
+// ---------------------------------------------------------------------------
+export const SIMULATION_EVALUATORS = {
+    goal_achieved: goalAchievedScorer,
+    criteria_met: criteriaMetScorer,
+    turn_efficiency: turnEfficiencyScorer,
+    conversation_quality: conversationQualityScorer,
+};
+/**
+ * Get a built-in simulation evaluator by name.
+ *
+ * @param name - Evaluator name (goal_achieved, criteria_met, etc.)
+ * @returns The scorer function
+ * @throws Error if evaluator not found
+ */
+export function getEvaluator(name) {
+    const evaluator = SIMULATION_EVALUATORS[name];
+    if (!evaluator) {
+        const available = Object.keys(SIMULATION_EVALUATORS).join(", ");
+        throw new Error(`Unknown evaluator: ${name}. Available: ${available}`);
+    }
+    return evaluator;
+}
+/**
+ * Get all built-in simulation evaluators.
+ *
+ * @returns Record of evaluator name to scorer function
+ */
+export function getAllEvaluators() {
+    return { ...SIMULATION_EVALUATORS };
+}

package/dist/lib/integrations/simulation/generators/datapoint-generator.d.ts

@@ -0,0 +1,60 @@
+/**
+ * Datapoint generator for creating test datasets.
+ *
+ * Combines personas and scenarios into complete datapoints with first messages.
+ */
+import type { Datapoint, Persona, Scenario } from "../types.js";
+/**
+ * Configuration for DatapointGenerator.
+ */
+export interface DatapointGeneratorConfig {
+    model?: string;
+    rateLimitDelay?: number;
+    maxConcurrentCalls?: number;
+}
+/**
+ * Generates complete datapoints for simulation.
+ *
+ * Orchestrates persona, scenario, and first message generation
+ * to produce ready-to-use test datapoints.
+ */
+export declare class DatapointGenerator {
+    private model;
+    private rateLimitDelay;
+    private semaphore;
+    private personaGenerator;
+    private scenarioGenerator;
+    private firstMessageGenerator;
+    constructor(config?: DatapointGeneratorConfig);
+    /**
+     * Generate datapoints from agent description.
+     *
+     * Creates personas and scenarios, then combines them into datapoints.
+     * Total datapoints = numPersonas x (numScenarios + boundary + security)
+     */
+    generateFromDescription(params: {
+        agentDescription: string;
+        context?: string;
+        numPersonas?: number;
+        numScenarios?: number;
+        edgeCasePercentage?: number;
+        perturbationRate?: number;
+        includeBoundary?: boolean;
+        numBoundary?: number;
+        includeSecurity?: boolean;
+        numSecurity?: number;
+        securitySeedExamples?: Record<string, unknown>[];
+        securityCategories?: string[];
+    }): Promise<Datapoint[]>;
+    /**
+     * Generate datapoints from persona-scenario combinations.
+     */
+    generateFromCombinations(personas: Persona[], scenarios: Scenario[]): Promise<Datapoint[]>;
+    /**
+     * Apply random input perturbations to first messages for robustness testing.
+     *
+     * Uses the shared message-perturbation module.
+     */
+    private static applyPerturbations;
+}
+//# sourceMappingURL=datapoint-generator.d.ts.map

package/dist/lib/integrations/simulation/generators/datapoint-generator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"datapoint-generator.d.ts","sourceRoot":"","sources":["../../../../../src/lib/integrations/simulation/generators/datapoint-generator.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAgDhE;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACvC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED;;;;;GAKG;AACH,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,SAAS,CAAY;IAC7B,OAAO,CAAC,gBAAgB,CAAmB;IAC3C,OAAO,CAAC,iBAAiB,CAAoB;IAC7C,OAAO,CAAC,qBAAqB,CAAwB;gBAEzC,MAAM,CAAC,EAAE,wBAAwB;IAa7C;;;;;OAKG;IACG,uBAAuB,CAAC,MAAM,EAAE;QACpC,gBAAgB,EAAE,MAAM,CAAC;QACzB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,kBAAkB,CAAC,EAAE,MAAM,CAAC;QAC5B,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAC1B,eAAe,CAAC,EAAE,OAAO,CAAC;QAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,eAAe,CAAC,EAAE,OAAO,CAAC;QAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,oBAAoB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;QACjD,kBAAkB,CAAC,EAAE,MAAM,EAAE,CAAC;KAC/B,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;IAyHxB;;OAEG;IACG,wBAAwB,CAC5B,QAAQ,EAAE,OAAO,EAAE,EACnB,SAAS,EAAE,QAAQ,EAAE,GACpB,OAAO,CAAC,SAAS,EAAE,CAAC;IAuCvB;;;;OAIG;IACH,OAAO,CAAC,MAAM,CAAC,kBAAkB;CA2BlC"}

package/dist/lib/integrations/simulation/generators/datapoint-generator.js

@@ -0,0 +1,223 @@
+/**
+ * Datapoint generator for creating test datasets.
+ *
+ * Combines personas and scenarios into complete datapoints with first messages.
+ */
+import { applyRandomPerturbation } from "../quality/message-perturbation.js";
+import { generateDatapoint } from "../utils/prompt-builders.js";
+import { FirstMessageGenerator } from "./first-message-generator.js";
+import { PersonaGenerator } from "./persona-generator.js";
+import { ScenarioGenerator } from "./scenario-generator.js";
+// Default rate limit settings
+const DEFAULT_RATE_LIMIT_DELAY = 100; // 100ms delay between API calls
+const DEFAULT_MAX_CONCURRENT_CALLS = 5;
+/**
+ * Simple semaphore for limiting concurrency.
+ */
+class Semaphore {
+    max;
+    queue = [];
+    running = 0;
+    constructor(max) {
+        this.max = max;
+    }
+    async acquire() {
+        if (this.running < this.max) {
+            this.running++;
+            return;
+        }
+        return new Promise((resolve) => {
+            this.queue.push(() => {
+                this.running++;
+                resolve();
+            });
+        });
+    }
+    release() {
+        this.running--;
+        const next = this.queue.shift();
+        if (next) {
+            next();
+        }
+    }
+}
+/**
+ * Sleep for a given number of milliseconds.
+ */
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+/**
+ * Generates complete datapoints for simulation.
+ *
+ * Orchestrates persona, scenario, and first message generation
+ * to produce ready-to-use test datapoints.
+ */
+export class DatapointGenerator {
+    model;
+    rateLimitDelay;
+    semaphore;
+    personaGenerator;
+    scenarioGenerator;
+    firstMessageGenerator;
+    constructor(config) {
+        this.model = config?.model ?? "azure/gpt-4o-mini";
+        this.rateLimitDelay = config?.rateLimitDelay ?? DEFAULT_RATE_LIMIT_DELAY;
+        this.semaphore = new Semaphore(config?.maxConcurrentCalls ?? DEFAULT_MAX_CONCURRENT_CALLS);
+        this.personaGenerator = new PersonaGenerator({ model: this.model });
+        this.scenarioGenerator = new ScenarioGenerator({ model: this.model });
+        this.firstMessageGenerator = new FirstMessageGenerator({
+            model: this.model,
+        });
+    }
+    /**
+     * Generate datapoints from agent description.
+     *
+     * Creates personas and scenarios, then combines them into datapoints.
+     * Total datapoints = numPersonas x (numScenarios + boundary + security)
+     */
+    async generateFromDescription(params) {
+        const { agentDescription, context = "", numPersonas = 3, numScenarios = 5, edgeCasePercentage = 0.2, perturbationRate = 0.0, includeBoundary = false, numBoundary = 5, includeSecurity = false, numSecurity = 5, securitySeedExamples, securityCategories, } = params;
+        console.log(`Generating ${numPersonas} personas and ${numScenarios} scenarios...`);
+        // Build named tasks for parallel generation
+        const namedTasks = {
+            personas: this.personaGenerator.generate({
+                agentDescription,
+                context,
+                numPersonas,
+                edgeCasePercentage,
+            }),
+            scenarios: this.scenarioGenerator.generate({
+                agentDescription,
+                context,
+                numScenarios,
+                edgeCasePercentage,
+            }),
+        };
+        if (includeBoundary) {
+            console.log(`Including ${numBoundary} boundary scenarios`);
+            namedTasks.boundary = this.scenarioGenerator.generateBoundaryScenarios({
+                agentDescription,
+                numScenarios: numBoundary,
+            });
+        }
+        if (includeSecurity) {
+            console.log(`Including ${numSecurity} security scenarios`);
+            namedTasks.security = this.scenarioGenerator.generateSecurityScenarios({
+                agentDescription,
+                seedExamples: securitySeedExamples,
+                categories: securityCategories,
+                numScenarios: numSecurity,
+            });
+        }
+        const taskKeys = Object.keys(namedTasks);
+        const taskPromises = Object.values(namedTasks);
+        const rawResults = await Promise.all(taskPromises);
+        const results = {};
+        for (let i = 0; i < taskKeys.length; i++) {
+            const key = taskKeys[i];
+            results[key] = rawResults[i];
+        }
+        let personas = results.personas;
+        let scenarios = results.scenarios;
+        // Merge additional scenario types
+        if (results.boundary) {
+            const boundary = results.boundary;
+            console.log(`Generated ${boundary.length} boundary scenarios`);
+            scenarios = [...scenarios, ...boundary];
+        }
+        if (results.security) {
+            const security = results.security;
+            console.log(`Generated ${security.length} security scenarios`);
+            scenarios = [...scenarios, ...security];
+        }
+        if (personas.length === 0) {
+            console.warn("No personas generated, using defaults");
+            personas = [
+                {
+                    name: "Default User",
+                    patience: 0.5,
+                    assertiveness: 0.5,
+                    politeness: 0.5,
+                    technical_level: 0.5,
+                    communication_style: "casual",
+                    background: "",
+                },
+            ];
+        }
+        if (scenarios.length === 0) {
+            console.warn("No scenarios generated, using defaults");
+            scenarios = [
+                {
+                    name: "Default Scenario",
+                    goal: "Get help",
+                    context: "User needs general assistance",
+                    starting_emotion: "neutral",
+                    criteria: [],
+                },
+            ];
+        }
+        // Generate datapoints from all combinations
+        let datapoints = await this.generateFromCombinations(personas, scenarios);
+        // Apply message perturbations for robustness testing
+        if (perturbationRate > 0.0) {
+            datapoints = DatapointGenerator.applyPerturbations(datapoints, perturbationRate);
+        }
+        return datapoints;
+    }
+    /**
+     * Generate datapoints from persona-scenario combinations.
+     */
+    async generateFromCombinations(personas, scenarios) {
+        // Build all combinations
+        const combinations = [];
+        for (const persona of personas) {
+            for (const scenario of scenarios) {
+                combinations.push([persona, scenario]);
+            }
+        }
+        console.log(`Generating ${combinations.length} datapoints from ${personas.length} personas x ${scenarios.length} scenarios`);
+        const generateSingle = async (persona, scenario) => {
+            await this.semaphore.acquire();
+            try {
+                const firstMessage = await this.firstMessageGenerator.generate(persona, scenario);
+                // Small delay to prevent overwhelming the API
+                await sleep(this.rateLimitDelay);
+                return generateDatapoint(persona, scenario, firstMessage);
+            }
+            finally {
+                this.semaphore.release();
+            }
+        };
+        // Generate all datapoints with bounded concurrency
+        const tasks = combinations.map(([p, s]) => generateSingle(p, s));
+        const datapoints = await Promise.all(tasks);
+        console.log(`Generated ${datapoints.length} datapoints`);
+        return datapoints;
+    }
+    /**
+     * Apply random input perturbations to first messages for robustness testing.
+     *
+     * Uses the shared message-perturbation module.
+     */
+    static applyPerturbations(datapoints, perturbationRate) {
+        let perturbedCount = 0;
+        const result = datapoints.map((dp) => {
+            if (dp.first_message && Math.random() < perturbationRate) {
+                const [perturbedMsg, pType] = applyRandomPerturbation(dp.first_message);
+                perturbedCount++;
+                console.debug(`Applied ${pType} perturbation to: ${dp.scenario.name}`);
+                // Create a new datapoint with the perturbed message (immutable update)
+                return {
+                    ...dp,
+                    first_message: perturbedMsg,
+                };
+            }
+            return dp;
+        });
+        if (perturbedCount > 0) {
+            console.log(`Applied perturbations to ${perturbedCount}/${datapoints.length} first messages`);
+        }
+        return result;
+    }
+}