@orq-ai/evaluatorq 1.2.2 → 1.2.3-rc.1

package/dist/lib/integrations/simulation/schemas.js

@@ -0,0 +1,76 @@
+/**
+ * Zod schemas for optional runtime validation of simulation types.
+ *
+ * Separated from types.ts so that importing the simulation module
+ * does not require zod to be installed. Only users who explicitly
+ * import these schemas need zod as a dependency.
+ */
+import { z } from "zod";
+export const PersonaSchema = z.object({
+    name: z.string(),
+    patience: z.number().min(0).max(1).default(0.5),
+    assertiveness: z.number().min(0).max(1).default(0.5),
+    politeness: z.number().min(0).max(1).default(0.5),
+    technical_level: z.number().min(0).max(1).default(0.5),
+    communication_style: z
+        .enum(["formal", "casual", "terse", "verbose"])
+        .default("casual"),
+    background: z.string().default(""),
+    emotional_arc: z
+        .enum([
+        "stable",
+        "escalating",
+        "de_escalating",
+        "volatile",
+        "manipulative",
+        "hostile",
+    ])
+        .optional(),
+    cultural_context: z
+        .enum([
+        "neutral",
+        "direct",
+        "indirect",
+        "high_context",
+        "low_context",
+        "hierarchical",
+    ])
+        .optional(),
+});
+export const CriterionSchema = z.object({
+    description: z.string(),
+    type: z.enum(["must_happen", "must_not_happen"]),
+    evaluator: z.string().nullable().optional(),
+});
+export const ScenarioSchema = z.object({
+    name: z.string(),
+    goal: z.string(),
+    context: z.string().optional(),
+    starting_emotion: z
+        .enum(["neutral", "frustrated", "confused", "happy", "urgent"])
+        .optional(),
+    criteria: z.array(CriterionSchema).optional(),
+    is_edge_case: z.boolean().optional(),
+    conversation_strategy: z
+        .enum([
+        "cooperative",
+        "topic_switching",
+        "contradictory",
+        "multi_intent",
+        "evasive",
+        "repetitive",
+        "ambiguous",
+    ])
+        .optional(),
+    ground_truth: z.string().optional(),
+    input_format: z
+        .enum([
+        "plain_text",
+        "with_url",
+        "with_attachment",
+        "form_data",
+        "code_block",
+        "mixed_media",
+    ])
+        .optional(),
+});

package/dist/lib/integrations/simulation/simulation/index.d.ts

@@ -0,0 +1,49 @@
+/**
+ * evaluatorq integration for agent simulation.
+ *
+ * Provides high-level functions to run simulations,
+ * either standalone or within the evaluatorq framework.
+ */
+import type { ChatMessage, Datapoint, Persona, Scenario, SimulationResult } from "../types.js";
+export interface SimulateParams {
+    evaluationName: string;
+    agentKey?: string;
+    targetCallback?: (messages: ChatMessage[]) => string | Promise<string>;
+    personas?: Persona[];
+    scenarios?: Scenario[];
+    datapoints?: Datapoint[];
+    maxTurns?: number;
+    model?: string;
+    evaluators?: string[];
+    parallelism?: number;
+}
+/**
+ * High-level function to run agent simulations.
+ *
+ * Handles:
+ * - Creating persona x scenario combinations
+ * - Generating first messages for each combination
+ * - Running simulations in parallel
+ * - Applying evaluators to results
+ */
+export declare function simulate(params: SimulateParams): Promise<SimulationResult[]>;
+export interface GenerateAndSimulateParams {
+    evaluationName: string;
+    agentKey?: string;
+    agentDescription: string;
+    targetCallback?: (messages: ChatMessage[]) => string | Promise<string>;
+    numPersonas?: number;
+    numScenarios?: number;
+    maxTurns?: number;
+    model?: string;
+    evaluators?: string[];
+    parallelism?: number;
+}
+/**
+ * Generate personas/scenarios and run simulations.
+ *
+ * Convenience function that combines generation and simulation.
+ */
+export declare function generateAndSimulate(params: GenerateAndSimulateParams): Promise<SimulationResult[]>;
+export { getAllEvaluators, getEvaluator, SIMULATION_EVALUATORS, } from "../evaluators/index.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/lib/integrations/simulation/simulation/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/lib/integrations/simulation/simulation/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAQH,OAAO,KAAK,EACV,WAAW,EACX,SAAS,EACT,OAAO,EACP,QAAQ,EACR,gBAAgB,EACjB,MAAM,aAAa,CAAC;AAOrB,MAAM,WAAW,cAAc;IAC7B,cAAc,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,CAAC,QAAQ,EAAE,WAAW,EAAE,KAAK,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACvE,QAAQ,CAAC,EAAE,OAAO,EAAE,CAAC;IACrB,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;IACvB,UAAU,CAAC,EAAE,SAAS,EAAE,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;;;;;;;GAQG;AACH,wBAAsB,QAAQ,CAC5B,MAAM,EAAE,cAAc,GACrB,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAkH7B;AAMD,MAAM,WAAW,yBAAyB;IACxC,cAAc,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,cAAc,CAAC,EAAE,CAAC,QAAQ,EAAE,WAAW,EAAE,KAAK,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACvE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;;;GAIG;AACH,wBAAsB,mBAAmB,CACvC,MAAM,EAAE,yBAAyB,GAChC,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAgF7B;AAGD,OAAO,EACL,gBAAgB,EAChB,YAAY,EACZ,qBAAqB,GACtB,MAAM,wBAAwB,CAAC"}

package/dist/lib/integrations/simulation/simulation/index.js

@@ -0,0 +1,159 @@
+/**
+ * evaluatorq integration for agent simulation.
+ *
+ * Provides high-level functions to run simulations,
+ * either standalone or within the evaluatorq framework.
+ */
+import OpenAI from "openai";
+import { fromOrqDeployment } from "../adapters.js";
+import { getEvaluator } from "../evaluators/index.js";
+import { FirstMessageGenerator } from "../generators/first-message-generator.js";
+import { SimulationRunner } from "../runner/simulation.js";
+import { generateDatapoint } from "../utils/prompt-builders.js";
+/**
+ * High-level function to run agent simulations.
+ *
+ * Handles:
+ * - Creating persona x scenario combinations
+ * - Generating first messages for each combination
+ * - Running simulations in parallel
+ * - Applying evaluators to results
+ */
+export async function simulate(params) {
+    const { targetCallback, personas, scenarios, maxTurns = 10, model = "azure/gpt-4o-mini", evaluators: evaluatorNames, parallelism = 5, } = params;
+    let { datapoints } = params;
+    // Validate evaluator names early — throw on unknown names
+    const resolvedEvaluatorNames = evaluatorNames ?? [
+        "goal_achieved",
+        "criteria_met",
+    ];
+    const scorers = resolvedEvaluatorNames.map((name) => ({
+        name,
+        fn: getEvaluator(name),
+    }));
+    // Build datapoints from personas x scenarios if not provided
+    if (!datapoints) {
+        if (!personas || !scenarios) {
+            throw new Error("Either provide 'datapoints' or both 'personas' and 'scenarios'");
+        }
+        if (personas.length === 0 || scenarios.length === 0) {
+            throw new Error("'personas' and 'scenarios' arrays must both be non-empty");
+        }
+        const apiKey = process.env.ORQ_API_KEY;
+        if (!apiKey) {
+            throw new Error("ORQ_API_KEY environment variable is not set. Set it before calling simulate().");
+        }
+        // Create a shared HTTP client so the generator doesn't leak its own pool
+        const sharedClient = new OpenAI({
+            apiKey,
+            baseURL: process.env.ROUTER_BASE_URL ?? "https://api.orq.ai/v2/router",
+        });
+        // Generate first messages for each combination (with bounded concurrency)
+        const firstMsgGen = new FirstMessageGenerator({
+            model,
+            client: sharedClient,
+        });
+        const pairs = personas.flatMap((persona) => scenarios.map((scenario) => ({ persona, scenario })));
+        const FIRST_MSG_CONCURRENCY = 5;
+        const generatedDatapoints = [];
+        for (let i = 0; i < pairs.length; i += FIRST_MSG_CONCURRENCY) {
+            const batch = pairs.slice(i, i + FIRST_MSG_CONCURRENCY);
+            const batchResults = await Promise.all(batch.map(async ({ persona, scenario }) => {
+                const firstMessage = await firstMsgGen.generate(persona, scenario);
+                return generateDatapoint(persona, scenario, firstMessage);
+            }));
+            generatedDatapoints.push(...batchResults);
+        }
+        datapoints = generatedDatapoints;
+    }
+    if (!datapoints || datapoints.length === 0) {
+        throw new Error("No datapoints to simulate — persona or scenario generation may have failed");
+    }
+    // Bridge agentKey to invoke() if no callback is provided
+    let resolvedCallback = targetCallback;
+    if (!resolvedCallback && params.agentKey) {
+        resolvedCallback = fromOrqDeployment(params.agentKey);
+    }
+    if (!resolvedCallback) {
+        throw new Error("Either targetCallback or agentKey is required");
+    }
+    // Create simulation runner
+    const runner = new SimulationRunner({
+        targetCallback: resolvedCallback,
+        model,
+        maxTurns,
+    });
+    try {
+        // Run simulations
+        const results = await runner.runBatch({
+            datapoints,
+            maxTurns,
+            maxConcurrency: parallelism,
+        });
+        // Apply evaluators to results
+        for (const result of results) {
+            const scores = {};
+            for (const { name, fn } of scorers) {
+                scores[name] = fn(result);
+            }
+            result.metadata.evaluator_scores = scores;
+        }
+        return results;
+    }
+    finally {
+        await runner.close();
+    }
+}
+/**
+ * Generate personas/scenarios and run simulations.
+ *
+ * Convenience function that combines generation and simulation.
+ */
+export async function generateAndSimulate(params) {
+    const { evaluationName, agentDescription, targetCallback, numPersonas = 5, numScenarios = 5, maxTurns = 10, model = "azure/gpt-4o-mini", evaluators, parallelism = 5, } = params;
+    // Bridge agentKey to invoke() if no callback is provided
+    let resolvedCallback = targetCallback;
+    if (!resolvedCallback && params.agentKey) {
+        resolvedCallback = fromOrqDeployment(params.agentKey);
+    }
+    if (!resolvedCallback) {
+        throw new Error("Either targetCallback or agentKey is required for generateAndSimulate");
+    }
+    // Dynamic import to avoid hard dependency on generators module
+    let PersonaGenerator;
+    let ScenarioGenerator;
+    try {
+        const generators = await import("../generators/index.js");
+        PersonaGenerator = generators.PersonaGenerator;
+        ScenarioGenerator = generators.ScenarioGenerator;
+    }
+    catch (err) {
+        throw new Error("Generators module not available. Install generators or provide pre-built datapoints using simulate() instead.", { cause: err });
+    }
+    // Generate personas and scenarios in parallel
+    const personaGen = new PersonaGenerator({ model });
+    const scenarioGen = new ScenarioGenerator({ model });
+    const [personas, scenarios] = await Promise.all([
+        personaGen.generate({
+            agentDescription,
+            numPersonas,
+        }),
+        scenarioGen.generate({
+            agentDescription,
+            numScenarios,
+        }),
+    ]);
+    // Run simulations
+    return simulate({
+        evaluationName,
+        targetCallback: resolvedCallback,
+        personas,
+        scenarios,
+        maxTurns,
+        model,
+        evaluators,
+        parallelism,
+    });
+}
+// Re-export evaluator utilities for convenience
+export { getAllEvaluators, getEvaluator, SIMULATION_EVALUATORS, } from "../evaluators/index.js";

package/dist/lib/integrations/simulation/types.d.ts

@@ -0,0 +1,101 @@
+/**
+ * Core type definitions for the agent simulation framework.
+ *
+ * Uses plain interfaces for maximum compatibility with generators/runner/agents.
+ * Zod schemas provided for optional runtime validation.
+ */
+export type CommunicationStyle = "formal" | "casual" | "terse" | "verbose";
+export type StartingEmotion = "neutral" | "frustrated" | "confused" | "happy" | "urgent";
+export type EmotionalArc = "stable" | "escalating" | "de_escalating" | "volatile" | "manipulative" | "hostile";
+export type CulturalContext = "neutral" | "direct" | "indirect" | "high_context" | "low_context" | "hierarchical";
+export type ConversationStrategy = "cooperative" | "topic_switching" | "contradictory" | "multi_intent" | "evasive" | "repetitive" | "ambiguous";
+export type InputFormat = "plain_text" | "with_url" | "with_attachment" | "form_data" | "code_block" | "mixed_media";
+export declare const EMOTIONAL_ARC_INSTRUCTIONS: Record<EmotionalArc, string>;
+export declare const CULTURAL_CONTEXT_INSTRUCTIONS: Record<CulturalContext, string>;
+export declare const STRATEGY_INSTRUCTIONS: Record<ConversationStrategy, string>;
+export declare const INPUT_FORMAT_INSTRUCTIONS: Record<InputFormat, string>;
+export interface ChatMessage {
+    role: "user" | "assistant" | "system";
+    content: string;
+}
+export interface TokenUsage {
+    prompt_tokens: number;
+    completion_tokens: number;
+    total_tokens: number;
+}
+export interface Persona {
+    name: string;
+    patience: number;
+    assertiveness: number;
+    politeness: number;
+    technical_level: number;
+    communication_style: CommunicationStyle;
+    background: string;
+    emotional_arc?: EmotionalArc;
+    cultural_context?: CulturalContext;
+}
+export interface Criterion {
+    description: string;
+    type: "must_happen" | "must_not_happen";
+    evaluator?: string | null;
+}
+export interface Scenario {
+    name: string;
+    goal: string;
+    context?: string;
+    starting_emotion?: StartingEmotion;
+    criteria?: Criterion[];
+    is_edge_case?: boolean;
+    conversation_strategy?: ConversationStrategy;
+    ground_truth?: string;
+    input_format?: InputFormat;
+}
+export interface Judgment {
+    should_terminate: boolean;
+    reason: string;
+    goal_achieved: boolean;
+    rules_broken: string[];
+    goal_completion_score: number;
+    response_quality?: number | null;
+    hallucination_risk?: number | null;
+    tone_appropriateness?: number | null;
+    factual_accuracy?: number | null;
+}
+export interface Message {
+    role: "user" | "assistant" | "system";
+    content: string;
+}
+export interface TurnMetrics {
+    turn_number: number;
+    token_usage: TokenUsage;
+    response_quality?: number | null;
+    hallucination_risk?: number | null;
+    tone_appropriateness?: number | null;
+    factual_accuracy?: number | null;
+    judge_reason: string;
+}
+export type TerminatedBy = "judge" | "max_turns" | "error" | "timeout";
+export interface SimulationResult {
+    messages: Message[];
+    terminated_by: TerminatedBy;
+    reason: string;
+    goal_achieved: boolean;
+    goal_completion_score: number;
+    rules_broken: string[];
+    turn_count: number;
+    token_usage: TokenUsage;
+    turn_metrics: TurnMetrics[];
+    metadata: Record<string, unknown>;
+    /** Convenience fields for evaluatorq integration */
+    criteria_results?: Record<string, boolean>;
+    total_turns?: number;
+}
+export interface Datapoint {
+    id: string;
+    persona: Persona;
+    scenario: Scenario;
+    user_system_prompt: string;
+    first_message: string;
+}
+export type { DataPoint, Job, Output } from "../../types.js";
+//# sourceMappingURL=types.d.ts.map

package/dist/lib/integrations/simulation/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../../src/lib/integrations/simulation/types.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAMH,MAAM,MAAM,kBAAkB,GAAG,QAAQ,GAAG,QAAQ,GAAG,OAAO,GAAG,SAAS,CAAC;AAE3E,MAAM,MAAM,eAAe,GACvB,SAAS,GACT,YAAY,GACZ,UAAU,GACV,OAAO,GACP,QAAQ,CAAC;AAEb,MAAM,MAAM,YAAY,GACpB,QAAQ,GACR,YAAY,GACZ,eAAe,GACf,UAAU,GACV,cAAc,GACd,SAAS,CAAC;AAEd,MAAM,MAAM,eAAe,GACvB,SAAS,GACT,QAAQ,GACR,UAAU,GACV,cAAc,GACd,aAAa,GACb,cAAc,CAAC;AAEnB,MAAM,MAAM,oBAAoB,GAC5B,aAAa,GACb,iBAAiB,GACjB,eAAe,GACf,cAAc,GACd,SAAS,GACT,YAAY,GACZ,WAAW,CAAC;AAEhB,MAAM,MAAM,WAAW,GACnB,YAAY,GACZ,UAAU,GACV,iBAAiB,GACjB,WAAW,GACX,YAAY,GACZ,aAAa,CAAC;AAMlB,eAAO,MAAM,0BAA0B,EAAE,MAAM,CAAC,YAAY,EAAE,MAAM,CA4BnE,CAAC;AAEF,eAAO,MAAM,6BAA6B,EAAE,MAAM,CAAC,eAAe,EAAE,MAAM,CA0BzE,CAAC;AAEF,eAAO,MAAM,qBAAqB,EAAE,MAAM,CAAC,oBAAoB,EAAE,MAAM,CA2BtE,CAAC;AAEF,eAAO,MAAM,yBAAyB,EAAE,MAAM,CAAC,WAAW,EAAE,MAAM,CAiBjE,CAAC;AAMF,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,GAAG,WAAW,GAAG,QAAQ,CAAC;IACtC,OAAO,EAAE,MAAM,CAAC;CACjB;AAMD,MAAM,WAAW,UAAU;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,YAAY,EAAE,MAAM,CAAC;CACtB;AAMD,MAAM,WAAW,OAAO;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,MAAM,CAAC;IACxB,mBAAmB,EAAE,kBAAkB,CAAC;IACxC,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,CAAC,EAAE,YAAY,CAAC;IAC7B,gBAAgB,CAAC,EAAE,eAAe,CAAC;CACpC;AAMD,MAAM,WAAW,SAAS;IACxB,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,aAAa,GAAG,iBAAiB,CAAC;IACxC,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC3B;AAMD,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,gBAAgB,CAAC,EAAE,eAAe,CAAC;IACnC,QAAQ,CAAC,EAAE,SAAS,EAAE,CAAC;IACvB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,qBAAqB,CAAC,EAAE,oBAAoB,CAAC;IAC7C,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,WAAW,CAAC;CAC5B;AAMD,MAAM,WAAW,QAAQ;IACvB,gBAAgB,EAAE,OAAO,CAAC;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,OAAO,CAAC;IACvB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,kBAAkB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,oBAAoB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAClC;AAMD,MAAM,WAAW,OAAO;IACtB,IAAI,EAAE,MAAM,GAAG,WAAW,GAAG,QAAQ,CAAC;IACtC,OAAO,EAAE,MAAM,CAAC;CACjB;AAMD,MAAM,WAAW,WAAW;IAC1B,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,UAAU,CAAC;IACxB,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,kBAAkB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,oBAAoB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,YAAY,EAAE,MAAM,CAAC;CACtB;AAMD,MAAM,MAAM,YAAY,GAAG,OAAO,GAAG,WAAW,GAAG,OAAO,GAAG,SAAS,CAAC;AAEvE,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,OAAO,EAAE,CAAC;IACpB,aAAa,EAAE,YAAY,CAAC;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,OAAO,CAAC;IACvB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,UAAU,CAAC;IACxB,YAAY,EAAE,WAAW,EAAE,CAAC;IAC5B,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClC,oDAAoD;IACpD,gBAAgB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC3C,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAMD,MAAM,WAAW,SAAS;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,OAAO,CAAC;IACjB,QAAQ,EAAE,QAAQ,CAAC;IACnB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,aAAa,EAAE,MAAM,CAAC;CACvB;AAMD,YAAY,EAAE,SAAS,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC"}

package/dist/lib/integrations/simulation/types.js

@@ -0,0 +1,90 @@
+/**
+ * Core type definitions for the agent simulation framework.
+ *
+ * Uses plain interfaces for maximum compatibility with generators/runner/agents.
+ * Zod schemas provided for optional runtime validation.
+ */
+// ---------------------------------------------------------------------------
+// Constant instruction maps (ported from Python)
+// ---------------------------------------------------------------------------
+export const EMOTIONAL_ARC_INSTRUCTIONS = {
+    stable: "",
+    escalating: "Your emotional intensity escalates over the course of the conversation. " +
+        "Start relatively calm but become progressively more frustrated, impatient, or upset " +
+        "with each turn, especially if the agent is slow to resolve your issue. " +
+        "By the end, you should be noticeably more emotional than at the start.",
+    de_escalating: "You start the conversation very upset, frustrated, or angry. As the agent responds " +
+        "and shows willingness to help, you gradually calm down. Your tone softens over time, " +
+        "and by the end you may even express gratitude. However, any misstep by the agent " +
+        "can temporarily reverse this trend.",
+    volatile: "Your emotional state is volatile and unpredictable. Swing between being calm, " +
+        "frustrated, grateful, and confused within the same conversation. A small positive " +
+        "response from the agent might suddenly improve your mood, while a minor setback " +
+        "could trigger strong frustration. Your reactions should feel disproportionate.",
+    manipulative: "You use emotional manipulation tactics to get what you want. Employ guilt-tripping " +
+        "('I've been a loyal customer for years and this is how you treat me?'), flattery " +
+        "('You seem so much more competent than the last agent'), threats to escalate " +
+        "('I'll leave a bad review'), or appeals to sympathy ('This is ruining my day'). " +
+        "Switch between these tactics based on what seems to be working.",
+    hostile: "You are genuinely hostile and confrontational. Use aggressive language, make " +
+        "unreasonable demands, and challenge the agent's competence. Express contempt and " +
+        "refuse to cooperate with standard procedures. This tests the agent's ability to " +
+        "maintain professionalism and de-escalate under pressure.",
+};
+export const CULTURAL_CONTEXT_INSTRUCTIONS = {
+    neutral: "",
+    direct: "You communicate in a very direct, low-context style typical of Northern European " +
+        "or North American cultures. Say exactly what you mean without hints or implication. " +
+        "Get straight to the point. 'No' means 'no' — you don't soften refusals.",
+    indirect: "You communicate indirectly, typical of many East Asian or Southeast Asian cultures. " +
+        "Avoid saying 'no' directly — instead use phrases like 'that might be difficult' or " +
+        "'I'll think about it.' Hint at problems rather than stating them outright. Use hedging " +
+        "language ('perhaps', 'maybe', 'it seems'). Preserving harmony is important.",
+    high_context: "You rely heavily on context and implication rather than explicit statements. " +
+        "You expect the agent to read between the lines and understand unstated needs. " +
+        "You may reference shared knowledge without explaining it. Use fewer words but " +
+        "expect more understanding. Silence can be meaningful.",
+    low_context: "You spell everything out explicitly and leave nothing to interpretation. " +
+        "Provide full context with every message. Repeat important details. " +
+        "Don't assume the agent remembers previous context. " +
+        "Be thorough and detailed in every message.",
+    hierarchical: "You approach the interaction with a strong sense of hierarchy, typical of many " +
+        "Middle Eastern, East Asian, or South Asian cultures. You may expect formal address, " +
+        "defer to authority ('can I speak to a manager?'), and be uncomfortable challenging " +
+        "the agent's statements directly. Status and titles matter to you.",
+};
+export const STRATEGY_INSTRUCTIONS = {
+    cooperative: "",
+    topic_switching: "You frequently switch topics mid-conversation. After a few exchanges about your main goal, " +
+        "bring up unrelated questions or concerns before returning to your original topic. " +
+        "This tests the agent's ability to handle context switching.",
+    contradictory: "You contradict yourself during the conversation. Say one thing, then later say the opposite " +
+        "or change your requirements. For example, first ask for a refund, then say you actually want " +
+        "a replacement. This tests the agent's ability to handle inconsistent user input.",
+    multi_intent: "You have multiple goals packed into each message. Combine questions, requests, and complaints " +
+        "in single messages. For example, ask about your order status while also requesting a password " +
+        "reset and complaining about a previous experience.",
+    evasive: "You are evasive and avoid directly answering the agent's questions. Give vague or incomplete " +
+        "responses when asked for details. The agent needs to work harder to extract the information " +
+        "it needs to help you.",
+    repetitive: "You repeat your requests and questions even after the agent has addressed them. Ask the same " +
+        "thing in slightly different ways, as if you didn't understand or weren't satisfied with " +
+        "the response. This tests the agent's patience and ability to rephrase explanations.",
+    ambiguous: "You are deliberately vague and unclear in your requests. Use imprecise language, " +
+        "avoid giving specific details, and make the agent work to understand what you actually need. " +
+        "When the agent asks clarifying questions, give partial or still-ambiguous answers. " +
+        "For example, say 'the thing isn't working' instead of specifying which product or error.",
+};
+export const INPUT_FORMAT_INSTRUCTIONS = {
+    plain_text: "",
+    with_url: "Include relevant URLs in your messages. Reference links to products, order pages, " +
+        "screenshots, or documentation.",
+    with_attachment: "Reference file attachments in your messages as if you're uploading them. " +
+        "Mention screenshots, receipts, photos, or documents.",
+    form_data: "Structure your messages like filled-out forms or structured data. Include labeled fields, " +
+        "order details in a structured format, or table-like information.",
+    code_block: "Include code snippets, error logs, stack traces, or technical output in your messages. " +
+        "Wrap technical content in code blocks.",
+    mixed_media: "Mix different input types in your messages. Combine plain text with URLs, " +
+        "attachment references, structured data, or code blocks.",
+};

package/dist/lib/integrations/simulation/utils/dataset-export.d.ts

@@ -0,0 +1,31 @@
+/**
+ * Dataset export/import utilities for JSONL format.
+ */
+import type { Datapoint, SimulationResult } from "../types.js";
+/**
+ * Export datapoints to JSONL format for orq.ai datasets.
+ */
+export declare function exportDatapointsToJsonl(datapoints: Datapoint[], outputPath: string): void;
+/**
+ * Export simulation results to JSONL format.
+ */
+export declare function exportResultsToJsonl(results: SimulationResult[], outputPath: string): void;
+/**
+ * Load datapoints from a JSONL file.
+ *
+ * Supports both the current format (with full persona/scenario objects) and a
+ * legacy format (with flat fields).
+ */
+export declare function loadDatapointsFromJsonl(inputPath: string): Datapoint[];
+/**
+ * Convert simulation results to JSONL string for dataset export.
+ */
+export declare function resultsToJsonl(results: {
+    datapoint: Datapoint;
+    result: SimulationResult;
+}[]): string;
+/**
+ * Parse a JSONL string into an array of objects.
+ */
+export declare function parseJsonl<T = Record<string, unknown>>(content: string): T[];
+//# sourceMappingURL=dataset-export.d.ts.map

package/dist/lib/integrations/simulation/utils/dataset-export.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dataset-export.d.ts","sourceRoot":"","sources":["../../../../../src/lib/integrations/simulation/utils/dataset-export.ts"],"names":[],"mappings":"AAAA;;GAEG;AAKH,OAAO,KAAK,EAEV,SAAS,EAGT,gBAAgB,EACjB,MAAM,aAAa,CAAC;AAMrB;;GAEG;AACH,wBAAgB,uBAAuB,CACrC,UAAU,EAAE,SAAS,EAAE,EACvB,UAAU,EAAE,MAAM,GACjB,IAAI,CAiBN;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAClC,OAAO,EAAE,gBAAgB,EAAE,EAC3B,UAAU,EAAE,MAAM,GACjB,IAAI,CAMN;AAMD;;;;;GAKG;AACH,wBAAgB,uBAAuB,CAAC,SAAS,EAAE,MAAM,GAAG,SAAS,EAAE,CAgEtE;AAMD;;GAEG;AACH,wBAAgB,cAAc,CAC5B,OAAO,EAAE;IAAE,SAAS,EAAE,SAAS,CAAC;IAAC,MAAM,EAAE,gBAAgB,CAAA;CAAE,EAAE,GAC5D,MAAM,CAoBR;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,EAAE,MAAM,GAAG,CAAC,EAAE,CAc5E"}

package/dist/lib/integrations/simulation/utils/dataset-export.js

@@ -0,0 +1,146 @@
+/**
+ * Dataset export/import utilities for JSONL format.
+ */
+import fs from "node:fs";
+import path from "node:path";
+// ---------------------------------------------------------------------------
+// Export
+// ---------------------------------------------------------------------------
+/**
+ * Export datapoints to JSONL format for orq.ai datasets.
+ */
+export function exportDatapointsToJsonl(datapoints, outputPath) {
+    const dir = path.dirname(outputPath);
+    fs.mkdirSync(dir, { recursive: true });
+    const lines = datapoints.map((dp) => JSON.stringify({
+        inputs: {
+            category: `${dp.persona.name} - ${dp.scenario.name}`,
+            first_message: dp.first_message,
+            user_system_prompt: dp.user_system_prompt,
+            persona: dp.persona,
+            scenario: dp.scenario,
+        },
+        expected_output: null,
+    }));
+    fs.writeFileSync(outputPath, `${lines.join("\n")}\n`, "utf-8");
+}
+/**
+ * Export simulation results to JSONL format.
+ */
+export function exportResultsToJsonl(results, outputPath) {
+    const dir = path.dirname(outputPath);
+    fs.mkdirSync(dir, { recursive: true });
+    const lines = results.map((r) => JSON.stringify(r));
+    fs.writeFileSync(outputPath, `${lines.join("\n")}\n`, "utf-8");
+}
+// ---------------------------------------------------------------------------
+// Import
+// ---------------------------------------------------------------------------
+/**
+ * Load datapoints from a JSONL file.
+ *
+ * Supports both the current format (with full persona/scenario objects) and a
+ * legacy format (with flat fields).
+ */
+export function loadDatapointsFromJsonl(inputPath) {
+    const content = fs.readFileSync(inputPath, "utf-8");
+    const datapoints = [];
+    for (const line of content.split("\n")) {
+        const trimmed = line.trim();
+        if (!trimmed)
+            continue;
+        let data;
+        try {
+            data = JSON.parse(trimmed);
+        }
+        catch {
+            console.warn(`loadDatapointsFromJsonl: skipping malformed line: ${trimmed.slice(0, 80)}`);
+            continue;
+        }
+        const inputs = data.inputs ?? {};
+        // Reconstruct persona
+        let persona;
+        if (inputs.persona && typeof inputs.persona === "object") {
+            persona = inputs.persona;
+        }
+        else {
+            persona = {
+                name: inputs.persona_name ?? "Unknown",
+                patience: 0.5,
+                assertiveness: 0.5,
+                politeness: 0.5,
+                technical_level: 0.5,
+                communication_style: "casual",
+                background: inputs.context ?? "",
+            };
+        }
+        // Reconstruct scenario
+        let scenario;
+        if (inputs.scenario && typeof inputs.scenario === "object") {
+            const raw = inputs.scenario;
+            const criteriaRaw = raw.criteria;
+            const criteria = Array.isArray(criteriaRaw)
+                ? criteriaRaw
+                : [];
+            scenario = { ...raw, criteria };
+        }
+        else {
+            scenario = {
+                name: inputs.scenario_name ?? "Unknown",
+                goal: inputs.goal ?? "",
+                context: inputs.context ?? "",
+            };
+        }
+        datapoints.push({
+            id: `dp_${crypto.randomUUID().replace(/-/g, "").slice(0, 12)}`,
+            persona,
+            scenario,
+            user_system_prompt: inputs.user_system_prompt ?? "",
+            first_message: inputs.first_message ?? "",
+        });
+    }
+    return datapoints;
+}
+// ---------------------------------------------------------------------------
+// String helpers
+// ---------------------------------------------------------------------------
+/**
+ * Convert simulation results to JSONL string for dataset export.
+ */
+export function resultsToJsonl(results) {
+    return results
+        .map((r) => JSON.stringify({
+        id: r.datapoint.id,
+        persona: r.datapoint.persona.name,
+        scenario: r.datapoint.scenario.name,
+        first_message: r.datapoint.first_message,
+        goal_achieved: r.result.goal_achieved,
+        goal_completion_score: r.result.goal_completion_score,
+        terminated_by: r.result.terminated_by,
+        turn_count: r.result.turn_count,
+        messages: r.result.messages,
+        rules_broken: r.result.rules_broken,
+        token_usage: r.result.token_usage,
+        turn_metrics: r.result.turn_metrics,
+        metadata: r.result.metadata,
+    }))
+        .join("\n");
+}
+/**
+ * Parse a JSONL string into an array of objects.
+ */
+export function parseJsonl(content) {
+    const results = [];
+    for (const line of content.split("\n")) {
+        const trimmed = line.trim();
+        if (!trimmed)
+            continue;
+        try {
+            results.push(JSON.parse(trimmed));
+        }
+        catch {
+            console.warn(`parseJsonl: skipping malformed line: ${trimmed.slice(0, 80)}`);
+        }
+    }
+    return results;
+}