@huydao/karrot 0.1.1

package/dist/reports/report.js

@@ -0,0 +1,366 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ScenarioExecutionError = void 0;
+exports.finalizeScenarioResult = finalizeScenarioResult;
+exports.getScenarioRunStatus = getScenarioRunStatus;
+exports.buildScenarioSlackRows = buildScenarioSlackRows;
+exports.writeScenarioRunReport = writeScenarioRunReport;
+const promises_1 = __importDefault(require("node:fs/promises"));
+const node_path_1 = __importDefault(require("node:path"));
+class ScenarioExecutionError extends Error {
+    result;
+    constructor(result) {
+        super(result.note ?? `${result.id} failed.`);
+        this.name = 'ScenarioExecutionError';
+        this.result = result;
+    }
+}
+exports.ScenarioExecutionError = ScenarioExecutionError;
+function sumNumbers(values) {
+    const definedValues = values.filter((value) => typeof value === 'number');
+    if (definedValues.length === 0) {
+        return undefined;
+    }
+    return Number(definedValues.reduce((total, value) => total + value, 0).toFixed(1));
+}
+function summarizeScenarioMetrics(turns) {
+    const protocolUsedKb = sumNumbers(turns.map((turn) => turn.metrics.protocolUsedKb));
+    const protocolTotalKb = sumNumbers(turns.map((turn) => turn.metrics.protocolTotalKb));
+    const efficiencyPercent = typeof protocolUsedKb === 'number' && typeof protocolTotalKb === 'number' && protocolTotalKb > 0
+        ? Math.round((protocolUsedKb / protocolTotalKb) * 100)
+        : undefined;
+    return {
+        ttfToolSeconds: sumNumbers(turns.map((turn) => turn.metrics.ttfToolSeconds)),
+        ttfTextSeconds: sumNumbers(turns.map((turn) => turn.metrics.ttfTextSeconds)),
+        totalSeconds: sumNumbers(turns.map((turn) => turn.metrics.totalSeconds)),
+        protocolUsedKb,
+        protocolTotalKb,
+        efficiencyPercent,
+    };
+}
+function buildScenarioNote(result) {
+    if (result.note) {
+        return result.note;
+    }
+    if (result.id === 'S3') {
+        return 'Correctly declined';
+    }
+    const toolCallCount = result.turns.reduce((total, turn) => total + turn.toolCallCount, 0);
+    if (toolCallCount === 0) {
+        return 'No tools';
+    }
+    if (result.turns.length > 1) {
+        return 'Multi-turn tool flow';
+    }
+    return 'Tool-assisted response';
+}
+function finalizeScenarioResult(result) {
+    result.metrics = summarizeScenarioMetrics(result.turns);
+    result.note = buildScenarioNote(result);
+    return result;
+}
+function formatSeconds(value) {
+    return typeof value === 'number' ? `${value.toFixed(1)}s` : '-';
+}
+function formatPercent(value) {
+    return typeof value === 'number' ? `${value}%` : '-';
+}
+function hasScenarioFailures(results) {
+    return results.some((result) => result.status === 'FAIL');
+}
+function getScenarioRunStatus(results) {
+    return hasScenarioFailures(results) ? 'FAIL' : 'PASS';
+}
+function buildScenarioSlackRows(results) {
+    return results.map((result) => ({
+        id: result.id,
+        name: result.name,
+        status: result.status,
+        ttfTool: formatSeconds(result.metrics.ttfToolSeconds),
+        ttfText: formatSeconds(result.metrics.ttfTextSeconds),
+        total: formatSeconds(result.metrics.totalSeconds),
+        efficiency: formatPercent(result.metrics.efficiencyPercent),
+        note: result.note,
+    }));
+}
+function buildScenarioRunSummary(results) {
+    const turns = results.flatMap((result) => result.turns);
+    const assertions = turns.flatMap((turn) => turn.assertionResults ?? []);
+    const evaluations = turns.flatMap((turn) => turn.evaluationResults ?? []);
+    const requestedEvalDimensions = [...new Set(turns.flatMap((turn) => turn.evalDimensions ?? []))].sort();
+    const scoresByDimension = evaluations.reduce((accumulator, evaluation) => {
+        accumulator[evaluation.dimension] ??= [];
+        accumulator[evaluation.dimension].push(evaluation.score);
+        return accumulator;
+    }, {});
+    const averageScoresByDimension = Object.fromEntries(Object.entries(scoresByDimension).map(([dimension, scores]) => [
+        dimension,
+        Number((scores.reduce((total, score) => total + score, 0) / scores.length).toFixed(1)),
+    ]));
+    return {
+        status: getScenarioRunStatus(results),
+        totalScenarios: results.length,
+        passedScenarios: results.filter((result) => result.status === 'PASS').length,
+        failedScenarios: results.filter((result) => result.status === 'FAIL').length,
+        skippedScenarios: results.filter((result) => result.status === 'SKIP').length,
+        totalTurns: turns.length,
+        totalAssertions: assertions.length,
+        passedAssertions: assertions.filter((assertion) => assertion.passed).length,
+        failedAssertions: assertions.filter((assertion) => !assertion.passed).length,
+        totalToolCalls: turns.reduce((total, turn) => total + turn.toolCallCount, 0),
+        totalEvaluations: evaluations.length,
+        averageScoresByDimension,
+        requestedEvalDimensions,
+    };
+}
+function escapeHtml(value) {
+    return value
+        .replaceAll('&', '&')
+        .replaceAll('<', '&lt;')
+        .replaceAll('>', '&gt;')
+        .replaceAll('"', '&quot;')
+        .replaceAll("'", '&#39;');
+}
+function renderOptionalText(value) {
+    if (!value?.trim()) {
+        return '<span class="muted">-</span>';
+    }
+    return escapeHtml(value);
+}
+function renderAssertionExpected(expected) {
+    if (Array.isArray(expected)) {
+        return expected.length > 0 ? escapeHtml(expected.join(', ')) : '<span class="muted">[]</span>';
+    }
+    return escapeHtml(expected);
+}
+function statusClass(status) {
+    return status.toLowerCase();
+}
+function renderMetrics(metrics) {
+    return [
+        `TTF Tool: ${formatSeconds(metrics.ttfToolSeconds)}`,
+        `TTF Text: ${formatSeconds(metrics.ttfTextSeconds)}`,
+        `Total: ${formatSeconds(metrics.totalSeconds)}`,
+        `Efficiency: ${formatPercent(metrics.efficiencyPercent)}`,
+    ].join(' | ');
+}
+function renderDimensionList(dimensions) {
+    if (!dimensions?.length) {
+        return '<span class="muted">-</span>';
+    }
+    return dimensions.map((dimension) => `<span class="dimension-chip">${escapeHtml(dimension)}</span>`).join(' ');
+}
+function renderAssertionRows(assertions) {
+    if (!assertions?.length) {
+        return '<p class="muted">No assertions for this turn.</p>';
+    }
+    return [
+        '<table class="assertions">',
+        '<thead><tr><th>Status</th><th>Type</th><th>Expectation</th><th>Description</th><th>Reason</th></tr></thead>',
+        '<tbody>',
+        ...assertions.map((assertion) => [
+            '<tr>',
+            `<td><span class="badge ${assertion.passed ? 'pass' : 'fail'}">${assertion.passed ? 'PASS' : 'FAIL'}</span></td>`,
+            `<td>${escapeHtml(`${assertion.kind}.${assertion.matcher}`)}</td>`,
+            `<td>${renderAssertionExpected(assertion.expected)}</td>`,
+            `<td>${renderOptionalText(assertion.description)}</td>`,
+            `<td>${escapeHtml(assertion.reason)}</td>`,
+            '</tr>',
+        ].join('')),
+        '</tbody>',
+        '</table>',
+    ].join('');
+}
+function renderEvaluationRows(evaluations) {
+    if (!evaluations?.length) {
+        return '<p class="muted">No evaluations for this turn.</p>';
+    }
+    return [
+        '<table class="evaluations">',
+        '<thead><tr><th>Dimension</th><th>Score</th><th>Why</th><th>Improve</th></tr></thead>',
+        '<tbody>',
+        ...evaluations.map((evaluation) => [
+            '<tr>',
+            `<td>${escapeHtml(evaluation.dimension)}</td>`,
+            `<td><span class="score-pill">${escapeHtml(`${evaluation.score}/10`)}</span></td>`,
+            `<td>${escapeHtml(evaluation.explanation)}</td>`,
+            `<td>${escapeHtml(evaluation.suggestion)}</td>`,
+            '</tr>',
+        ].join('')),
+        '</tbody>',
+        '</table>',
+    ].join('');
+}
+function renderEvaluationSummary(averageScoresByDimension) {
+    const entries = Object.entries(averageScoresByDimension);
+    if (entries.length === 0) {
+        return '<p class="muted">No turn evaluations were requested in this run.</p>';
+    }
+    return [
+        '<div class="score-grid">',
+        ...entries.map(([dimension, score]) => `<div class="score-card"><span class="label">${escapeHtml(dimension)}</span><span class="value">${escapeHtml(`${score}/10`)}</span></div>`),
+        '</div>',
+    ].join('');
+}
+function renderTurnCard(turn, index) {
+    return [
+        '<section class="turn-card">',
+        `<h4>${escapeHtml(`${turn.label} (${index + 1})`)}</h4>`,
+        '<div class="meta-grid">',
+        `<div><span class="meta-label">Thread</span><code>${renderOptionalText(turn.threadId)}</code></div>`,
+        `<div><span class="meta-label">Tools</span><code>${String(turn.toolCallCount)}</code></div>`,
+        `<div><span class="meta-label">Eval Dimensions</span><div>${renderDimensionList(turn.evalDimensions)}</div></div>`,
+        `<div><span class="meta-label">Metrics</span><code>${escapeHtml(renderMetrics(turn.metrics))}</code></div>`,
+        `<div><span class="meta-label">Artifact</span><code>${renderOptionalText(turn.outputPath)}</code></div>`,
+        '</div>',
+        '<div class="content-block">',
+        '<h5>Prompt</h5>',
+        `<pre>${escapeHtml(turn.message)}</pre>`,
+        '</div>',
+        '<div class="content-block">',
+        '<h5>Response</h5>',
+        `<pre>${renderOptionalText(turn.output)}</pre>`,
+        '</div>',
+        '<div class="content-block">',
+        '<h5>Assertions</h5>',
+        renderAssertionRows(turn.assertionResults),
+        '</div>',
+        '<div class="content-block">',
+        '<h5>Evaluations</h5>',
+        renderEvaluationRows(turn.evaluationResults),
+        '</div>',
+        '<div class="content-block">',
+        '<h5>Notes</h5>',
+        `<p>${renderOptionalText(turn.note)}</p>`,
+        '</div>',
+        '</section>',
+    ].join('');
+}
+function buildScenarioRunHtml(payload) {
+    const scenarioCards = payload.results
+        .map((result) => [
+        `<details class="scenario-card ${statusClass(result.status)}" ${result.status === 'FAIL' ? 'open' : ''}>`,
+        '<summary>',
+        `<span class="scenario-title">${escapeHtml(`${result.id} - ${result.name}`)}</span>`,
+        `<span class="badge ${statusClass(result.status)}">${result.status}</span>`,
+        `<span class="summary-note">${renderOptionalText(result.note)}</span>`,
+        '</summary>',
+        '<div class="scenario-body">',
+        '<div class="meta-grid">',
+        `<div><span class="meta-label">Thread</span><code>${renderOptionalText(result.threadId)}</code></div>`,
+        `<div><span class="meta-label">Turns</span><code>${String(result.turns.length)}</code></div>`,
+        `<div><span class="meta-label">Metrics</span><code>${escapeHtml(renderMetrics(result.metrics))}</code></div>`,
+        '</div>',
+        ...result.turns.map((turn, index) => renderTurnCard(turn, index)),
+        '</div>',
+        '</details>',
+    ].join(''))
+        .join('\n');
+    return [
+        '<!DOCTYPE html>',
+        '<html lang="en">',
+        '<head>',
+        '<meta charset="utf-8" />',
+        '<meta name="viewport" content="width=device-width, initial-scale=1" />',
+        `<title>${escapeHtml(`${payload.projectName} AI Scenario Report`)}</title>`,
+        '<style>',
+        'body{margin:0;font-family:ui-sans-serif,system-ui,-apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif;background:#f4f7fb;color:#142033;}',
+        '.page{max-width:1280px;margin:0 auto;padding:32px 24px 64px;}',
+        '.hero{background:linear-gradient(135deg,#0f172a,#1d4ed8);color:#fff;padding:28px;border-radius:20px;box-shadow:0 20px 50px rgba(15,23,42,.18);}',
+        '.hero h1{margin:0 0 8px;font-size:32px;}',
+        '.hero p{margin:4px 0;color:rgba(255,255,255,.88);}',
+        '.summary-grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(170px,1fr));gap:14px;margin:22px 0 28px;}',
+        '.summary-card,.panel,.turn-card,.scenario-card{background:#fff;border:1px solid #dbe4f0;border-radius:18px;box-shadow:0 10px 30px rgba(15,23,42,.06);}',
+        '.summary-card{padding:18px;}',
+        '.summary-card .label{display:block;font-size:12px;text-transform:uppercase;letter-spacing:.08em;color:#5b6b84;margin-bottom:8px;}',
+        '.summary-card .value{font-size:28px;font-weight:700;}',
+        '.summary-card .sub{font-size:13px;color:#61728d;}',
+        '.panels{display:grid;grid-template-columns:repeat(auto-fit,minmax(280px,1fr));gap:16px;margin-bottom:28px;}',
+        '.panel{padding:18px;}',
+        '.panel h2{margin:0 0 12px;font-size:18px;}',
+        '.panel pre{margin:0;white-space:pre-wrap;word-break:break-word;background:#f8fbff;border-radius:12px;padding:14px;font-size:13px;}',
+        '.scenario-list{display:grid;gap:18px;}',
+        '.scenario-card summary{list-style:none;display:flex;gap:12px;align-items:center;justify-content:space-between;padding:18px 20px;cursor:pointer;}',
+        '.scenario-card summary::-webkit-details-marker{display:none;}',
+        '.scenario-title{font-weight:700;font-size:18px;flex:1;}',
+        '.summary-note{color:#61728d;font-size:14px;max-width:40%;text-align:right;}',
+        '.scenario-body{padding:0 20px 20px;}',
+        '.meta-grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(220px,1fr));gap:12px;margin:8px 0 18px;}',
+        '.meta-grid>div{background:#f8fbff;border-radius:12px;padding:12px;}',
+        '.meta-label{display:block;font-size:12px;color:#61728d;text-transform:uppercase;letter-spacing:.08em;margin-bottom:6px;}',
+        '.turn-card{padding:18px;margin-top:16px;}',
+        '.turn-card h4,.content-block h5{margin:0 0 10px;}',
+        '.content-block{margin-top:14px;}',
+        '.content-block pre{margin:0;white-space:pre-wrap;word-break:break-word;background:#f8fbff;border-radius:12px;padding:14px;max-height:420px;overflow:auto;}',
+        '.content-block p{margin:0;background:#f8fbff;border-radius:12px;padding:14px;}',
+        '.assertions,.evaluations{width:100%;border-collapse:collapse;font-size:14px;}',
+        '.assertions th,.assertions td,.evaluations th,.evaluations td{padding:10px 12px;border-bottom:1px solid #e5edf7;vertical-align:top;text-align:left;}',
+        '.assertions th,.evaluations th{font-size:12px;text-transform:uppercase;letter-spacing:.08em;color:#61728d;}',
+        '.badge{display:inline-flex;align-items:center;justify-content:center;border-radius:999px;padding:5px 10px;font-size:12px;font-weight:700;min-width:56px;}',
+        '.badge.pass{background:#dcfce7;color:#166534;}',
+        '.badge.fail{background:#fee2e2;color:#991b1b;}',
+        '.badge.skip{background:#e2e8f0;color:#334155;}',
+        '.dimension-chip{display:inline-flex;align-items:center;justify-content:center;border-radius:999px;padding:4px 10px;margin:0 6px 6px 0;background:#eef2ff;color:#3730a3;font-size:12px;font-weight:600;}',
+        '.score-grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(160px,1fr));gap:12px;}',
+        '.score-card{background:#f8fbff;border-radius:12px;padding:14px;}',
+        '.score-card .label{display:block;font-size:12px;color:#61728d;text-transform:uppercase;letter-spacing:.08em;margin-bottom:8px;}',
+        '.score-card .value{font-size:24px;font-weight:700;}',
+        '.score-pill{display:inline-flex;align-items:center;justify-content:center;border-radius:999px;padding:4px 10px;background:#dbeafe;color:#1d4ed8;font-weight:700;min-width:58px;}',
+        '.muted{color:#7c8ba1;}',
+        'code{font-family:ui-monospace,SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono",monospace;font-size:12px;word-break:break-all;}',
+        '.footer{margin-top:28px;color:#61728d;font-size:13px;}',
+        '@media (max-width:900px){.scenario-card summary{flex-direction:column;align-items:flex-start;}.summary-note{max-width:none;text-align:left;}}',
+        '</style>',
+        '</head>',
+        '<body>',
+        '<div class="page">',
+        '<section class="hero">',
+        `<h1>${escapeHtml(payload.projectName)} AI Scenario Report</h1>`,
+        `<p>Environment: <strong>${escapeHtml(payload.environment)}</strong></p>`,
+        `<p>Generated: <strong>${escapeHtml(payload.generatedAt)}</strong></p>`,
+        `<p>Overall status: <span class="badge ${statusClass(payload.summary.status)}">${payload.summary.status}</span></p>`,
+        '</section>',
+        '<section class="summary-grid">',
+        `<article class="summary-card"><span class="label">Scenarios</span><div class="value">${payload.summary.totalScenarios}</div><div class="sub">${payload.summary.passedScenarios} pass / ${payload.summary.failedScenarios} fail / ${payload.summary.skippedScenarios} skip</div></article>`,
+        `<article class="summary-card"><span class="label">Turns</span><div class="value">${payload.summary.totalTurns}</div><div class="sub">${payload.summary.totalToolCalls} tool calls total</div></article>`,
+        `<article class="summary-card"><span class="label">Assertions</span><div class="value">${payload.summary.totalAssertions}</div><div class="sub">${payload.summary.passedAssertions} pass / ${payload.summary.failedAssertions} fail</div></article>`,
+        `<article class="summary-card"><span class="label">Evaluations</span><div class="value">${payload.summary.totalEvaluations}</div><div class="sub">LLM-scored dimensions</div></article>`,
+        '</section>',
+        '<section class="panels">',
+        `<article class="panel"><h2>Evaluation Averages</h2>${renderEvaluationSummary(payload.summary.averageScoresByDimension)}</article>`,
+        `<article class="panel"><h2>Requested Eval Dimensions</h2><div>${renderDimensionList(payload.summary.requestedEvalDimensions)}</div></article>`,
+        `<article class="panel"><h2>Scenario Context</h2><pre>${escapeHtml(JSON.stringify(payload.scenarioContext, null, 2))}</pre></article>`,
+        `<article class="panel"><h2>Runtime</h2><pre>${escapeHtml(JSON.stringify(payload.runtime, null, 2))}</pre></article>`,
+        '</section>',
+        `<section class="scenario-list">${scenarioCards}</section>`,
+        '<p class="footer">This report includes each tested scenario, each turn prompt, the captured assistant response, assertion outcomes, and the raw artifact path for follow-up debugging.</p>',
+        '</div>',
+        '</body>',
+        '</html>',
+    ].join('');
+}
+async function writeScenarioRunReport(options) {
+    await promises_1.default.mkdir(options.outputDirectory, { recursive: true });
+    const timestamp = new Date().toISOString();
+    const safeTimestamp = timestamp.replace(/[:.]/g, '-');
+    const jsonPath = node_path_1.default.join(options.outputDirectory, `${safeTimestamp}-run.json`);
+    const htmlPath = node_path_1.default.join(options.outputDirectory, `${safeTimestamp}-run.html`);
+    const payload = {
+        generatedAt: timestamp,
+        runtime: options.runtime,
+        environment: options.environment,
+        projectName: options.projectName,
+        scenarioContext: options.scenarioContext,
+        summary: buildScenarioRunSummary(options.results),
+        results: options.results,
+    };
+    await promises_1.default.writeFile(jsonPath, JSON.stringify(payload, null, 2), 'utf8');
+    await promises_1.default.writeFile(htmlPath, buildScenarioRunHtml(payload), 'utf8');
+    return {
+        jsonPath,
+        htmlPath,
+    };
+}

package/dist/scenarios/generated-message.d.ts

@@ -0,0 +1,15 @@
+import { type AiTurn, type BaseAiScenarioContext } from './scenario';
+type ConversationHistoryEntry = {
+    label: string;
+    message: string;
+    output?: string;
+};
+export declare function resolveTurnMessage<TContext extends BaseAiScenarioContext>(options: {
+    turn: AiTurn<TContext>;
+    context: TContext;
+    env: NodeJS.ProcessEnv;
+    scenarioId: string;
+    scenarioName: string;
+    history: ConversationHistoryEntry[];
+}): Promise<string>;
+export {};

package/dist/scenarios/generated-message.js

@@ -0,0 +1,116 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.resolveTurnMessage = resolveTurnMessage;
+const node_path_1 = __importDefault(require("node:path"));
+const openai_eval_1 = require("../utils/openai-eval");
+let cachedSystemPrompt;
+const DEFAULT_SYSTEM_PROMPT_PATH = node_path_1.default.resolve(__dirname, '..', 'prompts', 'turn-message-gen-system-prompt.md');
+async function getMessageGenSystemPrompt() {
+    if (cachedSystemPrompt) {
+        return cachedSystemPrompt;
+    }
+    const { readFile } = await import('node:fs/promises');
+    cachedSystemPrompt = await readFile(DEFAULT_SYSTEM_PROMPT_PATH, 'utf8');
+    return cachedSystemPrompt;
+}
+function isAiGeneratedMessageDefinition(value) {
+    return (typeof value === 'object' &&
+        value !== null &&
+        'type' in value &&
+        (((value.type === 'ai_gen_previous_context') ||
+            (value.type === 'ai_gen_guidance') ||
+            (value.type === 'ai_gen_content'))));
+}
+function cleanGeneratedMessage(value) {
+    const trimmed = value.trim();
+    if (!trimmed) {
+        return '';
+    }
+    const withoutCodeFence = trimmed.replace(/^```[\w-]*\s*|\s*```$/g, '').trim();
+    if ((withoutCodeFence.startsWith('"') && withoutCodeFence.endsWith('"')) ||
+        (withoutCodeFence.startsWith("'") && withoutCodeFence.endsWith("'"))) {
+        return withoutCodeFence.slice(1, -1).trim();
+    }
+    return withoutCodeFence;
+}
+async function generateMessageFromAi(options) {
+    if (options.definition.type === 'ai_gen_previous_context' && options.history.length === 0) {
+        throw new Error(`Turn ${options.turnLabel} uses aiGen.fromPreviousContext() but there is no previous conversation context.`);
+    }
+    const apiKey = (0, openai_eval_1.getRequiredOpenAiApiKey)(options.env);
+    const baseUrl = (0, openai_eval_1.getOpenAiBaseUrl)(options.env);
+    const model = (0, openai_eval_1.getOpenAiMessageGenModel)(options.env);
+    const systemPrompt = await getMessageGenSystemPrompt();
+    const response = await fetch(`${baseUrl}/responses`, {
+        method: 'POST',
+        headers: {
+            Authorization: `Bearer ${apiKey}`,
+            'Content-Type': 'application/json',
+        },
+        body: JSON.stringify({
+            model,
+            input: [
+                {
+                    role: 'system',
+                    content: [
+                        {
+                            type: 'input_text',
+                            text: systemPrompt,
+                        },
+                    ],
+                },
+                {
+                    role: 'user',
+                    content: [
+                        {
+                            type: 'input_text',
+                            text: JSON.stringify({
+                                scenarioId: options.scenarioId,
+                                scenarioName: options.scenarioName,
+                                turnLabel: options.turnLabel,
+                                mode: options.definition.type === 'ai_gen_previous_context'
+                                    ? 'fromPreviousContext'
+                                    : options.definition.type === 'ai_gen_guidance'
+                                        ? 'fromGuidance'
+                                        : 'fromContent',
+                                guidance: options.definition.type === 'ai_gen_guidance'
+                                    ? options.definition.guidance
+                                    : undefined,
+                                sourceContent: options.definition.type === 'ai_gen_content' ? options.definition.content : undefined,
+                                history: options.history,
+                            }, null, 2),
+                        },
+                    ],
+                },
+            ],
+        }),
+    });
+    if (!response.ok) {
+        throw new Error(`OpenAI message generation failed with HTTP ${response.status}: ${await response.text()}`);
+    }
+    const payload = (await response.json());
+    const generatedMessage = cleanGeneratedMessage((0, openai_eval_1.extractOutputText)(payload));
+    if (!generatedMessage) {
+        throw new Error(`OpenAI message generation returned no content for ${options.scenarioId} ${options.turnLabel}.`);
+    }
+    return generatedMessage;
+}
+async function resolveTurnMessage(options) {
+    if (typeof options.turn.message === 'function') {
+        return options.turn.message(options.context);
+    }
+    if (isAiGeneratedMessageDefinition(options.turn.message)) {
+        return await generateMessageFromAi({
+            definition: options.turn.message,
+            history: options.history,
+            env: options.env,
+            scenarioId: options.scenarioId,
+            scenarioName: options.scenarioName,
+            turnLabel: options.turn.label,
+        });
+    }
+    throw new Error(`Unsupported message definition for ${options.scenarioId} ${options.turn.label}.`);
+}

package/dist/scenarios/scenario-loader.d.ts

@@ -0,0 +1,12 @@
+import { AiScenarioSet, type BaseAiScenarioContext } from './scenario';
+type ScenarioContextBuilder<TContext extends BaseAiScenarioContext> = (projectId: string) => TContext;
+type LoadedScenarioModule<TContext extends BaseAiScenarioContext> = {
+    scenarioSet: AiScenarioSet<TContext>;
+    buildScenarioContext: ScenarioContextBuilder<TContext>;
+    scenarioFilePath: string;
+};
+export declare function loadScenarioModule<TContext extends BaseAiScenarioContext>(options: {
+    scenarioFile?: string;
+    defaultRelativePath: string;
+}): Promise<LoadedScenarioModule<TContext>>;
+export {};

package/dist/scenarios/scenario-loader.js

@@ -0,0 +1,103 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.loadScenarioModule = loadScenarioModule;
+const node_fs_1 = __importDefault(require("node:fs"));
+const node_path_1 = __importDefault(require("node:path"));
+const node_url_1 = require("node:url");
+function ensureTsExtension(value) {
+    return node_path_1.default.extname(value) ? value : `${value}.ts`;
+}
+function resolveScenarioFilePath(input, defaultRelativePath) {
+    const rawValue = (input?.trim() || defaultRelativePath).trim();
+    const withExtension = ensureTsExtension(rawValue);
+    if (node_path_1.default.isAbsolute(withExtension)) {
+        return withExtension;
+    }
+    if (withExtension.startsWith('.')) {
+        return node_path_1.default.resolve(process.cwd(), withExtension);
+    }
+    const directPath = node_path_1.default.resolve(process.cwd(), withExtension);
+    const scenariosPath = node_path_1.default.resolve(process.cwd(), 'data', 'ai-scenarios', withExtension);
+    if (node_fs_1.default.existsSync(directPath)) {
+        return directPath;
+    }
+    return scenariosPath;
+}
+function resolveExistingScenarioCandidate(input, defaultRelativePath) {
+    const rawValue = (input?.trim() || defaultRelativePath).trim();
+    const withExtension = ensureTsExtension(rawValue);
+    const candidates = [
+        node_path_1.default.isAbsolute(withExtension) ? withExtension : undefined,
+        withExtension.startsWith('.') ? node_path_1.default.resolve(process.cwd(), withExtension) : undefined,
+        node_path_1.default.resolve(process.cwd(), withExtension),
+        node_path_1.default.resolve(process.cwd(), 'data', 'ai-scenarios', withExtension),
+    ].filter((candidate) => Boolean(candidate));
+    const uniqueCandidates = [...new Set(candidates)];
+    for (const candidate of uniqueCandidates) {
+        try {
+            const stat = node_fs_1.default.statSync(candidate);
+            if (stat.isFile()) {
+                return candidate;
+            }
+        }
+        catch {
+            continue;
+        }
+    }
+    return resolveScenarioFilePath(input, defaultRelativePath);
+}
+function getScenarioSetExport(exportsObject, scenarioFilePath) {
+    for (const candidate of getExportCandidates(exportsObject)) {
+        if (isScenarioSetLike(candidate.scenarioSet)) {
+            return candidate.scenarioSet;
+        }
+        for (const value of Object.values(candidate)) {
+            if (isScenarioSetLike(value)) {
+                return value;
+            }
+        }
+    }
+    throw new Error(`Scenario file ${scenarioFilePath} does not export an AiScenarioSet. Add 'export const scenarioSet = new AiScenarioSet(...)' or an equivalent named export.`);
+}
+function isScenarioSetLike(value) {
+    if (!value || typeof value !== 'object') {
+        return false;
+    }
+    const candidate = value;
+    return Array.isArray(candidate.items) && typeof candidate.select === 'function';
+}
+function getScenarioContextBuilder(exportsObject) {
+    for (const candidate of getExportCandidates(exportsObject)) {
+        if (typeof candidate.buildScenarioContext === 'function') {
+            return candidate.buildScenarioContext;
+        }
+    }
+    return ((projectId) => ({ projectId }));
+}
+function getExportCandidates(exportsObject) {
+    const candidates = [
+        exportsObject,
+        unwrapModuleLikeExport(exportsObject.default),
+        unwrapModuleLikeExport(exportsObject['module.exports']),
+    ].filter((candidate) => Boolean(candidate));
+    return [...new Set(candidates)];
+}
+function unwrapModuleLikeExport(value) {
+    if (!value || typeof value !== 'object') {
+        return undefined;
+    }
+    return value;
+}
+async function loadScenarioModule(options) {
+    const scenarioFilePath = resolveExistingScenarioCandidate(options.scenarioFile, options.defaultRelativePath);
+    const moduleUrl = (0, node_url_1.pathToFileURL)(scenarioFilePath).href;
+    const exportsObject = (await import(moduleUrl));
+    return {
+        scenarioSet: getScenarioSetExport(exportsObject, scenarioFilePath),
+        buildScenarioContext: getScenarioContextBuilder(exportsObject),
+        scenarioFilePath,
+    };
+}