@goreal-ai/echo-pdk 0.7.1 → 0.8.0

package/dist/eval/loader.js

@@ -0,0 +1,170 @@
+/**
+ * @fileoverview Loader for .eval and .dset YAML files
+ *
+ * Parses and validates eval suite definitions and dataset files.
+ */
+import { readFile } from 'fs/promises';
+import { parse as parseYaml } from 'yaml';
+// =============================================================================
+// EVAL FILE LOADER
+// =============================================================================
+/**
+ * Load and validate an .eval file from disk.
+ */
+export async function loadEvalFile(filePath) {
+    const content = await readFile(filePath, 'utf-8');
+    return parseEvalContent(content, filePath);
+}
+/**
+ * Parse .eval YAML content into an EvalSuite.
+ */
+export function parseEvalContent(content, source) {
+    const raw = parseYaml(content);
+    if (!raw || typeof raw !== 'object') {
+        throw new EvalLoadError('Invalid .eval file: expected YAML object', source);
+    }
+    // Validate required fields
+    if (typeof raw.suite !== 'string') {
+        throw new EvalLoadError('Missing required field: suite', source);
+    }
+    if (!Array.isArray(raw.tests) || raw.tests.length === 0) {
+        throw new EvalLoadError('Missing or empty required field: tests', source);
+    }
+    // Parse config
+    const config = parseConfig(raw.config, source);
+    // Parse tests
+    const tests = raw.tests.map((t, i) => parseTest(t, i, source));
+    return {
+        suite: raw.suite,
+        config,
+        tests,
+    };
+}
+function parseConfig(raw, _source) {
+    if (!raw || typeof raw !== 'object') {
+        return { target: 'prompt.pdk' };
+    }
+    return {
+        target: typeof raw.target === 'string' ? raw.target : 'prompt.pdk',
+        model: typeof raw.model === 'string' ? raw.model : undefined,
+        timeout: typeof raw.timeout === 'number' ? raw.timeout : undefined,
+    };
+}
+function parseTest(raw, index, source) {
+    if (typeof raw.name !== 'string') {
+        throw new EvalLoadError(`Test at index ${index} missing required field: name`, source);
+    }
+    const test = {
+        name: raw.name,
+    };
+    if (raw.given && typeof raw.given === 'object') {
+        test.given = raw.given;
+    }
+    if (typeof raw.dataset === 'string') {
+        test.dataset = raw.dataset;
+    }
+    if (typeof raw.params === 'string') {
+        test.params = raw.params;
+    }
+    if (Array.isArray(raw.expect_render)) {
+        test.expect_render = raw.expect_render.map((a, i) => parseAssertion(a, i, raw.name, source));
+    }
+    if (Array.isArray(raw.expect_llm)) {
+        test.expect_llm = raw.expect_llm.map((a, i) => parseAssertion(a, i, raw.name, source));
+    }
+    if (!test.expect_render && !test.expect_llm) {
+        throw new EvalLoadError(`Test "${test.name}" has no assertions (need expect_render or expect_llm)`, source);
+    }
+    return test;
+}
+/**
+ * Parse a single assertion from YAML.
+ * Each assertion is a single-key object.
+ */
+function parseAssertion(raw, index, testName, source) {
+    const keys = Object.keys(raw);
+    if (keys.length === 0) {
+        throw new EvalLoadError(`Empty assertion at index ${index} in test "${testName}"`, source);
+    }
+    // The first key is the operator
+    const operator = keys[0];
+    const value = raw[operator];
+    // Validate known operators
+    const knownOperators = new Set([
+        'contains', 'not_contains', 'equals', 'matches',
+        'starts_with', 'ends_with', 'length', 'word_count',
+        'json_valid', 'json_schema', 'llm_judge', 'similar_to',
+        'sentiment', 'latency', 'token_count', 'cost',
+    ]);
+    if (!knownOperators.has(operator)) {
+        throw new EvalLoadError(`Unknown assertion operator "${operator}" in test "${testName}"`, source);
+    }
+    // Return as-is — the runner will handle type checking per operator
+    return { [operator]: value };
+}
+// =============================================================================
+// DATASET FILE LOADER
+// =============================================================================
+/**
+ * Load and validate a .dset file from disk.
+ */
+export async function loadDatasetFile(filePath) {
+    const content = await readFile(filePath, 'utf-8');
+    return parseDatasetContent(content, filePath);
+}
+/**
+ * Parse .dset YAML content into an EvalDataset.
+ */
+export function parseDatasetContent(content, source) {
+    const raw = parseYaml(content);
+    if (!raw || typeof raw !== 'object') {
+        throw new EvalLoadError('Invalid .dset file: expected YAML object', source);
+    }
+    if (typeof raw.name !== 'string') {
+        throw new EvalLoadError('Missing required field: name', source);
+    }
+    if (!Array.isArray(raw.parameters) || raw.parameters.length === 0) {
+        throw new EvalLoadError('Missing or empty required field: parameters', source);
+    }
+    // Parse golden
+    let golden;
+    if (raw.golden && typeof raw.golden === 'object') {
+        const g = raw.golden;
+        golden = {
+            response: typeof g.response === 'string' ? g.response : '',
+            model: typeof g.model === 'string' ? g.model : undefined,
+            recorded_at: typeof g.recorded_at === 'string' ? g.recorded_at : undefined,
+            metadata: g.metadata && typeof g.metadata === 'object'
+                ? g.metadata
+                : undefined,
+        };
+    }
+    // Parse parameters
+    const parameters = raw.parameters.map((p, i) => {
+        if (typeof p.name !== 'string') {
+            throw new EvalLoadError(`Parameter set at index ${i} missing required field: name`, source);
+        }
+        return p;
+    });
+    return {
+        name: raw.name,
+        description: typeof raw.description === 'string' ? raw.description : undefined,
+        golden,
+        parameters,
+    };
+}
+// =============================================================================
+// ERROR TYPE
+// =============================================================================
+/**
+ * Error thrown when loading/parsing eval or dataset files.
+ */
+export class EvalLoadError extends Error {
+    source;
+    constructor(message, source) {
+        super(source ? `${message} (in ${source})` : message);
+        this.source = source;
+        this.name = 'EvalLoadError';
+    }
+}
+//# sourceMappingURL=loader.js.map

package/dist/eval/loader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"loader.js","sourceRoot":"","sources":["../../src/eval/loader.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,MAAM,CAAC;AAW1C,gFAAgF;AAChF,mBAAmB;AACnB,gFAAgF;AAEhF;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,QAAgB;IACjD,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAClD,OAAO,gBAAgB,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;AAC7C,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAAe,EAAE,MAAe;IAC/D,MAAM,GAAG,GAAG,SAAS,CAAC,OAAO,CAA4B,CAAC;IAE1D,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QACpC,MAAM,IAAI,aAAa,CAAC,0CAA0C,EAAE,MAAM,CAAC,CAAC;IAC9E,CAAC;IAED,2BAA2B;IAC3B,IAAI,OAAO,GAAG,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;QAClC,MAAM,IAAI,aAAa,CAAC,+BAA+B,EAAE,MAAM,CAAC,CAAC;IACnE,CAAC;IAED,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxD,MAAM,IAAI,aAAa,CAAC,wCAAwC,EAAE,MAAM,CAAC,CAAC;IAC5E,CAAC;IAED,eAAe;IACf,MAAM,MAAM,GAAG,WAAW,CAAC,GAAG,CAAC,MAA6C,EAAE,MAAM,CAAC,CAAC;IAEtF,cAAc;IACd,MAAM,KAAK,GAAI,GAAG,CAAC,KAAmC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAClE,SAAS,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,CACxB,CAAC;IAEF,OAAO;QACL,KAAK,EAAE,GAAG,CAAC,KAAe;QAC1B,MAAM;QACN,KAAK;KACN,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAClB,GAAwC,EACxC,OAAgB;IAEhB,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QACpC,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC;IAClC,CAAC;IAED,OAAO;QACL,MAAM,EAAE,OAAO,GAAG,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,YAAY;QAClE,KAAK,EAAE,OAAO,GAAG,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;QAC5D,OAAO,EAAE,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS;KACnE,CAAC;AACJ,CAAC;AAED,SAAS,SAAS,CAChB,GAA4B,EAC5B,KAAa,EACb,MAAe;IAEf,IAAI,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QACjC,MAAM,IAAI,aAAa,CAAC,iBAAiB,KAAK,+BAA+B,EAAE,MAAM,CAAC,CAAC;IACzF,CAAC;IAED,MAAM,IAAI,GAAa;QACrB,IAAI,EAAE,GAAG,CAAC,IAAc;KACzB,CAAC;IAEF,IAAI,GAAG,CAAC,KAAK,IAAI,OAAO,GAAG,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC/C,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAgC,CAAC;IACpD,CAAC;IAED,IAAI,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;QACpC,IAAI,CAAC,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC;IAC7B,CAAC;IAED,IAAI,OAAO,GAAG,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;QACnC,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC;IAC3B,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,EAAE,CAAC;QACrC,IAAI,CAAC,aAAa,GAAG,GAAG,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAClD,cAAc,CAAC,CAA4B,EAAE,CAAC,EAAE,GAAG,CAAC,IAAc,EAAE,MAAM,CAAC,CAC5E,CAAC;IACJ,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;QAClC,IAAI,CAAC,UAAU,GAAG,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAC5C,cAAc,CAAC,CAA4B,EAAE,CAAC,EAAE,GAAG,CAAC,IAAc,EAAE,MAAM,CAAC,CAC5E,CAAC;IACJ,CAAC;IAED,IAAI,CAAC,IAAI,CAAC,aAAa,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;QAC5C,MAAM,IAAI,aAAa,CACrB,SAAS,IAAI,CAAC,IAAI,wDAAwD,EAC1E,MAAM,CACP,CAAC;IACJ,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,SAAS,cAAc,CACrB,GAA4B,EAC5B,KAAa,EACb,QAAgB,EAChB,MAAe;IAEf,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC9B,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtB,MAAM,IAAI,aAAa,CACrB,4BAA4B,KAAK,aAAa,QAAQ,GAAG,EACzD,MAAM,CACP,CAAC;IACJ,CAAC;IAED,gCAAgC;IAChC,MAAM,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAW,CAAC;IACnC,MAAM,KAAK,GAAG,GAAG,CAAC,QAAQ,CAAC,CAAC;IAE5B,2BAA2B;IAC3B,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC;QAC7B,UAAU,EAAE,cAAc,EAAE,QAAQ,EAAE,SAAS;QAC/C,aAAa,EAAE,WAAW,EAAE,QAAQ,EAAE,YAAY;QAClD,YAAY,EAAE,aAAa,EAAE,WAAW,EAAE,YAAY;QACtD,WAAW,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM;KAC9C,CAAC,CAAC;IAEH,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;QAClC,MAAM,IAAI,aAAa,CACrB,+BAA+B,QAAQ,cAAc,QAAQ,GAAG,EAChE,MAAM,CACP,CAAC;IACJ,CAAC;IAED,mEAAmE;IACnE,OAAO,EAAE,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAe,CAAC;AAC5C,CAAC;AAED,gFAAgF;AAChF,sBAAsB;AACtB,gFAAgF;AAEhF;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,QAAgB;IACpD,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAClD,OAAO,mBAAmB,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;AAChD,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,mBAAmB,CAAC,OAAe,EAAE,MAAe;IAClE,MAAM,GAAG,GAAG,SAAS,CAAC,OAAO,CAA4B,CAAC;IAE1D,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QACpC,MAAM,IAAI,aAAa,CAAC,0CAA0C,EAAE,MAAM,CAAC,CAAC;IAC9E,CAAC;IAED,IAAI,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QACjC,MAAM,IAAI,aAAa,CAAC,8BAA8B,EAAE,MAAM,CAAC,CAAC;IAClE,CAAC;IAED,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAClE,MAAM,IAAI,aAAa,CAAC,6CAA6C,EAAE,MAAM,CAAC,CAAC;IACjF,CAAC;IAED,eAAe;IACf,IAAI,MAA8B,CAAC;IACnC,IAAI,GAAG,CAAC,MAAM,IAAI,OAAO,GAAG,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;QACjD,MAAM,CAAC,GAAG,GAAG,CAAC,MAAiC,CAAC;QAChD,MAAM,GAAG;YACP,QAAQ,EAAE,OAAO,CAAC,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE;YAC1D,KAAK,EAAE,OAAO,CAAC,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;YACxD,WAAW,EAAE,OAAO,CAAC,CAAC,WAAW,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS;YAC1E,QAAQ,EAAE,CAAC,CAAC,QAAQ,IAAI,OAAO,CAAC,CAAC,QAAQ,KAAK,QAAQ;gBACpD,CAAC,CAAC,CAAC,CAAC,QAAmC;gBACvC,CAAC,CAAC,SAAS;SACd,CAAC;IACJ,CAAC;IAED,mBAAmB;IACnB,MAAM,UAAU,GAAwB,GAAG,CAAC,UAAwC,CAAC,GAAG,CACtF,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACP,IAAI,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC/B,MAAM,IAAI,aAAa,CAAC,0BAA0B,CAAC,+BAA+B,EAAE,MAAM,CAAC,CAAC;QAC9F,CAAC;QACD,OAAO,CAAqB,CAAC;IAC/B,CAAC,CACF,CAAC;IAEF,OAAO;QACL,IAAI,EAAE,GAAG,CAAC,IAAc;QACxB,WAAW,EAAE,OAAO,GAAG,CAAC,WAAW,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS;QAC9E,MAAM;QACN,UAAU;KACX,CAAC;AACJ,CAAC;AAED,gFAAgF;AAChF,aAAa;AACb,gFAAgF;AAEhF;;GAEG;AACH,MAAM,OAAO,aAAc,SAAQ,KAAK;IAG7B;IAFT,YACE,OAAe,EACR,MAAe;QAEtB,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,OAAO,QAAQ,MAAM,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAF/C,WAAM,GAAN,MAAM,CAAS;QAGtB,IAAI,CAAC,IAAI,GAAG,eAAe,CAAC;IAC9B,CAAC;CACF"}

package/dist/eval/reporter.d.ts

@@ -0,0 +1,26 @@
+/**
+ * @fileoverview Eval result reporters
+ *
+ * Formats EvalSuiteResult for different output targets:
+ * - Console: colorful human-readable output
+ * - JSON: structured data for programmatic use
+ * - JUnit XML: CI integration format
+ */
+import type { EvalSuiteResult } from './types.js';
+/**
+ * Format eval results for console output with colors (ANSI).
+ */
+export declare function formatConsole(result: EvalSuiteResult): string;
+/**
+ * Format eval results as JSON string.
+ */
+export declare function formatJson(result: EvalSuiteResult): string;
+/**
+ * Format eval results as JUnit XML for CI integration.
+ */
+export declare function formatJunit(result: EvalSuiteResult): string;
+/**
+ * Format eval results using the specified reporter.
+ */
+export declare function formatResults(result: EvalSuiteResult, reporter?: 'console' | 'json' | 'junit'): string;
+//# sourceMappingURL=reporter.d.ts.map

package/dist/eval/reporter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"reporter.d.ts","sourceRoot":"","sources":["../../src/eval/reporter.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAMlD;;GAEG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE,eAAe,GAAG,MAAM,CA4E7D;AAMD;;GAEG;AACH,wBAAgB,UAAU,CAAC,MAAM,EAAE,eAAe,GAAG,MAAM,CAE1D;AAMD;;GAEG;AACH,wBAAgB,WAAW,CAAC,MAAM,EAAE,eAAe,GAAG,MAAM,CAuC3D;AAMD;;GAEG;AACH,wBAAgB,aAAa,CAC3B,MAAM,EAAE,eAAe,EACvB,QAAQ,GAAE,SAAS,GAAG,MAAM,GAAG,OAAmB,GACjD,MAAM,CASR"}

package/dist/eval/reporter.js

@@ -0,0 +1,164 @@
+/**
+ * @fileoverview Eval result reporters
+ *
+ * Formats EvalSuiteResult for different output targets:
+ * - Console: colorful human-readable output
+ * - JSON: structured data for programmatic use
+ * - JUnit XML: CI integration format
+ */
+// =============================================================================
+// CONSOLE REPORTER
+// =============================================================================
+/**
+ * Format eval results for console output with colors (ANSI).
+ */
+export function formatConsole(result) {
+    const lines = [];
+    const { suiteName, tests, summary } = result;
+    // Header
+    const statusIcon = result.status === 'pass' ? '✓' : result.status === 'fail' ? '✗' : '⚠';
+    const statusColor = result.status === 'pass' ? '\x1b[32m' : result.status === 'fail' ? '\x1b[31m' : '\x1b[33m';
+    const reset = '\x1b[0m';
+    const dim = '\x1b[2m';
+    const bold = '\x1b[1m';
+    lines.push(`\n${bold}${statusColor}${statusIcon} ${suiteName}${reset}`);
+    lines.push(`${dim}${'─'.repeat(60)}${reset}`);
+    // Tests
+    for (const test of tests) {
+        const icon = test.status === 'pass' ? '\x1b[32m✓' : test.status === 'fail' ? '\x1b[31m✗' : '\x1b[33m⚠';
+        const duration = test.durationMs != null ? ` ${dim}(${test.durationMs}ms)${reset}` : '';
+        lines.push(`  ${icon} ${test.name}${reset}${duration}`);
+        if (test.error) {
+            lines.push(`    \x1b[31mError: ${test.error}${reset}`);
+        }
+        // Show rendered prompt if available
+        if (test.renderedOutput) {
+            lines.push(`    ${dim}Rendered prompt:${reset}`);
+            const renderedLines = test.renderedOutput.split('\n');
+            for (const rl of renderedLines) {
+                lines.push(`    ${dim}  ${rl}${reset}`);
+            }
+        }
+        // Show LLM response if available (use !== undefined to handle empty strings)
+        if (test.llmResponse !== undefined) {
+            lines.push(`    ${dim}LLM response:${reset}`);
+            if (test.llmResponse) {
+                const responseLines = test.llmResponse.split('\n');
+                for (const rl of responseLines) {
+                    lines.push(`    ${dim}  ${rl}${reset}`);
+                }
+            }
+            else {
+                lines.push(`    ${dim}  (empty — model may have returned an error)${reset}`);
+            }
+        }
+        // Show assertions
+        for (const assertion of test.assertions) {
+            if (assertion.status !== 'pass') {
+                const aIcon = assertion.status === 'fail' ? '\x1b[31m✗' : '\x1b[33m⚠';
+                const msg = assertion.message ?? `${assertion.operator} failed`;
+                lines.push(`    ${aIcon} ${assertion.operator}: ${msg}${reset}`);
+                if (assertion.expected) {
+                    lines.push(`      ${dim}expected: ${assertion.expected}${reset}`);
+                }
+                if (assertion.actual) {
+                    lines.push(`      ${dim}actual:   ${assertion.actual}${reset}`);
+                }
+            }
+            else if (assertion.operator === 'llm_judge' || assertion.operator === 'sentiment') {
+                // Always show reasoning for AI assertions, even on pass
+                lines.push(`    \x1b[32m✓ ${assertion.operator}:${reset} ${dim}${assertion.message ?? 'passed'}${reset}`);
+            }
+        }
+    }
+    // Summary
+    lines.push(`${dim}${'─'.repeat(60)}${reset}`);
+    const parts = [];
+    if (summary.passed > 0)
+        parts.push(`\x1b[32m${summary.passed} passed${reset}`);
+    if (summary.failed > 0)
+        parts.push(`\x1b[31m${summary.failed} failed${reset}`);
+    if (summary.errored > 0)
+        parts.push(`\x1b[33m${summary.errored} errored${reset}`);
+    parts.push(`${summary.total} total`);
+    lines.push(`  ${parts.join(', ')} ${dim}(${summary.durationMs}ms)${reset}`);
+    lines.push('');
+    return lines.join('\n');
+}
+// =============================================================================
+// JSON REPORTER
+// =============================================================================
+/**
+ * Format eval results as JSON string.
+ */
+export function formatJson(result) {
+    return JSON.stringify(result, null, 2);
+}
+// =============================================================================
+// JUNIT XML REPORTER
+// =============================================================================
+/**
+ * Format eval results as JUnit XML for CI integration.
+ */
+export function formatJunit(result) {
+    const { suiteName, tests, summary } = result;
+    const lines = [];
+    lines.push('<?xml version="1.0" encoding="UTF-8"?>');
+    lines.push(`<testsuite name="${escapeXml(suiteName)}" tests="${summary.total}" ` +
+        `failures="${summary.failed}" errors="${summary.errored}" ` +
+        `time="${(summary.durationMs / 1000).toFixed(3)}">`);
+    for (const test of tests) {
+        const time = test.durationMs != null ? ` time="${(test.durationMs / 1000).toFixed(3)}"` : '';
+        lines.push(`  <testcase name="${escapeXml(test.name)}"${time}>`);
+        if (test.status === 'fail') {
+            const failedAssertions = test.assertions.filter((a) => a.status === 'fail');
+            const message = failedAssertions
+                .map((a) => a.message ?? `${a.operator} failed`)
+                .join('; ');
+            lines.push(`    <failure message="${escapeXml(message)}">`);
+            for (const a of failedAssertions) {
+                lines.push(`      [${a.operator}] ${a.message ?? 'failed'}`);
+                if (a.expected)
+                    lines.push(`        expected: ${a.expected}`);
+                if (a.actual)
+                    lines.push(`        actual: ${a.actual}`);
+            }
+            lines.push('    </failure>');
+        }
+        if (test.status === 'error') {
+            const errorMsg = test.error ?? 'Unknown error';
+            lines.push(`    <error message="${escapeXml(errorMsg)}">${escapeXml(errorMsg)}</error>`);
+        }
+        lines.push('  </testcase>');
+    }
+    lines.push('</testsuite>');
+    return lines.join('\n');
+}
+// =============================================================================
+// FORMAT SELECTOR
+// =============================================================================
+/**
+ * Format eval results using the specified reporter.
+ */
+export function formatResults(result, reporter = 'console') {
+    switch (reporter) {
+        case 'console':
+            return formatConsole(result);
+        case 'json':
+            return formatJson(result);
+        case 'junit':
+            return formatJunit(result);
+    }
+}
+// =============================================================================
+// HELPERS
+// =============================================================================
+function escapeXml(text) {
+    return text
+        .replace(/&/g, '&amp;')
+        .replace(/</g, '&lt;')
+        .replace(/>/g, '&gt;')
+        .replace(/"/g, '&quot;')
+        .replace(/'/g, '&apos;');
+}
+//# sourceMappingURL=reporter.js.map

package/dist/eval/reporter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"reporter.js","sourceRoot":"","sources":["../../src/eval/reporter.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAIH,gFAAgF;AAChF,mBAAmB;AACnB,gFAAgF;AAEhF;;GAEG;AACH,MAAM,UAAU,aAAa,CAAC,MAAuB;IACnD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,MAAM,CAAC;IAE7C,SAAS;IACT,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;IACzF,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC;IAC/G,MAAM,KAAK,GAAG,SAAS,CAAC;IACxB,MAAM,GAAG,GAAG,SAAS,CAAC;IACtB,MAAM,IAAI,GAAG,SAAS,CAAC;IAEvB,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,GAAG,WAAW,GAAG,UAAU,IAAI,SAAS,GAAG,KAAK,EAAE,CAAC,CAAC;IACxE,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,CAAC;IAE9C,QAAQ;IACR,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC;QACvG,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,IAAI,IAAI,CAAC,UAAU,MAAM,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACxF,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,IAAI,CAAC,IAAI,GAAG,KAAK,GAAG,QAAQ,EAAE,CAAC,CAAC;QAExD,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,KAAK,CAAC,IAAI,CAAC,sBAAsB,IAAI,CAAC,KAAK,GAAG,KAAK,EAAE,CAAC,CAAC;QACzD,CAAC;QAED,oCAAoC;QACpC,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,KAAK,CAAC,IAAI,CAAC,OAAO,GAAG,mBAAmB,KAAK,EAAE,CAAC,CAAC;YACjD,MAAM,aAAa,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACtD,KAAK,MAAM,EAAE,IAAI,aAAa,EAAE,CAAC;gBAC/B,KAAK,CAAC,IAAI,CAAC,OAAO,GAAG,KAAK,EAAE,GAAG,KAAK,EAAE,CAAC,CAAC;YAC1C,CAAC;QACH,CAAC;QAED,6EAA6E;QAC7E,IAAI,IAAI,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;YACnC,KAAK,CAAC,IAAI,CAAC,OAAO,GAAG,gBAAgB,KAAK,EAAE,CAAC,CAAC;YAC9C,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;gBACrB,MAAM,aAAa,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACnD,KAAK,MAAM,EAAE,IAAI,aAAa,EAAE,CAAC;oBAC/B,KAAK,CAAC,IAAI,CAAC,OAAO,GAAG,KAAK,EAAE,GAAG,KAAK,EAAE,CAAC,CAAC;gBAC1C,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,KAAK,CAAC,IAAI,CAAC,OAAO,GAAG,+CAA+C,KAAK,EAAE,CAAC,CAAC;YAC/E,CAAC;QACH,CAAC;QAED,kBAAkB;QAClB,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACxC,IAAI,SAAS,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;gBAChC,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC;gBACtE,MAAM,GAAG,GAAG,SAAS,CAAC,OAAO,IAAI,GAAG,SAAS,CAAC,QAAQ,SAAS,CAAC;gBAChE,KAAK,CAAC,IAAI,CAAC,OAAO,KAAK,IAAI,SAAS,CAAC,QAAQ,KAAK,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC;gBACjE,IAAI,SAAS,CAAC,QAAQ,EAAE,CAAC;oBACvB,KAAK,CAAC,IAAI,CAAC,SAAS,GAAG,aAAa,SAAS,CAAC,QAAQ,GAAG,KAAK,EAAE,CAAC,CAAC;gBACpE,CAAC;gBACD,IAAI,SAAS,CAAC,MAAM,EAAE,CAAC;oBACrB,KAAK,CAAC,IAAI,CAAC,SAAS,GAAG,aAAa,SAAS,CAAC,MAAM,GAAG,KAAK,EAAE,CAAC,CAAC;gBAClE,CAAC;YACH,CAAC;iBAAM,IAAI,SAAS,CAAC,QAAQ,KAAK,WAAW,IAAI,SAAS,CAAC,QAAQ,KAAK,WAAW,EAAE,CAAC;gBACpF,wDAAwD;gBACxD,KAAK,CAAC,IAAI,CAAC,iBAAiB,SAAS,CAAC,QAAQ,IAAI,KAAK,IAAI,GAAG,GAAG,SAAS,CAAC,OAAO,IAAI,QAAQ,GAAG,KAAK,EAAE,CAAC,CAAC;YAC5G,CAAC;QACH,CAAC;IACH,CAAC;IAED,UAAU;IACV,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,CAAC;IAC9C,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,WAAW,OAAO,CAAC,MAAM,UAAU,KAAK,EAAE,CAAC,CAAC;IAC/E,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,WAAW,OAAO,CAAC,MAAM,UAAU,KAAK,EAAE,CAAC,CAAC;IAC/E,IAAI,OAAO,CAAC,OAAO,GAAG,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,WAAW,OAAO,CAAC,OAAO,WAAW,KAAK,EAAE,CAAC,CAAC;IAClF,KAAK,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC;IACrC,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,IAAI,OAAO,CAAC,UAAU,MAAM,KAAK,EAAE,CAAC,CAAC;IAC5E,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,gFAAgF;AAChF,gBAAgB;AAChB,gFAAgF;AAEhF;;GAEG;AACH,MAAM,UAAU,UAAU,CAAC,MAAuB;IAChD,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;AACzC,CAAC;AAED,gFAAgF;AAChF,qBAAqB;AACrB,gFAAgF;AAEhF;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,MAAuB;IACjD,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,MAAM,CAAC;IAE7C,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,CAAC,IAAI,CAAC,wCAAwC,CAAC,CAAC;IACrD,KAAK,CAAC,IAAI,CACR,oBAAoB,SAAS,CAAC,SAAS,CAAC,YAAY,OAAO,CAAC,KAAK,IAAI;QACnE,aAAa,OAAO,CAAC,MAAM,aAAa,OAAO,CAAC,OAAO,IAAI;QAC3D,SAAS,CAAC,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CACtD,CAAC;IAEF,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7F,KAAK,CAAC,IAAI,CAAC,qBAAqB,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,GAAG,CAAC,CAAC;QAEjE,IAAI,IAAI,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;YAC3B,MAAM,gBAAgB,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC;YAC5E,MAAM,OAAO,GAAG,gBAAgB;iBAC7B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,IAAI,GAAG,CAAC,CAAC,QAAQ,SAAS,CAAC;iBAC/C,IAAI,CAAC,IAAI,CAAC,CAAC;YACd,KAAK,CAAC,IAAI,CAAC,yBAAyB,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAC5D,KAAK,MAAM,CAAC,IAAI,gBAAgB,EAAE,CAAC;gBACjC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,OAAO,IAAI,QAAQ,EAAE,CAAC,CAAC;gBAC7D,IAAI,CAAC,CAAC,QAAQ;oBAAE,KAAK,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;gBAC9D,IAAI,CAAC,CAAC,MAAM;oBAAE,KAAK,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;YAC1D,CAAC;YACD,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAC/B,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;YAC5B,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,IAAI,eAAe,CAAC;YAC/C,KAAK,CAAC,IAAI,CAAC,uBAAuB,SAAS,CAAC,QAAQ,CAAC,KAAK,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QAC3F,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IAC9B,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IAC3B,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,gFAAgF;AAChF,kBAAkB;AAClB,gFAAgF;AAEhF;;GAEG;AACH,MAAM,UAAU,aAAa,CAC3B,MAAuB,EACvB,WAAyC,SAAS;IAElD,QAAQ,QAAQ,EAAE,CAAC;QACjB,KAAK,SAAS;YACZ,OAAO,aAAa,CAAC,MAAM,CAAC,CAAC;QAC/B,KAAK,MAAM;YACT,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC;QAC5B,KAAK,OAAO;YACV,OAAO,WAAW,CAAC,MAAM,CAAC,CAAC;IAC/B,CAAC;AACH,CAAC;AAED,gFAAgF;AAChF,UAAU;AACV,gFAAgF;AAEhF,SAAS,SAAS,CAAC,IAAY;IAC7B,OAAO,IAAI;SACR,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC;SACtB,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC;SACrB,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC;SACrB,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC;SACvB,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;AAC7B,CAAC"}

package/dist/eval/runner.d.ts

@@ -0,0 +1,28 @@
+/**
+ * @fileoverview Eval test runner
+ *
+ * Pipeline:
+ *   .eval file (YAML)
+ *     ↓ load + validate
+ *   EvalSuite { config, tests[] }
+ *     ↓ for each test
+ *     ├── Load variables (from given: or from dataset + params)
+ *     ├── Read target prompt.pdk
+ *     ├── Render with echo-pdk
+ *     ├── If expect_render: run assertions on rendered output
+ *     ├── If expect_llm: send to LLM → run assertions on response
+ *     ├── If record mode: save response as golden in .dset file
+ *     └── Collect results
+ *     ↓
+ *   EvalSuiteResult { tests[], summary }
+ */
+import type { EvalSuite, EvalSuiteResult, EvalRunnerConfig } from './types.js';
+/**
+ * Create and run an eval suite from a file.
+ */
+export declare function runEvalFile(evalFilePath: string, config: EvalRunnerConfig): Promise<EvalSuiteResult>;
+/**
+ * Run an eval suite.
+ */
+export declare function runEvalSuite(suite: EvalSuite, evalFilePath: string, config: EvalRunnerConfig): Promise<EvalSuiteResult>;
+//# sourceMappingURL=runner.d.ts.map

package/dist/eval/runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/eval/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAaH,OAAO,KAAK,EACV,SAAS,EAET,eAAe,EAIf,gBAAgB,EAGjB,MAAM,YAAY,CAAC;AAMpB;;GAEG;AACH,wBAAsB,WAAW,CAC/B,YAAY,EAAE,MAAM,EACpB,MAAM,EAAE,gBAAgB,GACvB,OAAO,CAAC,eAAe,CAAC,CAG1B;AAED;;GAEG;AACH,wBAAsB,YAAY,CAChC,KAAK,EAAE,SAAS,EAChB,YAAY,EAAE,MAAM,EACpB,MAAM,EAAE,gBAAgB,GACvB,OAAO,CAAC,eAAe,CAAC,CAkF1B"}

package/dist/eval/runner.js

@@ -0,0 +1,232 @@
+/**
+ * @fileoverview Eval test runner
+ *
+ * Pipeline:
+ *   .eval file (YAML)
+ *     ↓ load + validate
+ *   EvalSuite { config, tests[] }
+ *     ↓ for each test
+ *     ├── Load variables (from given: or from dataset + params)
+ *     ├── Read target prompt.pdk
+ *     ├── Render with echo-pdk
+ *     ├── If expect_render: run assertions on rendered output
+ *     ├── If expect_llm: send to LLM → run assertions on response
+ *     ├── If record mode: save response as golden in .dset file
+ *     └── Collect results
+ *     ↓
+ *   EvalSuiteResult { tests[], summary }
+ */
+import { readFile } from 'fs/promises';
+import { resolve, dirname } from 'path';
+import { createEcho } from '../index.js';
+import { loadEvalFile } from './loader.js';
+import { runAssertions } from './assertions.js';
+import { DatasetManager } from './dataset.js';
+import { createProvider } from '../providers/registry.js';
+import { toLLMProvider } from '../providers/base.js';
+import { createEmbeddingProvider } from '../embeddings/registry.js';
+import { cosineSimilarity } from '../embeddings/cosine.js';
+// =============================================================================
+// EVAL RUNNER
+// =============================================================================
+/**
+ * Create and run an eval suite from a file.
+ */
+export async function runEvalFile(evalFilePath, config) {
+    const suite = await loadEvalFile(evalFilePath);
+    return runEvalSuite(suite, evalFilePath, config);
+}
+/**
+ * Run an eval suite.
+ */
+export async function runEvalSuite(suite, evalFilePath, config) {
+    const suiteStart = Date.now();
+    // Determine prompt directory from eval file location
+    // Eval files live in <prompt>/eval/tests/ — go up to prompt dir
+    const evalDir = dirname(resolve(evalFilePath));
+    const promptDir = resolve(evalDir, '..', '..');
+    // Create dataset manager for this prompt
+    const datasetManager = new DatasetManager(promptDir);
+    // Load the target prompt template
+    const targetPath = resolve(promptDir, suite.config.target);
+    let template;
+    try {
+        template = await readFile(targetPath, 'utf-8');
+    }
+    catch {
+        return createErrorResult(suite.suite, `Failed to load target prompt: ${targetPath}`);
+    }
+    // Create echo instance
+    const echo = createEcho({ strict: false });
+    // Create embedding provider (explicit config or auto-detect from OpenAI)
+    let embeddingProvider;
+    if (config.embeddingProvider) {
+        embeddingProvider = createEmbeddingProvider(config.embeddingProvider);
+    }
+    else if (config.aiProvider?.type === 'openai') {
+        // Auto-detect: reuse OpenAI API key for embeddings
+        embeddingProvider = createEmbeddingProvider({
+            type: 'openai',
+            apiKey: config.aiProvider.apiKey,
+        });
+    }
+    // Build embeddings-based similarity closure
+    let embeddingSimilarity;
+    if (embeddingProvider) {
+        embeddingSimilarity = async (textA, textB) => {
+            const vectors = await embeddingProvider.embed([textA, textB]);
+            return cosineSimilarity(vectors[0], vectors[1]);
+        };
+    }
+    // Filter tests if needed
+    let tests = suite.tests;
+    if (config.filter) {
+        const pattern = config.filter.toLowerCase();
+        tests = tests.filter((t) => t.name.toLowerCase().includes(pattern));
+    }
+    // Run each test
+    const testResults = [];
+    for (const test of tests) {
+        const result = await runSingleTest(test, {
+            template,
+            echo,
+            datasetManager,
+            suiteConfig: suite.config,
+            runnerConfig: config,
+            promptDir,
+            embeddingSimilarity,
+        });
+        testResults.push(result);
+    }
+    // Compute summary
+    const summary = computeSummary(testResults, Date.now() - suiteStart);
+    // Determine overall status
+    const status = testResults.some((t) => t.status === 'error')
+        ? 'error'
+        : testResults.some((t) => t.status === 'fail')
+            ? 'fail'
+            : 'pass';
+    return {
+        suiteName: suite.suite,
+        status,
+        tests: testResults,
+        summary,
+    };
+}
+async function runSingleTest(test, ctx) {
+    const testStart = Date.now();
+    try {
+        // 1. Resolve variables
+        let variables;
+        if (test.given) {
+            variables = test.given;
+        }
+        else if (test.dataset && test.params) {
+            variables = await ctx.datasetManager.getParams(test.dataset, test.params);
+        }
+        else if (test.dataset) {
+            // Use first parameter set from dataset
+            const dataset = await ctx.datasetManager.load(test.dataset);
+            if (dataset.parameters.length === 0) {
+                throw new Error(`Dataset "${test.dataset}" has no parameter sets`);
+            }
+            const { name: _name, ...vars } = dataset.parameters[0];
+            variables = vars;
+        }
+        else {
+            variables = {};
+        }
+        const allAssertions = [];
+        let renderedOutput;
+        let llmResponseText;
+        // 2. Render the template
+        renderedOutput = await ctx.echo.render(ctx.template, variables);
+        // 3. Run expect_render assertions
+        if (test.expect_render) {
+            const renderCtx = { text: renderedOutput };
+            const results = await runAssertions(test.expect_render, renderCtx);
+            allAssertions.push(...results);
+        }
+        // 4. Run expect_llm assertions (requires LLM call)
+        if (test.expect_llm) {
+            // For now, LLM provider is optional.
+            // If not configured, LLM assertions will return 'error' status.
+            const llmProvider = ctx.runnerConfig.aiProvider
+                ? toLLMProvider(createProvider(ctx.runnerConfig.aiProvider))
+                : undefined;
+            let llmResponse;
+            if (llmProvider) {
+                const model = ctx.suiteConfig.model ?? ctx.runnerConfig.aiProvider?.model;
+                try {
+                    llmResponse = await llmProvider.complete(renderedOutput, model);
+                    llmResponseText = llmResponse.text;
+                }
+                catch (llmErr) {
+                    llmResponseText = '';
+                    allAssertions.push({
+                        operator: 'llm_call',
+                        status: 'error',
+                        message: `LLM call failed: ${llmErr.message}`,
+                    });
+                }
+                // Record mode — save golden
+                if (ctx.runnerConfig.record && test.dataset && llmResponse) {
+                    await ctx.datasetManager.recordGolden(test.dataset, llmResponse.text, llmResponse);
+                }
+            }
+            const llmCtx = {
+                text: llmResponseText ?? '',
+                llmResponse,
+                llmProvider,
+                loadGolden: (name) => ctx.datasetManager.getGolden(name),
+                embeddingSimilarity: ctx.embeddingSimilarity,
+            };
+            const results = await runAssertions(test.expect_llm, llmCtx);
+            allAssertions.push(...results);
+        }
+        // 5. Determine status
+        const status = allAssertions.some((a) => a.status === 'error')
+            ? 'error'
+            : allAssertions.some((a) => a.status === 'fail')
+                ? 'fail'
+                : 'pass';
+        return {
+            name: test.name,
+            status,
+            assertions: allAssertions,
+            durationMs: Date.now() - testStart,
+            renderedOutput,
+            llmResponse: llmResponseText,
+        };
+    }
+    catch (err) {
+        return {
+            name: test.name,
+            status: 'error',
+            assertions: [],
+            durationMs: Date.now() - testStart,
+            error: err.message,
+        };
+    }
+}
+// =============================================================================
+// HELPERS
+// =============================================================================
+function computeSummary(tests, durationMs) {
+    return {
+        total: tests.length,
+        passed: tests.filter((t) => t.status === 'pass').length,
+        failed: tests.filter((t) => t.status === 'fail').length,
+        errored: tests.filter((t) => t.status === 'error').length,
+        durationMs,
+    };
+}
+function createErrorResult(suiteName, _error) {
+    return {
+        suiteName,
+        status: 'error',
+        tests: [],
+        summary: { total: 0, passed: 0, failed: 0, errored: 0, durationMs: 0 },
+    };
+}
+//# sourceMappingURL=runner.js.map