@gnsx/genesys.agent.eval 1.0.0

package/dist/src/runner.d.ts

@@ -0,0 +1,75 @@
+/**
+ * Test runner orchestrates loading, execution, and judging of test cases.
+ *
+ * @module runner
+ */
+import type { EvalResults, RunnerConfig, TestCase, TestResult } from './types.js';
+/**
+ * Progress callback for test execution.
+ */
+export interface ProgressCallback {
+    /**
+     * Called when a test starts.
+     *
+     * @param testId - The test ID
+     * @param index - The test index (0-based)
+     * @param total - Total number of tests
+     */
+    onTestStart(testId: string, index: number, total: number): void;
+    /**
+     * Called when a test completes.
+     *
+     * @param result - The test result
+     * @param index - The test index (0-based)
+     * @param total - Total number of tests
+     */
+    onTestComplete(result: TestResult, index: number, total: number): void;
+    /**
+     * Called when a test fails with an error.
+     *
+     * @param testId - The test ID
+     * @param error - The error that occurred
+     * @param index - The test index (0-based)
+     * @param total - Total number of tests
+     */
+    onTestError(testId: string, error: string, index: number, total: number): void;
+}
+/**
+ * TestRunner orchestrates the evaluation process.
+ */
+export declare class TestRunner {
+    private _config;
+    constructor(config: RunnerConfig);
+    /**
+     * Run the evaluation.
+     *
+     * @param judge - Function to evaluate agent outputs
+     * @param progress - Optional progress callback
+     * @returns The evaluation results
+     */
+    run(judge: (test: TestCase, actualOutput: string) => Promise<{
+        score: number;
+        reasoning: string;
+        passed: boolean;
+    }>, progress?: ProgressCallback): Promise<EvalResults>;
+    /**
+     * Get the runner configuration.
+     */
+    get config(): RunnerConfig;
+}
+/**
+ * Run an evaluation with the given configuration.
+ *
+ * Convenience function that creates a TestRunner and executes it.
+ *
+ * @param config - Runner configuration
+ * @param judge - Judge function for evaluating responses
+ * @param progress - Optional progress callback
+ * @returns Evaluation results
+ */
+export declare function runEvaluation(config: RunnerConfig, judge: (test: TestCase, actualOutput: string) => Promise<{
+    score: number;
+    reasoning: string;
+    passed: boolean;
+}>, progress?: ProgressCallback): Promise<EvalResults>;
+//# sourceMappingURL=runner.d.ts.map

package/dist/src/runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/runner.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAKH,OAAO,KAAK,EACV,WAAW,EACX,YAAY,EACZ,QAAQ,EACR,UAAU,EAEX,MAAM,YAAY,CAAC;AAEpB;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B;;;;;;OAMG;IACH,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAEhE;;;;;;OAMG;IACH,cAAc,CAAC,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAEvE;;;;;;;OAOG;IACH,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;CAChF;AAmID;;GAEG;AACH,qBAAa,UAAU;IACrB,OAAO,CAAC,OAAO,CAAe;gBAElB,MAAM,EAAE,YAAY;IAIhC;;;;;;OAMG;IACG,GAAG,CACP,KAAK,EAAE,CAAC,IAAI,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,KAAK,OAAO,CAAC;QACvD,KAAK,EAAE,MAAM,CAAC;QACd,SAAS,EAAE,MAAM,CAAC;QAClB,MAAM,EAAE,OAAO,CAAC;KACjB,CAAC,EACF,QAAQ,CAAC,EAAE,gBAAgB,GAC1B,OAAO,CAAC,WAAW,CAAC;IA6CvB;;OAEG;IACH,IAAI,MAAM,IAAI,YAAY,CAEzB;CACF;AAED;;;;;;;;;GASG;AACH,wBAAsB,aAAa,CACjC,MAAM,EAAE,YAAY,EACpB,KAAK,EAAE,CAAC,IAAI,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,KAAK,OAAO,CAAC;IACvD,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,OAAO,CAAC;CACjB,CAAC,EACF,QAAQ,CAAC,EAAE,gBAAgB,GAC1B,OAAO,CAAC,WAAW,CAAC,CAGtB"}

package/dist/src/runner.js

@@ -0,0 +1,165 @@
+/**
+ * Test runner orchestrates loading, execution, and judging of test cases.
+ *
+ * @module runner
+ */
+import { runAgent } from './cli-runner.js';
+import { loadTestSuite } from './test-loader.js';
+/**
+ * Run a single test case.
+ *
+ * @param test - The test case to run
+ * @param suite - The test suite
+ * @param agent - The agent to use
+ * @param timeout - Timeout in milliseconds
+ * @param judge - Function to judge the response
+ * @param progress - Optional progress callback
+ * @param index - Test index
+ * @param total - Total tests
+ * @returns The test result
+ */
+async function runTest(test, suite, agent, timeout, judge, progress, index, total) {
+    progress?.onTestStart(test.id, index, total);
+    const startTime = Date.now();
+    // Build the full prompt with context
+    const parts = [];
+    if (suite.context) {
+        parts.push('Context:', suite.context, '');
+    }
+    if (test.context) {
+        parts.push('Specific Context:', test.context, '');
+    }
+    parts.push('Task:', test.input);
+    const prompt = parts.join('\n');
+    try {
+        // Run the agent
+        const response = await runAgent(agent, prompt, { cwd: process.cwd(), timeout });
+        // Judge the response
+        const { score, reasoning, passed } = await judge(test, response.output);
+        const result = {
+            testId: test.id,
+            input: test.input,
+            expectedOutput: test.expectedOutput,
+            actualOutput: response.output,
+            judgeScore: score,
+            judgeReasoning: reasoning,
+            durationMs: response.durationMs,
+            passed,
+        };
+        progress?.onTestComplete(result, index, total);
+        return result;
+    }
+    catch (error) {
+        const durationMs = Date.now() - startTime;
+        const errorMessage = error instanceof Error ? error.message : String(error);
+        progress?.onTestError(test.id, errorMessage, index, total);
+        return {
+            testId: test.id,
+            input: test.input,
+            expectedOutput: test.expectedOutput,
+            actualOutput: '',
+            judgeScore: 0,
+            judgeReasoning: `Error: ${errorMessage}`,
+            durationMs,
+            passed: false,
+            error: errorMessage,
+        };
+    }
+}
+/**
+ * Run tests in parallel with a concurrency limit.
+ *
+ * @param tests - Tests to run
+ * @param concurrency - Number of concurrent executions
+ * @param runner - Function to run a single test
+ * @returns Array of results in the same order as tests
+ */
+async function runInParallel(items, concurrency, runner) {
+    if (concurrency <= 1) {
+        // Sequential execution
+        const results = [];
+        for (let i = 0; i < items.length; i++) {
+            results.push(await runner(items[i], i));
+        }
+        return results;
+    }
+    // Parallel execution with concurrency limit
+    const results = new Array(items.length);
+    let index = 0;
+    async function worker() {
+        while (index < items.length) {
+            const currentIndex = index++;
+            results[currentIndex] = await runner(items[currentIndex], currentIndex);
+        }
+    }
+    // Start workers
+    const workers = Array(Math.min(concurrency, items.length))
+        .fill(null)
+        .map(() => worker());
+    await Promise.all(workers);
+    return results;
+}
+/**
+ * TestRunner orchestrates the evaluation process.
+ */
+export class TestRunner {
+    _config;
+    constructor(config) {
+        this._config = config;
+    }
+    /**
+     * Run the evaluation.
+     *
+     * @param judge - Function to evaluate agent outputs
+     * @param progress - Optional progress callback
+     * @returns The evaluation results
+     */
+    async run(judge, progress) {
+        // Load the test suite
+        const { suite } = await loadTestSuite(this._config.testsPath, this._config.cwd);
+        // Run all tests
+        const results = await runInParallel(suite.tests, this._config.parallel, async (test, index) => {
+            return runTest(test, suite, this._config.agent, this._config.timeout, judge, progress, index, suite.tests.length);
+        });
+        // Calculate summary
+        const totalDurationMs = results.reduce((sum, r) => sum + r.durationMs, 0);
+        const passed = results.filter(r => r.passed).length;
+        const failed = results.length - passed;
+        const avgScore = results.reduce((sum, r) => sum + r.judgeScore, 0) / results.length;
+        const evalResults = {
+            suite,
+            agent: this._config.agent,
+            timestamp: new Date().toISOString(),
+            results,
+            summary: {
+                total: results.length,
+                passed,
+                failed,
+                avgScore,
+                totalDurationMs,
+            },
+        };
+        return evalResults;
+    }
+    /**
+     * Get the runner configuration.
+     */
+    get config() {
+        return this._config;
+    }
+}
+/**
+ * Run an evaluation with the given configuration.
+ *
+ * Convenience function that creates a TestRunner and executes it.
+ *
+ * @param config - Runner configuration
+ * @param judge - Judge function for evaluating responses
+ * @param progress - Optional progress callback
+ * @returns Evaluation results
+ */
+export async function runEvaluation(config, judge, progress) {
+    const runner = new TestRunner(config);
+    return runner.run(judge, progress);
+}
+//# sourceMappingURL=runner.js.map

package/dist/src/runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/runner.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AA2CjD;;;;;;;;;;;;GAYG;AACH,KAAK,UAAU,OAAO,CACpB,IAAc,EACd,KAAgB,EAChB,KAAa,EACb,OAAe,EACf,KAIE,EACF,QAAsC,EACtC,KAAa,EACb,KAAa;IAEb,QAAQ,EAAE,WAAW,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IAE7C,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,qCAAqC;IACrC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;QAClB,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;IAC5C,CAAC;IACD,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;QACjB,KAAK,CAAC,IAAI,CAAC,mBAAmB,EAAE,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;IACpD,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;IAChC,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAEhC,IAAI,CAAC;QACH,gBAAgB;QAChB,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAC7B,KAAyB,EACzB,MAAM,EACN,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,CAChC,CAAC;QAEF,qBAAqB;QACrB,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,GAAG,MAAM,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;QAExE,MAAM,MAAM,GAAe;YACzB,MAAM,EAAE,IAAI,CAAC,EAAE;YACf,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,cAAc,EAAE,IAAI,CAAC,cAAc;YACnC,YAAY,EAAE,QAAQ,CAAC,MAAM;YAC7B,UAAU,EAAE,KAAK;YACjB,cAAc,EAAE,SAAS;YACzB,UAAU,EAAE,QAAQ,CAAC,UAAU;YAC/B,MAAM;SACP,CAAC;QAEF,QAAQ,EAAE,cAAc,CAAC,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;QAC/C,OAAO,MAAM,CAAC;IAChB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;QAC1C,MAAM,YAAY,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAE5E,QAAQ,EAAE,WAAW,CAAC,IAAI,CAAC,EAAE,EAAE,YAAY,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;QAE3D,OAAO;YACL,MAAM,EAAE,IAAI,CAAC,EAAE;YACf,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,cAAc,EAAE,IAAI,CAAC,cAAc;YACnC,YAAY,EAAE,EAAE;YAChB,UAAU,EAAE,CAAC;YACb,cAAc,EAAE,UAAU,YAAY,EAAE;YACxC,UAAU;YACV,MAAM,EAAE,KAAK;YACb,KAAK,EAAE,YAAY;SACpB,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;;;;;;GAOG;AACH,KAAK,UAAU,aAAa,CAC1B,KAAU,EACV,WAAmB,EACnB,MAA8C;IAE9C,IAAI,WAAW,IAAI,CAAC,EAAE,CAAC;QACrB,uBAAuB;QACvB,MAAM,OAAO,GAAQ,EAAE,CAAC;QACxB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,OAAO,CAAC,IAAI,CAAC,MAAM,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAC1C,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,4CAA4C;IAC5C,MAAM,OAAO,GAAG,IAAI,KAAK,CAAI,KAAK,CAAC,MAAM,CAAC,CAAC;IAC3C,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,KAAK,UAAU,MAAM;QACnB,OAAO,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YAC5B,MAAM,YAAY,GAAG,KAAK,EAAE,CAAC;YAC7B,OAAO,CAAC,YAAY,CAAC,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,EAAE,YAAY,CAAC,CAAC;QAC1E,CAAC;IACH,CAAC;IAED,gBAAgB;IAChB,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;SACvD,IAAI,CAAC,IAAI,CAAC;SACV,GAAG,CAAC,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC;IAEvB,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAE3B,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;GAEG;AACH,MAAM,OAAO,UAAU;IACb,OAAO,CAAe;IAE9B,YAAY,MAAoB;QAC9B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;IACxB,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,GAAG,CACP,KAIE,EACF,QAA2B;QAE3B,sBAAsB;QACtB,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAEhF,gBAAgB;QAChB,MAAM,OAAO,GAAG,MAAM,aAAa,CACjC,KAAK,CAAC,KAAK,EACX,IAAI,CAAC,OAAO,CAAC,QAAQ,EACrB,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE;YACpB,OAAO,OAAO,CACZ,IAAI,EACJ,KAAK,EACL,IAAI,CAAC,OAAO,CAAC,KAAK,EAClB,IAAI,CAAC,OAAO,CAAC,OAAO,EACpB,KAAK,EACL,QAAQ,EACR,KAAK,EACL,KAAK,CAAC,KAAK,CAAC,MAAM,CACnB,CAAC;QACJ,CAAC,CACF,CAAC;QAEF,oBAAoB;QACpB,MAAM,eAAe,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;QAC1E,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;QACpD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC;QACvC,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;QAEpF,MAAM,WAAW,GAAgB;YAC/B,KAAK;YACL,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK;YACzB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,OAAO;YACP,OAAO,EAAE;gBACP,KAAK,EAAE,OAAO,CAAC,MAAM;gBACrB,MAAM;gBACN,MAAM;gBACN,QAAQ;gBACR,eAAe;aAChB;SACF,CAAC;QAEF,OAAO,WAAW,CAAC;IACrB,CAAC;IAED;;OAEG;IACH,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;CACF;AAED;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,MAAoB,EACpB,KAIE,EACF,QAA2B;IAE3B,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,MAAM,CAAC,CAAC;IACtC,OAAO,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;AACrC,CAAC"}

package/dist/src/test-loader.d.ts

@@ -0,0 +1,66 @@
+/**
+ * Test suite loading and validation from YAML files.
+ *
+ * @module test-loader
+ */
+import { z } from 'zod';
+import type { TestSuite } from './types.js';
+/**
+ * Result of loading a test suite.
+ */
+export interface LoadResult {
+    /** The loaded test suite */
+    suite: TestSuite;
+    /** Absolute path to the test file */
+    path: string;
+}
+/**
+ * Error thrown when test file validation fails.
+ */
+export declare class TestValidationError extends Error {
+    readonly path: string;
+    readonly issues: z.ZodIssue[];
+    constructor(message: string, path: string, issues: z.ZodIssue[]);
+}
+/**
+ * Error thrown when a test file cannot be read.
+ */
+export declare class TestLoadError extends Error {
+    readonly path: string;
+    readonly cause?: unknown | undefined;
+    constructor(message: string, path: string, cause?: unknown | undefined);
+}
+/**
+ * Load and validate a test suite from a YAML file.
+ *
+ * @param filePath - Path to the YAML test file (relative or absolute)
+ * @param cwd - Working directory for resolving relative paths
+ * @returns The loaded and validated test suite
+ * @throws TestLoadError if the file cannot be read
+ * @throws TestValidationError if the YAML content is invalid
+ *
+ * @example
+ * ```typescript
+ * const { suite, path } = await loadTestSuite('./tests.yaml', process.cwd());
+ * console.log(`Loaded ${suite.tests.length} tests from ${path}`);
+ * ```
+ */
+export declare function loadTestSuite(filePath: string, cwd?: string): Promise<LoadResult>;
+/**
+ * Validate a test suite object without loading from file.
+ *
+ * @param data - The data to validate
+ * @returns The validated test suite
+ * @throws TestValidationError if validation fails
+ */
+export declare function validateTestSuite(data: unknown): TestSuite;
+/**
+ * Parse a test suite from a YAML string.
+ *
+ * @param yamlString - YAML content to parse
+ * @returns The parsed and validated test suite
+ * @throws TestLoadError if YAML parsing fails
+ * @throws TestValidationError if validation fails
+ */
+export declare function parseTestSuite(yamlString: string): TestSuite;
+//# sourceMappingURL=test-loader.d.ts.map

package/dist/src/test-loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"test-loader.d.ts","sourceRoot":"","sources":["../../src/test-loader.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAMH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,KAAK,EAAY,SAAS,EAAE,MAAM,YAAY,CAAC;AAsBtD;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,4BAA4B;IAC5B,KAAK,EAAE,SAAS,CAAC;IAEjB,qCAAqC;IACrC,IAAI,EAAE,MAAM,CAAC;CACd;AAED;;GAEG;AACH,qBAAa,mBAAoB,SAAQ,KAAK;aAG1B,IAAI,EAAE,MAAM;aACZ,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;gBAFpC,OAAO,EAAE,MAAM,EACC,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;CAKvC;AAED;;GAEG;AACH,qBAAa,aAAc,SAAQ,KAAK;aAGpB,IAAI,EAAE,MAAM;aACZ,KAAK,CAAC,EAAE,OAAO;gBAF/B,OAAO,EAAE,MAAM,EACC,IAAI,EAAE,MAAM,EACZ,KAAK,CAAC,EAAE,OAAO,YAAA;CAKlC;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,aAAa,CACjC,QAAQ,EAAE,MAAM,EAChB,GAAG,GAAE,MAAsB,GAC1B,OAAO,CAAC,UAAU,CAAC,CAmDrB;AAED;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,OAAO,GAAG,SAAS,CAwB1D;AAED;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS,CAa5D"}

package/dist/src/test-loader.js

@@ -0,0 +1,140 @@
+/**
+ * Test suite loading and validation from YAML files.
+ *
+ * @module test-loader
+ */
+import { readFile } from 'node:fs/promises';
+import { resolve } from 'node:path';
+import YAML from 'yaml';
+import { z } from 'zod';
+/**
+ * Zod schema for validating a test case.
+ */
+const testCaseSchema = z.object({
+    id: z.string().min(1, 'Test case ID is required'),
+    input: z.string().min(1, 'Test case input is required'),
+    context: z.string().optional(),
+    expectedOutput: z.string().min(1, 'Test case expectedOutput is required'),
+});
+/**
+ * Zod schema for validating a test suite.
+ */
+const testSuiteSchema = z.object({
+    name: z.string().min(1, 'Test suite name is required'),
+    description: z.string().optional(),
+    context: z.string().optional(),
+    tests: z.array(testCaseSchema).min(1, 'At least one test case is required'),
+});
+/**
+ * Error thrown when test file validation fails.
+ */
+export class TestValidationError extends Error {
+    path;
+    issues;
+    constructor(message, path, issues) {
+        super(message);
+        this.path = path;
+        this.issues = issues;
+        this.name = 'TestValidationError';
+    }
+}
+/**
+ * Error thrown when a test file cannot be read.
+ */
+export class TestLoadError extends Error {
+    path;
+    cause;
+    constructor(message, path, cause) {
+        super(message);
+        this.path = path;
+        this.cause = cause;
+        this.name = 'TestLoadError';
+    }
+}
+/**
+ * Load and validate a test suite from a YAML file.
+ *
+ * @param filePath - Path to the YAML test file (relative or absolute)
+ * @param cwd - Working directory for resolving relative paths
+ * @returns The loaded and validated test suite
+ * @throws TestLoadError if the file cannot be read
+ * @throws TestValidationError if the YAML content is invalid
+ *
+ * @example
+ * ```typescript
+ * const { suite, path } = await loadTestSuite('./tests.yaml', process.cwd());
+ * console.log(`Loaded ${suite.tests.length} tests from ${path}`);
+ * ```
+ */
+export async function loadTestSuite(filePath, cwd = process.cwd()) {
+    const absolutePath = resolve(cwd, filePath);
+    let content;
+    try {
+        content = await readFile(absolutePath, 'utf-8');
+    }
+    catch (error) {
+        throw new TestLoadError(`Failed to read test file: ${absolutePath}`, absolutePath, error);
+    }
+    let parsed;
+    try {
+        parsed = YAML.parse(content);
+    }
+    catch (error) {
+        throw new TestLoadError(`Failed to parse YAML: ${error instanceof Error ? error.message : String(error)}`, absolutePath, error);
+    }
+    const result = testSuiteSchema.safeParse(parsed);
+    if (!result.success) {
+        throw new TestValidationError(`Test suite validation failed: ${result.error.message}`, absolutePath, result.error.issues);
+    }
+    const validated = result.data;
+    // Check for duplicate test IDs
+    const ids = validated.tests.map((t) => t.id);
+    const duplicates = ids.filter((id, index) => ids.indexOf(id) !== index);
+    if (duplicates.length > 0) {
+        throw new TestValidationError(`Duplicate test IDs found: ${[...new Set(duplicates)].join(', ')}`, absolutePath, []);
+    }
+    return {
+        suite: validated,
+        path: absolutePath,
+    };
+}
+/**
+ * Validate a test suite object without loading from file.
+ *
+ * @param data - The data to validate
+ * @returns The validated test suite
+ * @throws TestValidationError if validation fails
+ */
+export function validateTestSuite(data) {
+    const result = testSuiteSchema.safeParse(data);
+    if (!result.success) {
+        throw new TestValidationError(`Test suite validation failed: ${result.error.message}`, '<inline>', result.error.issues);
+    }
+    const validated = result.data;
+    // Check for duplicate test IDs
+    const ids = validated.tests.map((t) => t.id);
+    const duplicates = ids.filter((id, index) => ids.indexOf(id) !== index);
+    if (duplicates.length > 0) {
+        throw new TestValidationError(`Duplicate test IDs found: ${[...new Set(duplicates)].join(', ')}`, '<inline>', []);
+    }
+    return validated;
+}
+/**
+ * Parse a test suite from a YAML string.
+ *
+ * @param yamlString - YAML content to parse
+ * @returns The parsed and validated test suite
+ * @throws TestLoadError if YAML parsing fails
+ * @throws TestValidationError if validation fails
+ */
+export function parseTestSuite(yamlString) {
+    let parsed;
+    try {
+        parsed = YAML.parse(yamlString);
+    }
+    catch (error) {
+        throw new TestLoadError(`Failed to parse YAML: ${error instanceof Error ? error.message : String(error)}`, '<string>', error);
+    }
+    return validateTestSuite(parsed);
+}
+//# sourceMappingURL=test-loader.js.map

package/dist/src/test-loader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"test-loader.js","sourceRoot":"","sources":["../../src/test-loader.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB;;GAEG;AACH,MAAM,cAAc,GAAG,CAAC,CAAC,MAAM,CAAC;IAC9B,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,0BAA0B,CAAC;IACjD,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,6BAA6B,CAAC;IACvD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC9B,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,sCAAsC,CAAC;CAC1E,CAAC,CAAC;AAEH;;GAEG;AACH,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC;IAC/B,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,6BAA6B,CAAC;IACtD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAClC,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC9B,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,oCAAoC,CAAC;CAC5E,CAAC,CAAC;AAaH;;GAEG;AACH,MAAM,OAAO,mBAAoB,SAAQ,KAAK;IAG1B;IACA;IAHlB,YACE,OAAe,EACC,IAAY,EACZ,MAAoB;QAEpC,KAAK,CAAC,OAAO,CAAC,CAAC;QAHC,SAAI,GAAJ,IAAI,CAAQ;QACZ,WAAM,GAAN,MAAM,CAAc;QAGpC,IAAI,CAAC,IAAI,GAAG,qBAAqB,CAAC;IACpC,CAAC;CACF;AAED;;GAEG;AACH,MAAM,OAAO,aAAc,SAAQ,KAAK;IAGpB;IACA;IAHlB,YACE,OAAe,EACC,IAAY,EACZ,KAAe;QAE/B,KAAK,CAAC,OAAO,CAAC,CAAC;QAHC,SAAI,GAAJ,IAAI,CAAQ;QACZ,UAAK,GAAL,KAAK,CAAU;QAG/B,IAAI,CAAC,IAAI,GAAG,eAAe,CAAC;IAC9B,CAAC;CACF;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,QAAgB,EAChB,MAAc,OAAO,CAAC,GAAG,EAAE;IAE3B,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAE5C,IAAI,OAAe,CAAC;IACpB,IAAI,CAAC;QACH,OAAO,GAAG,MAAM,QAAQ,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;IAClD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,aAAa,CACrB,6BAA6B,YAAY,EAAE,EAC3C,YAAY,EACZ,KAAK,CACN,CAAC;IACJ,CAAC;IAED,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC/B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,aAAa,CACrB,yBAAyB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,EACjF,YAAY,EACZ,KAAK,CACN,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,eAAe,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IACjD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,IAAI,mBAAmB,CAC3B,iCAAiC,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,EACvD,YAAY,EACZ,MAAM,CAAC,KAAK,CAAC,MAAM,CACpB,CAAC;IACJ,CAAC;IAED,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC;IAE9B,+BAA+B;IAC/B,MAAM,GAAG,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAW,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACvD,MAAM,UAAU,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,EAAU,EAAE,KAAa,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,KAAK,KAAK,CAAC,CAAC;IACxF,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,mBAAmB,CAC3B,6BAA6B,CAAC,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAClE,YAAY,EACZ,EAAE,CACH,CAAC;IACJ,CAAC;IAED,OAAO;QACL,KAAK,EAAE,SAAS;QAChB,IAAI,EAAE,YAAY;KACnB,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAAa;IAC7C,MAAM,MAAM,GAAG,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IAC/C,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,IAAI,mBAAmB,CAC3B,iCAAiC,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,EACvD,UAAU,EACV,MAAM,CAAC,KAAK,CAAC,MAAM,CACpB,CAAC;IACJ,CAAC;IAED,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC;IAE9B,+BAA+B;IAC/B,MAAM,GAAG,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAW,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACvD,MAAM,UAAU,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,EAAU,EAAE,KAAa,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,KAAK,KAAK,CAAC,CAAC;IACxF,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,mBAAmB,CAC3B,6BAA6B,CAAC,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAClE,UAAU,EACV,EAAE,CACH,CAAC;IACJ,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,cAAc,CAAC,UAAkB;IAC/C,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IAClC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,aAAa,CACrB,yBAAyB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,EACjF,UAAU,EACV,KAAK,CACN,CAAC;IACJ,CAAC;IAED,OAAO,iBAAiB,CAAC,MAAM,CAAC,CAAC;AACnC,CAAC"}

package/dist/src/types.d.ts

@@ -0,0 +1,161 @@
+/**
+ * Core type definitions for the agent evaluation harness.
+ *
+ * @module types
+ */
+/**
+ * A single test case for evaluating an agent.
+ */
+export interface TestCase {
+    /** Unique identifier for this test case */
+    id: string;
+    /** The input prompt to send to the agent */
+    input: string;
+    /** Optional additional context for this specific test case */
+    context?: string;
+    /** Natural language description of expected output */
+    expectedOutput: string;
+}
+/**
+ * A suite of test cases for evaluation.
+ */
+export interface TestSuite {
+    /** Name of the test suite */
+    name: string;
+    /** Optional description of what this suite tests */
+    description?: string;
+    /** Optional global context available to all tests in this suite */
+    context?: string;
+    /** Array of test cases */
+    tests: TestCase[];
+}
+/**
+ * Response from running a CLI agent.
+ */
+export interface AgentResponse {
+    /** The actual output from the agent */
+    output: string;
+    /** Exit code from the process */
+    exitCode: number;
+    /** Any stderr output */
+    stderr: string;
+    /** Duration in milliseconds */
+    durationMs: number;
+}
+/**
+ * Result of running a single test case.
+ */
+export interface TestResult {
+    /** Test case identifier */
+    testId: string;
+    /** Input that was sent to the agent */
+    input: string;
+    /** Expected output description */
+    expectedOutput: string;
+    /** Actual output from the agent */
+    actualOutput: string;
+    /** Judge score from 0 to 1 */
+    judgeScore: number;
+    /** Judge's reasoning for the score */
+    judgeReasoning: string;
+    /** Duration in milliseconds */
+    durationMs: number;
+    /** Whether the test passed (score >= threshold) */
+    passed: boolean;
+    /** Any error that occurred during execution */
+    error?: string;
+}
+/**
+ * Summary statistics for an evaluation run.
+ */
+export interface EvalSummary {
+    /** Total number of tests */
+    total: number;
+    /** Number of tests that passed */
+    passed: number;
+    /** Number of tests that failed */
+    failed: number;
+    /** Average judge score across all tests */
+    avgScore: number;
+    /** Total duration in milliseconds */
+    totalDurationMs: number;
+}
+/**
+ * Complete results from an evaluation run.
+ */
+export interface EvalResults {
+    /** The test suite that was run */
+    suite: TestSuite;
+    /** Agent name that was tested */
+    agent: string;
+    /** Timestamp of the evaluation */
+    timestamp: string;
+    /** Individual test results */
+    results: TestResult[];
+    /** Summary statistics */
+    summary: EvalSummary;
+}
+/**
+ * Configuration for the judge LLM.
+ */
+export interface JudgeConfig {
+    /** Provider name (e.g., 'anthropic', 'openai') */
+    provider: string;
+    /** Model identifier */
+    model: string;
+    /** Score threshold for passing (0-1, default: 0.7) */
+    passThreshold?: number;
+    /** Temperature for judge LLM (default: 0) */
+    temperature?: number;
+}
+/**
+ * Configuration for the test runner.
+ */
+export interface RunnerConfig {
+    /** Path to test file */
+    testsPath: string;
+    /** Agent to test: 'pi', 'genesys', or any custom command */
+    agent: string;
+    /** Working directory */
+    cwd: string;
+    /** Timeout in milliseconds (default: 120000) */
+    timeout: number;
+    /** Output file path (optional) */
+    outputPath?: string;
+    /** Output format */
+    format: 'console' | 'json' | 'html';
+    /** Number of parallel test executions */
+    parallel: number;
+    /** Judge configuration */
+    judge: JudgeConfig;
+}
+/**
+ * CLI argument structure.
+ */
+export interface Args {
+    /** Path to YAML test file */
+    tests: string;
+    /** Agent to test: 'pi' or 'genesys' */
+    agent: string;
+    /** Working directory */
+    cwd: string;
+    /** Timeout in seconds */
+    timeout: number;
+    /** Output file path */
+    output?: string;
+    /** Output format */
+    format: 'console' | 'json' | 'html';
+    /** Parallelism level */
+    parallel: number;
+    /** Judge type: 'embedding' (default) or 'llm' */
+    judgeType: 'embedding' | 'llm';
+    /** Judge model (for LLM judge) */
+    judgeModel: string;
+    /** Judge provider (for LLM judge) */
+    judgeProvider: string;
+    /** Show help */
+    help: boolean;
+    /** Show version */
+    version: boolean;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/src/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/types.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH;;GAEG;AACH,MAAM,WAAW,QAAQ;IACvB,2CAA2C;IAC3C,EAAE,EAAE,MAAM,CAAC;IAEX,4CAA4C;IAC5C,KAAK,EAAE,MAAM,CAAC;IAEd,8DAA8D;IAC9D,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,sDAAsD;IACtD,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,6BAA6B;IAC7B,IAAI,EAAE,MAAM,CAAC;IAEb,oDAAoD;IACpD,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB,mEAAmE;IACnE,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,0BAA0B;IAC1B,KAAK,EAAE,QAAQ,EAAE,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,uCAAuC;IACvC,MAAM,EAAE,MAAM,CAAC;IAEf,iCAAiC;IACjC,QAAQ,EAAE,MAAM,CAAC;IAEjB,wBAAwB;IACxB,MAAM,EAAE,MAAM,CAAC;IAEf,+BAA+B;IAC/B,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,2BAA2B;IAC3B,MAAM,EAAE,MAAM,CAAC;IAEf,uCAAuC;IACvC,KAAK,EAAE,MAAM,CAAC;IAEd,kCAAkC;IAClC,cAAc,EAAE,MAAM,CAAC;IAEvB,mCAAmC;IACnC,YAAY,EAAE,MAAM,CAAC;IAErB,8BAA8B;IAC9B,UAAU,EAAE,MAAM,CAAC;IAEnB,sCAAsC;IACtC,cAAc,EAAE,MAAM,CAAC;IAEvB,+BAA+B;IAC/B,UAAU,EAAE,MAAM,CAAC;IAEnB,mDAAmD;IACnD,MAAM,EAAE,OAAO,CAAC;IAEhB,+CAA+C;IAC/C,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,4BAA4B;IAC5B,KAAK,EAAE,MAAM,CAAC;IAEd,kCAAkC;IAClC,MAAM,EAAE,MAAM,CAAC;IAEf,kCAAkC;IAClC,MAAM,EAAE,MAAM,CAAC;IAEf,2CAA2C;IAC3C,QAAQ,EAAE,MAAM,CAAC;IAEjB,qCAAqC;IACrC,eAAe,EAAE,MAAM,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,kCAAkC;IAClC,KAAK,EAAE,SAAS,CAAC;IAEjB,iCAAiC;IACjC,KAAK,EAAE,MAAM,CAAC;IAEd,kCAAkC;IAClC,SAAS,EAAE,MAAM,CAAC;IAElB,8BAA8B;IAC9B,OAAO,EAAE,UAAU,EAAE,CAAC;IAEtB,yBAAyB;IACzB,OAAO,EAAE,WAAW,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,kDAAkD;IAClD,QAAQ,EAAE,MAAM,CAAC;IAEjB,uBAAuB;IACvB,KAAK,EAAE,MAAM,CAAC;IAEd,sDAAsD;IACtD,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB,6CAA6C;IAC7C,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,wBAAwB;IACxB,SAAS,EAAE,MAAM,CAAC;IAElB,4DAA4D;IAC5D,KAAK,EAAE,MAAM,CAAC;IAEd,wBAAwB;IACxB,GAAG,EAAE,MAAM,CAAC;IAEZ,gDAAgD;IAChD,OAAO,EAAE,MAAM,CAAC;IAEhB,kCAAkC;IAClC,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,oBAAoB;IACpB,MAAM,EAAE,SAAS,GAAG,MAAM,GAAG,MAAM,CAAC;IAEpC,yCAAyC;IACzC,QAAQ,EAAE,MAAM,CAAC;IAEjB,0BAA0B;IAC1B,KAAK,EAAE,WAAW,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,IAAI;IACnB,6BAA6B;IAC7B,KAAK,EAAE,MAAM,CAAC;IAEd,uCAAuC;IACvC,KAAK,EAAE,MAAM,CAAC;IAEd,wBAAwB;IACxB,GAAG,EAAE,MAAM,CAAC;IAEZ,yBAAyB;IACzB,OAAO,EAAE,MAAM,CAAC;IAEhB,uBAAuB;IACvB,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB,oBAAoB;IACpB,MAAM,EAAE,SAAS,GAAG,MAAM,GAAG,MAAM,CAAC;IAEpC,wBAAwB;IACxB,QAAQ,EAAE,MAAM,CAAC;IAEjB,iDAAiD;IACjD,SAAS,EAAE,WAAW,GAAG,KAAK,CAAC;IAE/B,kCAAkC;IAClC,UAAU,EAAE,MAAM,CAAC;IAEnB,qCAAqC;IACrC,aAAa,EAAE,MAAM,CAAC;IAEtB,gBAAgB;IAChB,IAAI,EAAE,OAAO,CAAC;IAEd,mBAAmB;IACnB,OAAO,EAAE,OAAO,CAAC;CAClB"}

package/dist/src/types.js

@@ -0,0 +1,7 @@
+/**
+ * Core type definitions for the agent evaluation harness.
+ *
+ * @module types
+ */
+export {};
+//# sourceMappingURL=types.js.map

package/dist/src/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/types.ts"],"names":[],"mappings":"AAAA;;;;GAIG"}

package/dist/src/utils/package.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Package metadata utilities.
+ *
+ * @module utils/package
+ */
+/**
+ * Get package.json contents.
+ *
+ * @returns Package.json as an object
+ */
+export declare function getPackageJson(): {
+    version: string;
+    name: string;
+    [key: string]: unknown;
+};
+//# sourceMappingURL=package.d.ts.map

package/dist/src/utils/package.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"package.d.ts","sourceRoot":"","sources":["../../../src/utils/package.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAMH;;;;GAIG;AACH,wBAAgB,cAAc,IAAI;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE,CAe1F"}