@ozperium/agentspec 0.1.0

package/dist/diff.js

@@ -0,0 +1,167 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ensureStoreDir = ensureStoreDir;
+exports.saveRun = saveRun;
+exports.loadPreviousRun = loadPreviousRun;
+exports.computeDiffs = computeDiffs;
+exports.formatDiffs = formatDiffs;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const STORE_DIR = '.agentspec';
+function ensureStoreDir() {
+    const dir = path.join(process.cwd(), STORE_DIR);
+    if (!fs.existsSync(dir)) {
+        fs.mkdirSync(dir, { recursive: true });
+    }
+    return dir;
+}
+function saveRun(results) {
+    const dir = ensureStoreDir();
+    const timestamp = new Date().toISOString();
+    for (const result of results) {
+        const file = path.join(dir, `${sanitize(result.suite)}__${sanitize(result.test)}.json`);
+        const stored = {
+            test: result.test,
+            suite: result.suite,
+            passed: result.passed,
+            output: result.output || '',
+            timestamp,
+        };
+        fs.writeFileSync(file, JSON.stringify(stored, null, 2));
+    }
+}
+function loadPreviousRun(suite, test) {
+    const dir = ensureStoreDir();
+    const file = path.join(dir, `${sanitize(suite)}__${sanitize(test)}.json`);
+    if (!fs.existsSync(file))
+        return null;
+    try {
+        return JSON.parse(fs.readFileSync(file, 'utf-8'));
+    }
+    catch {
+        return null;
+    }
+}
+function computeDiffs(currentResults) {
+    const diffs = [];
+    for (const result of currentResults) {
+        const previous = loadPreviousRun(result.suite, result.test);
+        if (!previous) {
+            // New test, no diff
+            continue;
+        }
+        if (previous.output === result.output && previous.passed === result.passed) {
+            // No change
+            continue;
+        }
+        let status;
+        if (!previous.passed && result.passed) {
+            status = 'improvement';
+        }
+        else if (previous.passed && !result.passed) {
+            status = 'regression';
+        }
+        else {
+            status = 'changed';
+        }
+        const summary = generateDiffSummary(previous.output, result.output || '', status);
+        diffs.push({
+            test: result.test,
+            suite: result.suite,
+            status,
+            old_output: previous.output,
+            new_output: result.output || '',
+            old_passed: previous.passed,
+            new_passed: result.passed,
+            summary,
+        });
+    }
+    return diffs;
+}
+function formatDiffs(diffs) {
+    if (diffs.length === 0) {
+        return '  No behavior changes detected.';
+    }
+    const lines = [''];
+    lines.push('  AgentSpec — Behavior Diff Report');
+    lines.push('  ' + '═'.repeat(50));
+    for (const diff of diffs) {
+        const icon = diff.status === 'regression' ? '⚠️' : diff.status === 'improvement' ? '✨' : '📝';
+        const statusLabel = diff.status.toUpperCase();
+        lines.push(`  ${icon} ${statusLabel} ${diff.suite} > ${diff.test}`);
+        lines.push(`  ${'─'.repeat(50)}`);
+        lines.push(`  ${diff.summary}`);
+        lines.push('');
+    }
+    const regressions = diffs.filter(d => d.status === 'regression').length;
+    const improvements = diffs.filter(d => d.status === 'improvement').length;
+    const changed = diffs.filter(d => d.status === 'changed').length;
+    lines.push('  ' + '═'.repeat(50));
+    lines.push(`  ${regressions} regression(s), ${improvements} improvement(s), ${changed} changed`);
+    lines.push('');
+    return lines.join('\n');
+}
+function generateDiffSummary(oldOutput, newOutput, status) {
+    const oldWords = oldOutput.toLowerCase().split(/\s+/).filter(w => w.length > 2);
+    const newWords = newOutput.toLowerCase().split(/\s+/).filter(w => w.length > 2);
+    const oldSet = new Set(oldWords);
+    const newSet = new Set(newWords);
+    const removed = oldWords.filter(w => !newSet.has(w));
+    const added = newWords.filter(w => !oldSet.has(w));
+    const lines = [];
+    if (status === 'regression') {
+        lines.push('  ⚠  Behavior REGRESSED — test was passing, now failing');
+    }
+    else if (status === 'improvement') {
+        lines.push('  ✨  Behavior IMPROVED — test was failing, now passing');
+    }
+    else {
+        lines.push('  📝  Behavior CHANGED — output differs but pass/fail status unchanged');
+    }
+    if (added.length > 0) {
+        lines.push(`  + added: ${added.slice(0, 10).join(', ')}${added.length > 10 ? '...' : ''}`);
+    }
+    if (removed.length > 0) {
+        lines.push(`  - removed: ${removed.slice(0, 10).join(', ')}${removed.length > 10 ? '...' : ''}`);
+    }
+    if (added.length === 0 && removed.length === 0) {
+        lines.push('  (pass/fail status changed but output text is identical — likely a metadata assertion)');
+    }
+    return lines.join('\n');
+}
+function sanitize(s) {
+    return s.replace(/[^a-zA-Z0-9_-]/g, '_');
+}

package/dist/http-agent.d.ts

@@ -0,0 +1,14 @@
+import { AgentRunner, AgentOutput, TestSuite } from './runner';
+export interface HTTPAgentConfig {
+    endpoint: string;
+    headers?: Record<string, string>;
+    timeout_ms?: number;
+    input_field?: string;
+    output_field?: string;
+}
+export declare class HTTPAgent implements AgentRunner {
+    config: HTTPAgentConfig;
+    constructor(config: HTTPAgentConfig);
+    run(input: string, suite: TestSuite): Promise<AgentOutput>;
+    private httpRequest;
+}

package/dist/http-agent.js

@@ -0,0 +1,102 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.HTTPAgent = void 0;
+const http = __importStar(require("http"));
+const https = __importStar(require("https"));
+const url_1 = require("url");
+class HTTPAgent {
+    config;
+    constructor(config) {
+        this.config = config;
+    }
+    async run(input, suite) {
+        const endpoint = this.config.endpoint;
+        const timeout = this.config.timeout_ms || 60000;
+        const inputField = this.config.input_field || 'input';
+        const body = JSON.stringify({ [inputField]: input, ...(suite.agent || {}) });
+        const headers = {
+            'Content-Type': 'application/json',
+            ...this.config.headers,
+        };
+        try {
+            const response = await this.httpRequest(endpoint, body, headers, timeout);
+            const outputField = this.config.output_field || 'output';
+            const text = response[outputField] || response.text || response.response || JSON.stringify(response);
+            return {
+                text: typeof text === 'string' ? text : JSON.stringify(text),
+                tokens: response.tokens || response.usage?.total_tokens,
+                latencyMs: response.latency_ms,
+                toolsCalled: response.tools_called || response.toolsCalled || [],
+            };
+        }
+        catch (e) {
+            throw new Error(`HTTP agent error: ${e instanceof Error ? e.message : String(e)}`);
+        }
+    }
+    httpRequest(url, body, headers, timeoutMs) {
+        return new Promise((resolve, reject) => {
+            const parsed = new url_1.URL(url);
+            const isHttps = parsed.protocol === 'https:';
+            const lib = isHttps ? https : http;
+            const req = lib.request({
+                hostname: parsed.hostname,
+                port: parsed.port || (isHttps ? 443 : 80),
+                path: parsed.pathname + parsed.search,
+                method: 'POST',
+                headers: { ...headers, 'Content-Length': Buffer.byteLength(body) },
+            }, (res) => {
+                let data = '';
+                res.on('data', (chunk) => (data += chunk));
+                res.on('end', () => {
+                    try {
+                        resolve(JSON.parse(data));
+                    }
+                    catch {
+                        // Non-JSON response — return as text
+                        resolve({ output: data });
+                    }
+                });
+            });
+            req.on('error', reject);
+            req.setTimeout(timeoutMs, () => {
+                req.destroy(new Error('Agent request timeout'));
+            });
+            req.write(body);
+            req.end();
+        });
+    }
+}
+exports.HTTPAgent = HTTPAgent;

package/dist/index.d.ts

@@ -0,0 +1,10 @@
+export { runAll, runSuite, MockAgent } from './runner';
+export { AgentRunner, AgentOutput } from './runner';
+export { HTTPAgent, HTTPAgentConfig } from './http-agent';
+export { evaluateAssertions, evaluateAssertionsAsync, setLLMJudgeConfig, getLLMJudgeConfig } from './assertions';
+export { llmJudge, LLMJudgeConfig } from './llm-judge';
+export { loadTestSuite, findTestSuites } from './loader';
+export { formatResults, formatResultsJSON, formatResultsJUnit } from './reporter';
+export { saveRun, computeDiffs, formatDiffs } from './diff';
+export * from './types';
+export * from './types/diff';

package/dist/index.js

@@ -0,0 +1,43 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.formatDiffs = exports.computeDiffs = exports.saveRun = exports.formatResultsJUnit = exports.formatResultsJSON = exports.formatResults = exports.findTestSuites = exports.loadTestSuite = exports.llmJudge = exports.getLLMJudgeConfig = exports.setLLMJudgeConfig = exports.evaluateAssertionsAsync = exports.evaluateAssertions = exports.HTTPAgent = exports.MockAgent = exports.runSuite = exports.runAll = void 0;
+var runner_1 = require("./runner");
+Object.defineProperty(exports, "runAll", { enumerable: true, get: function () { return runner_1.runAll; } });
+Object.defineProperty(exports, "runSuite", { enumerable: true, get: function () { return runner_1.runSuite; } });
+Object.defineProperty(exports, "MockAgent", { enumerable: true, get: function () { return runner_1.MockAgent; } });
+var http_agent_1 = require("./http-agent");
+Object.defineProperty(exports, "HTTPAgent", { enumerable: true, get: function () { return http_agent_1.HTTPAgent; } });
+var assertions_1 = require("./assertions");
+Object.defineProperty(exports, "evaluateAssertions", { enumerable: true, get: function () { return assertions_1.evaluateAssertions; } });
+Object.defineProperty(exports, "evaluateAssertionsAsync", { enumerable: true, get: function () { return assertions_1.evaluateAssertionsAsync; } });
+Object.defineProperty(exports, "setLLMJudgeConfig", { enumerable: true, get: function () { return assertions_1.setLLMJudgeConfig; } });
+Object.defineProperty(exports, "getLLMJudgeConfig", { enumerable: true, get: function () { return assertions_1.getLLMJudgeConfig; } });
+var llm_judge_1 = require("./llm-judge");
+Object.defineProperty(exports, "llmJudge", { enumerable: true, get: function () { return llm_judge_1.llmJudge; } });
+var loader_1 = require("./loader");
+Object.defineProperty(exports, "loadTestSuite", { enumerable: true, get: function () { return loader_1.loadTestSuite; } });
+Object.defineProperty(exports, "findTestSuites", { enumerable: true, get: function () { return loader_1.findTestSuites; } });
+var reporter_1 = require("./reporter");
+Object.defineProperty(exports, "formatResults", { enumerable: true, get: function () { return reporter_1.formatResults; } });
+Object.defineProperty(exports, "formatResultsJSON", { enumerable: true, get: function () { return reporter_1.formatResultsJSON; } });
+Object.defineProperty(exports, "formatResultsJUnit", { enumerable: true, get: function () { return reporter_1.formatResultsJUnit; } });
+var diff_1 = require("./diff");
+Object.defineProperty(exports, "saveRun", { enumerable: true, get: function () { return diff_1.saveRun; } });
+Object.defineProperty(exports, "computeDiffs", { enumerable: true, get: function () { return diff_1.computeDiffs; } });
+Object.defineProperty(exports, "formatDiffs", { enumerable: true, get: function () { return diff_1.formatDiffs; } });
+__exportStar(require("./types"), exports);
+__exportStar(require("./types/diff"), exports);

package/dist/llm-judge.d.ts

@@ -0,0 +1,11 @@
+export interface LLMJudgeConfig {
+    endpoint?: string;
+    model?: string;
+    api_key?: string;
+    timeout_ms?: number;
+}
+export declare function llmJudge(input: string, output: string, criteria: string, config?: LLMJudgeConfig): Promise<{
+    passed: boolean;
+    score: number;
+    reasoning: string;
+}>;

package/dist/llm-judge.js

@@ -0,0 +1,124 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.llmJudge = llmJudge;
+const http = __importStar(require("http"));
+const https = __importStar(require("https"));
+const url_1 = require("url");
+const DEFAULT_ENDPOINT = 'http://127.0.0.1:11434/v1/chat/completions';
+const DEFAULT_MODEL = 'qwen2.5:7b';
+const DEFAULT_TIMEOUT = 30000;
+async function llmJudge(input, output, criteria, config) {
+    const endpoint = config?.endpoint || DEFAULT_ENDPOINT;
+    const model = config?.model || DEFAULT_MODEL;
+    const timeout = config?.timeout_ms || DEFAULT_TIMEOUT;
+    const apiKey = config?.api_key;
+    const systemPrompt = `You are a test judge. Evaluate whether the AI agent's response meets the criteria.
+
+Input given to agent: "${input}"
+Agent's response: "${output}"
+Criteria: "${criteria}"
+
+Respond with ONLY a JSON object:
+{"passed": true/false, "score": 0.0-1.0, "reasoning": "brief explanation"}`;
+    const body = JSON.stringify({
+        model,
+        messages: [
+            { role: 'system', content: systemPrompt },
+            { role: 'user', content: `Evaluate: does the response meet the criteria?` }
+        ],
+        temperature: 0.1,
+        max_tokens: 200,
+    });
+    try {
+        const response = await httpRequest(endpoint, body, apiKey, timeout);
+        const text = response.choices?.[0]?.message?.content || '';
+        // Parse JSON from response (LLMs sometimes wrap in markdown)
+        const jsonMatch = text.match(/\{[\s\S]*\}/);
+        if (!jsonMatch) {
+            return { passed: false, score: 0, reasoning: `Judge returned non-JSON: ${text.substring(0, 100)}` };
+        }
+        const result = JSON.parse(jsonMatch[0]);
+        return {
+            passed: !!result.passed,
+            score: typeof result.score === 'number' ? result.score : (result.passed ? 1.0 : 0.0),
+            reasoning: result.reasoning || 'No reasoning provided',
+        };
+    }
+    catch (e) {
+        return {
+            passed: false,
+            score: 0,
+            reasoning: `Judge error: ${e instanceof Error ? e.message : String(e)}`,
+        };
+    }
+}
+function httpRequest(url, body, apiKey, timeoutMs) {
+    return new Promise((resolve, reject) => {
+        const parsed = new url_1.URL(url);
+        const isHttps = parsed.protocol === 'https:';
+        const lib = isHttps ? https : http;
+        const headers = {
+            'Content-Type': 'application/json',
+        };
+        if (apiKey) {
+            headers['Authorization'] = `Bearer ${apiKey}`;
+        }
+        const req = lib.request({
+            hostname: parsed.hostname,
+            port: parsed.port || (isHttps ? 443 : 80),
+            path: parsed.pathname + parsed.search,
+            method: 'POST',
+            headers,
+        }, (res) => {
+            let data = '';
+            res.on('data', (chunk) => (data += chunk));
+            res.on('end', () => {
+                try {
+                    resolve(JSON.parse(data));
+                }
+                catch (e) {
+                    reject(new Error(`Invalid JSON response: ${data.substring(0, 200)}`));
+                }
+            });
+        });
+        req.on('error', reject);
+        req.setTimeout(timeoutMs, () => {
+            req.destroy(new Error('Request timeout'));
+        });
+        req.write(body);
+        req.end();
+    });
+}

package/dist/loader.d.ts

@@ -0,0 +1,3 @@
+import { TestSuite } from './types';
+export declare function loadTestSuite(filePath: string): TestSuite;
+export declare function findTestSuites(dir: string): string[];

package/dist/loader.js

@@ -0,0 +1,64 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.loadTestSuite = loadTestSuite;
+exports.findTestSuites = findTestSuites;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const yaml_1 = require("yaml");
+function loadTestSuite(filePath) {
+    const content = fs.readFileSync(filePath, 'utf-8');
+    const parsed = (0, yaml_1.parse)(content);
+    if (!parsed.name || !parsed.tests) {
+        throw new Error(`Invalid test suite: missing name or tests in ${filePath}`);
+    }
+    return parsed;
+}
+function findTestSuites(dir) {
+    if (!fs.existsSync(dir)) {
+        return [];
+    }
+    const files = [];
+    for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+        const fullPath = path.join(dir, entry.name);
+        if (entry.isDirectory()) {
+            files.push(...findTestSuites(fullPath));
+        }
+        else if (entry.name.endsWith('.yaml') || entry.name.endsWith('.yml')) {
+            files.push(fullPath);
+        }
+    }
+    return files;
+}

package/dist/reporter.d.ts

@@ -0,0 +1,4 @@
+import { RunResult } from './types';
+export declare function formatResults(result: RunResult): string;
+export declare function formatResultsJSON(result: RunResult): string;
+export declare function formatResultsJUnit(result: RunResult): string;

package/dist/reporter.js

@@ -0,0 +1,80 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.formatResults = formatResults;
+exports.formatResultsJSON = formatResultsJSON;
+exports.formatResultsJUnit = formatResultsJUnit;
+function formatResults(result) {
+    const lines = [];
+    lines.push('');
+    lines.push('  AgentSpec — Test Results');
+    lines.push('  ' + '─'.repeat(50));
+    for (const test of result.results) {
+        const status = test.passed ? '✓' : '✗';
+        const statusColor = test.passed ? '\x1b[32m' : '\x1b[31m';
+        const resetColor = '\x1b[0m';
+        lines.push(`  ${statusColor}${status}${resetColor} ${test.suite} > ${test.test} (${test.duration_ms}ms)`);
+        if (!test.passed) {
+            if (test.error) {
+                lines.push(`      Error: ${test.error}`);
+            }
+            for (const a of test.assertions) {
+                if (!a.passed) {
+                    lines.push(`      ${a.assertion}: ${a.message}`);
+                    if (a.expected && a.actual) {
+                        lines.push(`        expected: ${a.expected}`);
+                        lines.push(`        actual:   ${a.actual}`);
+                    }
+                }
+            }
+        }
+    }
+    lines.push('  ' + '─'.repeat(50));
+    const totalColor = result.failed > 0 ? '\x1b[31m' : '\x1b[32m';
+    const resetColor2 = '\x1b[0m';
+    lines.push(`  ${totalColor}${result.passed} passed${resetColor2}, ${result.failed} failed (${result.total} total)`);
+    lines.push(`  Duration: ${result.duration_ms}ms`);
+    lines.push('');
+    return lines.join('\n');
+}
+function formatResultsJSON(result) {
+    return JSON.stringify(result, null, 2);
+}
+function formatResultsJUnit(result) {
+    const lines = ['<?xml version="1.0" encoding="UTF-8"?>'];
+    lines.push(`<testsuites tests="${result.total}" failures="${result.failed}" time="${result.duration_ms / 1000}">`);
+    const bySuite = new Map();
+    for (const test of result.results) {
+        if (!bySuite.has(test.suite))
+            bySuite.set(test.suite, []);
+        bySuite.get(test.suite).push(test);
+    }
+    const suiteEntries = Array.from(bySuite.entries());
+    for (const [suiteName, tests] of suiteEntries) {
+        const suiteFailures = tests.filter(t => !t.passed).length;
+        const suiteTime = tests.reduce((sum, t) => sum + t.duration_ms, 0) / 1000;
+        lines.push(`  <testsuite name="${escapeXml(suiteName)}" tests="${tests.length}" failures="${suiteFailures}" time="${suiteTime}">`);
+        for (const test of tests) {
+            lines.push(`    <testcase name="${escapeXml(test.test)}" classname="${escapeXml(suiteName)}" time="${test.duration_ms / 1000}"${test.passed ? ' />' : '>'}`);
+            if (!test.passed) {
+                const failureMsg = test.error || test.assertions.filter(a => !a.passed).map(a => a.message).join('; ');
+                lines.push(`      <failure message="${escapeXml(failureMsg)}">${escapeXml(failureMsg)}</failure>`);
+                lines.push(`    </testcase>`);
+            }
+        }
+        lines.push(`  </testsuite>`);
+    }
+    lines.push('</testsuites>');
+    return lines.join('\n');
+}
+function escapeXml(s) {
+    return s.replace(/[<>&'"]/g, (c) => {
+        switch (c) {
+            case '<': return '&lt;';
+            case '>': return '&gt;';
+            case '&': return '&amp;';
+            case '\'': return '&apos;';
+            case '"': return '&quot;';
+            default: return c;
+        }
+    });
+}

package/dist/runner.d.ts

@@ -0,0 +1,19 @@
+import { TestResult, RunResult, TestSuite } from './types';
+export { TestSuite } from './types';
+export interface AgentRunner {
+    run(input: string, suite: TestSuite): Promise<AgentOutput>;
+}
+export interface AgentOutput {
+    text: string;
+    tokens?: number;
+    latencyMs?: number;
+    toolsCalled?: string[];
+}
+export declare function runSuite(suitePath: string, agent: AgentRunner): Promise<TestResult[]>;
+export declare function runAll(dir: string, agent: AgentRunner, filter?: string | null): Promise<RunResult>;
+export declare class MockAgent implements AgentRunner {
+    responses: Record<string, string>;
+    echo: boolean;
+    constructor(responses?: Record<string, string>, echo?: boolean);
+    run(input: string, suite: TestSuite): Promise<AgentOutput>;
+}