@prompd/test 0.5.0-beta.9

package/dist/EvaluatorEngine.d.ts

@@ -0,0 +1,32 @@
+/**
+ * Routes assertions to the correct evaluator and manages execution order.
+ *
+ * Execution order: nlp -> script -> prmd (cheap to expensive).
+ * Fail-fast by default — stops on first failure unless runAll is set.
+ */
+import type { AssertionDef, AssertionResult, EvaluatorType } from './types';
+import type { EvaluatorContext } from './evaluators/types';
+import type { CompilerModule } from './cli-types';
+export interface EvaluatorEngineOptions {
+    testFileDir: string;
+    evaluatorPrompt?: string;
+    workspaceRoot?: string;
+    registryUrl?: string;
+    allowedEvaluators?: EvaluatorType[];
+    failFast?: boolean;
+    cliModule?: CompilerModule;
+    provider?: string;
+    model?: string;
+}
+export declare class EvaluatorEngine {
+    private evaluators;
+    private allowedEvaluators;
+    private failFast;
+    constructor(options: EvaluatorEngineOptions);
+    /**
+     * Evaluate all assertions in cost-priority order.
+     * Returns results for each assertion.
+     */
+    evaluate(assertions: AssertionDef[], context: EvaluatorContext, onResult?: (result: AssertionResult) => void): Promise<AssertionResult[]>;
+}
+//# sourceMappingURL=EvaluatorEngine.d.ts.map

package/dist/EvaluatorEngine.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"EvaluatorEngine.d.ts","sourceRoot":"","sources":["../src/EvaluatorEngine.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAC5E,OAAO,KAAK,EAAa,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAYlD,MAAM,WAAW,sBAAsB;IACrC,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,iBAAiB,CAAC,EAAE,aAAa,EAAE,CAAC;IACpC,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,SAAS,CAAC,EAAE,cAAc,CAAC;IAC3B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,qBAAa,eAAe;IAC1B,OAAO,CAAC,UAAU,CAAgC;IAClD,OAAO,CAAC,iBAAiB,CAAqB;IAC9C,OAAO,CAAC,QAAQ,CAAU;gBAEd,OAAO,EAAE,sBAAsB;IAqB3C;;;OAGG;IACG,QAAQ,CACZ,UAAU,EAAE,YAAY,EAAE,EAC1B,OAAO,EAAE,gBAAgB,EACzB,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,eAAe,KAAK,IAAI,GAC3C,OAAO,CAAC,eAAe,EAAE,CAAC;CA8D9B"}

package/dist/EvaluatorEngine.js

@@ -0,0 +1,97 @@
+"use strict";
+/**
+ * Routes assertions to the correct evaluator and manages execution order.
+ *
+ * Execution order: nlp -> script -> prmd (cheap to expensive).
+ * Fail-fast by default — stops on first failure unless runAll is set.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.EvaluatorEngine = void 0;
+const NlpEvaluator_1 = require("./evaluators/NlpEvaluator");
+const ScriptEvaluator_1 = require("./evaluators/ScriptEvaluator");
+const PrmdEvaluator_1 = require("./evaluators/PrmdEvaluator");
+/** Execution priority — lower number runs first */
+const EVALUATOR_PRIORITY = {
+    nlp: 0,
+    script: 1,
+    prmd: 2,
+};
+class EvaluatorEngine {
+    constructor(options) {
+        this.failFast = options.failFast !== false;
+        this.allowedEvaluators = new Set(options.allowedEvaluators || ['nlp', 'script', 'prmd']);
+        const prmdOptions = {
+            testFileDir: options.testFileDir,
+            evaluatorPrompt: options.evaluatorPrompt,
+            workspaceRoot: options.workspaceRoot,
+            registryUrl: options.registryUrl,
+            cliModule: options.cliModule,
+            provider: options.provider,
+            model: options.model,
+        };
+        this.evaluators = new Map([
+            ['nlp', new NlpEvaluator_1.NlpEvaluator()],
+            ['script', new ScriptEvaluator_1.ScriptEvaluator(options.testFileDir)],
+            ['prmd', new PrmdEvaluator_1.PrmdEvaluator(prmdOptions)],
+        ]);
+    }
+    /**
+     * Evaluate all assertions in cost-priority order.
+     * Returns results for each assertion.
+     */
+    async evaluate(assertions, context, onResult) {
+        const results = [];
+        // Sort by evaluator priority (nlp first, prmd last)
+        const sorted = [...assertions].sort((a, b) => EVALUATOR_PRIORITY[a.evaluator] - EVALUATOR_PRIORITY[b.evaluator]);
+        for (const assertion of sorted) {
+            // Skip evaluators that aren't allowed
+            if (!this.allowedEvaluators.has(assertion.evaluator)) {
+                const skipped = {
+                    evaluator: assertion.evaluator,
+                    check: assertion.check,
+                    status: 'skip',
+                    reason: `Evaluator type "${assertion.evaluator}" skipped by filter`,
+                    duration: 0,
+                };
+                results.push(skipped);
+                onResult?.(skipped);
+                continue;
+            }
+            const evaluator = this.evaluators.get(assertion.evaluator);
+            if (!evaluator) {
+                const errorResult = {
+                    evaluator: assertion.evaluator,
+                    check: assertion.check,
+                    status: 'error',
+                    reason: `No evaluator registered for type "${assertion.evaluator}"`,
+                    duration: 0,
+                };
+                results.push(errorResult);
+                onResult?.(errorResult);
+                continue;
+            }
+            const result = await evaluator.evaluate(assertion, context);
+            results.push(result);
+            onResult?.(result);
+            // Fail-fast: stop on first failure
+            if (this.failFast && result.status !== 'pass') {
+                // Mark remaining assertions as skipped
+                const remaining = sorted.slice(sorted.indexOf(assertion) + 1);
+                for (const rem of remaining) {
+                    const skipped = {
+                        evaluator: rem.evaluator,
+                        check: rem.check,
+                        status: 'skip',
+                        reason: 'Skipped due to prior failure (fail-fast)',
+                        duration: 0,
+                    };
+                    results.push(skipped);
+                    onResult?.(skipped);
+                }
+                break;
+            }
+        }
+        return results;
+    }
+}
+exports.EvaluatorEngine = EvaluatorEngine;

package/dist/TestDiscovery.d.ts

@@ -0,0 +1,28 @@
+/**
+ * Discovers .test.prmd files and pairs them with their source .prmd files.
+ */
+import type { TestSuite } from './types';
+export interface DiscoveryResult {
+    suites: TestSuite[];
+    errors: DiscoveryError[];
+}
+export interface DiscoveryError {
+    filePath: string;
+    message: string;
+}
+export declare class TestDiscovery {
+    private parser;
+    constructor();
+    /**
+     * Discover test suites from a target path.
+     *
+     * - If targetPath is a .test.prmd file, parse it directly.
+     * - If targetPath is a .prmd file, look for a colocated .test.prmd sidecar.
+     * - If targetPath is a directory, glob for all .test.prmd files recursively.
+     */
+    discover(targetPath: string): Promise<DiscoveryResult>;
+    private discoverDirectory;
+    private discoverTestFile;
+    private discoverFromSource;
+}
+//# sourceMappingURL=TestDiscovery.d.ts.map

package/dist/TestDiscovery.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"TestDiscovery.d.ts","sourceRoot":"","sources":["../src/TestDiscovery.ts"],"names":[],"mappings":"AAAA;;GAEG;AAMH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAEzC,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,SAAS,EAAE,CAAC;IACpB,MAAM,EAAE,cAAc,EAAE,CAAC;CAC1B;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,qBAAa,aAAa;IACxB,OAAO,CAAC,MAAM,CAAa;;IAM3B;;;;;;OAMG;IACG,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;YA+B9C,iBAAiB;YAsBjB,gBAAgB;YA4BhB,kBAAkB;CAiBjC"}

package/dist/TestDiscovery.js

@@ -0,0 +1,137 @@
+"use strict";
+/**
+ * Discovers .test.prmd files and pairs them with their source .prmd files.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TestDiscovery = void 0;
+const path = __importStar(require("path"));
+const fs = __importStar(require("fs"));
+const glob_1 = require("glob");
+const TestParser_1 = require("./TestParser");
+class TestDiscovery {
+    constructor() {
+        this.parser = new TestParser_1.TestParser();
+    }
+    /**
+     * Discover test suites from a target path.
+     *
+     * - If targetPath is a .test.prmd file, parse it directly.
+     * - If targetPath is a .prmd file, look for a colocated .test.prmd sidecar.
+     * - If targetPath is a directory, glob for all .test.prmd files recursively.
+     */
+    async discover(targetPath) {
+        const resolved = path.resolve(targetPath);
+        const suites = [];
+        const errors = [];
+        if (!fs.existsSync(resolved)) {
+            errors.push({ filePath: resolved, message: 'Path does not exist' });
+            return { suites, errors };
+        }
+        const stat = fs.statSync(resolved);
+        if (stat.isDirectory()) {
+            return this.discoverDirectory(resolved);
+        }
+        if (resolved.endsWith('.test.prmd')) {
+            return this.discoverTestFile(resolved);
+        }
+        if (resolved.endsWith('.prmd')) {
+            return this.discoverFromSource(resolved);
+        }
+        errors.push({
+            filePath: resolved,
+            message: 'Target must be a .prmd file, .test.prmd file, or directory',
+        });
+        return { suites, errors };
+    }
+    async discoverDirectory(dirPath) {
+        const suites = [];
+        const errors = [];
+        const pattern = '**/*.test.prmd';
+        const testFiles = await (0, glob_1.glob)(pattern, {
+            cwd: dirPath,
+            absolute: true,
+            nodir: true,
+            windowsPathsNoEscape: true,
+        });
+        for (const testFile of testFiles) {
+            const normalized = testFile.replace(/\\/g, '/');
+            const result = await this.discoverTestFile(normalized);
+            suites.push(...result.suites);
+            errors.push(...result.errors);
+        }
+        return { suites, errors };
+    }
+    async discoverTestFile(testFilePath) {
+        const suites = [];
+        const errors = [];
+        try {
+            const content = fs.readFileSync(testFilePath, 'utf-8');
+            const suite = this.parser.parse(content, testFilePath);
+            // Validate that the target .prmd file exists
+            if (!fs.existsSync(suite.target)) {
+                errors.push({
+                    filePath: testFilePath,
+                    message: `Target prompt file not found: ${suite.target}`,
+                });
+                return { suites, errors };
+            }
+            suites.push(suite);
+        }
+        catch (err) {
+            errors.push({
+                filePath: testFilePath,
+                message: err instanceof Error ? err.message : String(err),
+            });
+        }
+        return { suites, errors };
+    }
+    async discoverFromSource(sourcePath) {
+        const dir = path.dirname(sourcePath);
+        const base = path.basename(sourcePath, '.prmd');
+        const testFilePath = path.join(dir, `${base}.test.prmd`);
+        if (!fs.existsSync(testFilePath)) {
+            return {
+                suites: [],
+                errors: [{
+                        filePath: sourcePath,
+                        message: `No colocated test file found: ${testFilePath}`,
+                    }],
+            };
+        }
+        return this.discoverTestFile(testFilePath);
+    }
+}
+exports.TestDiscovery = TestDiscovery;

package/dist/TestParser.d.ts

@@ -0,0 +1,25 @@
+/**
+ * Parses .test.prmd files into TestSuite structures.
+ *
+ * A .test.prmd file has YAML frontmatter (test definitions) and
+ * an optional content block (evaluator prompt for prmd evaluators).
+ */
+import type { TestSuite } from './types';
+export declare class TestParser {
+    /**
+     * Parse a .test.prmd file's raw content into a TestSuite.
+     */
+    parse(content: string, testFilePath: string): TestSuite;
+    private splitFrontmatter;
+    private resolveTarget;
+    private parseTests;
+    private parseAssertions;
+    private validateNlpAssertion;
+    private validateScriptAssertion;
+    private validatePrmdAssertion;
+}
+export declare class TestParseError extends Error {
+    readonly filePath: string;
+    constructor(message: string, filePath: string);
+}
+//# sourceMappingURL=TestParser.d.ts.map

package/dist/TestParser.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"TestParser.d.ts","sourceRoot":"","sources":["../src/TestParser.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH,OAAO,KAAK,EAAE,SAAS,EAAmD,MAAM,SAAS,CAAC;AAgC1F,qBAAa,UAAU;IACrB;;OAEG;IACH,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,GAAG,SAAS;IAmCvD,OAAO,CAAC,gBAAgB;IAWxB,OAAO,CAAC,aAAa;IAarB,OAAO,CAAC,UAAU;IA2BlB,OAAO,CAAC,eAAe;IAgCvB,OAAO,CAAC,oBAAoB;IA4B5B,OAAO,CAAC,uBAAuB;IAmB/B,OAAO,CAAC,qBAAqB;CAc9B;AAED,qBAAa,cAAe,SAAQ,KAAK;IACvC,SAAgB,QAAQ,EAAE,MAAM,CAAC;gBAErB,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM;CAK9C"}

package/dist/TestParser.js

@@ -0,0 +1,187 @@
+"use strict";
+/**
+ * Parses .test.prmd files into TestSuite structures.
+ *
+ * A .test.prmd file has YAML frontmatter (test definitions) and
+ * an optional content block (evaluator prompt for prmd evaluators).
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TestParseError = exports.TestParser = void 0;
+const path = __importStar(require("path"));
+const YAML = __importStar(require("yaml"));
+const VALID_EVALUATOR_TYPES = ['nlp', 'script', 'prmd'];
+const VALID_NLP_CHECKS = [
+    'contains', 'not_contains', 'matches',
+    'max_tokens', 'min_tokens', 'starts_with', 'ends_with'
+];
+class TestParser {
+    /**
+     * Parse a .test.prmd file's raw content into a TestSuite.
+     */
+    parse(content, testFilePath) {
+        const normalized = content.replace(/\r\n/g, '\n');
+        const { frontmatter, body } = this.splitFrontmatter(normalized);
+        if (!frontmatter) {
+            throw new TestParseError('Missing YAML frontmatter in .test.prmd file', testFilePath);
+        }
+        let parsed;
+        try {
+            parsed = YAML.parse(frontmatter);
+        }
+        catch (err) {
+            const message = err instanceof Error ? err.message : String(err);
+            throw new TestParseError(`Invalid YAML frontmatter: ${message}`, testFilePath);
+        }
+        if (!parsed || typeof parsed !== 'object') {
+            throw new TestParseError('Frontmatter must be a YAML object', testFilePath);
+        }
+        const name = parsed.name || path.basename(testFilePath, '.test.prmd');
+        const target = this.resolveTarget(parsed.target, testFilePath);
+        const tests = this.parseTests(parsed.tests, testFilePath);
+        const evaluatorPrompt = body.trim() || undefined;
+        return {
+            name,
+            description: parsed.description,
+            target,
+            testFilePath,
+            tests,
+            evaluatorPrompt,
+        };
+    }
+    splitFrontmatter(content) {
+        const match = content.match(/^---\n([\s\S]*?)\n---\n?([\s\S]*)$/);
+        if (!match) {
+            return { frontmatter: null, body: content };
+        }
+        return {
+            frontmatter: match[1],
+            body: match[2],
+        };
+    }
+    resolveTarget(target, testFilePath) {
+        if (target) {
+            const dir = path.dirname(testFilePath);
+            return path.resolve(dir, target);
+        }
+        // Auto-discover: summarize.test.prmd -> summarize.prmd
+        const dir = path.dirname(testFilePath);
+        const base = path.basename(testFilePath);
+        const sourceBase = base.replace(/\.test\.prmd$/, '.prmd');
+        return path.resolve(dir, sourceBase);
+    }
+    parseTests(rawTests, filePath) {
+        if (!rawTests || !Array.isArray(rawTests)) {
+            throw new TestParseError('Frontmatter must contain a "tests" array', filePath);
+        }
+        if (rawTests.length === 0) {
+            throw new TestParseError('"tests" array must not be empty', filePath);
+        }
+        return rawTests.map((raw, index) => {
+            const name = raw.name || `test_${index + 1}`;
+            const params = raw.params && typeof raw.params === 'object' ? raw.params : {};
+            if (raw.expect_error) {
+                return {
+                    name,
+                    params,
+                    assert: [],
+                    expect_error: true,
+                };
+            }
+            const assertions = this.parseAssertions(raw.assert, name, filePath);
+            return { name, params, assert: assertions };
+        });
+    }
+    parseAssertions(rawAssertions, testName, filePath) {
+        if (!rawAssertions || !Array.isArray(rawAssertions)) {
+            return [];
+        }
+        return rawAssertions.map((raw, index) => {
+            if (!raw.evaluator || !VALID_EVALUATOR_TYPES.includes(raw.evaluator)) {
+                throw new TestParseError(`Test "${testName}", assertion ${index + 1}: invalid evaluator "${raw.evaluator}". ` +
+                    `Must be one of: ${VALID_EVALUATOR_TYPES.join(', ')}`, filePath);
+            }
+            const evaluator = raw.evaluator;
+            if (evaluator === 'nlp') {
+                return this.validateNlpAssertion(raw, testName, index, filePath);
+            }
+            if (evaluator === 'script') {
+                return this.validateScriptAssertion(raw, testName, index, filePath);
+            }
+            return this.validatePrmdAssertion(raw, testName, index, filePath);
+        });
+    }
+    validateNlpAssertion(raw, testName, index, filePath) {
+        if (!raw.check || !VALID_NLP_CHECKS.includes(raw.check)) {
+            throw new TestParseError(`Test "${testName}", assertion ${index + 1}: NLP evaluator requires a valid "check". ` +
+                `Must be one of: ${VALID_NLP_CHECKS.join(', ')}`, filePath);
+        }
+        if (raw.value === undefined || raw.value === null) {
+            throw new TestParseError(`Test "${testName}", assertion ${index + 1}: NLP evaluator requires a "value"`, filePath);
+        }
+        return {
+            evaluator: 'nlp',
+            check: raw.check,
+            value: raw.value,
+        };
+    }
+    validateScriptAssertion(raw, testName, index, filePath) {
+        if (!raw.run || typeof raw.run !== 'string') {
+            throw new TestParseError(`Test "${testName}", assertion ${index + 1}: script evaluator requires a "run" path`, filePath);
+        }
+        return {
+            evaluator: 'script',
+            run: raw.run,
+        };
+    }
+    validatePrmdAssertion(raw, _testName, _index, _filePath) {
+        // prompt: is optional — if omitted, uses the content block of the .test.prmd
+        return {
+            evaluator: 'prmd',
+            prompt: raw.prompt || undefined,
+            provider: raw.provider || undefined,
+            model: raw.model || undefined,
+        };
+    }
+}
+exports.TestParser = TestParser;
+class TestParseError extends Error {
+    constructor(message, filePath) {
+        super(`${message} (${filePath})`);
+        this.name = 'TestParseError';
+        this.filePath = filePath;
+    }
+}
+exports.TestParseError = TestParseError;

package/dist/TestRunner.d.ts

@@ -0,0 +1,57 @@
+/**
+ * TestRunner - orchestrates the full test lifecycle:
+ * discovery -> compile -> execute -> evaluate -> report
+ *
+ * Consumes @prompd/cli for compilation and execution.
+ * This is the primary public API for @prompd/test.
+ */
+import type { CompilerModule } from './cli-types';
+import type { TestHarness } from '@prompd/cli';
+import type { TestRunResult, TestRunOptions, TestProgressCallback } from './types';
+export declare class TestRunner implements TestHarness {
+    private discovery;
+    private cliModule;
+    private configLoaded;
+    /**
+     * @param cli - Optional pre-loaded @prompd/cli module. If provided, skips dynamic import.
+     *              This is the recommended approach when running inside Electron where the CLI
+     *              is already loaded by the main process.
+     */
+    constructor(cli?: CompilerModule);
+    /**
+     * Ensure CLI config is loaded (API keys, provider settings).
+     * Called once before any execution.
+     */
+    private ensureConfig;
+    /**
+     * Run tests for a target path (file or directory).
+     * Returns structured results and an exit code (0 = all pass, 1 = failures).
+     */
+    run(targetPath: string, options?: TestRunOptions, onProgress?: TestProgressCallback): Promise<TestRunResult>;
+    /**
+     * Run tests and return formatted output string.
+     */
+    runAndReport(targetPath: string, options?: TestRunOptions, onProgress?: TestProgressCallback): Promise<{
+        output: string;
+        exitCode: number;
+    }>;
+    private runSuite;
+    private runTestCase;
+    /**
+     * Compile a .prmd file and return both the compiled text and metadata
+     * (provider, model, temperature, max_tokens from frontmatter).
+     */
+    private compileTarget;
+    /**
+     * Execute compiled prompt text against an LLM using the executor's callLLM directly.
+     * This avoids re-compilation through executeRawText which loses metadata.
+     */
+    private executePrompt;
+    private resolveAllowedEvaluators;
+    private getReporter;
+    private buildSummary;
+    private buildErrorResult;
+    private getDefaultModel;
+    private getCli;
+}
+//# sourceMappingURL=TestRunner.d.ts.map

package/dist/TestRunner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"TestRunner.d.ts","sourceRoot":"","sources":["../src/TestRunner.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAWH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAClD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC/C,OAAO,KAAK,EAIV,aAAa,EAGb,cAAc,EACd,oBAAoB,EAErB,MAAM,SAAS,CAAC;AAEjB,qBAAa,UAAW,YAAW,WAAW;IAC5C,OAAO,CAAC,SAAS,CAAgB;IACjC,OAAO,CAAC,SAAS,CAA+B;IAChD,OAAO,CAAC,YAAY,CAAS;IAE7B;;;;OAIG;gBACS,GAAG,CAAC,EAAE,cAAc;IAOhC;;;OAGG;YACW,YAAY;IAyB1B;;;OAGG;IACG,GAAG,CACP,UAAU,EAAE,MAAM,EAClB,OAAO,GAAE,cAAmB,EAC5B,UAAU,CAAC,EAAE,oBAAoB,GAChC,OAAO,CAAC,aAAa,CAAC;IAgCzB;;OAEG;IACG,YAAY,CAChB,UAAU,EAAE,MAAM,EAClB,OAAO,GAAE,cAAmB,EAC5B,UAAU,CAAC,EAAE,oBAAoB,GAChC,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC;YAQlC,QAAQ;YAoBR,WAAW;IAuJzB;;;OAGG;YACW,aAAa;IA+C3B;;;OAGG;YACW,aAAa;IAuD3B,OAAO,CAAC,wBAAwB;IAUhC,OAAO,CAAC,WAAW;IAYnB,OAAO,CAAC,YAAY;IA8CpB,OAAO,CAAC,gBAAgB;IAcxB,OAAO,CAAC,eAAe;YAYT,MAAM;CAQrB"}