@prompd/test 0.5.0-beta.9

package/dist/index.d.ts

@@ -0,0 +1,25 @@
+/**
+ * @prompd/test - Prompt testing and evaluation framework
+ *
+ * Provides test discovery, assertion evaluation, and reporting for .prmd files.
+ * Consumes @prompd/cli for compilation and execution.
+ */
+export { TestRunner } from './TestRunner';
+export { TestParser, TestParseError } from './TestParser';
+export { TestDiscovery } from './TestDiscovery';
+export { EvaluatorEngine } from './EvaluatorEngine';
+export { NlpEvaluator } from './evaluators/NlpEvaluator';
+export { ScriptEvaluator } from './evaluators/ScriptEvaluator';
+export { PrmdEvaluator } from './evaluators/PrmdEvaluator';
+export { ConsoleReporter } from './reporters/ConsoleReporter';
+export { JsonReporter } from './reporters/JsonReporter';
+export { JunitReporter } from './reporters/JunitReporter';
+export type { TestSuite, TestCase, AssertionDef, TestResult, TestRunResult, TestSuiteResult, TestRunSummary, TestRunOptions, TestProgressEvent, TestProgressCallback, TestStatus, AssertionStatus, AssertionResult, EvaluatorType, NlpCheck, } from './types';
+export type { Evaluator, EvaluatorContext, } from './evaluators/types';
+export type { Reporter, } from './reporters/types';
+export type { DiscoveryResult, DiscoveryError, } from './TestDiscovery';
+export type { EvaluatorEngineOptions, } from './EvaluatorEngine';
+export type { TestHarness, TestHarnessResult, TestHarnessOptions, TestHarnessProgressEvent, TestHarnessProgressCallback, } from '@prompd/cli';
+export type { PrmdEvaluatorOptions, } from './evaluators/PrmdEvaluator';
+export type { CompilerModule, } from './cli-types';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAChD,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAGpD,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AACzD,OAAO,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAG3D,OAAO,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AAC9D,OAAO,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AACxD,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAG1D,YAAY,EACV,SAAS,EACT,QAAQ,EACR,YAAY,EACZ,UAAU,EACV,aAAa,EACb,eAAe,EACf,cAAc,EACd,cAAc,EACd,iBAAiB,EACjB,oBAAoB,EACpB,UAAU,EACV,eAAe,EACf,eAAe,EACf,aAAa,EACb,QAAQ,GACT,MAAM,SAAS,CAAC;AAEjB,YAAY,EACV,SAAS,EACT,gBAAgB,GACjB,MAAM,oBAAoB,CAAC;AAE5B,YAAY,EACV,QAAQ,GACT,MAAM,mBAAmB,CAAC;AAE3B,YAAY,EACV,eAAe,EACf,cAAc,GACf,MAAM,iBAAiB,CAAC;AAEzB,YAAY,EACV,sBAAsB,GACvB,MAAM,mBAAmB,CAAC;AAG3B,YAAY,EACV,WAAW,EACX,iBAAiB,EACjB,kBAAkB,EAClB,wBAAwB,EACxB,2BAA2B,GAC5B,MAAM,aAAa,CAAC;AAErB,YAAY,EACV,oBAAoB,GACrB,MAAM,4BAA4B,CAAC;AAEpC,YAAY,EACV,cAAc,GACf,MAAM,aAAa,CAAC"}

package/dist/index.js

@@ -0,0 +1,33 @@
+"use strict";
+/**
+ * @prompd/test - Prompt testing and evaluation framework
+ *
+ * Provides test discovery, assertion evaluation, and reporting for .prmd files.
+ * Consumes @prompd/cli for compilation and execution.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.JunitReporter = exports.JsonReporter = exports.ConsoleReporter = exports.PrmdEvaluator = exports.ScriptEvaluator = exports.NlpEvaluator = exports.EvaluatorEngine = exports.TestDiscovery = exports.TestParseError = exports.TestParser = exports.TestRunner = void 0;
+// Core classes
+var TestRunner_1 = require("./TestRunner");
+Object.defineProperty(exports, "TestRunner", { enumerable: true, get: function () { return TestRunner_1.TestRunner; } });
+var TestParser_1 = require("./TestParser");
+Object.defineProperty(exports, "TestParser", { enumerable: true, get: function () { return TestParser_1.TestParser; } });
+Object.defineProperty(exports, "TestParseError", { enumerable: true, get: function () { return TestParser_1.TestParseError; } });
+var TestDiscovery_1 = require("./TestDiscovery");
+Object.defineProperty(exports, "TestDiscovery", { enumerable: true, get: function () { return TestDiscovery_1.TestDiscovery; } });
+var EvaluatorEngine_1 = require("./EvaluatorEngine");
+Object.defineProperty(exports, "EvaluatorEngine", { enumerable: true, get: function () { return EvaluatorEngine_1.EvaluatorEngine; } });
+// Evaluators
+var NlpEvaluator_1 = require("./evaluators/NlpEvaluator");
+Object.defineProperty(exports, "NlpEvaluator", { enumerable: true, get: function () { return NlpEvaluator_1.NlpEvaluator; } });
+var ScriptEvaluator_1 = require("./evaluators/ScriptEvaluator");
+Object.defineProperty(exports, "ScriptEvaluator", { enumerable: true, get: function () { return ScriptEvaluator_1.ScriptEvaluator; } });
+var PrmdEvaluator_1 = require("./evaluators/PrmdEvaluator");
+Object.defineProperty(exports, "PrmdEvaluator", { enumerable: true, get: function () { return PrmdEvaluator_1.PrmdEvaluator; } });
+// Reporters
+var ConsoleReporter_1 = require("./reporters/ConsoleReporter");
+Object.defineProperty(exports, "ConsoleReporter", { enumerable: true, get: function () { return ConsoleReporter_1.ConsoleReporter; } });
+var JsonReporter_1 = require("./reporters/JsonReporter");
+Object.defineProperty(exports, "JsonReporter", { enumerable: true, get: function () { return JsonReporter_1.JsonReporter; } });
+var JunitReporter_1 = require("./reporters/JunitReporter");
+Object.defineProperty(exports, "JunitReporter", { enumerable: true, get: function () { return JunitReporter_1.JunitReporter; } });

package/dist/reporters/ConsoleReporter.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Console Reporter - terminal output with pass/fail formatting.
+ *
+ * Does NOT use emojis (breaks things per project rules).
+ * Uses simple text markers: [PASS], [FAIL], [ERROR], [SKIP].
+ */
+import type { Reporter } from './types';
+import type { TestRunResult } from '../types';
+export declare class ConsoleReporter implements Reporter {
+    private verbose;
+    constructor(verbose?: boolean);
+    report(result: TestRunResult): string;
+    private appendAssertionDetail;
+    private statusMarker;
+    private formatDuration;
+}
+//# sourceMappingURL=ConsoleReporter.d.ts.map

package/dist/reporters/ConsoleReporter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ConsoleReporter.d.ts","sourceRoot":"","sources":["../../src/reporters/ConsoleReporter.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AACxC,OAAO,KAAK,EAAE,aAAa,EAA+B,MAAM,UAAU,CAAC;AAE3E,qBAAa,eAAgB,YAAW,QAAQ;IAC9C,OAAO,CAAC,OAAO,CAAU;gBAEb,OAAO,UAAQ;IAI3B,MAAM,CAAC,MAAM,EAAE,aAAa,GAAG,MAAM;IAwDrC,OAAO,CAAC,qBAAqB;IAW7B,OAAO,CAAC,YAAY;IAUpB,OAAO,CAAC,cAAc;CAKvB"}

package/dist/reporters/ConsoleReporter.js

@@ -0,0 +1,85 @@
+"use strict";
+/**
+ * Console Reporter - terminal output with pass/fail formatting.
+ *
+ * Does NOT use emojis (breaks things per project rules).
+ * Uses simple text markers: [PASS], [FAIL], [ERROR], [SKIP].
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ConsoleReporter = void 0;
+class ConsoleReporter {
+    constructor(verbose = false) {
+        this.verbose = verbose;
+    }
+    report(result) {
+        const lines = [];
+        lines.push('');
+        lines.push('=== Prompd Test Results ===');
+        lines.push('');
+        for (const suite of result.suites) {
+            lines.push(`  ${suite.suite}`);
+            for (const test of suite.results) {
+                const marker = this.statusMarker(test.status);
+                const duration = this.formatDuration(test.duration);
+                const meta = test.execution
+                    ? ` [${test.execution.provider}/${test.execution.model}${test.execution.usage?.totalTokens ? ` ${test.execution.usage.totalTokens}tok` : ''}]`
+                    : '';
+                lines.push(`    ${marker} ${test.testName} (${duration})${meta}`);
+                if (test.status === 'error' && test.error) {
+                    lines.push(`      Error: ${test.error}`);
+                }
+                if (this.verbose || test.status === 'fail' || test.status === 'error') {
+                    for (const assertion of test.assertions) {
+                        this.appendAssertionDetail(lines, assertion);
+                    }
+                }
+            }
+            lines.push('');
+        }
+        // Summary
+        const s = result.summary;
+        lines.push('---');
+        lines.push(`Tests: ${s.passed} passed, ${s.failed} failed, ${s.errors} errors, ${s.skipped} skipped, ${s.total} total`);
+        lines.push(`Time:  ${this.formatDuration(s.duration)}`);
+        if (s.totalTokens) {
+            lines.push(`Tokens: ${s.totalTokens.toLocaleString()}`);
+        }
+        if (s.models && s.models.length > 0) {
+            lines.push(`Models: ${s.models.join(', ')}`);
+        }
+        if (s.failed > 0 || s.errors > 0) {
+            lines.push('Result: FAIL');
+        }
+        else {
+            lines.push('Result: PASS');
+        }
+        lines.push('');
+        return lines.join('\n');
+    }
+    appendAssertionDetail(lines, assertion) {
+        const marker = this.statusMarker(assertion.status);
+        const check = assertion.check ? ` (${assertion.check})` : '';
+        const duration = this.formatDuration(assertion.duration);
+        lines.push(`      ${marker} ${assertion.evaluator}${check} [${duration}]`);
+        if (assertion.reason && (assertion.status !== 'pass' || this.verbose)) {
+            lines.push(`        ${assertion.reason}`);
+        }
+    }
+    statusMarker(status) {
+        switch (status) {
+            case 'pass': return '[PASS]';
+            case 'fail': return '[FAIL]';
+            case 'error': return '[ERR ]';
+            case 'skip': return '[SKIP]';
+            default: return '[????]';
+        }
+    }
+    formatDuration(ms) {
+        if (ms < 1000)
+            return `${ms}ms`;
+        if (ms < 60000)
+            return `${(ms / 1000).toFixed(1)}s`;
+        return `${(ms / 60000).toFixed(1)}m`;
+    }
+}
+exports.ConsoleReporter = ConsoleReporter;

package/dist/reporters/JsonReporter.d.ts

@@ -0,0 +1,11 @@
+/**
+ * JSON Reporter - structured output for programmatic consumption and CI.
+ */
+import type { Reporter } from './types';
+import type { TestRunResult } from '../types';
+export declare class JsonReporter implements Reporter {
+    private pretty;
+    constructor(pretty?: boolean);
+    report(result: TestRunResult): string;
+}
+//# sourceMappingURL=JsonReporter.d.ts.map

package/dist/reporters/JsonReporter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"JsonReporter.d.ts","sourceRoot":"","sources":["../../src/reporters/JsonReporter.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AACxC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAE9C,qBAAa,YAAa,YAAW,QAAQ;IAC3C,OAAO,CAAC,MAAM,CAAU;gBAEZ,MAAM,UAAO;IAIzB,MAAM,CAAC,MAAM,EAAE,aAAa,GAAG,MAAM;CAMtC"}

package/dist/reporters/JsonReporter.js

@@ -0,0 +1,18 @@
+"use strict";
+/**
+ * JSON Reporter - structured output for programmatic consumption and CI.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.JsonReporter = void 0;
+class JsonReporter {
+    constructor(pretty = true) {
+        this.pretty = pretty;
+    }
+    report(result) {
+        if (this.pretty) {
+            return JSON.stringify(result, null, 2);
+        }
+        return JSON.stringify(result);
+    }
+}
+exports.JsonReporter = JsonReporter;

package/dist/reporters/JunitReporter.d.ts

@@ -0,0 +1,15 @@
+/**
+ * JUnit XML Reporter - generates JUnit-compatible XML for CI systems.
+ *
+ * Output format follows the JUnit XML schema used by Jenkins, GitHub Actions,
+ * Azure DevOps, and most CI platforms.
+ */
+import type { Reporter } from './types';
+import type { TestRunResult } from '../types';
+export declare class JunitReporter implements Reporter {
+    report(result: TestRunResult): string;
+    private appendTestCase;
+    private buildFailureDetail;
+    private escapeXml;
+}
+//# sourceMappingURL=JunitReporter.d.ts.map

package/dist/reporters/JunitReporter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"JunitReporter.d.ts","sourceRoot":"","sources":["../../src/reporters/JunitReporter.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AACxC,OAAO,KAAK,EAAE,aAAa,EAAc,MAAM,UAAU,CAAC;AAE1D,qBAAa,aAAc,YAAW,QAAQ;IAC5C,MAAM,CAAC,MAAM,EAAE,aAAa,GAAG,MAAM;IAwCrC,OAAO,CAAC,cAAc;IAyCtB,OAAO,CAAC,kBAAkB;IAY1B,OAAO,CAAC,SAAS;CAQlB"}

package/dist/reporters/JunitReporter.js

@@ -0,0 +1,89 @@
+"use strict";
+/**
+ * JUnit XML Reporter - generates JUnit-compatible XML for CI systems.
+ *
+ * Output format follows the JUnit XML schema used by Jenkins, GitHub Actions,
+ * Azure DevOps, and most CI platforms.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.JunitReporter = void 0;
+class JunitReporter {
+    report(result) {
+        const lines = [];
+        lines.push('<?xml version="1.0" encoding="UTF-8"?>');
+        lines.push(`<testsuites tests="${result.summary.total}" ` +
+            `failures="${result.summary.failed}" ` +
+            `errors="${result.summary.errors}" ` +
+            `skipped="${result.summary.skipped}" ` +
+            `time="${(result.summary.duration / 1000).toFixed(3)}">`);
+        for (const suite of result.suites) {
+            const suiteTests = suite.results.length;
+            const suiteFailures = suite.results.filter(r => r.status === 'fail').length;
+            const suiteErrors = suite.results.filter(r => r.status === 'error').length;
+            const suiteSkipped = suite.results.filter(r => r.status === 'skip').length;
+            const suiteDuration = suite.results.reduce((sum, r) => sum + r.duration, 0);
+            lines.push(`  <testsuite name="${this.escapeXml(suite.suite)}" ` +
+                `tests="${suiteTests}" ` +
+                `failures="${suiteFailures}" ` +
+                `errors="${suiteErrors}" ` +
+                `skipped="${suiteSkipped}" ` +
+                `time="${(suiteDuration / 1000).toFixed(3)}" ` +
+                `file="${this.escapeXml(suite.testFilePath)}">`);
+            for (const test of suite.results) {
+                this.appendTestCase(lines, suite.suite, test);
+            }
+            lines.push('  </testsuite>');
+        }
+        lines.push('</testsuites>');
+        return lines.join('\n');
+    }
+    appendTestCase(lines, suiteName, test) {
+        const time = (test.duration / 1000).toFixed(3);
+        lines.push(`    <testcase name="${this.escapeXml(test.testName)}" ` +
+            `classname="${this.escapeXml(suiteName)}" ` +
+            `time="${time}">`);
+        if (test.status === 'fail') {
+            const failedAssertions = test.assertions.filter(a => a.status === 'fail');
+            const message = failedAssertions
+                .map(a => `${a.evaluator}${a.check ? `(${a.check})` : ''}: ${a.reason || 'failed'}`)
+                .join('; ');
+            lines.push(`      <failure message="${this.escapeXml(message)}">`);
+            lines.push(this.escapeXml(this.buildFailureDetail(test)));
+            lines.push('      </failure>');
+        }
+        if (test.status === 'error') {
+            const errorMessage = test.error || 'Unknown error';
+            lines.push(`      <error message="${this.escapeXml(errorMessage)}">`);
+            lines.push(this.escapeXml(errorMessage));
+            lines.push('      </error>');
+        }
+        if (test.status === 'skip') {
+            lines.push('      <skipped/>');
+        }
+        // Include output as system-out if available
+        if (test.output) {
+            lines.push('      <system-out>');
+            lines.push(this.escapeXml(test.output.substring(0, 10000)));
+            lines.push('      </system-out>');
+        }
+        lines.push('    </testcase>');
+    }
+    buildFailureDetail(test) {
+        const details = [];
+        for (const assertion of test.assertions) {
+            const prefix = assertion.status === 'pass' ? '[PASS]' : '[FAIL]';
+            const check = assertion.check ? ` (${assertion.check})` : '';
+            details.push(`${prefix} ${assertion.evaluator}${check}: ${assertion.reason || ''}`);
+        }
+        return details.join('\n');
+    }
+    escapeXml(str) {
+        return str
+            .replace(/&/g, '&amp;')
+            .replace(/</g, '&lt;')
+            .replace(/>/g, '&gt;')
+            .replace(/"/g, '&quot;')
+            .replace(/'/g, '&apos;');
+    }
+}
+exports.JunitReporter = JunitReporter;

package/dist/reporters/types.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Reporter interface for @prompd/test
+ */
+import type { TestRunResult } from '../types';
+export interface Reporter {
+    report(result: TestRunResult): string;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/reporters/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/reporters/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAE9C,MAAM,WAAW,QAAQ;IACvB,MAAM,CAAC,MAAM,EAAE,aAAa,GAAG,MAAM,CAAC;CACvC"}

package/dist/reporters/types.js

@@ -0,0 +1,5 @@
+"use strict";
+/**
+ * Reporter interface for @prompd/test
+ */
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/types.d.ts

@@ -0,0 +1,115 @@
+/**
+ * Core type definitions for @prompd/test
+ */
+export type EvaluatorType = 'nlp' | 'script' | 'prmd';
+export type NlpCheck = 'contains' | 'not_contains' | 'matches' | 'max_tokens' | 'min_tokens' | 'starts_with' | 'ends_with';
+export interface AssertionDef {
+    evaluator: EvaluatorType;
+    check?: NlpCheck;
+    value?: string | string[] | number;
+    run?: string;
+    prompt?: string;
+    provider?: string;
+    model?: string;
+}
+export interface TestCase {
+    name: string;
+    params: Record<string, unknown>;
+    assert: AssertionDef[];
+    expect_error?: boolean;
+}
+export interface TestSuite {
+    name: string;
+    description?: string;
+    target: string;
+    testFilePath: string;
+    tests: TestCase[];
+    evaluatorPrompt?: string;
+}
+export type TestStatus = 'pass' | 'fail' | 'error' | 'skip';
+export type AssertionStatus = 'pass' | 'fail' | 'error' | 'skip';
+export interface AssertionResult {
+    evaluator: EvaluatorType;
+    check?: string;
+    status: AssertionStatus;
+    reason?: string;
+    duration: number;
+}
+export interface TestExecutionMetadata {
+    provider: string;
+    model: string;
+    duration: number;
+    usage?: {
+        promptTokens?: number;
+        completionTokens?: number;
+        totalTokens?: number;
+    };
+}
+export interface TestResult {
+    suite: string;
+    testName: string;
+    status: TestStatus;
+    duration: number;
+    assertions: AssertionResult[];
+    output?: string;
+    compiledInput?: string;
+    error?: string;
+    execution?: TestExecutionMetadata;
+}
+export interface TestRunSummary {
+    total: number;
+    passed: number;
+    failed: number;
+    errors: number;
+    skipped: number;
+    duration: number;
+    totalTokens?: number;
+    providers?: string[];
+    models?: string[];
+}
+export interface TestRunResult {
+    suites: TestSuiteResult[];
+    summary: TestRunSummary;
+}
+export interface TestSuiteResult {
+    suite: string;
+    testFilePath: string;
+    results: TestResult[];
+}
+export interface TestRunOptions {
+    evaluators?: EvaluatorType[];
+    noLlm?: boolean;
+    reporter?: 'console' | 'json' | 'junit';
+    failFast?: boolean;
+    runAll?: boolean;
+    verbose?: boolean;
+    workspaceRoot?: string;
+    registryUrl?: string;
+    provider?: string;
+    model?: string;
+}
+export type TestProgressEvent = {
+    type: 'suite_start';
+    suite: string;
+    testCount: number;
+} | {
+    type: 'test_start';
+    suite: string;
+    testName: string;
+} | {
+    type: 'test_complete';
+    suite: string;
+    testName: string;
+    result: TestResult;
+} | {
+    type: 'suite_complete';
+    suite: string;
+    results: TestResult[];
+} | {
+    type: 'assertion_complete';
+    suite: string;
+    testName: string;
+    assertion: AssertionResult;
+};
+export type TestProgressCallback = (event: TestProgressEvent) => void;
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,MAAM,MAAM,aAAa,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;AAEtD,MAAM,MAAM,QAAQ,GAChB,UAAU,GACV,cAAc,GACd,SAAS,GACT,YAAY,GACZ,YAAY,GACZ,aAAa,GACb,WAAW,CAAC;AAIhB,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,aAAa,CAAC;IAEzB,KAAK,CAAC,EAAE,QAAQ,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,MAAM,CAAC;IAEnC,GAAG,CAAC,EAAE,MAAM,CAAC;IAEb,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAChC,MAAM,EAAE,YAAY,EAAE,CAAC;IACvB,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,QAAQ,EAAE,CAAC;IAClB,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAID,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC;AAC5D,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC;AAEjE,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,aAAa,CAAC;IACzB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,eAAe,CAAC;IACxB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,qBAAqB;IACpC,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE;QACN,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;KACtB,CAAC;CACH;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,UAAU,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,qBAAqB,CAAC;CACnC;AAED,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;CACnB;AAED,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,eAAe,EAAE,CAAC;IAC1B,OAAO,EAAE,cAAc,CAAC;CACzB;AAED,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,UAAU,EAAE,CAAC;CACvB;AAID,MAAM,WAAW,cAAc;IAC7B,UAAU,CAAC,EAAE,aAAa,EAAE,CAAC;IAC7B,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,QAAQ,CAAC,EAAE,SAAS,GAAG,MAAM,GAAG,OAAO,CAAC;IACxC,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAID,MAAM,MAAM,iBAAiB,GACzB;IAAE,IAAI,EAAE,aAAa,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GACzD;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,GACvD;IAAE,IAAI,EAAE,eAAe,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,UAAU,CAAA;CAAE,GAC9E;IAAE,IAAI,EAAE,gBAAgB,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,UAAU,EAAE,CAAA;CAAE,GAChE;IAAE,IAAI,EAAE,oBAAoB,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,eAAe,CAAA;CAAE,CAAC;AAEhG,MAAM,MAAM,oBAAoB,GAAG,CAAC,KAAK,EAAE,iBAAiB,KAAK,IAAI,CAAC"}

package/dist/types.js

@@ -0,0 +1,5 @@
+"use strict";
+/**
+ * Core type definitions for @prompd/test
+ */
+Object.defineProperty(exports, "__esModule", { value: true });

package/package.json

@@ -0,0 +1,34 @@
+{
+  "name": "@prompd/test",
+  "version": "0.5.0-beta.9",
+  "description": "Prompt testing and evaluation framework for Prompd",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "scripts": {
+    "build": "tsc",
+    "dev": "tsc --watch",
+    "typecheck": "tsc --noEmit",
+    "clean": "rm -rf dist"
+  },
+  "keywords": [
+    "test",
+    "eval",
+    "evaluator",
+    "prompt",
+    "prompd"
+  ],
+  "author": "Prompd Team",
+  "license": "Elastic-2.0",
+  "dependencies": {
+    "glob": "^10.3.10",
+    "yaml": "^2.7.1"
+  },
+  "peerDependencies": {
+    "@prompd/cli": ">=0.5.0-beta.9"
+  },
+  "devDependencies": {
+    "@prompd/cli": "^0.5.0-beta.9",
+    "@types/node": "^18.19.17",
+    "typescript": "^5.7.3"
+  }
+}

package/src/EvaluatorEngine.ts

@@ -0,0 +1,130 @@
+/**
+ * Routes assertions to the correct evaluator and manages execution order.
+ *
+ * Execution order: nlp -> script -> prmd (cheap to expensive).
+ * Fail-fast by default — stops on first failure unless runAll is set.
+ */
+
+import type { AssertionDef, AssertionResult, EvaluatorType } from './types';
+import type { Evaluator, EvaluatorContext } from './evaluators/types';
+import type { CompilerModule } from './cli-types';
+import { NlpEvaluator } from './evaluators/NlpEvaluator';
+import { ScriptEvaluator } from './evaluators/ScriptEvaluator';
+import { PrmdEvaluator, type PrmdEvaluatorOptions } from './evaluators/PrmdEvaluator';
+
+/** Execution priority — lower number runs first */
+const EVALUATOR_PRIORITY: Record<EvaluatorType, number> = {
+  nlp: 0,
+  script: 1,
+  prmd: 2,
+};
+
+export interface EvaluatorEngineOptions {
+  testFileDir: string;
+  evaluatorPrompt?: string;
+  workspaceRoot?: string;
+  registryUrl?: string;
+  allowedEvaluators?: EvaluatorType[];
+  failFast?: boolean;
+  cliModule?: CompilerModule;
+  provider?: string;
+  model?: string;
+}
+
+export class EvaluatorEngine {
+  private evaluators: Map<EvaluatorType, Evaluator>;
+  private allowedEvaluators: Set<EvaluatorType>;
+  private failFast: boolean;
+
+  constructor(options: EvaluatorEngineOptions) {
+    this.failFast = options.failFast !== false;
+    this.allowedEvaluators = new Set(options.allowedEvaluators || ['nlp', 'script', 'prmd']);
+
+    const prmdOptions: PrmdEvaluatorOptions = {
+      testFileDir: options.testFileDir,
+      evaluatorPrompt: options.evaluatorPrompt,
+      workspaceRoot: options.workspaceRoot,
+      registryUrl: options.registryUrl,
+      cliModule: options.cliModule,
+      provider: options.provider,
+      model: options.model,
+    };
+
+    this.evaluators = new Map<EvaluatorType, Evaluator>([
+      ['nlp', new NlpEvaluator()],
+      ['script', new ScriptEvaluator(options.testFileDir)],
+      ['prmd', new PrmdEvaluator(prmdOptions)],
+    ]);
+  }
+
+  /**
+   * Evaluate all assertions in cost-priority order.
+   * Returns results for each assertion.
+   */
+  async evaluate(
+    assertions: AssertionDef[],
+    context: EvaluatorContext,
+    onResult?: (result: AssertionResult) => void
+  ): Promise<AssertionResult[]> {
+    const results: AssertionResult[] = [];
+
+    // Sort by evaluator priority (nlp first, prmd last)
+    const sorted = [...assertions].sort(
+      (a, b) => EVALUATOR_PRIORITY[a.evaluator] - EVALUATOR_PRIORITY[b.evaluator]
+    );
+
+    for (const assertion of sorted) {
+      // Skip evaluators that aren't allowed
+      if (!this.allowedEvaluators.has(assertion.evaluator)) {
+        const skipped: AssertionResult = {
+          evaluator: assertion.evaluator,
+          check: assertion.check,
+          status: 'skip',
+          reason: `Evaluator type "${assertion.evaluator}" skipped by filter`,
+          duration: 0,
+        };
+        results.push(skipped);
+        onResult?.(skipped);
+        continue;
+      }
+
+      const evaluator = this.evaluators.get(assertion.evaluator);
+      if (!evaluator) {
+        const errorResult: AssertionResult = {
+          evaluator: assertion.evaluator,
+          check: assertion.check,
+          status: 'error',
+          reason: `No evaluator registered for type "${assertion.evaluator}"`,
+          duration: 0,
+        };
+        results.push(errorResult);
+        onResult?.(errorResult);
+        continue;
+      }
+
+      const result = await evaluator.evaluate(assertion, context);
+      results.push(result);
+      onResult?.(result);
+
+      // Fail-fast: stop on first failure
+      if (this.failFast && result.status !== 'pass') {
+        // Mark remaining assertions as skipped
+        const remaining = sorted.slice(sorted.indexOf(assertion) + 1);
+        for (const rem of remaining) {
+          const skipped: AssertionResult = {
+            evaluator: rem.evaluator,
+            check: rem.check,
+            status: 'skip',
+            reason: 'Skipped due to prior failure (fail-fast)',
+            duration: 0,
+          };
+          results.push(skipped);
+          onResult?.(skipped);
+        }
+        break;
+      }
+    }
+
+    return results;
+  }
+}