@prompd/test 0.5.0-beta.10

package/src/EvaluatorEngine.ts

@@ -0,0 +1,130 @@
+/**
+ * Routes assertions to the correct evaluator and manages execution order.
+ *
+ * Execution order: nlp -> script -> prmd (cheap to expensive).
+ * Fail-fast by default — stops on first failure unless runAll is set.
+ */
+
+import type { AssertionDef, AssertionResult, EvaluatorType } from './types';
+import type { Evaluator, EvaluatorContext } from './evaluators/types';
+import type { CompilerModule } from './cli-types';
+import { NlpEvaluator } from './evaluators/NlpEvaluator';
+import { ScriptEvaluator } from './evaluators/ScriptEvaluator';
+import { PrmdEvaluator, type PrmdEvaluatorOptions } from './evaluators/PrmdEvaluator';
+
+/** Execution priority — lower number runs first */
+const EVALUATOR_PRIORITY: Record<EvaluatorType, number> = {
+  nlp: 0,
+  script: 1,
+  prmd: 2,
+};
+
+export interface EvaluatorEngineOptions {
+  testFileDir: string;
+  evaluatorPrompt?: string;
+  workspaceRoot?: string;
+  registryUrl?: string;
+  allowedEvaluators?: EvaluatorType[];
+  failFast?: boolean;
+  cliModule?: CompilerModule;
+  provider?: string;
+  model?: string;
+}
+
+export class EvaluatorEngine {
+  private evaluators: Map<EvaluatorType, Evaluator>;
+  private allowedEvaluators: Set<EvaluatorType>;
+  private failFast: boolean;
+
+  constructor(options: EvaluatorEngineOptions) {
+    this.failFast = options.failFast !== false;
+    this.allowedEvaluators = new Set(options.allowedEvaluators || ['nlp', 'script', 'prmd']);
+
+    const prmdOptions: PrmdEvaluatorOptions = {
+      testFileDir: options.testFileDir,
+      evaluatorPrompt: options.evaluatorPrompt,
+      workspaceRoot: options.workspaceRoot,
+      registryUrl: options.registryUrl,
+      cliModule: options.cliModule,
+      provider: options.provider,
+      model: options.model,
+    };
+
+    this.evaluators = new Map<EvaluatorType, Evaluator>([
+      ['nlp', new NlpEvaluator()],
+      ['script', new ScriptEvaluator(options.testFileDir)],
+      ['prmd', new PrmdEvaluator(prmdOptions)],
+    ]);
+  }
+
+  /**
+   * Evaluate all assertions in cost-priority order.
+   * Returns results for each assertion.
+   */
+  async evaluate(
+    assertions: AssertionDef[],
+    context: EvaluatorContext,
+    onResult?: (result: AssertionResult) => void
+  ): Promise<AssertionResult[]> {
+    const results: AssertionResult[] = [];
+
+    // Sort by evaluator priority (nlp first, prmd last)
+    const sorted = [...assertions].sort(
+      (a, b) => EVALUATOR_PRIORITY[a.evaluator] - EVALUATOR_PRIORITY[b.evaluator]
+    );
+
+    for (const assertion of sorted) {
+      // Skip evaluators that aren't allowed
+      if (!this.allowedEvaluators.has(assertion.evaluator)) {
+        const skipped: AssertionResult = {
+          evaluator: assertion.evaluator,
+          check: assertion.check,
+          status: 'skip',
+          reason: `Evaluator type "${assertion.evaluator}" skipped by filter`,
+          duration: 0,
+        };
+        results.push(skipped);
+        onResult?.(skipped);
+        continue;
+      }
+
+      const evaluator = this.evaluators.get(assertion.evaluator);
+      if (!evaluator) {
+        const errorResult: AssertionResult = {
+          evaluator: assertion.evaluator,
+          check: assertion.check,
+          status: 'error',
+          reason: `No evaluator registered for type "${assertion.evaluator}"`,
+          duration: 0,
+        };
+        results.push(errorResult);
+        onResult?.(errorResult);
+        continue;
+      }
+
+      const result = await evaluator.evaluate(assertion, context);
+      results.push(result);
+      onResult?.(result);
+
+      // Fail-fast: stop on first failure
+      if (this.failFast && result.status !== 'pass') {
+        // Mark remaining assertions as skipped
+        const remaining = sorted.slice(sorted.indexOf(assertion) + 1);
+        for (const rem of remaining) {
+          const skipped: AssertionResult = {
+            evaluator: rem.evaluator,
+            check: rem.check,
+            status: 'skip',
+            reason: 'Skipped due to prior failure (fail-fast)',
+            duration: 0,
+          };
+          results.push(skipped);
+          onResult?.(skipped);
+        }
+        break;
+      }
+    }
+
+    return results;
+  }
+}

package/src/TestDiscovery.ts

@@ -0,0 +1,133 @@
+/**
+ * Discovers .test.prmd files and pairs them with their source .prmd files.
+ */
+
+import * as path from 'path';
+import * as fs from 'fs';
+import { glob } from 'glob';
+import { TestParser } from './TestParser';
+import type { TestSuite } from './types';
+
+export interface DiscoveryResult {
+  suites: TestSuite[];
+  errors: DiscoveryError[];
+}
+
+export interface DiscoveryError {
+  filePath: string;
+  message: string;
+}
+
+export class TestDiscovery {
+  private parser: TestParser;
+
+  constructor() {
+    this.parser = new TestParser();
+  }
+
+  /**
+   * Discover test suites from a target path.
+   *
+   * - If targetPath is a .test.prmd file, parse it directly.
+   * - If targetPath is a .prmd file, look for a colocated .test.prmd sidecar.
+   * - If targetPath is a directory, glob for all .test.prmd files recursively.
+   */
+  async discover(targetPath: string): Promise<DiscoveryResult> {
+    const resolved = path.resolve(targetPath);
+    const suites: TestSuite[] = [];
+    const errors: DiscoveryError[] = [];
+
+    if (!fs.existsSync(resolved)) {
+      errors.push({ filePath: resolved, message: 'Path does not exist' });
+      return { suites, errors };
+    }
+
+    const stat = fs.statSync(resolved);
+
+    if (stat.isDirectory()) {
+      return this.discoverDirectory(resolved);
+    }
+
+    if (resolved.endsWith('.test.prmd')) {
+      return this.discoverTestFile(resolved);
+    }
+
+    if (resolved.endsWith('.prmd')) {
+      return this.discoverFromSource(resolved);
+    }
+
+    errors.push({
+      filePath: resolved,
+      message: 'Target must be a .prmd file, .test.prmd file, or directory',
+    });
+    return { suites, errors };
+  }
+
+  private async discoverDirectory(dirPath: string): Promise<DiscoveryResult> {
+    const suites: TestSuite[] = [];
+    const errors: DiscoveryError[] = [];
+
+    const pattern = '**/*.test.prmd';
+    const testFiles = await glob(pattern, {
+      cwd: dirPath,
+      absolute: true,
+      nodir: true,
+      windowsPathsNoEscape: true,
+    });
+
+    for (const testFile of testFiles) {
+      const normalized = testFile.replace(/\\/g, '/');
+      const result = await this.discoverTestFile(normalized);
+      suites.push(...result.suites);
+      errors.push(...result.errors);
+    }
+
+    return { suites, errors };
+  }
+
+  private async discoverTestFile(testFilePath: string): Promise<DiscoveryResult> {
+    const suites: TestSuite[] = [];
+    const errors: DiscoveryError[] = [];
+
+    try {
+      const content = fs.readFileSync(testFilePath, 'utf-8');
+      const suite = this.parser.parse(content, testFilePath);
+
+      // Validate that the target .prmd file exists
+      if (!fs.existsSync(suite.target)) {
+        errors.push({
+          filePath: testFilePath,
+          message: `Target prompt file not found: ${suite.target}`,
+        });
+        return { suites, errors };
+      }
+
+      suites.push(suite);
+    } catch (err) {
+      errors.push({
+        filePath: testFilePath,
+        message: err instanceof Error ? err.message : String(err),
+      });
+    }
+
+    return { suites, errors };
+  }
+
+  private async discoverFromSource(sourcePath: string): Promise<DiscoveryResult> {
+    const dir = path.dirname(sourcePath);
+    const base = path.basename(sourcePath, '.prmd');
+    const testFilePath = path.join(dir, `${base}.test.prmd`);
+
+    if (!fs.existsSync(testFilePath)) {
+      return {
+        suites: [],
+        errors: [{
+          filePath: sourcePath,
+          message: `No colocated test file found: ${testFilePath}`,
+        }],
+      };
+    }
+
+    return this.discoverTestFile(testFilePath);
+  }
+}

package/src/TestParser.ts

@@ -0,0 +1,235 @@
+/**
+ * Parses .test.prmd files into TestSuite structures.
+ *
+ * A .test.prmd file has YAML frontmatter (test definitions) and
+ * an optional content block (evaluator prompt for prmd evaluators).
+ */
+
+import * as path from 'path';
+import * as YAML from 'yaml';
+import type { TestSuite, TestCase, AssertionDef, EvaluatorType, NlpCheck } from './types';
+
+const VALID_EVALUATOR_TYPES: EvaluatorType[] = ['nlp', 'script', 'prmd'];
+const VALID_NLP_CHECKS: NlpCheck[] = [
+  'contains', 'not_contains', 'matches',
+  'max_tokens', 'min_tokens', 'starts_with', 'ends_with'
+];
+
+interface ParsedFrontmatter {
+  name?: string;
+  description?: string;
+  target?: string;
+  tests?: RawTestCase[];
+}
+
+interface RawTestCase {
+  name?: string;
+  params?: Record<string, unknown>;
+  assert?: RawAssertionDef[];
+  expect_error?: boolean;
+}
+
+interface RawAssertionDef {
+  evaluator?: string;
+  check?: string;
+  value?: unknown;
+  run?: string;
+  prompt?: string;
+  provider?: string;
+  model?: string;
+}
+
+export class TestParser {
+  /**
+   * Parse a .test.prmd file's raw content into a TestSuite.
+   */
+  parse(content: string, testFilePath: string): TestSuite {
+    const normalized = content.replace(/\r\n/g, '\n');
+    const { frontmatter, body } = this.splitFrontmatter(normalized);
+
+    if (!frontmatter) {
+      throw new TestParseError('Missing YAML frontmatter in .test.prmd file', testFilePath);
+    }
+
+    let parsed: ParsedFrontmatter;
+    try {
+      parsed = YAML.parse(frontmatter) as ParsedFrontmatter;
+    } catch (err) {
+      const message = err instanceof Error ? err.message : String(err);
+      throw new TestParseError(`Invalid YAML frontmatter: ${message}`, testFilePath);
+    }
+
+    if (!parsed || typeof parsed !== 'object') {
+      throw new TestParseError('Frontmatter must be a YAML object', testFilePath);
+    }
+
+    const name = parsed.name || path.basename(testFilePath, '.test.prmd');
+    const target = this.resolveTarget(parsed.target, testFilePath);
+    const tests = this.parseTests(parsed.tests, testFilePath);
+    const evaluatorPrompt = body.trim() || undefined;
+
+    return {
+      name,
+      description: parsed.description,
+      target,
+      testFilePath,
+      tests,
+      evaluatorPrompt,
+    };
+  }
+
+  private splitFrontmatter(content: string): { frontmatter: string | null; body: string } {
+    const match = content.match(/^---\n([\s\S]*?)\n---\n?([\s\S]*)$/);
+    if (!match) {
+      return { frontmatter: null, body: content };
+    }
+    return {
+      frontmatter: match[1],
+      body: match[2],
+    };
+  }
+
+  private resolveTarget(target: string | undefined, testFilePath: string): string {
+    if (target) {
+      const dir = path.dirname(testFilePath);
+      return path.resolve(dir, target);
+    }
+
+    // Auto-discover: summarize.test.prmd -> summarize.prmd
+    const dir = path.dirname(testFilePath);
+    const base = path.basename(testFilePath);
+    const sourceBase = base.replace(/\.test\.prmd$/, '.prmd');
+    return path.resolve(dir, sourceBase);
+  }
+
+  private parseTests(rawTests: RawTestCase[] | undefined, filePath: string): TestCase[] {
+    if (!rawTests || !Array.isArray(rawTests)) {
+      throw new TestParseError('Frontmatter must contain a "tests" array', filePath);
+    }
+
+    if (rawTests.length === 0) {
+      throw new TestParseError('"tests" array must not be empty', filePath);
+    }
+
+    return rawTests.map((raw, index) => {
+      const name = raw.name || `test_${index + 1}`;
+      const params = raw.params && typeof raw.params === 'object' ? raw.params : {};
+
+      if (raw.expect_error) {
+        return {
+          name,
+          params,
+          assert: [],
+          expect_error: true,
+        };
+      }
+
+      const assertions = this.parseAssertions(raw.assert, name, filePath);
+      return { name, params, assert: assertions };
+    });
+  }
+
+  private parseAssertions(
+    rawAssertions: RawAssertionDef[] | undefined,
+    testName: string,
+    filePath: string
+  ): AssertionDef[] {
+    if (!rawAssertions || !Array.isArray(rawAssertions)) {
+      return [];
+    }
+
+    return rawAssertions.map((raw, index) => {
+      if (!raw.evaluator || !VALID_EVALUATOR_TYPES.includes(raw.evaluator as EvaluatorType)) {
+        throw new TestParseError(
+          `Test "${testName}", assertion ${index + 1}: invalid evaluator "${raw.evaluator}". ` +
+          `Must be one of: ${VALID_EVALUATOR_TYPES.join(', ')}`,
+          filePath
+        );
+      }
+
+      const evaluator = raw.evaluator as EvaluatorType;
+
+      if (evaluator === 'nlp') {
+        return this.validateNlpAssertion(raw, testName, index, filePath);
+      }
+
+      if (evaluator === 'script') {
+        return this.validateScriptAssertion(raw, testName, index, filePath);
+      }
+
+      return this.validatePrmdAssertion(raw, testName, index, filePath);
+    });
+  }
+
+  private validateNlpAssertion(
+    raw: RawAssertionDef,
+    testName: string,
+    index: number,
+    filePath: string
+  ): AssertionDef {
+    if (!raw.check || !VALID_NLP_CHECKS.includes(raw.check as NlpCheck)) {
+      throw new TestParseError(
+        `Test "${testName}", assertion ${index + 1}: NLP evaluator requires a valid "check". ` +
+        `Must be one of: ${VALID_NLP_CHECKS.join(', ')}`,
+        filePath
+      );
+    }
+
+    if (raw.value === undefined || raw.value === null) {
+      throw new TestParseError(
+        `Test "${testName}", assertion ${index + 1}: NLP evaluator requires a "value"`,
+        filePath
+      );
+    }
+
+    return {
+      evaluator: 'nlp',
+      check: raw.check as NlpCheck,
+      value: raw.value as string | string[] | number,
+    };
+  }
+
+  private validateScriptAssertion(
+    raw: RawAssertionDef,
+    testName: string,
+    index: number,
+    filePath: string
+  ): AssertionDef {
+    if (!raw.run || typeof raw.run !== 'string') {
+      throw new TestParseError(
+        `Test "${testName}", assertion ${index + 1}: script evaluator requires a "run" path`,
+        filePath
+      );
+    }
+
+    return {
+      evaluator: 'script',
+      run: raw.run,
+    };
+  }
+
+  private validatePrmdAssertion(
+    raw: RawAssertionDef,
+    _testName: string,
+    _index: number,
+    _filePath: string
+  ): AssertionDef {
+    // prompt: is optional — if omitted, uses the content block of the .test.prmd
+    return {
+      evaluator: 'prmd',
+      prompt: raw.prompt || undefined,
+      provider: raw.provider || undefined,
+      model: raw.model || undefined,
+    };
+  }
+}
+
+export class TestParseError extends Error {
+  public readonly filePath: string;
+
+  constructor(message: string, filePath: string) {
+    super(`${message} (${filePath})`);
+    this.name = 'TestParseError';
+    this.filePath = filePath;
+  }
+}