@prompd/test 0.5.0-beta.9

package/src/index.ts

@@ -0,0 +1,76 @@
+/**
+ * @prompd/test - Prompt testing and evaluation framework
+ *
+ * Provides test discovery, assertion evaluation, and reporting for .prmd files.
+ * Consumes @prompd/cli for compilation and execution.
+ */
+
+// Core classes
+export { TestRunner } from './TestRunner';
+export { TestParser, TestParseError } from './TestParser';
+export { TestDiscovery } from './TestDiscovery';
+export { EvaluatorEngine } from './EvaluatorEngine';
+
+// Evaluators
+export { NlpEvaluator } from './evaluators/NlpEvaluator';
+export { ScriptEvaluator } from './evaluators/ScriptEvaluator';
+export { PrmdEvaluator } from './evaluators/PrmdEvaluator';
+
+// Reporters
+export { ConsoleReporter } from './reporters/ConsoleReporter';
+export { JsonReporter } from './reporters/JsonReporter';
+export { JunitReporter } from './reporters/JunitReporter';
+
+// Types
+export type {
+  TestSuite,
+  TestCase,
+  AssertionDef,
+  TestResult,
+  TestRunResult,
+  TestSuiteResult,
+  TestRunSummary,
+  TestRunOptions,
+  TestProgressEvent,
+  TestProgressCallback,
+  TestStatus,
+  AssertionStatus,
+  AssertionResult,
+  EvaluatorType,
+  NlpCheck,
+} from './types';
+
+export type {
+  Evaluator,
+  EvaluatorContext,
+} from './evaluators/types';
+
+export type {
+  Reporter,
+} from './reporters/types';
+
+export type {
+  DiscoveryResult,
+  DiscoveryError,
+} from './TestDiscovery';
+
+export type {
+  EvaluatorEngineOptions,
+} from './EvaluatorEngine';
+
+// Re-export TestHarness interface from @prompd/cli for convenience
+export type {
+  TestHarness,
+  TestHarnessResult,
+  TestHarnessOptions,
+  TestHarnessProgressEvent,
+  TestHarnessProgressCallback,
+} from '@prompd/cli';
+
+export type {
+  PrmdEvaluatorOptions,
+} from './evaluators/PrmdEvaluator';
+
+export type {
+  CompilerModule,
+} from './cli-types';

package/src/reporters/ConsoleReporter.ts

@@ -0,0 +1,100 @@
+/**
+ * Console Reporter - terminal output with pass/fail formatting.
+ *
+ * Does NOT use emojis (breaks things per project rules).
+ * Uses simple text markers: [PASS], [FAIL], [ERROR], [SKIP].
+ */
+
+import type { Reporter } from './types';
+import type { TestRunResult, TestResult, AssertionResult } from '../types';
+
+export class ConsoleReporter implements Reporter {
+  private verbose: boolean;
+
+  constructor(verbose = false) {
+    this.verbose = verbose;
+  }
+
+  report(result: TestRunResult): string {
+    const lines: string[] = [];
+
+    lines.push('');
+    lines.push('=== Prompd Test Results ===');
+    lines.push('');
+
+    for (const suite of result.suites) {
+      lines.push(`  ${suite.suite}`);
+
+      for (const test of suite.results) {
+        const marker = this.statusMarker(test.status);
+        const duration = this.formatDuration(test.duration);
+        const meta = test.execution
+          ? ` [${test.execution.provider}/${test.execution.model}${test.execution.usage?.totalTokens ? ` ${test.execution.usage.totalTokens}tok` : ''}]`
+          : '';
+        lines.push(`    ${marker} ${test.testName} (${duration})${meta}`);
+
+        if (test.status === 'error' && test.error) {
+          lines.push(`      Error: ${test.error}`);
+        }
+
+        if (this.verbose || test.status === 'fail' || test.status === 'error') {
+          for (const assertion of test.assertions) {
+            this.appendAssertionDetail(lines, assertion);
+          }
+        }
+      }
+
+      lines.push('');
+    }
+
+    // Summary
+    const s = result.summary;
+    lines.push('---');
+    lines.push(
+      `Tests: ${s.passed} passed, ${s.failed} failed, ${s.errors} errors, ${s.skipped} skipped, ${s.total} total`
+    );
+    lines.push(`Time:  ${this.formatDuration(s.duration)}`);
+    if (s.totalTokens) {
+      lines.push(`Tokens: ${s.totalTokens.toLocaleString()}`);
+    }
+    if (s.models && s.models.length > 0) {
+      lines.push(`Models: ${s.models.join(', ')}`);
+    }
+
+    if (s.failed > 0 || s.errors > 0) {
+      lines.push('Result: FAIL');
+    } else {
+      lines.push('Result: PASS');
+    }
+
+    lines.push('');
+    return lines.join('\n');
+  }
+
+  private appendAssertionDetail(lines: string[], assertion: AssertionResult): void {
+    const marker = this.statusMarker(assertion.status);
+    const check = assertion.check ? ` (${assertion.check})` : '';
+    const duration = this.formatDuration(assertion.duration);
+    lines.push(`      ${marker} ${assertion.evaluator}${check} [${duration}]`);
+
+    if (assertion.reason && (assertion.status !== 'pass' || this.verbose)) {
+      lines.push(`        ${assertion.reason}`);
+    }
+  }
+
+  private statusMarker(status: string): string {
+    switch (status) {
+      case 'pass': return '[PASS]';
+      case 'fail': return '[FAIL]';
+      case 'error': return '[ERR ]';
+      case 'skip': return '[SKIP]';
+      default: return '[????]';
+    }
+  }
+
+  private formatDuration(ms: number): string {
+    if (ms < 1000) return `${ms}ms`;
+    if (ms < 60000) return `${(ms / 1000).toFixed(1)}s`;
+    return `${(ms / 60000).toFixed(1)}m`;
+  }
+}

package/src/reporters/JsonReporter.ts

@@ -0,0 +1,21 @@
+/**
+ * JSON Reporter - structured output for programmatic consumption and CI.
+ */
+
+import type { Reporter } from './types';
+import type { TestRunResult } from '../types';
+
+export class JsonReporter implements Reporter {
+  private pretty: boolean;
+
+  constructor(pretty = true) {
+    this.pretty = pretty;
+  }
+
+  report(result: TestRunResult): string {
+    if (this.pretty) {
+      return JSON.stringify(result, null, 2);
+    }
+    return JSON.stringify(result);
+  }
+}

package/src/reporters/JunitReporter.ts

@@ -0,0 +1,113 @@
+/**
+ * JUnit XML Reporter - generates JUnit-compatible XML for CI systems.
+ *
+ * Output format follows the JUnit XML schema used by Jenkins, GitHub Actions,
+ * Azure DevOps, and most CI platforms.
+ */
+
+import type { Reporter } from './types';
+import type { TestRunResult, TestResult } from '../types';
+
+export class JunitReporter implements Reporter {
+  report(result: TestRunResult): string {
+    const lines: string[] = [];
+
+    lines.push('<?xml version="1.0" encoding="UTF-8"?>');
+    lines.push(
+      `<testsuites tests="${result.summary.total}" ` +
+      `failures="${result.summary.failed}" ` +
+      `errors="${result.summary.errors}" ` +
+      `skipped="${result.summary.skipped}" ` +
+      `time="${(result.summary.duration / 1000).toFixed(3)}">`
+    );
+
+    for (const suite of result.suites) {
+      const suiteTests = suite.results.length;
+      const suiteFailures = suite.results.filter(r => r.status === 'fail').length;
+      const suiteErrors = suite.results.filter(r => r.status === 'error').length;
+      const suiteSkipped = suite.results.filter(r => r.status === 'skip').length;
+      const suiteDuration = suite.results.reduce((sum, r) => sum + r.duration, 0);
+
+      lines.push(
+        `  <testsuite name="${this.escapeXml(suite.suite)}" ` +
+        `tests="${suiteTests}" ` +
+        `failures="${suiteFailures}" ` +
+        `errors="${suiteErrors}" ` +
+        `skipped="${suiteSkipped}" ` +
+        `time="${(suiteDuration / 1000).toFixed(3)}" ` +
+        `file="${this.escapeXml(suite.testFilePath)}">`
+      );
+
+      for (const test of suite.results) {
+        this.appendTestCase(lines, suite.suite, test);
+      }
+
+      lines.push('  </testsuite>');
+    }
+
+    lines.push('</testsuites>');
+    return lines.join('\n');
+  }
+
+  private appendTestCase(lines: string[], suiteName: string, test: TestResult): void {
+    const time = (test.duration / 1000).toFixed(3);
+
+    lines.push(
+      `    <testcase name="${this.escapeXml(test.testName)}" ` +
+      `classname="${this.escapeXml(suiteName)}" ` +
+      `time="${time}">`
+    );
+
+    if (test.status === 'fail') {
+      const failedAssertions = test.assertions.filter(a => a.status === 'fail');
+      const message = failedAssertions
+        .map(a => `${a.evaluator}${a.check ? `(${a.check})` : ''}: ${a.reason || 'failed'}`)
+        .join('; ');
+
+      lines.push(`      <failure message="${this.escapeXml(message)}">`);
+      lines.push(this.escapeXml(this.buildFailureDetail(test)));
+      lines.push('      </failure>');
+    }
+
+    if (test.status === 'error') {
+      const errorMessage = test.error || 'Unknown error';
+      lines.push(`      <error message="${this.escapeXml(errorMessage)}">`);
+      lines.push(this.escapeXml(errorMessage));
+      lines.push('      </error>');
+    }
+
+    if (test.status === 'skip') {
+      lines.push('      <skipped/>');
+    }
+
+    // Include output as system-out if available
+    if (test.output) {
+      lines.push('      <system-out>');
+      lines.push(this.escapeXml(test.output.substring(0, 10000)));
+      lines.push('      </system-out>');
+    }
+
+    lines.push('    </testcase>');
+  }
+
+  private buildFailureDetail(test: TestResult): string {
+    const details: string[] = [];
+
+    for (const assertion of test.assertions) {
+      const prefix = assertion.status === 'pass' ? '[PASS]' : '[FAIL]';
+      const check = assertion.check ? ` (${assertion.check})` : '';
+      details.push(`${prefix} ${assertion.evaluator}${check}: ${assertion.reason || ''}`);
+    }
+
+    return details.join('\n');
+  }
+
+  private escapeXml(str: string): string {
+    return str
+      .replace(/&/g, '&amp;')
+      .replace(/</g, '&lt;')
+      .replace(/>/g, '&gt;')
+      .replace(/"/g, '&quot;')
+      .replace(/'/g, '&apos;');
+  }
+}

package/src/reporters/types.ts

@@ -0,0 +1,9 @@
+/**
+ * Reporter interface for @prompd/test
+ */
+
+import type { TestRunResult } from '../types';
+
+export interface Reporter {
+  report(result: TestRunResult): string;
+}

package/src/types.ts

@@ -0,0 +1,133 @@
+/**
+ * Core type definitions for @prompd/test
+ */
+
+// --- Evaluator taxonomy ---
+
+export type EvaluatorType = 'nlp' | 'script' | 'prmd';
+
+export type NlpCheck =
+  | 'contains'
+  | 'not_contains'
+  | 'matches'
+  | 'max_tokens'
+  | 'min_tokens'
+  | 'starts_with'
+  | 'ends_with';
+
+// --- Test definition types (parsed from .test.prmd frontmatter) ---
+
+export interface AssertionDef {
+  evaluator: EvaluatorType;
+  // NLP fields
+  check?: NlpCheck;
+  value?: string | string[] | number;
+  // Script fields
+  run?: string;
+  // Prmd fields — prompt: registry ref, local file, or omit to use content block
+  prompt?: string;
+  provider?: string;
+  model?: string;
+}
+
+export interface TestCase {
+  name: string;
+  params: Record<string, unknown>;
+  assert: AssertionDef[];
+  expect_error?: boolean;
+}
+
+export interface TestSuite {
+  name: string;
+  description?: string;
+  target: string;
+  testFilePath: string;
+  tests: TestCase[];
+  evaluatorPrompt?: string;
+}
+
+// --- Test result types ---
+
+export type TestStatus = 'pass' | 'fail' | 'error' | 'skip';
+export type AssertionStatus = 'pass' | 'fail' | 'error' | 'skip';
+
+export interface AssertionResult {
+  evaluator: EvaluatorType;
+  check?: string;
+  status: AssertionStatus;
+  reason?: string;
+  duration: number;
+}
+
+export interface TestExecutionMetadata {
+  provider: string;
+  model: string;
+  duration: number;
+  usage?: {
+    promptTokens?: number;
+    completionTokens?: number;
+    totalTokens?: number;
+  };
+}
+
+export interface TestResult {
+  suite: string;
+  testName: string;
+  status: TestStatus;
+  duration: number;
+  assertions: AssertionResult[];
+  output?: string;
+  compiledInput?: string;
+  error?: string;
+  execution?: TestExecutionMetadata;
+}
+
+export interface TestRunSummary {
+  total: number;
+  passed: number;
+  failed: number;
+  errors: number;
+  skipped: number;
+  duration: number;
+  totalTokens?: number;
+  providers?: string[];
+  models?: string[];
+}
+
+export interface TestRunResult {
+  suites: TestSuiteResult[];
+  summary: TestRunSummary;
+}
+
+export interface TestSuiteResult {
+  suite: string;
+  testFilePath: string;
+  results: TestResult[];
+}
+
+// --- Options ---
+
+export interface TestRunOptions {
+  evaluators?: EvaluatorType[];
+  noLlm?: boolean;
+  reporter?: 'console' | 'json' | 'junit';
+  failFast?: boolean;
+  runAll?: boolean;
+  verbose?: boolean;
+  workspaceRoot?: string;
+  registryUrl?: string;
+  // Default provider/model for test execution (overridden by .prmd frontmatter)
+  provider?: string;
+  model?: string;
+}
+
+// --- Progress callback ---
+
+export type TestProgressEvent =
+  | { type: 'suite_start'; suite: string; testCount: number }
+  | { type: 'test_start'; suite: string; testName: string }
+  | { type: 'test_complete'; suite: string; testName: string; result: TestResult }
+  | { type: 'suite_complete'; suite: string; results: TestResult[] }
+  | { type: 'assertion_complete'; suite: string; testName: string; assertion: AssertionResult };
+
+export type TestProgressCallback = (event: TestProgressEvent) => void;

package/tsconfig.json

@@ -0,0 +1,20 @@
+{
+  "compilerOptions": {
+    "target": "ES2020",
+    "module": "commonjs",
+    "lib": ["ES2020"],
+    "declaration": true,
+    "declarationMap": true,
+    "outDir": "./dist",
+    "rootDir": "./src",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "resolveJsonModule": true,
+    "moduleResolution": "node",
+    "types": ["node"]
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist"]
+}