@plaited/agent-eval-harness 0.5.0

package/src/schemas/tests/fixtures/grader-exec.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env python3
+"""
+Test fixture: Python grader script using stdin/stdout JSON protocol.
+"""
+
+import json
+import sys
+
+def main():
+    data = json.load(sys.stdin)
+
+    output = data.get("output", "").lower()
+    hint = (data.get("hint") or "").lower()
+
+    if hint:
+        pass_result = hint in output
+    else:
+        pass_result = True
+
+    result = {
+        "pass": pass_result,
+        "score": 1.0 if pass_result else 0.0,
+        "reasoning": "Contains expected" if pass_result else "Missing expected"
+    }
+
+    print(json.dumps(result))
+
+if __name__ == "__main__":
+    main()

package/src/schemas/tests/fixtures/grader-module.ts

@@ -0,0 +1,14 @@
+/**
+ * Test fixture: TypeScript grader module.
+ */
+
+import type { Grader } from '../../schemas.ts'
+
+export const grade: Grader = async ({ input: _input, output, hint }) => {
+  const pass = hint ? output.toLowerCase().includes(hint.toLowerCase()) : true
+  return {
+    pass,
+    score: pass ? 1.0 : 0.0,
+    reasoning: pass ? 'Contains expected text' : 'Missing expected text',
+  }
+}

package/src/schemas/tests/grader-loader.spec.ts

@@ -0,0 +1,153 @@
+import { describe, expect, test } from 'bun:test'
+import { join } from 'node:path'
+import { loadGrader } from '../grader-loader.ts'
+
+const fixturesDir = join(import.meta.dir, 'fixtures')
+
+// ============================================================================
+// Module Graders (TypeScript/JavaScript)
+// ============================================================================
+
+describe('loadGrader - module graders', () => {
+  test('loads TypeScript grader module', async () => {
+    const grader = await loadGrader(join(fixturesDir, 'grader-module.ts'))
+
+    const result = await grader({
+      input: 'What is 2+2?',
+      output: 'The answer is 4',
+      hint: '4',
+    })
+
+    expect(result.pass).toBe(true)
+    expect(result.score).toBe(1.0)
+    expect(result.reasoning).toBe('Contains expected text')
+  })
+
+  test('fails when module does not export grade function', async () => {
+    await expect(loadGrader(join(fixturesDir, 'grader-bad-module.ts'))).rejects.toThrow(
+      "Grader module must export a 'grade' function",
+    )
+  })
+
+  test('fails when module does not exist', async () => {
+    await expect(loadGrader(join(fixturesDir, 'nonexistent.ts'))).rejects.toThrow('Grader not found')
+  })
+})
+
+// ============================================================================
+// Executable Graders (Python, etc.)
+// ============================================================================
+
+describe('loadGrader - executable graders', () => {
+  test('loads and executes Python grader', async () => {
+    const grader = await loadGrader(join(fixturesDir, 'grader-exec.py'))
+
+    const result = await grader({
+      input: 'What is 2+2?',
+      output: 'The answer is 4',
+      hint: '4',
+    })
+
+    expect(result.pass).toBe(true)
+    expect(result.score).toBe(1.0)
+    expect(result.reasoning).toBe('Contains expected')
+  })
+
+  test('Python grader returns pass=false when expected not in output', async () => {
+    const grader = await loadGrader(join(fixturesDir, 'grader-exec.py'))
+
+    const result = await grader({
+      input: 'What is 2+2?',
+      output: 'I do not know',
+      hint: '4',
+    })
+
+    expect(result.pass).toBe(false)
+    expect(result.score).toBe(0.0)
+  })
+
+  test('throws when executable exits with non-zero code', async () => {
+    const grader = await loadGrader(join(fixturesDir, 'grader-exec-fail.py'))
+
+    await expect(
+      grader({
+        input: 'test',
+        output: 'test',
+      }),
+    ).rejects.toThrow('Grader exited with code 1')
+  })
+
+  test('throws when executable outputs invalid JSON', async () => {
+    const grader = await loadGrader(join(fixturesDir, 'grader-exec-invalid.py'))
+
+    await expect(
+      grader({
+        input: 'test',
+        output: 'test',
+      }),
+    ).rejects.toThrow('Grader output is not valid JSON')
+  })
+
+  test('fails when executable does not exist', async () => {
+    await expect(loadGrader(join(fixturesDir, 'nonexistent.py'))).rejects.toThrow('Grader not found')
+  })
+})
+
+// ============================================================================
+// Extension Detection
+// ============================================================================
+
+describe('loadGrader - extension detection', () => {
+  test('detects .ts as module', async () => {
+    const grader = await loadGrader(join(fixturesDir, 'grader-module.ts'))
+    // If this doesn't throw, it was loaded as a module (not executed)
+    expect(grader).toBeInstanceOf(Function)
+  })
+
+  test('detects .py as executable', async () => {
+    const grader = await loadGrader(join(fixturesDir, 'grader-exec.py'))
+    expect(grader).toBeInstanceOf(Function)
+  })
+})
+
+// ============================================================================
+// Trajectory Support
+// ============================================================================
+
+describe('loadGrader - trajectory support', () => {
+  test('passes trajectory to module grader', async () => {
+    const grader = await loadGrader(join(fixturesDir, 'grader-module.ts'))
+
+    const trajectory = [
+      { type: 'message' as const, content: 'Hello', timestamp: 0 },
+      { type: 'tool_call' as const, name: 'read', status: 'completed', timestamp: 100 },
+    ]
+
+    const result = await grader({
+      input: 'test',
+      output: 'The answer is 4',
+      hint: '4',
+      trajectory,
+    })
+
+    expect(result.pass).toBe(true)
+  })
+
+  test('passes trajectory to executable grader', async () => {
+    const grader = await loadGrader(join(fixturesDir, 'grader-exec.py'))
+
+    const trajectory = [
+      { type: 'message' as const, content: 'Hello', timestamp: 0 },
+      { type: 'tool_call' as const, name: 'read', status: 'completed', timestamp: 100 },
+    ]
+
+    const result = await grader({
+      input: 'test',
+      output: 'The answer is 4',
+      hint: '4',
+      trajectory,
+    })
+
+    expect(result.pass).toBe(true)
+  })
+})

package/src/schemas/tests/schemas-cli.spec.ts

@@ -0,0 +1,142 @@
+import { afterEach, beforeEach, describe, expect, test } from 'bun:test'
+import { runSchemas } from '../schemas-cli.ts'
+
+// ============================================================================
+// runSchemas
+// ============================================================================
+
+describe('runSchemas', () => {
+  const testOutputDir = '/tmp/agent-eval-harness-test-schemas'
+
+  beforeEach(async () => {
+    // Clean up test directory
+    await Bun.$`rm -rf ${testOutputDir}`.nothrow()
+  })
+
+  afterEach(async () => {
+    // Clean up test directory
+    await Bun.$`rm -rf ${testOutputDir}`.nothrow()
+  })
+
+  describe('list mode', () => {
+    test('returns array of schema names', async () => {
+      const result = await runSchemas({ list: true })
+      expect(Array.isArray(result)).toBe(true)
+      const names = result as string[]
+      expect(names).toContain('PromptCase')
+      expect(names).toContain('CaptureResult')
+      expect(names).toContain('GraderResult')
+    })
+  })
+
+  describe('single schema mode', () => {
+    test('returns single schema by name', async () => {
+      const result = await runSchemas({ schemaName: 'PromptCase', json: true })
+      expect(typeof result).toBe('object')
+      const schemas = result as Record<string, object>
+      expect(schemas.PromptCase).toBeDefined()
+      expect(schemas.PromptCase).toHaveProperty('$schema')
+      expect(schemas.PromptCase).toHaveProperty('title', 'PromptCase')
+    })
+
+    test('writes schema to file when outputPath provided', async () => {
+      const outputPath = `${testOutputDir}/prompt-case.json`
+      await Bun.$`mkdir -p ${testOutputDir}`
+
+      await runSchemas({
+        schemaName: 'GraderResult',
+        outputPath,
+      })
+
+      const content = await Bun.file(outputPath).text()
+      const schema = JSON.parse(content)
+      expect(schema.title).toBe('GraderResult')
+      expect(schema.$schema).toBe('https://json-schema.org/draft/2020-12/schema')
+    })
+  })
+
+  describe('all schemas mode', () => {
+    test('returns all schemas as object', async () => {
+      const result = await runSchemas({ json: true })
+      expect(typeof result).toBe('object')
+      const schemas = result as Record<string, object>
+
+      // Check a sampling of expected schemas
+      expect(schemas.PromptCase).toBeDefined()
+      expect(schemas.CaptureResult).toBeDefined()
+      expect(schemas.GraderResult).toBeDefined()
+      expect(schemas.TrajectoryStep).toBeDefined()
+      expect(schemas.Session).toBeDefined()
+    })
+
+    test('writes all schemas to single file', async () => {
+      const outputPath = `${testOutputDir}/all-schemas.json`
+      await Bun.$`mkdir -p ${testOutputDir}`
+
+      await runSchemas({
+        json: true,
+        outputPath,
+      })
+
+      const content = await Bun.file(outputPath).text()
+      const schemas = JSON.parse(content)
+      expect(schemas.PromptCase).toBeDefined()
+      expect(schemas.CaptureResult).toBeDefined()
+    })
+
+    test('splits schemas into separate files', async () => {
+      await runSchemas({
+        json: true,
+        split: true,
+        outputPath: testOutputDir,
+      })
+
+      // Check that individual files were created
+      const promptCaseExists = await Bun.file(`${testOutputDir}/PromptCase.json`).exists()
+      const captureResultExists = await Bun.file(`${testOutputDir}/CaptureResult.json`).exists()
+      const graderResultExists = await Bun.file(`${testOutputDir}/GraderResult.json`).exists()
+
+      expect(promptCaseExists).toBe(true)
+      expect(captureResultExists).toBe(true)
+      expect(graderResultExists).toBe(true)
+
+      // Verify content
+      const promptCaseContent = await Bun.file(`${testOutputDir}/PromptCase.json`).text()
+      const promptCaseSchema = JSON.parse(promptCaseContent)
+      expect(promptCaseSchema.title).toBe('PromptCase')
+    })
+  })
+
+  describe('schema content validation', () => {
+    test('PromptCase schema has correct structure', async () => {
+      const result = await runSchemas({ schemaName: 'PromptCase', json: true })
+      const schemas = result as Record<string, object>
+      const schema = schemas.PromptCase as Record<string, unknown>
+
+      expect(schema.$schema).toBe('https://json-schema.org/draft/2020-12/schema')
+      expect(schema.title).toBe('PromptCase')
+      expect(schema.type).toBe('object')
+
+      // Check properties exist
+      const properties = schema.properties as Record<string, unknown>
+      expect(properties).toBeDefined()
+      expect(properties.id).toBeDefined()
+      expect(properties.input).toBeDefined()
+    })
+
+    test('GraderResult schema has correct constraints', async () => {
+      const result = await runSchemas({ schemaName: 'GraderResult', json: true })
+      const schemas = result as Record<string, object>
+      const schema = schemas.GraderResult as Record<string, unknown>
+
+      expect(schema.type).toBe('object')
+      const properties = schema.properties as Record<string, Record<string, unknown>>
+      expect(properties.pass).toBeDefined()
+      expect(properties.score).toBeDefined()
+      expect(properties.pass?.type).toBe('boolean')
+      expect(properties.score?.type).toBe('number')
+      expect(properties.score?.minimum).toBe(0)
+      expect(properties.score?.maximum).toBe(1)
+    })
+  })
+})