@plaited/acp-harness 0.2.6 → 0.3.1

package/src/grader-loader.ts

@@ -0,0 +1,174 @@
+/**
+ * Polyglot grader loader module.
+ *
+ * @remarks
+ * Supports loading graders from:
+ * - TypeScript/JavaScript modules (import as ES module)
+ * - Executable scripts (Python, Ruby, shell, etc. via subprocess)
+ *
+ * Executable graders use stdin/stdout JSON protocol:
+ * - Input: `{"input": "...", "output": "...", "expected": "...", "trajectory": [...]}`
+ * - Output: `{"pass": true, "score": 1.0, "reasoning": "..."}`
+ *
+ * @packageDocumentation
+ */
+
+import type { Grader, TrajectoryStep } from './schemas.ts'
+import { GraderResultSchema } from './schemas.ts'
+
+// ============================================================================
+// Constants
+// ============================================================================
+
+/** File extensions that are imported as ES modules */
+const JS_EXTENSIONS = ['.ts', '.js', '.mjs', '.cjs']
+
+// ============================================================================
+// Helpers
+// ============================================================================
+
+/** Check if a file path is a JavaScript/TypeScript module */
+const isJsModule = (path: string): boolean => JS_EXTENSIONS.some((ext) => path.endsWith(ext))
+
+/** Resolve path relative to process.cwd() */
+const resolvePath = (path: string): string => {
+  if (path.startsWith('/')) return path
+  return `${process.cwd()}/${path}`
+}
+
+// ============================================================================
+// Executable Grader
+// ============================================================================
+
+/** Input format for executable graders (stdin JSON) */
+type ExecGraderInput = {
+  input: string
+  output: string
+  expected?: string
+  trajectory?: TrajectoryStep[]
+}
+
+/**
+ * Create a grader function that executes an external script.
+ *
+ * @remarks
+ * The script receives JSON on stdin and must output JSON on stdout.
+ * Non-zero exit codes are treated as errors.
+ *
+ * @param execPath - Absolute path to the executable script
+ * @returns Grader function
+ */
+const createExecGrader = (execPath: string): Grader => {
+  return async (params) => {
+    const input: ExecGraderInput = {
+      input: params.input,
+      output: params.output,
+      expected: params.expected,
+      trajectory: params.trajectory,
+    }
+
+    const inputJson = JSON.stringify(input)
+
+    const proc = Bun.spawn([execPath], {
+      stdin: new TextEncoder().encode(inputJson),
+      stdout: 'pipe',
+      stderr: 'pipe',
+    })
+
+    const [stdout, stderr, exitCode] = await Promise.all([
+      new Response(proc.stdout).text(),
+      new Response(proc.stderr).text(),
+      proc.exited,
+    ])
+
+    if (exitCode !== 0) {
+      throw new Error(`Grader exited with code ${exitCode}: ${stderr.trim() || 'No error output'}`)
+    }
+
+    const trimmedStdout = stdout.trim()
+    if (!trimmedStdout) {
+      throw new Error('Grader produced no output')
+    }
+
+    let parsed: unknown
+    try {
+      parsed = JSON.parse(trimmedStdout)
+    } catch {
+      throw new Error(`Grader output is not valid JSON: ${trimmedStdout.slice(0, 100)}`)
+    }
+
+    const result = GraderResultSchema.safeParse(parsed)
+    if (!result.success) {
+      throw new Error(`Invalid grader result: ${result.error.message}`)
+    }
+
+    return result.data
+  }
+}
+
+// ============================================================================
+// Module Grader
+// ============================================================================
+
+/**
+ * Load a grader from a JavaScript/TypeScript module.
+ *
+ * @remarks
+ * The module must export a `grade` function matching the `Grader` type.
+ *
+ * @param modulePath - Absolute path to the module
+ * @returns Grader function
+ */
+const loadModuleGrader = async (modulePath: string): Promise<Grader> => {
+  const graderModule = await import(modulePath)
+
+  if (typeof graderModule.grade !== 'function') {
+    throw new Error(`Grader module must export a 'grade' function`)
+  }
+
+  return graderModule.grade as Grader
+}
+
+// ============================================================================
+// Public API
+// ============================================================================
+
+/**
+ * Load a grader from a file path.
+ *
+ * @remarks
+ * Detection logic:
+ * - `.ts`, `.js`, `.mjs`, `.cjs` → Import as ES module
+ * - Everything else → Execute as subprocess
+ *
+ * @param graderPath - Path to the grader (relative or absolute)
+ * @returns Grader function
+ * @throws Error if grader not found or invalid
+ *
+ * @example
+ * ```typescript
+ * // TypeScript grader
+ * const grader = await loadGrader('./grader.ts')
+ *
+ * // Python grader
+ * const grader = await loadGrader('./grader.py')
+ *
+ * // Any executable
+ * const grader = await loadGrader('./my-grader')
+ * ```
+ */
+export const loadGrader = async (graderPath: string): Promise<Grader> => {
+  const resolvedPath = resolvePath(graderPath)
+
+  // Check file exists
+  const file = Bun.file(resolvedPath)
+  if (!(await file.exists())) {
+    throw new Error(`Grader not found: ${resolvedPath}`)
+  }
+
+  if (isJsModule(resolvedPath)) {
+    return loadModuleGrader(resolvedPath)
+  }
+
+  return createExecGrader(resolvedPath)
+}

package/src/harness.ts

@@ -0,0 +1,35 @@
+/**
+ * Harness commands for agent evaluation.
+ *
+ * @remarks
+ * Re-exports all harness command modules for programmatic use.
+ * For CLI usage, run `acp-harness <command> --help`.
+ *
+ * **Commands:**
+ * - `capture` - Core trajectory capture
+ * - `trials` - Multi-run pass@k/pass^k analysis
+ * - `summarize` - Derive compact views from results
+ * - `calibrate` - Sample failures for grader review
+ * - `validateRefs` - Check reference solutions
+ * - `balance` - Analyze test set coverage
+ * - `schemasCli` - Export JSON schemas
+ *
+ * @packageDocumentation
+ */
+
+export type { BalanceConfig } from './balance.ts'
+export { balance, runBalance } from './balance.ts'
+export type { CalibrateConfig } from './calibrate.ts'
+export { calibrate, runCalibrate } from './calibrate.ts'
+// Config types
+export type { CaptureConfig } from './capture.ts'
+// Command implementations (for programmatic use)
+export { capture, extractOutput, extractTrajectory, hasToolErrors, loadPrompts, runCapture } from './capture.ts'
+export type { SchemasConfig } from './schemas-cli.ts'
+export { runSchemas, schemasCli } from './schemas-cli.ts'
+export type { SummarizeConfig } from './summarize.ts'
+export { runSummarize, summarize } from './summarize.ts'
+export type { TrialsConfig } from './trials.ts'
+export { runTrials, trials } from './trials.ts'
+export type { ValidateRefsConfig } from './validate-refs.ts'
+export { runValidateRefs, validateRefs } from './validate-refs.ts'

package/src/schemas-cli.ts

@@ -0,0 +1,239 @@
+/**
+ * Schemas command - export JSON schemas for non-TypeScript users.
+ *
+ * @remarks
+ * Uses Zod 4's native `z.toJSONSchema()` to generate JSON Schema from
+ * the harness schemas. Useful for validation in other languages/tools.
+ *
+ * @packageDocumentation
+ */
+
+import { parseArgs } from 'node:util'
+import { z } from 'zod'
+import * as schemas from './schemas.ts'
+
+// ============================================================================
+// Schema Registry
+// ============================================================================
+
+/** Available schemas for export */
+const SCHEMA_REGISTRY: Record<string, z.ZodSchema> = {
+  PromptCase: schemas.PromptCaseSchema,
+  GraderResult: schemas.GraderResultSchema,
+  TrajectoryStep: schemas.TrajectoryStepSchema,
+  CaptureResult: schemas.CaptureResultSchema,
+  SummaryResult: schemas.SummaryResultSchema,
+  TrialEntry: schemas.TrialEntrySchema,
+  TrialResult: schemas.TrialResultSchema,
+  CalibrationSample: schemas.CalibrationSampleSchema,
+  BalanceAnalysis: schemas.BalanceAnalysisSchema,
+  ValidationResult: schemas.ValidationResultSchema,
+  McpServerConfig: schemas.McpServerSchema,
+  Session: schemas.SessionSchema,
+  JsonRpcRequest: schemas.JsonRpcRequestSchema,
+  JsonRpcResponse: schemas.JsonRpcResponseSchema,
+  JsonRpcError: schemas.JsonRpcErrorSchema,
+}
+
+// ============================================================================
+// Types
+// ============================================================================
+
+/** Configuration for schemas command */
+export type SchemasConfig = {
+  /** Specific schema name to export (undefined = all) */
+  schemaName?: string
+  /** Output file path */
+  outputPath?: string
+  /** Output as JSON (vs list) */
+  json?: boolean
+  /** Split into separate files */
+  split?: boolean
+  /** List available schemas */
+  list?: boolean
+}
+
+// ============================================================================
+// Helpers
+// ============================================================================
+
+/** Resolve path relative to process.cwd() */
+const resolvePath = (path: string): string => {
+  if (path.startsWith('/')) return path
+  return `${process.cwd()}/${path}`
+}
+
+/** Generate JSON Schema from Zod schema */
+const toJsonSchema = (schema: z.ZodSchema, name: string): object => {
+  try {
+    // Zod 4's native JSON Schema generation
+    const jsonSchema = z.toJSONSchema(schema)
+    return {
+      $schema: 'https://json-schema.org/draft/2020-12/schema',
+      title: name,
+      ...jsonSchema,
+    }
+  } catch (error) {
+    // Fallback for schemas that can't be converted
+    return {
+      $schema: 'https://json-schema.org/draft/2020-12/schema',
+      title: name,
+      description: `Schema for ${name} (auto-generation failed: ${error instanceof Error ? error.message : 'unknown error'})`,
+    }
+  }
+}
+
+// ============================================================================
+// Schemas Implementation
+// ============================================================================
+
+/**
+ * Execute schemas command with configuration object.
+ *
+ * @param config - Schemas configuration
+ * @returns Generated JSON schemas
+ */
+export const runSchemas = async (config: SchemasConfig): Promise<Record<string, object> | string[]> => {
+  const { schemaName, outputPath, json = false, split = false, list = false } = config
+
+  // List mode
+  if (list) {
+    const names = Object.keys(SCHEMA_REGISTRY)
+    // biome-ignore lint/suspicious/noConsole: CLI stdout output
+    console.log('Available schemas:')
+    for (const name of names) {
+      // biome-ignore lint/suspicious/noConsole: CLI stdout output
+      console.log(`  - ${name}`)
+    }
+    return names
+  }
+
+  // Single schema mode
+  if (schemaName) {
+    const schema = SCHEMA_REGISTRY[schemaName]
+    if (!schema) {
+      console.error(`Error: Unknown schema '${schemaName}'`)
+      console.error(`Available: ${Object.keys(SCHEMA_REGISTRY).join(', ')}`)
+      process.exit(1)
+    }
+
+    const jsonSchema = toJsonSchema(schema, schemaName)
+    const output = JSON.stringify(jsonSchema, null, 2)
+
+    if (outputPath) {
+      await Bun.write(resolvePath(outputPath), output)
+    } else {
+      // biome-ignore lint/suspicious/noConsole: CLI stdout output
+      console.log(output)
+    }
+
+    return { [schemaName]: jsonSchema }
+  }
+
+  // All schemas mode
+  const allSchemas: Record<string, object> = {}
+
+  for (const [name, schema] of Object.entries(SCHEMA_REGISTRY)) {
+    allSchemas[name] = toJsonSchema(schema, name)
+  }
+
+  if (split && outputPath) {
+    // Create directory and write separate files
+    const dir = resolvePath(outputPath)
+    await Bun.$`mkdir -p ${dir}`
+
+    for (const [name, jsonSchema] of Object.entries(allSchemas)) {
+      const filePath = `${dir}/${name}.json`
+      await Bun.write(filePath, JSON.stringify(jsonSchema, null, 2))
+    }
+
+    console.error(`Wrote ${Object.keys(allSchemas).length} schema files to ${dir}/`)
+  } else if (json) {
+    const output = JSON.stringify(allSchemas, null, 2)
+
+    if (outputPath) {
+      await Bun.write(resolvePath(outputPath), output)
+    } else {
+      // biome-ignore lint/suspicious/noConsole: CLI stdout output
+      console.log(output)
+    }
+  } else {
+    // Default: list schemas
+    // biome-ignore lint/suspicious/noConsole: CLI stdout output
+    console.log('Available schemas (use --json to export):')
+    for (const name of Object.keys(allSchemas)) {
+      // biome-ignore lint/suspicious/noConsole: CLI stdout output
+      console.log(`  - ${name}`)
+    }
+  }
+
+  return allSchemas
+}
+
+// ============================================================================
+// CLI Entry Point
+// ============================================================================
+
+/**
+ * Schemas command CLI handler.
+ *
+ * @param args - Command line arguments (after 'schemas')
+ */
+export const schemasCli = async (args: string[]): Promise<void> => {
+  const { values, positionals } = parseArgs({
+    args,
+    options: {
+      output: { type: 'string', short: 'o' },
+      json: { type: 'boolean', short: 'j', default: false },
+      split: { type: 'boolean', short: 's', default: false },
+      list: { type: 'boolean', short: 'l', default: false },
+      help: { type: 'boolean', short: 'h' },
+    },
+    allowPositionals: true,
+  })
+
+  if (values.help) {
+    // biome-ignore lint/suspicious/noConsole: CLI help output
+    console.log(`
+Usage: acp-harness schemas [schema-name] [options]
+
+Arguments:
+  schema-name       Specific schema to export (optional)
+
+Options:
+  -o, --output      Output file or directory (with --split)
+  -j, --json        Export as JSON (default: list names)
+  -s, --split       Split into separate files (requires --output dir)
+  -l, --list        List available schemas
+  -h, --help        Show this help message
+
+Available Schemas:
+  PromptCase, GraderResult, TrajectoryStep, CaptureResult, SummaryResult,
+  TrialEntry, TrialResult, CalibrationSample, BalanceAnalysis, ValidationResult,
+  McpServerConfig, Session, JsonRpcRequest, JsonRpcResponse, JsonRpcError
+
+Examples:
+  # List available schemas
+  acp-harness schemas --list
+
+  # Export all schemas as single JSON file
+  acp-harness schemas --json -o schemas.json
+
+  # Export specific schema
+  acp-harness schemas CaptureResult --json
+  acp-harness schemas TrialResult --json -o trial-schema.json
+
+  # Export all schemas as separate files
+  acp-harness schemas --json --split -o schemas/
+`)
+    return
+  }
+
+  await runSchemas({
+    schemaName: positionals[0],
+    outputPath: values.output,
+    json: values.json ?? false,
+    split: values.split ?? false,
+    list: values.list ?? false,
+  })
+}