cognitive-core 0.2.0 → 0.2.2

package/tests/search/evaluator.test.ts

@@ -0,0 +1,257 @@
+import { describe, it, expect, beforeEach } from 'vitest';
+import {
+  SolutionEvaluator,
+  createSolutionEvaluator,
+  type VerificationResult,
+} from '../../src/search/evaluator.js';
+import { createTrajectory } from '../../src/types/trajectory.js';
+import { createTask } from '../../src/types/task.js';
+import { createStep } from '../../src/types/step.js';
+import { successOutcome, failureOutcome } from '../../src/types/outcome.js';
+import type { Trajectory } from '../../src/types/trajectory.js';
+import type { Task } from '../../src/types/task.js';
+
+function makeTask(overrides?: Partial<Task>): Task {
+  return createTask({
+    domain: 'code',
+    description: 'fix a bug in the login module',
+    ...overrides,
+  });
+}
+
+function makeTrajectory(opts?: {
+  success?: boolean;
+  stepCount?: number;
+  errorSteps?: number;
+  withAttribution?: boolean;
+}): Trajectory {
+  const steps = [];
+  const count = opts?.stepCount ?? 3;
+  const errorCount = opts?.errorSteps ?? 0;
+
+  for (let i = 0; i < count; i++) {
+    steps.push(
+      createStep({
+        action: `Step ${i + 1}`,
+        observation: i < errorCount ? 'error: something failed' : 'ok',
+        attributionScore: opts?.withAttribution ? 0.2 : undefined,
+      })
+    );
+  }
+
+  return createTrajectory({
+    task: makeTask(),
+    steps,
+    outcome: (opts?.success ?? true) ? successOutcome('done') : failureOutcome('failed'),
+    agentId: 'agent-1',
+  });
+}
+
+describe('SolutionEvaluator', () => {
+  let evaluator: SolutionEvaluator;
+
+  beforeEach(() => {
+    evaluator = createSolutionEvaluator(null);
+  });
+
+  describe('heuristic evaluation (fallback)', () => {
+    it('should rate successful trajectory with few steps well', async () => {
+      const result = await evaluator.evaluate(
+        makeTrajectory({ success: true, stepCount: 3 }),
+        makeTask()
+      );
+
+      expect(result.method).toBe('heuristic');
+      expect(result.score).toBeGreaterThanOrEqual(0.7);
+      expect(result.acceptable).toBe(true);
+      expect(result.issues).toHaveLength(0);
+    });
+
+    it('should penalize failed trajectory', async () => {
+      const result = await evaluator.evaluate(
+        makeTrajectory({ success: false }),
+        makeTask()
+      );
+
+      expect(result.score).toBeLessThan(0.6);
+      expect(result.acceptable).toBe(false);
+      expect(result.issues.length).toBeGreaterThan(0);
+      expect(result.issues[0].type).toBe('error');
+    });
+
+    it('should penalize empty trajectory', async () => {
+      const trajectory = createTrajectory({
+        task: makeTask(),
+        steps: [],
+        outcome: failureOutcome('no steps taken'),
+        agentId: 'a',
+      });
+
+      const result = await evaluator.evaluate(trajectory, makeTask());
+      expect(result.score).toBeLessThan(0.3);
+      expect(result.issues.some((i) => i.type === 'incomplete')).toBe(true);
+    });
+
+    it('should penalize very long trajectories', async () => {
+      const result = await evaluator.evaluate(
+        makeTrajectory({ success: true, stepCount: 25 }),
+        makeTask()
+      );
+
+      expect(result.issues.some((i) => i.type === 'inefficient')).toBe(true);
+    });
+
+    it('should penalize trajectories with error steps', async () => {
+      const result = await evaluator.evaluate(
+        makeTrajectory({ success: true, stepCount: 5, errorSteps: 3 }),
+        makeTask()
+      );
+
+      expect(result.score).toBeLessThan(
+        // Compare with clean trajectory
+        (await evaluator.evaluate(makeTrajectory({ success: true, stepCount: 5 }), makeTask())).score
+      );
+    });
+
+    it('should boost score for steps with high attribution', async () => {
+      const withAttr = await evaluator.evaluate(
+        makeTrajectory({ success: true, withAttribution: true }),
+        makeTask()
+      );
+      const withoutAttr = await evaluator.evaluate(
+        makeTrajectory({ success: true }),
+        makeTask()
+      );
+
+      expect(withAttr.score).toBeGreaterThanOrEqual(withoutAttr.score);
+    });
+
+    it('should clamp score between 0 and 1', async () => {
+      // Very bad: failed + empty + errors
+      const trajectory = createTrajectory({
+        task: makeTask(),
+        steps: [],
+        outcome: failureOutcome('total failure'),
+        agentId: 'a',
+      });
+
+      const result = await evaluator.evaluate(trajectory, makeTask());
+      expect(result.score).toBeGreaterThanOrEqual(0);
+      expect(result.score).toBeLessThanOrEqual(1);
+    });
+  });
+
+  describe('verification-based evaluation', () => {
+    it('should use registered verifier for matching domain', async () => {
+      const verifier = async (_t: Trajectory, _task: Task): Promise<VerificationResult> => ({
+        passed: true,
+        confidence: 0.95,
+        issues: [],
+        details: 'all tests pass',
+      });
+
+      evaluator.registerVerifier('code', verifier);
+
+      const result = await evaluator.evaluate(makeTrajectory(), makeTask({ domain: 'code' }));
+
+      expect(result.method).toBe('verification');
+      expect(result.acceptable).toBe(true);
+      expect(result.score).toBeGreaterThanOrEqual(0.7);
+    });
+
+    it('should fall through to heuristic when verification confidence too low', async () => {
+      const verifier = async (): Promise<VerificationResult> => ({
+        passed: true,
+        confidence: 0.3, // Below default threshold of 0.8
+      });
+
+      evaluator.registerVerifier('code', verifier);
+
+      const result = await evaluator.evaluate(makeTrajectory(), makeTask({ domain: 'code' }));
+      expect(result.method).toBe('heuristic');
+    });
+
+    it('should fall through when verifier throws', async () => {
+      const verifier = async (): Promise<VerificationResult> => {
+        throw new Error('verification crashed');
+      };
+
+      evaluator.registerVerifier('code', verifier);
+
+      const result = await evaluator.evaluate(makeTrajectory(), makeTask({ domain: 'code' }));
+      expect(result.method).toBe('heuristic');
+    });
+
+    it('should handle failed verification with issues', async () => {
+      const verifier = async (): Promise<VerificationResult> => ({
+        passed: false,
+        confidence: 0.9,
+        issues: [
+          { type: 'incorrect', description: 'test failed: should return 42', severity: 'major' },
+        ],
+      });
+
+      evaluator.registerVerifier('code', verifier);
+
+      const result = await evaluator.evaluate(makeTrajectory(), makeTask({ domain: 'code' }));
+
+      expect(result.method).toBe('verification');
+      expect(result.acceptable).toBe(false);
+      expect(result.score).toBeLessThan(0.5);
+      expect(result.issues).toHaveLength(1);
+    });
+
+    it('should not use verifier for wrong domain', async () => {
+      const verifier = async (): Promise<VerificationResult> => ({
+        passed: true,
+        confidence: 0.95,
+      });
+
+      evaluator.registerVerifier('python', verifier);
+
+      // Task domain is 'code', not 'python'
+      const result = await evaluator.evaluate(makeTrajectory(), makeTask({ domain: 'code' }));
+      expect(result.method).toBe('heuristic');
+    });
+  });
+
+  describe('alwaysUseAgent config', () => {
+    it('should skip verification shortcut when alwaysUseAgent=true', async () => {
+      const alwaysAgentEval = createSolutionEvaluator(null, { alwaysUseAgent: true });
+
+      const verifier = async (): Promise<VerificationResult> => ({
+        passed: true,
+        confidence: 0.99,
+      });
+      alwaysAgentEval.registerVerifier('code', verifier);
+
+      // With no agent or taskRunner, falls through to heuristic
+      const result = await alwaysAgentEval.evaluate(makeTrajectory(), makeTask({ domain: 'code' }));
+      expect(result.method).toBe('heuristic');
+    });
+  });
+
+  describe('parseQuality edge cases', () => {
+    // Test through parseAgentEvaluation indirectly using mock agent
+    it('should handle unknown quality strings gracefully', async () => {
+      // Falls through to heuristic since no agent available
+      const result = await evaluator.evaluate(makeTrajectory(), makeTask());
+      expect(['excellent', 'good', 'needs_work', 'poor']).toContain(result.quality);
+    });
+  });
+});
+
+describe('scoreToQuality (used by evaluator)', () => {
+  it('should map scores to correct quality levels', async () => {
+    const { scoreToQuality } = await import('../../src/search/refinement-types.js');
+
+    expect(scoreToQuality(0.9)).toBe('excellent');
+    expect(scoreToQuality(0.85)).toBe('excellent');
+    expect(scoreToQuality(0.75)).toBe('good');
+    expect(scoreToQuality(0.7)).toBe('good');
+    expect(scoreToQuality(0.5)).toBe('needs_work');
+    expect(scoreToQuality(0.4)).toBe('needs_work');
+    expect(scoreToQuality(0.3)).toBe('poor');
+    expect(scoreToQuality(0.0)).toBe('poor');
+  });
+});

package/tests/search/verification-runner.test.ts

@@ -0,0 +1,357 @@
+import { describe, it, expect, beforeEach } from 'vitest';
+import {
+  VerificationRunner,
+  createVerificationRunner,
+  TestRunners,
+  type CommandVerificationConfig,
+} from '../../src/search/verification-runner.js';
+import { createTrajectory } from '../../src/types/trajectory.js';
+import { createTask } from '../../src/types/task.js';
+import { createStep } from '../../src/types/step.js';
+import { successOutcome } from '../../src/types/outcome.js';
+import type { Trajectory } from '../../src/types/trajectory.js';
+import type { Task } from '../../src/types/task.js';
+
+function makeTask(): Task {
+  return createTask({
+    domain: 'test-domain',
+    description: 'test task',
+  });
+}
+
+function makeTrajectory(): Trajectory {
+  return createTrajectory({
+    task: makeTask(),
+    steps: [createStep({ action: 'echo hello', observation: 'hello' })],
+    outcome: successOutcome('hello world'),
+    agentId: 'agent',
+  });
+}
+
+describe('VerificationRunner', () => {
+  let runner: VerificationRunner;
+
+  beforeEach(() => {
+    runner = createVerificationRunner({ cwd: '/tmp', timeout: 5000 });
+  });
+
+  describe('runCommand', () => {
+    it('should execute a simple command and capture stdout', async () => {
+      const result = await runner.runCommand('echo hello', {});
+      expect(result.stdout.trim()).toBe('hello');
+      expect(result.exitCode).toBe(0);
+      expect(result.timedOut).toBe(false);
+    });
+
+    it('should capture stderr', async () => {
+      const result = await runner.runCommand('echo error >&2', {});
+      expect(result.stderr.trim()).toBe('error');
+    });
+
+    it('should return non-zero exit code for failing commands', async () => {
+      const result = await runner.runCommand('exit 42', {});
+      expect(result.exitCode).toBe(42);
+    });
+
+    it('should handle command timeout', { timeout: 20000 }, async () => {
+      // Use a bash loop that responds to SIGTERM better than plain sleep
+      const result = await runner.runCommand('while true; do sleep 0.1; done', { timeout: 500 });
+      expect(result.timedOut).toBe(true);
+    });
+
+    it('should handle command not found', async () => {
+      const result = await runner.runCommand('nonexistent_command_xyz_12345', {});
+      expect(result.exitCode).not.toBe(0);
+    });
+
+    it('should pass environment variables', async () => {
+      const result = await runner.runCommand('echo $MY_TEST_VAR', {
+        env: { MY_TEST_VAR: 'hello123' },
+      });
+      expect(result.stdout.trim()).toBe('hello123');
+    });
+
+    it('should use specified working directory', async () => {
+      const result = await runner.runCommand('pwd', { cwd: '/tmp' });
+      expect(result.stdout.trim()).toMatch(/\/tmp/);
+    });
+  });
+
+  describe('runCommandSync', () => {
+    it('should execute command synchronously', () => {
+      const result = runner.runCommandSync('echo sync-test');
+      expect(result.stdout.trim()).toBe('sync-test');
+      expect(result.exitCode).toBe(0);
+    });
+
+    it('should handle sync command failure', () => {
+      const result = runner.runCommandSync('exit 1');
+      expect(result.exitCode).toBe(1);
+    });
+
+    it('should detect sync command failure on timeout', () => {
+      const result = runner.runCommandSync('sleep 30', { timeout: 500 });
+      // execSync throws on timeout; killed + SIGTERM detection may vary by platform
+      // The important thing is that the command does not succeed
+      expect(result.exitCode).not.toBe(0);
+    });
+  });
+
+  describe('createVerifier', () => {
+    it('should create verifier that passes on exit code 0', async () => {
+      const config: CommandVerificationConfig = {
+        command: 'echo "all tests pass"',
+        timeout: 5000,
+      };
+
+      const verifier = runner.createVerifier(config);
+      const result = await verifier(makeTrajectory(), makeTask());
+
+      expect(result.passed).toBe(true);
+      expect(result.confidence).toBe(0.7); // No pattern/extractor → 0.7
+    });
+
+    it('should create verifier that fails on non-zero exit', async () => {
+      const config: CommandVerificationConfig = {
+        command: 'exit 1',
+        timeout: 5000,
+      };
+
+      const verifier = runner.createVerifier(config);
+      const result = await verifier(makeTrajectory(), makeTask());
+
+      expect(result.passed).toBe(false);
+      expect(result.issues!.length).toBeGreaterThan(0);
+    });
+
+    it('should check success pattern when provided', async () => {
+      const config: CommandVerificationConfig = {
+        command: 'echo "Tests: 5 passed, 0 failed"',
+        successPattern: /\d+ passed, 0 failed/,
+        timeout: 5000,
+      };
+
+      const verifier = runner.createVerifier(config);
+      const result = await verifier(makeTrajectory(), makeTask());
+
+      expect(result.passed).toBe(true);
+      expect(result.confidence).toBe(0.9); // Has successPattern → 0.9
+    });
+
+    it('should check failure pattern when provided', async () => {
+      const config: CommandVerificationConfig = {
+        command: 'echo "FAIL test_login"',
+        failurePattern: /FAIL/,
+        successOnZeroExit: false,
+        timeout: 5000,
+      };
+
+      const verifier = runner.createVerifier(config);
+      const result = await verifier(makeTrajectory(), makeTask());
+
+      expect(result.passed).toBe(false);
+    });
+
+    it('should use custom issue extractor', async () => {
+      const config: CommandVerificationConfig = {
+        command: 'echo "ERROR: missing return"',
+        timeout: 5000,
+        successOnZeroExit: false,
+        issueExtractor: (output) => {
+          const match = output.match(/ERROR:\s*(.+)/);
+          return match
+            ? [{ type: 'error' as const, description: match[1], severity: 'critical' as const }]
+            : [];
+        },
+      };
+
+      const verifier = runner.createVerifier(config);
+      const result = await verifier(makeTrajectory(), makeTask());
+
+      expect(result.confidence).toBe(0.9); // Has issueExtractor
+    });
+
+    it('should handle timeout in verifier', { timeout: 20000 }, async () => {
+      const config: CommandVerificationConfig = {
+        command: 'while true; do sleep 0.1; done',
+        timeout: 500,
+      };
+
+      const verifier = runner.createVerifier(config);
+      const result = await verifier(makeTrajectory(), makeTask());
+
+      expect(result.passed).toBe(false);
+      expect(result.confidence).toBe(0.5); // Timeout → 0.5
+      expect(result.details).toContain('timed out');
+    });
+
+    it('should interpolate {{taskId}} and {{trajectoryId}} in command', async () => {
+      const config: CommandVerificationConfig = {
+        command: 'echo "task={{taskId}} traj={{trajectoryId}}"',
+        timeout: 5000,
+      };
+
+      const trajectory = makeTrajectory();
+      const task = makeTask();
+      const verifier = runner.createVerifier(config);
+      const result = await verifier(trajectory, task);
+
+      expect(result.details).toContain(`task=${task.id}`);
+      expect(result.details).toContain(`traj=${trajectory.id}`);
+    });
+  });
+
+  describe('createCompositeVerifier', () => {
+    it('should pass when all verifiers pass', async () => {
+      const composite = runner.createCompositeVerifier([
+        { command: 'echo pass1', timeout: 5000 },
+        { command: 'echo pass2', timeout: 5000 },
+      ]);
+
+      const result = await composite(makeTrajectory(), makeTask());
+      expect(result.passed).toBe(true);
+    });
+
+    it('should fail when any verifier fails', async () => {
+      const composite = runner.createCompositeVerifier([
+        { command: 'echo pass', timeout: 5000 },
+        { command: 'exit 1', timeout: 5000 },
+      ]);
+
+      const result = await composite(makeTrajectory(), makeTask());
+      expect(result.passed).toBe(false);
+    });
+
+    it('should aggregate issues from all verifiers', async () => {
+      const composite = runner.createCompositeVerifier([
+        { command: 'exit 1', timeout: 5000 },
+        { command: 'exit 2', timeout: 5000 },
+      ]);
+
+      const result = await composite(makeTrajectory(), makeTask());
+      expect(result.issues!.length).toBeGreaterThanOrEqual(1);
+    });
+
+    it('should fail fast on critical issues', async () => {
+      let secondRan = false;
+      const configs: CommandVerificationConfig[] = [
+        {
+          command: 'exit 1',
+          timeout: 5000,
+          issueExtractor: () => [
+            { type: 'error', description: 'critical failure', severity: 'critical' },
+          ],
+        },
+        {
+          command: 'echo should-not-run',
+          timeout: 5000,
+          issueExtractor: () => {
+            secondRan = true;
+            return [];
+          },
+        },
+      ];
+
+      const composite = runner.createCompositeVerifier(configs);
+      await composite(makeTrajectory(), makeTask());
+
+      // Second verifier should not have run due to fail-fast
+      expect(secondRan).toBe(false);
+    });
+
+    it('should average confidence across verifiers', async () => {
+      const composite = runner.createCompositeVerifier([
+        { command: 'echo pass', timeout: 5000 }, // confidence 0.7
+        { command: 'echo pass', timeout: 5000 }, // confidence 0.7
+      ]);
+
+      const result = await composite(makeTrajectory(), makeTask());
+      expect(result.confidence).toBeCloseTo(0.7, 1);
+    });
+  });
+
+  describe('TestRunners presets', () => {
+    it('should create vitest configuration', () => {
+      const config = TestRunners.vitest('tests/foo.test.ts');
+      expect(config.command).toContain('vitest');
+      expect(config.command).toContain('tests/foo.test.ts');
+      expect(config.timeout).toBe(60000);
+      expect(config.issueExtractor).toBeDefined();
+    });
+
+    it('should create jest configuration', () => {
+      const config = TestRunners.jest();
+      expect(config.command).toContain('jest');
+      expect(config.issueExtractor).toBeDefined();
+    });
+
+    it('should create typescript configuration', () => {
+      const config = TestRunners.typescript();
+      expect(config.command).toContain('tsc --noEmit');
+      expect(config.issueExtractor).toBeDefined();
+    });
+
+    it('should create eslint configuration', () => {
+      const config = TestRunners.eslint('src/');
+      expect(config.command).toContain('eslint');
+      expect(config.command).toContain('src/');
+    });
+
+    it('should create pytest configuration', () => {
+      const config = TestRunners.pytest();
+      expect(config.command).toContain('pytest');
+      expect(config.failurePattern).toBeDefined();
+    });
+
+    it('should create go test configuration', () => {
+      const config = TestRunners.goTest();
+      expect(config.command).toContain('go test');
+    });
+
+    it('should create cargo test configuration', () => {
+      const config = TestRunners.cargoTest();
+      expect(config.command).toContain('cargo test');
+      expect(config.timeout).toBe(120000);
+    });
+  });
+
+  describe('default issue extraction', () => {
+    it('should extract error messages from output', async () => {
+      const config: CommandVerificationConfig = {
+        command: 'echo "Error: cannot read file" && exit 1',
+        timeout: 5000,
+      };
+
+      const verifier = runner.createVerifier(config);
+      const result = await verifier(makeTrajectory(), makeTask());
+
+      expect(result.issues!.length).toBeGreaterThan(0);
+      expect(result.issues![0].description).toContain('cannot read file');
+    });
+
+    it('should extract fail messages from output', async () => {
+      const config: CommandVerificationConfig = {
+        command: 'echo "Fail: test_login" && exit 1',
+        timeout: 5000,
+      };
+
+      const verifier = runner.createVerifier(config);
+      const result = await verifier(makeTrajectory(), makeTask());
+
+      expect(result.issues!.length).toBeGreaterThan(0);
+    });
+
+    it('should generate generic issue when no pattern matches', async () => {
+      const config: CommandVerificationConfig = {
+        command: 'exit 99',
+        timeout: 5000,
+      };
+
+      const verifier = runner.createVerifier(config);
+      const result = await verifier(makeTrajectory(), makeTask());
+
+      expect(result.issues!.length).toBeGreaterThan(0);
+      expect(result.issues![0].description).toContain('exit code 99');
+    });
+  });
+});