cognitive-core 0.2.5 → 0.3.1

package/tests/search/evaluator.test.ts

@@ -1,257 +0,0 @@
-import { describe, it, expect, beforeEach } from 'vitest';
-import {
-  SolutionEvaluator,
-  createSolutionEvaluator,
-  type VerificationResult,
-} from '../../src/search/evaluator.js';
-import { createTrajectory } from '../../src/types/trajectory.js';
-import { createTask } from '../../src/types/task.js';
-import { createStep } from '../../src/types/step.js';
-import { successOutcome, failureOutcome } from '../../src/types/outcome.js';
-import type { Trajectory } from '../../src/types/trajectory.js';
-import type { Task } from '../../src/types/task.js';
-
-function makeTask(overrides?: Partial<Task>): Task {
-  return createTask({
-    domain: 'code',
-    description: 'fix a bug in the login module',
-    ...overrides,
-  });
-}
-
-function makeTrajectory(opts?: {
-  success?: boolean;
-  stepCount?: number;
-  errorSteps?: number;
-  withAttribution?: boolean;
-}): Trajectory {
-  const steps = [];
-  const count = opts?.stepCount ?? 3;
-  const errorCount = opts?.errorSteps ?? 0;
-
-  for (let i = 0; i < count; i++) {
-    steps.push(
-      createStep({
-        action: `Step ${i + 1}`,
-        observation: i < errorCount ? 'error: something failed' : 'ok',
-        attributionScore: opts?.withAttribution ? 0.2 : undefined,
-      })
-    );
-  }
-
-  return createTrajectory({
-    task: makeTask(),
-    steps,
-    outcome: (opts?.success ?? true) ? successOutcome('done') : failureOutcome('failed'),
-    agentId: 'agent-1',
-  });
-}
-
-describe('SolutionEvaluator', () => {
-  let evaluator: SolutionEvaluator;
-
-  beforeEach(() => {
-    evaluator = createSolutionEvaluator(null);
-  });
-
-  describe('heuristic evaluation (fallback)', () => {
-    it('should rate successful trajectory with few steps well', async () => {
-      const result = await evaluator.evaluate(
-        makeTrajectory({ success: true, stepCount: 3 }),
-        makeTask()
-      );
-
-      expect(result.method).toBe('heuristic');
-      expect(result.score).toBeGreaterThanOrEqual(0.7);
-      expect(result.acceptable).toBe(true);
-      expect(result.issues).toHaveLength(0);
-    });
-
-    it('should penalize failed trajectory', async () => {
-      const result = await evaluator.evaluate(
-        makeTrajectory({ success: false }),
-        makeTask()
-      );
-
-      expect(result.score).toBeLessThan(0.6);
-      expect(result.acceptable).toBe(false);
-      expect(result.issues.length).toBeGreaterThan(0);
-      expect(result.issues[0].type).toBe('error');
-    });
-
-    it('should penalize empty trajectory', async () => {
-      const trajectory = createTrajectory({
-        task: makeTask(),
-        steps: [],
-        outcome: failureOutcome('no steps taken'),
-        agentId: 'a',
-      });
-
-      const result = await evaluator.evaluate(trajectory, makeTask());
-      expect(result.score).toBeLessThan(0.3);
-      expect(result.issues.some((i) => i.type === 'incomplete')).toBe(true);
-    });
-
-    it('should penalize very long trajectories', async () => {
-      const result = await evaluator.evaluate(
-        makeTrajectory({ success: true, stepCount: 25 }),
-        makeTask()
-      );
-
-      expect(result.issues.some((i) => i.type === 'inefficient')).toBe(true);
-    });
-
-    it('should penalize trajectories with error steps', async () => {
-      const result = await evaluator.evaluate(
-        makeTrajectory({ success: true, stepCount: 5, errorSteps: 3 }),
-        makeTask()
-      );
-
-      expect(result.score).toBeLessThan(
-        // Compare with clean trajectory
-        (await evaluator.evaluate(makeTrajectory({ success: true, stepCount: 5 }), makeTask())).score
-      );
-    });
-
-    it('should boost score for steps with high attribution', async () => {
-      const withAttr = await evaluator.evaluate(
-        makeTrajectory({ success: true, withAttribution: true }),
-        makeTask()
-      );
-      const withoutAttr = await evaluator.evaluate(
-        makeTrajectory({ success: true }),
-        makeTask()
-      );
-
-      expect(withAttr.score).toBeGreaterThanOrEqual(withoutAttr.score);
-    });
-
-    it('should clamp score between 0 and 1', async () => {
-      // Very bad: failed + empty + errors
-      const trajectory = createTrajectory({
-        task: makeTask(),
-        steps: [],
-        outcome: failureOutcome('total failure'),
-        agentId: 'a',
-      });
-
-      const result = await evaluator.evaluate(trajectory, makeTask());
-      expect(result.score).toBeGreaterThanOrEqual(0);
-      expect(result.score).toBeLessThanOrEqual(1);
-    });
-  });
-
-  describe('verification-based evaluation', () => {
-    it('should use registered verifier for matching domain', async () => {
-      const verifier = async (_t: Trajectory, _task: Task): Promise<VerificationResult> => ({
-        passed: true,
-        confidence: 0.95,
-        issues: [],
-        details: 'all tests pass',
-      });
-
-      evaluator.registerVerifier('code', verifier);
-
-      const result = await evaluator.evaluate(makeTrajectory(), makeTask({ domain: 'code' }));
-
-      expect(result.method).toBe('verification');
-      expect(result.acceptable).toBe(true);
-      expect(result.score).toBeGreaterThanOrEqual(0.7);
-    });
-
-    it('should fall through to heuristic when verification confidence too low', async () => {
-      const verifier = async (): Promise<VerificationResult> => ({
-        passed: true,
-        confidence: 0.3, // Below default threshold of 0.8
-      });
-
-      evaluator.registerVerifier('code', verifier);
-
-      const result = await evaluator.evaluate(makeTrajectory(), makeTask({ domain: 'code' }));
-      expect(result.method).toBe('heuristic');
-    });
-
-    it('should fall through when verifier throws', async () => {
-      const verifier = async (): Promise<VerificationResult> => {
-        throw new Error('verification crashed');
-      };
-
-      evaluator.registerVerifier('code', verifier);
-
-      const result = await evaluator.evaluate(makeTrajectory(), makeTask({ domain: 'code' }));
-      expect(result.method).toBe('heuristic');
-    });
-
-    it('should handle failed verification with issues', async () => {
-      const verifier = async (): Promise<VerificationResult> => ({
-        passed: false,
-        confidence: 0.9,
-        issues: [
-          { type: 'incorrect', description: 'test failed: should return 42', severity: 'major' },
-        ],
-      });
-
-      evaluator.registerVerifier('code', verifier);
-
-      const result = await evaluator.evaluate(makeTrajectory(), makeTask({ domain: 'code' }));
-
-      expect(result.method).toBe('verification');
-      expect(result.acceptable).toBe(false);
-      expect(result.score).toBeLessThan(0.5);
-      expect(result.issues).toHaveLength(1);
-    });
-
-    it('should not use verifier for wrong domain', async () => {
-      const verifier = async (): Promise<VerificationResult> => ({
-        passed: true,
-        confidence: 0.95,
-      });
-
-      evaluator.registerVerifier('python', verifier);
-
-      // Task domain is 'code', not 'python'
-      const result = await evaluator.evaluate(makeTrajectory(), makeTask({ domain: 'code' }));
-      expect(result.method).toBe('heuristic');
-    });
-  });
-
-  describe('alwaysUseAgent config', () => {
-    it('should skip verification shortcut when alwaysUseAgent=true', async () => {
-      const alwaysAgentEval = createSolutionEvaluator(null, { alwaysUseAgent: true });
-
-      const verifier = async (): Promise<VerificationResult> => ({
-        passed: true,
-        confidence: 0.99,
-      });
-      alwaysAgentEval.registerVerifier('code', verifier);
-
-      // With no agent or taskRunner, falls through to heuristic
-      const result = await alwaysAgentEval.evaluate(makeTrajectory(), makeTask({ domain: 'code' }));
-      expect(result.method).toBe('heuristic');
-    });
-  });
-
-  describe('parseQuality edge cases', () => {
-    // Test through parseAgentEvaluation indirectly using mock agent
-    it('should handle unknown quality strings gracefully', async () => {
-      // Falls through to heuristic since no agent available
-      const result = await evaluator.evaluate(makeTrajectory(), makeTask());
-      expect(['excellent', 'good', 'needs_work', 'poor']).toContain(result.quality);
-    });
-  });
-});
-
-describe('scoreToQuality (used by evaluator)', () => {
-  it('should map scores to correct quality levels', async () => {
-    const { scoreToQuality } = await import('../../src/search/refinement-types.js');
-
-    expect(scoreToQuality(0.9)).toBe('excellent');
-    expect(scoreToQuality(0.85)).toBe('excellent');
-    expect(scoreToQuality(0.75)).toBe('good');
-    expect(scoreToQuality(0.7)).toBe('good');
-    expect(scoreToQuality(0.5)).toBe('needs_work');
-    expect(scoreToQuality(0.4)).toBe('needs_work');
-    expect(scoreToQuality(0.3)).toBe('poor');
-    expect(scoreToQuality(0.0)).toBe('poor');
-  });
-});

package/tests/search/moe-gate.test.ts

@@ -1,250 +0,0 @@
-import { describe, it, expect } from 'vitest';
-import {
-  MoEGate,
-  extractTaskFeatures,
-  MOE_FEATURE_SIZE,
-  MOE_STRATEGIES,
-} from '../../src/search/moe-gate.js';
-
-describe('MoEGate', () => {
-  const strategies = [...MOE_STRATEGIES];
-  const featureSize = MOE_FEATURE_SIZE;
-
-  describe('gate produces valid probability distribution', () => {
-    it('should produce probabilities that sum to 1', () => {
-      const gate = new MoEGate(strategies, featureSize);
-      const features = [0.5, 0.3, 0.8, 0.6, 0.9, 1, 0, 0.2];
-
-      const probs = gate.gate(features);
-
-      let sum = 0;
-      for (const [, prob] of probs) {
-        expect(prob).toBeGreaterThanOrEqual(0);
-        expect(prob).toBeLessThanOrEqual(1);
-        sum += prob;
-      }
-      expect(sum).toBeCloseTo(1.0, 5);
-    });
-
-    it('should produce one weight per strategy', () => {
-      const gate = new MoEGate(strategies, featureSize);
-      const features = Array(featureSize).fill(0.5);
-
-      const probs = gate.gate(features);
-      expect(probs.size).toBe(strategies.length);
-
-      for (const strategy of strategies) {
-        expect(probs.has(strategy)).toBe(true);
-      }
-    });
-  });
-
-  describe('cold start produces uniform weights', () => {
-    it('should produce near-uniform weights before any updates', () => {
-      const gate = new MoEGate(strategies, featureSize);
-      const features = Array(featureSize).fill(0.5);
-
-      const probs = gate.gate(features);
-
-      const expected = 1 / strategies.length;
-      for (const [, prob] of probs) {
-        // Near-uniform (within 10% of expected)
-        expect(Math.abs(prob - expected)).toBeLessThan(0.1);
-      }
-    });
-
-    it('should not be marked as trained initially', () => {
-      const gate = new MoEGate(strategies, featureSize);
-      expect(gate.isTrained).toBe(false);
-      expect(gate.totalUpdates).toBe(0);
-    });
-  });
-
-  describe('update shifts weights toward successful strategy', () => {
-    it('should increase probability of rewarded strategy', () => {
-      const gate = new MoEGate(strategies, featureSize);
-      const features = [0.5, 0.3, 0.8, 0.6, 0.9, 1, 0, 0.2];
-
-      const before = gate.gate(features).get('direct')!;
-
-      // Repeatedly reward 'direct' strategy
-      for (let i = 0; i < 50; i++) {
-        gate.update(features, 'direct', 1.0, 0.05);
-      }
-
-      const after = gate.gate(features).get('direct')!;
-      expect(after).toBeGreaterThan(before);
-    });
-
-    it('should decrease probability of failing strategy', () => {
-      const gate = new MoEGate(strategies, featureSize);
-      const features = [0.5, 0.3, 0.8, 0.6, 0.9, 1, 0, 0.2];
-
-      // First train 'explore' to have high weight
-      for (let i = 0; i < 20; i++) {
-        gate.update(features, 'explore', 1.0, 0.05);
-      }
-      const beforeFail = gate.gate(features).get('explore')!;
-
-      // Then penalize 'explore' (reward=0)
-      for (let i = 0; i < 30; i++) {
-        gate.update(features, 'explore', 0.0, 0.05);
-      }
-      const afterFail = gate.gate(features).get('explore')!;
-
-      expect(afterFail).toBeLessThan(beforeFail);
-    });
-
-    it('should be marked as trained after updates', () => {
-      const gate = new MoEGate(strategies, featureSize);
-      gate.update(Array(featureSize).fill(0.5), 'direct', 1.0);
-      expect(gate.isTrained).toBe(true);
-      expect(gate.totalUpdates).toBe(1);
-    });
-  });
-
-  describe('repeated updates converge', () => {
-    it('should converge on the consistently rewarded strategy', () => {
-      const gate = new MoEGate(strategies, featureSize);
-      const features = [0.5, 0.3, 0.8, 0.6, 0.9, 1, 0, 0.2];
-
-      // Consistently reward 'adapt'
-      for (let i = 0; i < 100; i++) {
-        gate.update(features, 'adapt', 1.0, 0.02);
-      }
-
-      const selected = gate.selectStrategy(features);
-      expect(selected).toBe('adapt');
-    });
-  });
-
-  describe('domain-specific weights', () => {
-    it('should learn different strategies for different domains', () => {
-      const gate = new MoEGate(strategies, featureSize);
-      const features = [0.5, 0.3, 0.8, 0.6, 0.9, 1, 0, 0.2];
-
-      // TypeScript domain prefers 'direct'
-      for (let i = 0; i < 50; i++) {
-        gate.update(features, 'direct', 1.0, 0.05, 'typescript');
-      }
-
-      // Devops domain prefers 'explore'
-      for (let i = 0; i < 50; i++) {
-        gate.update(features, 'explore', 1.0, 0.05, 'devops');
-      }
-
-      const tsStrategy = gate.selectStrategy(features, 'typescript');
-      const devopsStrategy = gate.selectStrategy(features, 'devops');
-
-      expect(tsStrategy).toBe('direct');
-      expect(devopsStrategy).toBe('explore');
-    });
-  });
-
-  describe('serialization round-trips correctly', () => {
-    it('should produce identical gate output after serialization', () => {
-      const gate = new MoEGate(strategies, featureSize);
-      const features = [0.5, 0.3, 0.8, 0.6, 0.9, 1, 0, 0.2];
-
-      // Train it
-      for (let i = 0; i < 20; i++) {
-        gate.update(features, 'adapt', 1.0, 0.05);
-        gate.update(features, 'direct', 0.5, 0.05, 'typescript');
-      }
-
-      // Serialize and deserialize
-      const json = gate.toJSON();
-      const restored = MoEGate.fromJSON(json);
-
-      // Compare outputs
-      const originalProbs = gate.gate(features);
-      const restoredProbs = restored.gate(features);
-
-      for (const strategy of strategies) {
-        expect(restoredProbs.get(strategy)).toBeCloseTo(
-          originalProbs.get(strategy)!,
-          10
-        );
-      }
-
-      expect(restored.isTrained).toBe(gate.isTrained);
-      expect(restored.totalUpdates).toBe(gate.totalUpdates);
-    });
-
-    it('should preserve domain weights across serialization', () => {
-      const gate = new MoEGate(strategies, featureSize);
-      const features = [0.5, 0.3, 0.8, 0.6, 0.9, 1, 0, 0.2];
-
-      for (let i = 0; i < 30; i++) {
-        gate.update(features, 'direct', 1.0, 0.05, 'typescript');
-      }
-
-      const json = gate.toJSON();
-      const restored = MoEGate.fromJSON(json);
-
-      const originalTs = gate.gate(features, 'typescript');
-      const restoredTs = restored.gate(features, 'typescript');
-
-      for (const strategy of strategies) {
-        expect(restoredTs.get(strategy)).toBeCloseTo(
-          originalTs.get(strategy)!,
-          10
-        );
-      }
-    });
-  });
-});
-
-describe('extractTaskFeatures', () => {
-  it('should produce feature vector of correct size', () => {
-    const features = extractTaskFeatures(
-      'Fix TypeScript type error in user module',
-      { experiences: [], playbooks: [] },
-    );
-
-    expect(features).toHaveLength(MOE_FEATURE_SIZE);
-  });
-
-  it('should set bug fix indicator for bug-related tasks', () => {
-    const features = extractTaskFeatures(
-      'Fix the authentication bug in login flow',
-      { experiences: [], playbooks: [] },
-    );
-
-    expect(features[5]).toBe(1); // isBugFix
-  });
-
-  it('should set new feature indicator for implementation tasks', () => {
-    const features = extractTaskFeatures(
-      'Implement user profile page with avatar upload',
-      { experiences: [], playbooks: [] },
-    );
-
-    expect(features[6]).toBe(1); // isNewFeature
-  });
-
-  it('should reflect memory context scores', () => {
-    const features = extractTaskFeatures(
-      'Test task',
-      {
-        experiences: [{ score: 0.85 }],
-        playbooks: [{ score: 0.9, playbook: { confidence: 0.95 } }],
-      },
-    );
-
-    expect(features[2]).toBe(0.85); // best experience match
-    expect(features[3]).toBe(0.9);  // best playbook match
-    expect(features[4]).toBe(0.95); // best playbook confidence
-  });
-
-  it('should handle empty memory context', () => {
-    const features = extractTaskFeatures(
-      'Novel task',
-      { experiences: [], playbooks: [] },
-    );
-
-    expect(features[2]).toBe(0); // no experiences
-    expect(features[3]).toBe(0); // no playbooks
-    expect(features[4]).toBe(0); // no playbook confidence
-  });
-});