cognitive-core 0.0.2 → 0.1.0

package/tests/learning/analyzer.test.ts

@@ -0,0 +1,426 @@
+import { describe, it, expect } from 'vitest';
+import {
+  TrajectoryAnalyzer,
+  simpleCreditAssignment,
+  outcomeCreditAssignment,
+  getCreditAssignmentFn,
+  createAnalyzer,
+  type AnalyzerConfig,
+} from '../../src/learning/analyzer.js';
+import { createTrajectory } from '../../src/types/trajectory.js';
+import { createTask } from '../../src/types/task.js';
+import { createStep } from '../../src/types/step.js';
+import { successOutcome, failureOutcome } from '../../src/types/outcome.js';
+
+// Default config for tests
+const DEFAULT_CONFIG: AnalyzerConfig = {
+  creditDecayFactor: 0.5,
+  actionRepetitionThreshold: 3,
+  attributionThreshold: 0.15,
+};
+
+describe('simpleCreditAssignment', () => {
+  it('should assign exponentially decaying credit from end', async () => {
+    const task = createTask({ domain: 'test', description: 'test' });
+    const steps = [
+      createStep({ action: 'a1', observation: 'o1' }),
+      createStep({ action: 'a2', observation: 'o2' }),
+      createStep({ action: 'a3', observation: 'o3' }),
+    ];
+    const trajectory = createTrajectory({
+      task,
+      steps,
+      outcome: successOutcome('done'),
+      agentId: 'agent',
+    });
+
+    const scores = await simpleCreditAssignment(trajectory, DEFAULT_CONFIG);
+
+    expect(scores).toHaveLength(3);
+    expect(scores[2]).toBe(1.0); // Last step gets full credit
+    expect(scores[1]).toBe(0.5); // Second to last
+    expect(scores[0]).toBe(0.25); // Third to last
+  });
+
+  it('should handle single step trajectory', async () => {
+    const trajectory = createTrajectory({
+      task: createTask({ domain: 'test', description: 'test' }),
+      steps: [createStep({ action: 'a', observation: 'o' })],
+      outcome: successOutcome('x'),
+      agentId: 'agent',
+    });
+
+    const scores = await simpleCreditAssignment(trajectory, DEFAULT_CONFIG);
+    expect(scores).toEqual([1.0]);
+  });
+
+  it('should handle empty steps', async () => {
+    const trajectory = createTrajectory({
+      task: createTask({ domain: 'test', description: 'test' }),
+      steps: [],
+      outcome: successOutcome('x'),
+      agentId: 'agent',
+    });
+
+    const scores = await simpleCreditAssignment(trajectory, DEFAULT_CONFIG);
+    expect(scores).toEqual([]);
+  });
+
+  it('should use configurable decay factor', async () => {
+    const trajectory = createTrajectory({
+      task: createTask({ domain: 'test', description: 'test' }),
+      steps: [
+        createStep({ action: 'a1', observation: 'o1' }),
+        createStep({ action: 'a2', observation: 'o2' }),
+        createStep({ action: 'a3', observation: 'o3' }),
+      ],
+      outcome: successOutcome('done'),
+      agentId: 'agent',
+    });
+
+    const customConfig = { ...DEFAULT_CONFIG, creditDecayFactor: 0.8 };
+    const scores = await simpleCreditAssignment(trajectory, customConfig);
+
+    expect(scores[0]).toBeCloseTo(0.64); // 0.8^2
+    expect(scores[1]).toBeCloseTo(0.8);  // 0.8^1
+    expect(scores[2]).toBe(1.0);          // 0.8^0
+  });
+});
+
+describe('outcomeCreditAssignment', () => {
+  it('should assign higher credit for successful trajectories', async () => {
+    const successTrajectory = createTrajectory({
+      task: createTask({ domain: 'test', description: 'test' }),
+      steps: [
+        createStep({ action: 'a1', observation: 'o1' }),
+        createStep({ action: 'a2', observation: 'o2' }),
+      ],
+      outcome: successOutcome('done'),
+      agentId: 'agent',
+    });
+
+    const failTrajectory = createTrajectory({
+      task: createTask({ domain: 'test', description: 'test' }),
+      steps: [
+        createStep({ action: 'a1', observation: 'o1' }),
+        createStep({ action: 'a2', observation: 'o2' }),
+      ],
+      outcome: failureOutcome('error'),
+      agentId: 'agent',
+    });
+
+    const successScores = await outcomeCreditAssignment(successTrajectory, DEFAULT_CONFIG);
+    const failScores = await outcomeCreditAssignment(failTrajectory, DEFAULT_CONFIG);
+
+    expect(successScores[0]).toBe(0.7);
+    expect(successScores[1]).toBe(1.0); // Last step boosted
+    expect(failScores[0]).toBe(0.3);
+    expect(failScores[1]).toBe(0.3);
+  });
+});
+
+describe('getCreditAssignmentFn', () => {
+  it('should return correct function for strategy', () => {
+    expect(getCreditAssignmentFn('simple')).toBe(simpleCreditAssignment);
+    expect(getCreditAssignmentFn('llm')).toBe(simpleCreditAssignment); // Falls back
+    expect(getCreditAssignmentFn('counterfactual')).toBe(outcomeCreditAssignment);
+  });
+});
+
+describe('TrajectoryAnalyzer', () => {
+  const task = createTask({
+    domain: 'code',
+    description: 'Write a test function',
+  });
+
+  describe('analyze', () => {
+    it('should analyze a successful trajectory', async () => {
+      const steps = [
+        createStep({ thought: 'First, read the code', action: 'read', observation: 'Code content' }),
+        createStep({ thought: 'Now write the test', action: 'write', observation: 'Test written' }),
+        createStep({ action: 'run_test', observation: 'Tests passed' }),
+      ];
+      const trajectory = createTrajectory({
+        task,
+        steps,
+        outcome: successOutcome('Test implemented'),
+        agentId: 'agent',
+      });
+
+      const analyzer = new TrajectoryAnalyzer('simple');
+      const result = await analyzer.analyze(trajectory);
+
+      expect(result.success).toBe(true);
+      expect(result.stepAttribution).toHaveLength(3);
+      expect(result.keySteps.length).toBeGreaterThan(0);
+      expect(result.abstractable).toBe(true);
+    });
+
+    it('should detect error patterns in failed trajectories', async () => {
+      const trajectory = createTrajectory({
+        task,
+        steps: [createStep({ action: 'try', observation: 'error' })],
+        outcome: failureOutcome('Type error: null is not an object'),
+        agentId: 'agent',
+      });
+
+      const analyzer = createAnalyzer();
+      const result = await analyzer.analyze(trajectory);
+
+      expect(result.success).toBe(false);
+      expect(result.errorPatterns.length).toBeGreaterThan(0);
+      // Should match type error pattern
+      expect(result.errorPatterns.some(p => p.type === 'type' || p.type === 'general')).toBe(true);
+    });
+
+    it('should detect timeout errors', async () => {
+      const trajectory = createTrajectory({
+        task,
+        steps: [],
+        outcome: failureOutcome('Operation timeout exceeded'),
+        agentId: 'agent',
+      });
+
+      const analyzer = createAnalyzer();
+      const result = await analyzer.analyze(trajectory);
+
+      expect(result.errorPatterns.some(p => p.type === 'timeout')).toBe(true);
+    });
+
+    it('should detect repeated actions', async () => {
+      const steps = [
+        createStep({ action: 'retry_connection', observation: 'failed' }),
+        createStep({ action: 'retry_connection', observation: 'failed' }),
+        createStep({ action: 'retry_connection', observation: 'failed' }),
+        createStep({ action: 'retry_connection', observation: 'failed' }),
+      ];
+      const trajectory = createTrajectory({
+        task,
+        steps,
+        outcome: failureOutcome('Max retries exceeded'),
+        agentId: 'agent',
+      });
+
+      const analyzer = createAnalyzer();
+      const result = await analyzer.analyze(trajectory);
+
+      expect(result.errorPatterns.some(p => p.type === 'repetition')).toBe(true);
+    });
+
+    it('should extract training examples', async () => {
+      const steps = [
+        createStep({
+          thought: 'I should check the input type',
+          action: 'validate_input(data)',
+          observation: 'Input is valid',
+        }),
+        createStep({
+          thought: 'Now process the data',
+          action: 'process(data)',
+          observation: 'Processing complete',
+        }),
+      ];
+      const trajectory = createTrajectory({
+        task,
+        steps,
+        outcome: successOutcome({ result: 'done' }),
+        agentId: 'agent',
+      });
+
+      const analyzer = createAnalyzer();
+      const result = await analyzer.analyze(trajectory);
+
+      expect(result.trainingExamples.length).toBeGreaterThan(0);
+      // Should include task -> solution example
+      expect(result.trainingExamples.some(e =>
+        e.input === task.description
+      )).toBe(true);
+    });
+
+    it('should apply attribution scores to steps', async () => {
+      const steps = [
+        createStep({ action: 'a1', observation: 'o1' }),
+        createStep({ action: 'a2', observation: 'o2' }),
+      ];
+      const trajectory = createTrajectory({
+        task,
+        steps,
+        outcome: successOutcome('x'),
+        agentId: 'agent',
+      });
+
+      const analyzer = createAnalyzer();
+      await analyzer.analyze(trajectory);
+
+      // Steps should now have attribution scores
+      expect(trajectory.steps[0].attributionScore).toBeDefined();
+      expect(trajectory.steps[1].attributionScore).toBeDefined();
+    });
+  });
+
+  describe('setStrategy and setThreshold', () => {
+    it('should change credit assignment strategy', async () => {
+      const analyzer = createAnalyzer('simple');
+
+      const trajectory = createTrajectory({
+        task,
+        steps: [
+          createStep({ action: 'a', observation: 'o' }),
+          createStep({ action: 'b', observation: 'o' }),
+        ],
+        outcome: successOutcome('x'),
+        agentId: 'agent',
+      });
+
+      const result1 = await analyzer.analyze(trajectory);
+
+      analyzer.setStrategy('counterfactual');
+      const result2 = await analyzer.analyze(trajectory);
+
+      // Different strategies may produce different attributions
+      expect(result1.stepAttribution).not.toEqual(result2.stepAttribution);
+    });
+
+    it('should change attribution threshold', async () => {
+      const analyzer = createAnalyzer();
+
+      const steps = [
+        { ...createStep({ action: 'a', observation: 'o' }), attributionScore: 0.2 },
+        { ...createStep({ action: 'b', observation: 'o' }), attributionScore: 0.1 },
+      ];
+      const trajectory = createTrajectory({
+        task,
+        steps,
+        outcome: successOutcome('x'),
+        agentId: 'agent',
+      });
+
+      analyzer.setThreshold(0.15);
+      const result1 = await analyzer.analyze(trajectory);
+
+      analyzer.setThreshold(0.05);
+      const result2 = await analyzer.analyze(trajectory);
+
+      // Lower threshold should include more key steps
+      expect(result2.keySteps.length).toBeGreaterThanOrEqual(result1.keySteps.length);
+    });
+  });
+
+  describe('configuration', () => {
+    it('should get current configuration', () => {
+      const analyzer = createAnalyzer('simple', {
+        creditDecayFactor: 0.7,
+        actionRepetitionThreshold: 5,
+      });
+
+      const config = analyzer.getConfig();
+      expect(config.creditDecayFactor).toBe(0.7);
+      expect(config.actionRepetitionThreshold).toBe(5);
+      expect(config.attributionThreshold).toBe(0.15); // Default
+    });
+
+    it('should update configuration', () => {
+      const analyzer = createAnalyzer();
+      analyzer.updateConfig({ creditDecayFactor: 0.9 });
+
+      const config = analyzer.getConfig();
+      expect(config.creditDecayFactor).toBe(0.9);
+    });
+
+    it('should preserve unchanged values when updating', () => {
+      const analyzer = createAnalyzer('simple', {
+        creditDecayFactor: 0.6,
+        actionRepetitionThreshold: 4,
+      });
+
+      analyzer.updateConfig({ creditDecayFactor: 0.8 });
+
+      const config = analyzer.getConfig();
+      expect(config.creditDecayFactor).toBe(0.8);
+      expect(config.actionRepetitionThreshold).toBe(4); // Unchanged
+    });
+
+    it('should use configurable repetition threshold', async () => {
+      const analyzer = createAnalyzer('simple', {
+        actionRepetitionThreshold: 5, // Higher threshold
+      });
+
+      // Only 3 repetitions - shouldn't trigger with threshold of 5
+      const steps = [
+        createStep({ action: 'same_action', observation: 'obs1' }),
+        createStep({ action: 'same_action', observation: 'obs2' }),
+        createStep({ action: 'same_action', observation: 'obs3' }),
+      ];
+
+      const trajectory = createTrajectory({
+        task,
+        steps,
+        outcome: failureOutcome('Failed'),
+        agentId: 'agent',
+      });
+
+      const result = await analyzer.analyze(trajectory);
+      // Should NOT detect repetition with threshold of 5
+      expect(result.errorPatterns.some(p => p.type === 'repetition')).toBe(false);
+    });
+  });
+
+  describe('abstractability assessment', () => {
+    it('should mark successful trajectories with key steps as abstractable', async () => {
+      const steps = [
+        createStep({ action: 'a1', observation: 'o1' }),
+        createStep({ action: 'a2', observation: 'o2' }),
+        createStep({ action: 'a3', observation: 'o3' }),
+      ];
+      const trajectory = createTrajectory({
+        task,
+        steps,
+        outcome: successOutcome('done'),
+        agentId: 'agent',
+      });
+
+      const analyzer = createAnalyzer();
+      const result = await analyzer.analyze(trajectory);
+
+      expect(result.abstractable).toBe(true);
+    });
+
+    it('should not mark simple failed trajectories as abstractable', async () => {
+      const trajectory = createTrajectory({
+        task,
+        steps: [createStep({ action: 'fail', observation: 'error' })],
+        outcome: failureOutcome('Unknown error'),
+        agentId: 'agent',
+      });
+
+      const analyzer = createAnalyzer();
+      const result = await analyzer.analyze(trajectory);
+
+      // No clear error pattern, not abstractable
+      expect(result.abstractable).toBe(false);
+    });
+
+    it('should mark failed trajectories with partial progress as potentially abstractable', async () => {
+      const trajectory = createTrajectory({
+        task,
+        steps: [
+          createStep({ action: 'setup', observation: 'done' }),
+          createStep({ action: 'execute', observation: 'partial' }),
+        ],
+        outcome: {
+          success: false,
+          partialScore: 0.5,
+          errorInfo: 'Timeout after partial completion',
+          verificationDetails: {},
+        },
+        agentId: 'agent',
+      });
+
+      const analyzer = createAnalyzer();
+      const result = await analyzer.analyze(trajectory);
+
+      // Has error pattern with partial progress, might be abstractable
+      expect(result.errorPatterns.length).toBeGreaterThan(0);
+    });
+  });
+});