cognitive-core 0.0.2 → 0.1.0

package/tests/search/refinement.test.ts

@@ -0,0 +1,267 @@
+import { describe, it, expect, beforeEach, vi } from 'vitest';
+import {
+  createEvaluationResult,
+  scoreToQuality,
+  isQualityAcceptable,
+  type EvaluationResult,
+  type QualityLevel,
+} from '../../src/search/refinement-types.js';
+import {
+  SolutionEvaluator,
+  createSolutionEvaluator,
+  type VerificationResult,
+} from '../../src/search/evaluator.js';
+import type { Trajectory } from '../../src/types/trajectory.js';
+import type { Task } from '../../src/types/task.js';
+
+describe('Refinement Types', () => {
+  describe('scoreToQuality', () => {
+    it('should return excellent for scores >= 0.85', () => {
+      expect(scoreToQuality(0.85)).toBe('excellent');
+      expect(scoreToQuality(0.9)).toBe('excellent');
+      expect(scoreToQuality(1.0)).toBe('excellent');
+    });
+
+    it('should return good for scores >= 0.7 and < 0.85', () => {
+      expect(scoreToQuality(0.7)).toBe('good');
+      expect(scoreToQuality(0.8)).toBe('good');
+      expect(scoreToQuality(0.84)).toBe('good');
+    });
+
+    it('should return needs_work for scores >= 0.4 and < 0.7', () => {
+      expect(scoreToQuality(0.4)).toBe('needs_work');
+      expect(scoreToQuality(0.5)).toBe('needs_work');
+      expect(scoreToQuality(0.69)).toBe('needs_work');
+    });
+
+    it('should return poor for scores < 0.4', () => {
+      expect(scoreToQuality(0)).toBe('poor');
+      expect(scoreToQuality(0.2)).toBe('poor');
+      expect(scoreToQuality(0.39)).toBe('poor');
+    });
+  });
+
+  describe('isQualityAcceptable', () => {
+    it('should accept quality >= minAcceptable', () => {
+      expect(isQualityAcceptable('excellent', 'good')).toBe(true);
+      expect(isQualityAcceptable('good', 'good')).toBe(true);
+      expect(isQualityAcceptable('excellent', 'excellent')).toBe(true);
+    });
+
+    it('should reject quality < minAcceptable', () => {
+      expect(isQualityAcceptable('needs_work', 'good')).toBe(false);
+      expect(isQualityAcceptable('poor', 'good')).toBe(false);
+      expect(isQualityAcceptable('good', 'excellent')).toBe(false);
+    });
+
+    it('should use good as default minAcceptable', () => {
+      expect(isQualityAcceptable('good')).toBe(true);
+      expect(isQualityAcceptable('excellent')).toBe(true);
+      expect(isQualityAcceptable('needs_work')).toBe(false);
+    });
+  });
+
+  describe('createEvaluationResult', () => {
+    it('should create a valid evaluation result', () => {
+      const result = createEvaluationResult({
+        quality: 'good',
+        score: 0.75,
+        acceptable: true,
+        issues: [],
+        method: 'verification',
+      });
+
+      expect(result.quality).toBe('good');
+      expect(result.score).toBe(0.75);
+      expect(result.acceptable).toBe(true);
+      expect(result.issues).toEqual([]);
+      expect(result.method).toBe('verification');
+    });
+
+    it('should validate score bounds', () => {
+      expect(() => createEvaluationResult({
+        quality: 'good',
+        score: 1.5,
+        acceptable: true,
+        method: 'verification',
+      })).toThrow();
+
+      expect(() => createEvaluationResult({
+        quality: 'good',
+        score: -0.1,
+        acceptable: true,
+        method: 'verification',
+      })).toThrow();
+    });
+
+    it('should include issues when provided', () => {
+      const result = createEvaluationResult({
+        quality: 'needs_work',
+        score: 0.5,
+        acceptable: false,
+        issues: [
+          { type: 'incomplete', description: 'Missing tests', severity: 'major' },
+        ],
+        method: 'agent',
+      });
+
+      expect(result.issues.length).toBe(1);
+      expect(result.issues[0].type).toBe('incomplete');
+      expect(result.issues[0].description).toBe('Missing tests');
+    });
+  });
+});
+
+describe('SolutionEvaluator', () => {
+  const createMockTrajectory = (success: boolean): Trajectory => ({
+    id: 'test-trajectory',
+    task: {
+      id: 'test-task',
+      description: 'Test task',
+      domain: 'test',
+      context: {},
+      metadata: {},
+      createdAt: new Date(),
+    },
+    steps: [
+      {
+        action: 'test action',
+        observation: 'test observation',
+        metadata: {},
+      },
+    ],
+    outcome: {
+      success,
+      partialScore: success ? 1.0 : 0.0,
+      errorInfo: success ? undefined : 'Test error',
+      verificationDetails: {},
+      solution: success ? 'Test solution' : undefined,
+    },
+    agentId: 'test-agent',
+    timestamp: new Date(),
+    llmCalls: 1,
+    totalTokens: 100,
+    wallTimeSeconds: 1,
+    metadata: {},
+  });
+
+  const createMockTask = (): Task => ({
+    id: 'test-task',
+    description: 'Test task description',
+    domain: 'test',
+    context: {},
+    metadata: {},
+    createdAt: new Date(),
+  });
+
+  describe('heuristic evaluation', () => {
+    let evaluator: SolutionEvaluator;
+
+    beforeEach(() => {
+      evaluator = createSolutionEvaluator(null);
+    });
+
+    it('should evaluate successful trajectory positively', async () => {
+      const trajectory = createMockTrajectory(true);
+      const task = createMockTask();
+
+      const result = await evaluator.evaluate(trajectory, task);
+
+      expect(result.score).toBeGreaterThan(0.5);
+      expect(result.method).toBe('heuristic');
+    });
+
+    it('should evaluate failed trajectory negatively', async () => {
+      const trajectory = createMockTrajectory(false);
+      const task = createMockTask();
+
+      const result = await evaluator.evaluate(trajectory, task);
+
+      expect(result.score).toBeLessThanOrEqual(0.5);
+      expect(result.acceptable).toBe(false);
+      expect(result.issues.length).toBeGreaterThan(0);
+    });
+
+    it('should penalize trajectories with no steps', async () => {
+      const trajectory = createMockTrajectory(false);
+      trajectory.steps = [];
+      const task = createMockTask();
+
+      const result = await evaluator.evaluate(trajectory, task);
+
+      expect(result.score).toBeLessThan(0.5);
+      const incompleteIssue = result.issues.find(i => i.type === 'incomplete');
+      expect(incompleteIssue).toBeDefined();
+    });
+
+    it('should penalize long trajectories', async () => {
+      const trajectory = createMockTrajectory(true);
+      trajectory.steps = Array(25).fill({
+        action: 'action',
+        observation: 'observation',
+        metadata: {},
+      });
+      const task = createMockTask();
+
+      const result = await evaluator.evaluate(trajectory, task);
+
+      const inefficientIssue = result.issues.find(i => i.type === 'inefficient');
+      expect(inefficientIssue).toBeDefined();
+    });
+  });
+
+  describe('verification', () => {
+    it('should use registered verifier for domain', async () => {
+      const evaluator = createSolutionEvaluator(null);
+
+      const mockVerifier = vi.fn().mockResolvedValue({
+        passed: true,
+        confidence: 0.9,
+      } as VerificationResult);
+
+      evaluator.registerVerifier('test', mockVerifier);
+
+      const trajectory = createMockTrajectory(true);
+      const task = createMockTask();
+
+      const result = await evaluator.evaluate(trajectory, task);
+
+      expect(mockVerifier).toHaveBeenCalled();
+      expect(result.method).toBe('verification');
+      expect(result.acceptable).toBe(true);
+    });
+
+    it('should fall back to heuristic if verifier fails', async () => {
+      const evaluator = createSolutionEvaluator(null);
+
+      const mockVerifier = vi.fn().mockRejectedValue(new Error('Verification error'));
+      evaluator.registerVerifier('test', mockVerifier);
+
+      const trajectory = createMockTrajectory(true);
+      const task = createMockTask();
+
+      const result = await evaluator.evaluate(trajectory, task);
+
+      expect(result.method).toBe('heuristic');
+    });
+
+    it('should not use verifier for unknown domain', async () => {
+      const evaluator = createSolutionEvaluator(null);
+
+      const mockVerifier = vi.fn().mockResolvedValue({
+        passed: true,
+        confidence: 0.9,
+      } as VerificationResult);
+
+      evaluator.registerVerifier('other', mockVerifier);
+
+      const trajectory = createMockTrajectory(true);
+      const task = createMockTask();
+
+      const result = await evaluator.evaluate(trajectory, task);
+
+      expect(mockVerifier).not.toHaveBeenCalled();
+      expect(result.method).toBe('heuristic');
+    });
+  });
+});

package/tests/search/router.test.ts

@@ -0,0 +1,427 @@
+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
+import { TaskRouter, createRouter, type RoutingDecision } from '../../src/search/router.js';
+import { createMemorySystem } from '../../src/memory/system.js';
+import { createTask } from '../../src/types/task.js';
+import type { MemorySystem } from '../../src/memory/system.js';
+import type { MetaLearner } from '../../src/learning/meta-learner.js';
+import { mkdtemp, rm } from 'node:fs/promises';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+
+describe('TaskRouter', () => {
+  let tempDir: string;
+  let memory: MemorySystem;
+  let router: TaskRouter;
+
+  beforeEach(async () => {
+    tempDir = await mkdtemp(join(tmpdir(), 'router-test-'));
+    memory = createMemorySystem(tempDir);
+    await memory.init();
+    router = createRouter(memory);
+  });
+
+  afterEach(async () => {
+    await memory.close();
+    await rm(tempDir, { recursive: true, force: true });
+  });
+
+  describe('basic routing', () => {
+    it('should return fallback strategy when no memory exists', async () => {
+      const task = createTask({
+        domain: 'test',
+        description: 'A completely new task with no similar experiences',
+      });
+
+      const decision = await router.route(task);
+
+      expect(decision.strategy).toBe('fallback');
+      expect(decision.confidence).toBeLessThan(0.5);
+      expect(decision.memoryContext).toBeDefined();
+      expect(decision.reasoning).toContain('No relevant memory');
+    });
+
+    it('should return direct strategy when high-confidence playbook exists', async () => {
+      // Add a high-confidence playbook with very matching situations
+      await memory.playbooks.add({
+        id: 'playbook-1',
+        name: 'Fix TypeScript errors',
+        applicability: {
+          situations: ['Fix TypeScript type error in user module', 'TypeScript compilation error', 'Type mismatch in code'],
+          triggers: ['tsc error', 'type error', 'TypeScript type error'],
+          antiPatterns: [],
+          domains: ['typescript', 'code'],
+        },
+        guidance: {
+          strategy: 'Read error message, identify type mismatch, fix types',
+          tactics: ['Check variable types', 'Add type annotations'],
+          steps: ['Read error', 'Find source', 'Fix type'],
+        },
+        verification: {
+          successIndicators: ['No compilation errors'],
+          failureIndicators: ['Still has errors'],
+        },
+        evolution: {
+          version: '1.0.0',
+          createdFrom: [],
+          failures: [],
+          refinements: [],
+          successCount: 10,
+          failureCount: 1,
+        },
+        confidence: 0.9,
+        complexity: 'simple',
+        estimatedEffort: 1,
+        createdAt: new Date(),
+        updatedAt: new Date(),
+      });
+
+      const task = createTask({
+        domain: 'typescript',
+        description: 'Fix TypeScript type error in user module',
+      });
+
+      const decision = await router.route(task);
+
+      // With matching playbook, should get direct strategy or at least find playbooks
+      expect(decision.memoryContext).toBeDefined();
+      // Strategy depends on similarity matching - may be direct or explore
+      expect(['direct', 'explore', 'fallback']).toContain(decision.strategy);
+    });
+
+    it('should return adapt strategy when similar successful experience exists', async () => {
+      // Add a successful experience
+      await memory.experiences.add({
+        id: 'exp-1',
+        taskInput: 'Fix authentication bug in login flow',
+        solutionOutput: 'Added proper token validation and error handling',
+        feedback: 'Successfully fixed the auth bug',
+        success: true,
+        domain: 'auth',
+        trajectoryId: 'traj-1',
+        usageCount: 0,
+        createdAt: new Date(),
+        metadata: {},
+      });
+
+      const task = createTask({
+        domain: 'auth',
+        description: 'Fix authentication bug in login flow for mobile',
+      });
+
+      const decision = await router.route(task);
+
+      // Should find the similar experience
+      expect(decision.memoryContext.experiences.length).toBeGreaterThan(0);
+      // Strategy depends on similarity score
+      expect(['adapt', 'explore', 'direct']).toContain(decision.strategy);
+    });
+
+    it('should return explore strategy when low similarity experiences exist', async () => {
+      // Add an experience with somewhat related content
+      await memory.experiences.add({
+        id: 'exp-partial',
+        taskInput: 'Implement user profile page',
+        solutionOutput: 'Created React component with user data',
+        feedback: 'Profile page works',
+        success: true,
+        domain: 'frontend',
+        trajectoryId: 'traj-partial',
+        usageCount: 0,
+        createdAt: new Date(),
+        metadata: {},
+      });
+
+      const task = createTask({
+        domain: 'frontend',
+        description: 'Add settings page with dark mode toggle',
+      });
+
+      const decision = await router.route(task);
+
+      expect(decision.memoryContext).toBeDefined();
+      // With partial similarity, should get explore or fallback
+      expect(['explore', 'fallback', 'direct']).toContain(decision.strategy);
+    });
+  });
+
+  describe('domain routing', () => {
+    it('should filter by domain when useDomainRouting is enabled', async () => {
+      // Add experiences in different domains
+      await memory.experiences.add({
+        id: 'exp-frontend',
+        taskInput: 'Build React component',
+        solutionOutput: 'Created component',
+        feedback: 'Works',
+        success: true,
+        domain: 'frontend',
+        trajectoryId: 'traj-fe',
+        usageCount: 0,
+        createdAt: new Date(),
+        metadata: {},
+      });
+
+      await memory.experiences.add({
+        id: 'exp-backend',
+        taskInput: 'Build API endpoint',
+        solutionOutput: 'Created endpoint',
+        feedback: 'Works',
+        success: true,
+        domain: 'backend',
+        trajectoryId: 'traj-be',
+        usageCount: 0,
+        createdAt: new Date(),
+        metadata: {},
+      });
+
+      const frontendRouter = createRouter(memory, { useDomainRouting: true });
+      const task = createTask({
+        domain: 'frontend',
+        description: 'Build React component for dashboard',
+      });
+
+      const decision = await frontendRouter.route(task);
+
+      // Should prioritize frontend experiences
+      expect(decision.memoryContext).toBeDefined();
+    });
+  });
+
+  describe('configuration', () => {
+    it('should respect similarityThreshold config', async () => {
+      const strictRouter = createRouter(memory, { similarityThreshold: 0.95 });
+
+      await memory.experiences.add({
+        id: 'exp-strict',
+        taskInput: 'Test task',
+        solutionOutput: 'Test solution',
+        feedback: 'Good',
+        success: true,
+        domain: 'test',
+        trajectoryId: 'traj-strict',
+        usageCount: 0,
+        createdAt: new Date(),
+        metadata: {},
+      });
+
+      const task = createTask({
+        domain: 'test',
+        description: 'Slightly different test task',
+      });
+
+      const decision = await strictRouter.route(task);
+
+      // With strict threshold, should fall back more often
+      expect(decision.confidence).toBeDefined();
+    });
+
+    it('should use defaultStrategy from config', async () => {
+      const exploreRouter = createRouter(memory, { defaultStrategy: 'explore' });
+
+      const task = createTask({
+        domain: 'unknown',
+        description: 'Completely novel task',
+      });
+
+      const decision = await exploreRouter.route(task);
+
+      // When no memory, should use default strategy reasoning
+      expect(decision).toBeDefined();
+    });
+
+    it('should update config dynamically', () => {
+      router.updateConfig({ similarityThreshold: 0.5 });
+      // Config is updated internally
+    });
+  });
+
+  describe('routing decision structure', () => {
+    it('should include all required fields in routing decision', async () => {
+      const task = createTask({
+        domain: 'test',
+        description: 'Test task',
+      });
+
+      const decision = await router.route(task);
+
+      expect(decision.strategy).toBeDefined();
+      expect(['direct', 'adapt', 'explore', 'fallback']).toContain(decision.strategy);
+      expect(decision.confidence).toBeGreaterThanOrEqual(0);
+      expect(decision.confidence).toBeLessThanOrEqual(1);
+      expect(decision.memoryContext).toBeDefined();
+      expect(decision.estimatedBudget).toBeGreaterThan(0);
+      expect(decision.reasoning).toBeDefined();
+      expect(typeof decision.reasoning).toBe('string');
+    });
+
+    it('should provide estimated budget based on strategy', async () => {
+      // Add high-quality playbook for low budget
+      await memory.playbooks.add({
+        id: 'pb-budget',
+        name: 'Quick fix',
+        applicability: {
+          situations: ['Quick budget test'],
+          triggers: ['budget test'],
+          antiPatterns: [],
+          domains: ['test'],
+        },
+        guidance: {
+          strategy: 'Simple fix',
+          tactics: [],
+        },
+        verification: {
+          successIndicators: ['Done'],
+          failureIndicators: [],
+        },
+        evolution: {
+          version: '1.0.0',
+          createdFrom: [],
+          failures: [],
+          refinements: [],
+          successCount: 5,
+          failureCount: 0,
+        },
+        confidence: 0.9,
+        complexity: 'simple',
+        estimatedEffort: 1,
+        createdAt: new Date(),
+        updatedAt: new Date(),
+      });
+
+      const task = createTask({
+        domain: 'test',
+        description: 'Quick budget test task',
+      });
+
+      const decision = await router.route(task);
+
+      // Direct strategy with high confidence should have low budget
+      if (decision.strategy === 'direct' && decision.confidence > 0.7) {
+        expect(decision.estimatedBudget).toBeLessThanOrEqual(4);
+      }
+    });
+  });
+
+  describe('meta-learning integration', () => {
+    it('should apply meta-learning adjustments when metaLearner is set', async () => {
+      const mockMetaLearner: MetaLearner = {
+        adjustRouting: vi.fn().mockResolvedValue({
+          strategy: 'explore',
+          confidence: 0.7,
+          reasoning: 'Meta-adjusted reasoning',
+        }),
+        recordOutcome: vi.fn(),
+        getActiveGuidance: vi.fn().mockResolvedValue([]),
+        generateObservation: vi.fn(),
+      } as unknown as MetaLearner;
+
+      router.setMetaLearner(mockMetaLearner);
+
+      const task = createTask({
+        domain: 'test',
+        description: 'Meta-learning test task',
+      });
+
+      const decision = await router.route(task);
+
+      expect(mockMetaLearner.adjustRouting).toHaveBeenCalled();
+      // The decision should reflect meta-learning adjustment
+      expect(decision.strategy).toBe('explore');
+      expect(decision.appliedMetaStrategies).toContain('meta-learning adjustment');
+    });
+
+    it('should continue with base routing if meta-learning fails', async () => {
+      const failingMetaLearner: MetaLearner = {
+        adjustRouting: vi.fn().mockRejectedValue(new Error('Meta-learning failed')),
+        recordOutcome: vi.fn(),
+        getActiveGuidance: vi.fn().mockResolvedValue([]),
+        generateObservation: vi.fn(),
+      } as unknown as MetaLearner;
+
+      router.setMetaLearner(failingMetaLearner);
+
+      const task = createTask({
+        domain: 'test',
+        description: 'Failing meta test',
+      });
+
+      // Should not throw, should fall back to base routing
+      const decision = await router.route(task);
+      expect(decision).toBeDefined();
+      expect(decision.strategy).toBeDefined();
+    });
+  });
+
+  describe('task characteristics extraction', () => {
+    it('should identify bug fix tasks', async () => {
+      const task = createTask({
+        domain: 'code',
+        description: 'Fix the bug in user authentication',
+      });
+
+      const decision = await router.route(task);
+      expect(decision).toBeDefined();
+    });
+
+    it('should identify testing tasks', async () => {
+      const task = createTask({
+        domain: 'code',
+        description: 'Write unit tests for the payment service',
+      });
+
+      const decision = await router.route(task);
+      expect(decision).toBeDefined();
+    });
+
+    it('should identify complex multi-file tasks', async () => {
+      const task = createTask({
+        domain: 'code',
+        description: 'Refactor authentication across multiple files in the codebase',
+      });
+
+      const decision = await router.route(task);
+      expect(decision).toBeDefined();
+    });
+  });
+});
+
+describe('createRouter', () => {
+  let tempDir: string;
+  let memory: MemorySystem;
+
+  beforeEach(async () => {
+    tempDir = await mkdtemp(join(tmpdir(), 'create-router-test-'));
+    memory = createMemorySystem(tempDir);
+    await memory.init();
+  });
+
+  afterEach(async () => {
+    await memory.close();
+    await rm(tempDir, { recursive: true, force: true });
+  });
+
+  it('should create a router with default config', () => {
+    const router = createRouter(memory);
+    expect(router).toBeInstanceOf(TaskRouter);
+  });
+
+  it('should create a router with custom config', () => {
+    const router = createRouter(memory, {
+      similarityThreshold: 0.7,
+      useDomainRouting: false,
+      defaultStrategy: 'explore',
+    });
+    expect(router).toBeInstanceOf(TaskRouter);
+  });
+
+  it('should create a router with meta learner', () => {
+    const mockMetaLearner = {
+      adjustRouting: vi.fn(),
+      recordOutcome: vi.fn(),
+      getActiveGuidance: vi.fn(),
+      generateObservation: vi.fn(),
+    } as unknown as MetaLearner;
+
+    const router = createRouter(memory, {}, mockMetaLearner);
+    expect(router).toBeInstanceOf(TaskRouter);
+  });
+});