cognitive-core 0.2.0 → 0.2.1

package/tests/memory/meta.test.ts

@@ -0,0 +1,399 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { MetaMemory, createMetaMemory } from '../../src/memory/meta.js';
+import { createSqlitePersistence } from '../../src/persistence/index.js';
+import { createMetaObservation, createMetaStrategy } from '../../src/types/meta.js';
+import type { MetaObservation, RoutingDecisionType } from '../../src/types/meta.js';
+import { mkdtemp, rm } from 'node:fs/promises';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+
+function makeObservation(overrides?: {
+  trajectoryId?: string;
+  decision?: RoutingDecisionType;
+  success?: boolean;
+  confidence?: number;
+  playbooksApplied?: string[];
+  retrievalQuality?: 'helpful' | 'neutral' | 'misleading';
+  timestamp?: Date;
+}): MetaObservation {
+  return createMetaObservation({
+    trajectoryId: overrides?.trajectoryId ?? `traj-${Math.random().toString(36).slice(2, 6)}`,
+    routing: {
+      decision: overrides?.decision ?? 'direct',
+      confidence: overrides?.confidence ?? 0.8,
+      reasoning: 'test routing',
+    },
+    memoryUsage: {
+      experiencesRetrieved: 3,
+      playbooksApplied: overrides?.playbooksApplied ?? [],
+      retrievalQuality: overrides?.retrievalQuality ?? 'neutral',
+    },
+    execution: {
+      decompositionUsed: false,
+      refinementIterations: 0,
+      backtrackingOccurred: false,
+      toolsUsed: ['Read', 'Edit'],
+    },
+    outcome: {
+      success: overrides?.success ?? true,
+      quality: overrides?.success === false ? 'poor' : 'good',
+      effortActual: 5,
+      effortEstimate: 8,
+    },
+    lessons: {
+      whatWorked: ['direct approach'],
+      whatFailed: [],
+      suggestions: [],
+    },
+    timestamp: overrides?.timestamp,
+  });
+}
+
+describe('MetaMemory', () => {
+  let tempDir: string;
+  let persistence: any;
+  let meta: MetaMemory;
+
+  beforeEach(async () => {
+    tempDir = await mkdtemp(join(tmpdir(), 'meta-mem-test-'));
+    persistence = createSqlitePersistence({ baseDir: tempDir });
+    await persistence.init();
+    meta = createMetaMemory(persistence);
+    await meta.init();
+  });
+
+  afterEach(async () => {
+    persistence.close();
+    await rm(tempDir, { recursive: true, force: true });
+  });
+
+  describe('recordObservation and retrieval', () => {
+    it('should store and retrieve observations', async () => {
+      const obs = makeObservation({ trajectoryId: 'traj-1' });
+      await meta.recordObservation(obs);
+
+      const all = await meta.getObservations();
+      expect(all).toHaveLength(1);
+      expect(all[0].trajectoryId).toBe('traj-1');
+    });
+
+    it('should retrieve by trajectory ID', async () => {
+      await meta.recordObservation(makeObservation({ trajectoryId: 'traj-A' }));
+      await meta.recordObservation(makeObservation({ trajectoryId: 'traj-B' }));
+
+      const result = await meta.getByTrajectoryId('traj-A');
+      expect(result).toBeDefined();
+      expect(result!.trajectoryId).toBe('traj-A');
+    });
+
+    it('should return undefined for non-existent trajectory ID', async () => {
+      const result = await meta.getByTrajectoryId('nonexistent');
+      expect(result).toBeUndefined();
+    });
+  });
+
+  describe('observation filtering', () => {
+    it('should filter by routing decision', async () => {
+      await meta.recordObservation(makeObservation({ decision: 'direct' }));
+      await meta.recordObservation(makeObservation({ decision: 'explore' }));
+      await meta.recordObservation(makeObservation({ decision: 'direct' }));
+
+      const results = await meta.getObservations({ routingDecision: 'direct' });
+      expect(results).toHaveLength(2);
+      expect(results.every((o) => o.routing.decision === 'direct')).toBe(true);
+    });
+
+    it('should filter by outcome', async () => {
+      await meta.recordObservation(makeObservation({ success: true }));
+      await meta.recordObservation(makeObservation({ success: false }));
+      await meta.recordObservation(makeObservation({ success: true }));
+
+      const successes = await meta.getObservations({ outcome: 'success' });
+      expect(successes).toHaveLength(2);
+
+      const failures = await meta.getObservations({ outcome: 'failure' });
+      expect(failures).toHaveLength(1);
+    });
+
+    it('should filter by date range', async () => {
+      const oldDate = new Date('2025-01-01');
+      const newDate = new Date('2026-03-01');
+      await meta.recordObservation(makeObservation({ timestamp: oldDate }));
+      await meta.recordObservation(makeObservation({ timestamp: newDate }));
+
+      const results = await meta.getObservations({ minDate: new Date('2026-01-01') });
+      expect(results).toHaveLength(1);
+    });
+
+    it('should limit results', async () => {
+      for (let i = 0; i < 5; i++) {
+        await meta.recordObservation(makeObservation());
+      }
+
+      const results = await meta.getObservations({ limit: 3 });
+      expect(results).toHaveLength(3);
+    });
+  });
+
+  describe('strategy management', () => {
+    it('should add and retrieve a strategy', async () => {
+      const strategy = createMetaStrategy({
+        name: 'test-strategy',
+        condition: { taskCharacteristics: ['complex'], memoryState: [] },
+        adjustment: {
+          routingBias: { explore: 0.2 },
+          retrievalModification: 'expand',
+          executionHint: 'decompose',
+        },
+      });
+
+      await meta.addStrategy(strategy);
+      const retrieved = await meta.getStrategy(strategy.id);
+
+      expect(retrieved).toBeDefined();
+      expect(retrieved!.name).toBe('test-strategy');
+      expect(retrieved!.condition.taskCharacteristics).toContain('complex');
+    });
+
+    it('should list all strategies', async () => {
+      for (let i = 0; i < 3; i++) {
+        await meta.addStrategy(
+          createMetaStrategy({
+            name: `strategy-${i}`,
+            condition: { taskCharacteristics: [], memoryState: [] },
+            adjustment: { routingBias: {}, retrievalModification: '', executionHint: '' },
+          })
+        );
+      }
+
+      const all = await meta.getAllStrategies();
+      expect(all).toHaveLength(3);
+    });
+
+    it('should return undefined for non-existent strategy', async () => {
+      const result = await meta.getStrategy('nonexistent');
+      expect(result).toBeUndefined();
+    });
+  });
+
+  describe('recordStrategyApplication', () => {
+    it('should update application count and success rate', async () => {
+      const strategy = createMetaStrategy({
+        name: 'test',
+        condition: { taskCharacteristics: [], memoryState: [] },
+        adjustment: { routingBias: {}, retrievalModification: '', executionHint: '' },
+      });
+      await meta.addStrategy(strategy);
+
+      await meta.recordStrategyApplication(strategy.id, true);
+      const updated = await meta.getStrategy(strategy.id);
+
+      expect(updated!.applicationCount).toBe(1);
+      expect(updated!.successRate).toBeGreaterThan(0);
+    });
+
+    it('should use exponential moving average for success rate', async () => {
+      const strategy = createMetaStrategy({
+        name: 'ema-test',
+        condition: { taskCharacteristics: [], memoryState: [] },
+        adjustment: { routingBias: {}, retrievalModification: '', executionHint: '' },
+      });
+      strategy.successRate = 0.5; // Set initial
+      await meta.addStrategy(strategy);
+
+      await meta.recordStrategyApplication(strategy.id, true);
+      const after = await meta.getStrategy(strategy.id);
+
+      // EMA: 0.5 * (1 - 0.1) + 1 * 0.1 = 0.55
+      expect(after!.successRate).toBeCloseTo(0.55, 2);
+    });
+
+    it('should no-op for nonexistent strategy', async () => {
+      // Should not throw
+      await meta.recordStrategyApplication('nonexistent', true);
+    });
+  });
+
+  describe('getRoutingAdjustments', () => {
+    it('should return empty when no strategies match', async () => {
+      await meta.addStrategy(
+        createMetaStrategy({
+          name: 'specific',
+          condition: {
+            taskCharacteristics: ['database migration'],
+            memoryState: ['no experiences'],
+          },
+          adjustment: { routingBias: {}, retrievalModification: '', executionHint: '' },
+        })
+      );
+
+      const results = await meta.getRoutingAdjustments({
+        taskCharacteristics: ['UI change'],
+        memoryState: ['plenty of experiences'],
+      });
+      expect(results).toHaveLength(0);
+    });
+
+    it('should match strategies with empty conditions (always applies)', async () => {
+      await meta.addStrategy(
+        createMetaStrategy({
+          name: 'universal',
+          condition: { taskCharacteristics: [], memoryState: [] },
+          adjustment: { routingBias: {}, retrievalModification: '', executionHint: 'always' },
+        })
+      );
+
+      const results = await meta.getRoutingAdjustments({
+        taskCharacteristics: ['anything'],
+        memoryState: ['anything'],
+      });
+      expect(results).toHaveLength(1);
+    });
+
+    it('should match via case-insensitive substring', async () => {
+      await meta.addStrategy(
+        createMetaStrategy({
+          name: 'multi-file',
+          condition: {
+            taskCharacteristics: ['multi-file'],
+            memoryState: [],
+          },
+          adjustment: { routingBias: {}, retrievalModification: '', executionHint: '' },
+        })
+      );
+
+      const results = await meta.getRoutingAdjustments({
+        taskCharacteristics: ['This is a Multi-File Change'],
+        memoryState: [],
+      });
+      expect(results).toHaveLength(1);
+    });
+  });
+
+  describe('analyzeRoutingEffectiveness', () => {
+    it('should aggregate success rates by routing decision', async () => {
+      // 3 successful direct, 2 failed direct
+      for (let i = 0; i < 5; i++) {
+        await meta.recordObservation(makeObservation({ decision: 'direct', success: i < 3 }));
+      }
+
+      const analysis = await meta.analyzeRoutingEffectiveness();
+      expect(analysis.byDecision.direct.total).toBeGreaterThanOrEqual(5);
+      expect(analysis.byDecision.direct.successful).toBeGreaterThanOrEqual(3);
+    });
+
+    it('should generate recommendations for low success rate', async () => {
+      // 5 failed explore decisions
+      for (let i = 0; i < 5; i++) {
+        await meta.recordObservation(makeObservation({ decision: 'explore', success: false }));
+      }
+
+      const analysis = await meta.analyzeRoutingEffectiveness();
+      expect(analysis.recommendations.length).toBeGreaterThan(0);
+      expect(analysis.recommendations[0]).toContain('explore');
+    });
+  });
+
+  describe('analyzeRetrievalQuality', () => {
+    it('should track helpful and misleading playbook patterns', async () => {
+      await meta.recordObservation(
+        makeObservation({ playbooksApplied: ['pb-1'], retrievalQuality: 'helpful' })
+      );
+      await meta.recordObservation(
+        makeObservation({ playbooksApplied: ['pb-1'], retrievalQuality: 'helpful' })
+      );
+      await meta.recordObservation(
+        makeObservation({ playbooksApplied: ['pb-2'], retrievalQuality: 'misleading' })
+      );
+
+      const analysis = await meta.analyzeRetrievalQuality();
+      expect(analysis.helpfulPatterns).toContain('pb-1');
+      expect(analysis.misleadingPatterns).toContain('pb-2');
+    });
+
+    it('should recommend refinement for consistently misleading playbooks', async () => {
+      for (let i = 0; i < 4; i++) {
+        await meta.recordObservation(
+          makeObservation({ playbooksApplied: ['pb-bad'], retrievalQuality: 'misleading' })
+        );
+      }
+
+      const analysis = await meta.analyzeRetrievalQuality();
+      expect(analysis.recommendations.length).toBeGreaterThan(0);
+      expect(analysis.recommendations[0]).toContain('pb-bad');
+    });
+  });
+
+  describe('generateMetaStrategies', () => {
+    it('should generate low-confidence-explore strategy when pattern detected', async () => {
+      // 5+ low-confidence observations with low success
+      for (let i = 0; i < 6; i++) {
+        await meta.recordObservation(
+          makeObservation({ confidence: 0.3, success: i === 0 }) // 1/6 success
+        );
+      }
+
+      const strategies = await meta.generateMetaStrategies();
+      const lowConf = strategies.find((s) => s.name === 'low-confidence-explore');
+      expect(lowConf).toBeDefined();
+      expect(lowConf!.adjustment.routingBias.explore).toBe(0.2);
+    });
+
+    it('should not duplicate existing strategies', async () => {
+      for (let i = 0; i < 6; i++) {
+        await meta.recordObservation(makeObservation({ confidence: 0.3, success: false }));
+      }
+
+      // Generate once
+      await meta.generateMetaStrategies();
+      // Generate again — should not duplicate
+      const second = await meta.generateMetaStrategies();
+      const lowConf = second.filter((s) => s.name === 'low-confidence-explore');
+      expect(lowConf).toHaveLength(0); // Already existed, not re-created
+    });
+  });
+
+  describe('getStats', () => {
+    it('should return zeroes for empty store', async () => {
+      const stats = await meta.getStats();
+      expect(stats.totalObservations).toBe(0);
+      expect(stats.totalStrategies).toBe(0);
+      expect(stats.averageSuccessRate).toBe(0);
+      expect(stats.recentTrend).toBe('stable');
+    });
+
+    it('should count observations by routing decision', async () => {
+      await meta.recordObservation(makeObservation({ decision: 'direct' }));
+      await meta.recordObservation(makeObservation({ decision: 'direct' }));
+      await meta.recordObservation(makeObservation({ decision: 'explore' }));
+
+      const stats = await meta.getStats();
+      expect(stats.totalObservations).toBe(3);
+      expect(stats.observationsByRouting.direct).toBe(2);
+      expect(stats.observationsByRouting.explore).toBe(1);
+    });
+
+    it('should compute average success rate', async () => {
+      await meta.recordObservation(makeObservation({ success: true }));
+      await meta.recordObservation(makeObservation({ success: true }));
+      await meta.recordObservation(makeObservation({ success: false }));
+
+      const stats = await meta.getStats();
+      expect(stats.averageSuccessRate).toBeCloseTo(2 / 3, 2);
+    });
+  });
+
+  describe('pruneOldObservations', () => {
+    it('should remove observations older than maxAge', async () => {
+      const oldDate = new Date(Date.now() - 60 * 24 * 60 * 60 * 1000); // 60 days ago
+      await meta.recordObservation(makeObservation({ timestamp: oldDate }));
+      await meta.recordObservation(makeObservation()); // now
+
+      const removed = await meta.pruneOldObservations(30 * 24 * 60 * 60 * 1000); // 30 day cutoff
+      expect(removed).toBeGreaterThanOrEqual(1);
+
+      const remaining = await meta.getObservations();
+      expect(remaining).toHaveLength(1);
+    });
+  });
+});

package/tests/search/evaluator.test.ts

@@ -0,0 +1,257 @@
+import { describe, it, expect, beforeEach } from 'vitest';
+import {
+  SolutionEvaluator,
+  createSolutionEvaluator,
+  type VerificationResult,
+} from '../../src/search/evaluator.js';
+import { createTrajectory } from '../../src/types/trajectory.js';
+import { createTask } from '../../src/types/task.js';
+import { createStep } from '../../src/types/step.js';
+import { successOutcome, failureOutcome } from '../../src/types/outcome.js';
+import type { Trajectory } from '../../src/types/trajectory.js';
+import type { Task } from '../../src/types/task.js';
+
+function makeTask(overrides?: Partial<Task>): Task {
+  return createTask({
+    domain: 'code',
+    description: 'fix a bug in the login module',
+    ...overrides,
+  });
+}
+
+function makeTrajectory(opts?: {
+  success?: boolean;
+  stepCount?: number;
+  errorSteps?: number;
+  withAttribution?: boolean;
+}): Trajectory {
+  const steps = [];
+  const count = opts?.stepCount ?? 3;
+  const errorCount = opts?.errorSteps ?? 0;
+
+  for (let i = 0; i < count; i++) {
+    steps.push(
+      createStep({
+        action: `Step ${i + 1}`,
+        observation: i < errorCount ? 'error: something failed' : 'ok',
+        attributionScore: opts?.withAttribution ? 0.2 : undefined,
+      })
+    );
+  }
+
+  return createTrajectory({
+    task: makeTask(),
+    steps,
+    outcome: (opts?.success ?? true) ? successOutcome('done') : failureOutcome('failed'),
+    agentId: 'agent-1',
+  });
+}
+
+describe('SolutionEvaluator', () => {
+  let evaluator: SolutionEvaluator;
+
+  beforeEach(() => {
+    evaluator = createSolutionEvaluator(null);
+  });
+
+  describe('heuristic evaluation (fallback)', () => {
+    it('should rate successful trajectory with few steps well', async () => {
+      const result = await evaluator.evaluate(
+        makeTrajectory({ success: true, stepCount: 3 }),
+        makeTask()
+      );
+
+      expect(result.method).toBe('heuristic');
+      expect(result.score).toBeGreaterThanOrEqual(0.7);
+      expect(result.acceptable).toBe(true);
+      expect(result.issues).toHaveLength(0);
+    });
+
+    it('should penalize failed trajectory', async () => {
+      const result = await evaluator.evaluate(
+        makeTrajectory({ success: false }),
+        makeTask()
+      );
+
+      expect(result.score).toBeLessThan(0.6);
+      expect(result.acceptable).toBe(false);
+      expect(result.issues.length).toBeGreaterThan(0);
+      expect(result.issues[0].type).toBe('error');
+    });
+
+    it('should penalize empty trajectory', async () => {
+      const trajectory = createTrajectory({
+        task: makeTask(),
+        steps: [],
+        outcome: failureOutcome('no steps taken'),
+        agentId: 'a',
+      });
+
+      const result = await evaluator.evaluate(trajectory, makeTask());
+      expect(result.score).toBeLessThan(0.3);
+      expect(result.issues.some((i) => i.type === 'incomplete')).toBe(true);
+    });
+
+    it('should penalize very long trajectories', async () => {
+      const result = await evaluator.evaluate(
+        makeTrajectory({ success: true, stepCount: 25 }),
+        makeTask()
+      );
+
+      expect(result.issues.some((i) => i.type === 'inefficient')).toBe(true);
+    });
+
+    it('should penalize trajectories with error steps', async () => {
+      const result = await evaluator.evaluate(
+        makeTrajectory({ success: true, stepCount: 5, errorSteps: 3 }),
+        makeTask()
+      );
+
+      expect(result.score).toBeLessThan(
+        // Compare with clean trajectory
+        (await evaluator.evaluate(makeTrajectory({ success: true, stepCount: 5 }), makeTask())).score
+      );
+    });
+
+    it('should boost score for steps with high attribution', async () => {
+      const withAttr = await evaluator.evaluate(
+        makeTrajectory({ success: true, withAttribution: true }),
+        makeTask()
+      );
+      const withoutAttr = await evaluator.evaluate(
+        makeTrajectory({ success: true }),
+        makeTask()
+      );
+
+      expect(withAttr.score).toBeGreaterThanOrEqual(withoutAttr.score);
+    });
+
+    it('should clamp score between 0 and 1', async () => {
+      // Very bad: failed + empty + errors
+      const trajectory = createTrajectory({
+        task: makeTask(),
+        steps: [],
+        outcome: failureOutcome('total failure'),
+        agentId: 'a',
+      });
+
+      const result = await evaluator.evaluate(trajectory, makeTask());
+      expect(result.score).toBeGreaterThanOrEqual(0);
+      expect(result.score).toBeLessThanOrEqual(1);
+    });
+  });
+
+  describe('verification-based evaluation', () => {
+    it('should use registered verifier for matching domain', async () => {
+      const verifier = async (_t: Trajectory, _task: Task): Promise<VerificationResult> => ({
+        passed: true,
+        confidence: 0.95,
+        issues: [],
+        details: 'all tests pass',
+      });
+
+      evaluator.registerVerifier('code', verifier);
+
+      const result = await evaluator.evaluate(makeTrajectory(), makeTask({ domain: 'code' }));
+
+      expect(result.method).toBe('verification');
+      expect(result.acceptable).toBe(true);
+      expect(result.score).toBeGreaterThanOrEqual(0.7);
+    });
+
+    it('should fall through to heuristic when verification confidence too low', async () => {
+      const verifier = async (): Promise<VerificationResult> => ({
+        passed: true,
+        confidence: 0.3, // Below default threshold of 0.8
+      });
+
+      evaluator.registerVerifier('code', verifier);
+
+      const result = await evaluator.evaluate(makeTrajectory(), makeTask({ domain: 'code' }));
+      expect(result.method).toBe('heuristic');
+    });
+
+    it('should fall through when verifier throws', async () => {
+      const verifier = async (): Promise<VerificationResult> => {
+        throw new Error('verification crashed');
+      };
+
+      evaluator.registerVerifier('code', verifier);
+
+      const result = await evaluator.evaluate(makeTrajectory(), makeTask({ domain: 'code' }));
+      expect(result.method).toBe('heuristic');
+    });
+
+    it('should handle failed verification with issues', async () => {
+      const verifier = async (): Promise<VerificationResult> => ({
+        passed: false,
+        confidence: 0.9,
+        issues: [
+          { type: 'incorrect', description: 'test failed: should return 42', severity: 'major' },
+        ],
+      });
+
+      evaluator.registerVerifier('code', verifier);
+
+      const result = await evaluator.evaluate(makeTrajectory(), makeTask({ domain: 'code' }));
+
+      expect(result.method).toBe('verification');
+      expect(result.acceptable).toBe(false);
+      expect(result.score).toBeLessThan(0.5);
+      expect(result.issues).toHaveLength(1);
+    });
+
+    it('should not use verifier for wrong domain', async () => {
+      const verifier = async (): Promise<VerificationResult> => ({
+        passed: true,
+        confidence: 0.95,
+      });
+
+      evaluator.registerVerifier('python', verifier);
+
+      // Task domain is 'code', not 'python'
+      const result = await evaluator.evaluate(makeTrajectory(), makeTask({ domain: 'code' }));
+      expect(result.method).toBe('heuristic');
+    });
+  });
+
+  describe('alwaysUseAgent config', () => {
+    it('should skip verification shortcut when alwaysUseAgent=true', async () => {
+      const alwaysAgentEval = createSolutionEvaluator(null, { alwaysUseAgent: true });
+
+      const verifier = async (): Promise<VerificationResult> => ({
+        passed: true,
+        confidence: 0.99,
+      });
+      alwaysAgentEval.registerVerifier('code', verifier);
+
+      // With no agent or taskRunner, falls through to heuristic
+      const result = await alwaysAgentEval.evaluate(makeTrajectory(), makeTask({ domain: 'code' }));
+      expect(result.method).toBe('heuristic');
+    });
+  });
+
+  describe('parseQuality edge cases', () => {
+    // Test through parseAgentEvaluation indirectly using mock agent
+    it('should handle unknown quality strings gracefully', async () => {
+      // Falls through to heuristic since no agent available
+      const result = await evaluator.evaluate(makeTrajectory(), makeTask());
+      expect(['excellent', 'good', 'needs_work', 'poor']).toContain(result.quality);
+    });
+  });
+});
+
+describe('scoreToQuality (used by evaluator)', () => {
+  it('should map scores to correct quality levels', async () => {
+    const { scoreToQuality } = await import('../../src/search/refinement-types.js');
+
+    expect(scoreToQuality(0.9)).toBe('excellent');
+    expect(scoreToQuality(0.85)).toBe('excellent');
+    expect(scoreToQuality(0.75)).toBe('good');
+    expect(scoreToQuality(0.7)).toBe('good');
+    expect(scoreToQuality(0.5)).toBe('needs_work');
+    expect(scoreToQuality(0.4)).toBe('needs_work');
+    expect(scoreToQuality(0.3)).toBe('poor');
+    expect(scoreToQuality(0.0)).toBe('poor');
+  });
+});