cognitive-core 0.2.0 → 0.2.2

package/tests/learning/meta-learner.test.ts

@@ -0,0 +1,418 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { MetaLearner, createMetaLearner } from '../../src/learning/meta-learner.js';
+import { MetaMemory, createMetaMemory } from '../../src/memory/meta.js';
+import { createSqlitePersistence } from '../../src/persistence/index.js';
+import { createTrajectory } from '../../src/types/trajectory.js';
+import { createTask } from '../../src/types/task.js';
+import { createStep } from '../../src/types/step.js';
+import { successOutcome, failureOutcome } from '../../src/types/outcome.js';
+import type { Trajectory } from '../../src/types/trajectory.js';
+import type { PlaybookMatch } from '../../src/memory/playbook.js';
+import type { RoutingDecision } from '../../src/learning/meta-learner.js';
+import { mkdtemp, rm } from 'node:fs/promises';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+
+function makeRouting(overrides?: Partial<RoutingDecision>): RoutingDecision {
+  return {
+    strategy: 'direct',
+    confidence: 0.8,
+    estimatedBudget: 10,
+    reasoning: 'direct match',
+    ...overrides,
+  };
+}
+
+function makeTrajectory(overrides?: {
+  success?: boolean;
+  steps?: Array<{ action: string; observation?: string; thought?: string }>;
+  llmCalls?: number;
+}): Trajectory {
+  const success = overrides?.success ?? true;
+  const rawSteps = overrides?.steps ?? [
+    { action: 'Read src/index.ts', observation: 'file contents' },
+    { action: 'Edit src/index.ts', observation: 'edited' },
+  ];
+  const steps = rawSteps.map((s) =>
+    createStep({
+      action: s.action,
+      observation: s.observation ?? '',
+      thought: s.thought,
+    })
+  );
+  return createTrajectory({
+    task: createTask({ domain: 'code', description: 'fix bug' }),
+    steps,
+    outcome: success ? successOutcome('fixed') : failureOutcome('failed'),
+    agentId: 'agent-1',
+    llmCalls: overrides?.llmCalls,
+  });
+}
+
+function makePlaybookMatch(name: string, tactics: string[] = []): PlaybookMatch {
+  return {
+    playbook: {
+      id: `pb-${name}`,
+      name,
+      domain: 'code',
+      trigger: { pattern: 'test', examples: [] },
+      guidance: { tactics, avoidances: [], context: '' },
+      confidence: 0.8,
+      successCount: 5,
+      failureCount: 1,
+      createdAt: new Date(),
+      updatedAt: new Date(),
+    },
+    score: 0.9,
+    source: 'text-similarity',
+  } as PlaybookMatch;
+}
+
+describe('MetaLearner', () => {
+  let tempDir: string;
+  let persistence: any;
+  let metaMemory: MetaMemory;
+  let learner: MetaLearner;
+
+  beforeEach(async () => {
+    tempDir = await mkdtemp(join(tmpdir(), 'meta-learner-test-'));
+    persistence = createSqlitePersistence({ baseDir: tempDir });
+    await persistence.init();
+    metaMemory = createMetaMemory(persistence);
+    await metaMemory.init();
+    learner = createMetaLearner(metaMemory);
+  });
+
+  afterEach(async () => {
+    persistence.close();
+    await rm(tempDir, { recursive: true, force: true });
+  });
+
+  describe('generateReflection', () => {
+    it('should create and store an observation for a successful trajectory', async () => {
+      const trajectory = makeTrajectory({ success: true });
+      const routing = makeRouting();
+      const playbooks = [makePlaybookMatch('fix-bugs', ['read file', 'edit file'])];
+
+      const observation = await learner.generateReflection(trajectory, routing, playbooks);
+
+      expect(observation).toBeDefined();
+      expect(observation.trajectoryId).toBe(trajectory.id);
+      expect(observation.routing.decision).toBe('direct');
+      expect(observation.routing.confidence).toBe(0.8);
+      expect(observation.outcome.success).toBe(true);
+      expect(observation.lessons.whatWorked.length).toBeGreaterThan(0);
+    });
+
+    it('should create observation for a failed trajectory', async () => {
+      const trajectory = makeTrajectory({
+        success: false,
+        steps: [
+          { action: 'Read src/broken.ts', observation: 'error: file not found' },
+          { action: 'Bash npm test', observation: 'error: tests failed' },
+        ],
+      });
+      const routing = makeRouting({ confidence: 0.9 });
+
+      const observation = await learner.generateReflection(trajectory, routing, []);
+
+      expect(observation.outcome.success).toBe(false);
+      expect(observation.outcome.quality).toBe('poor');
+      expect(observation.lessons.whatFailed.length).toBeGreaterThan(0);
+      expect(observation.lessons.suggestions.length).toBeGreaterThan(0);
+    });
+
+    it('should persist observation to MetaMemory', async () => {
+      const trajectory = makeTrajectory();
+      await learner.generateReflection(trajectory, makeRouting(), []);
+
+      const stored = await metaMemory.getByTrajectoryId(trajectory.id);
+      expect(stored).toBeDefined();
+      expect(stored!.trajectoryId).toBe(trajectory.id);
+    });
+
+    it('should detect backtracking when actions repeat', async () => {
+      const trajectory = makeTrajectory({
+        steps: [
+          { action: 'Read foo.ts' },
+          { action: 'Edit foo.ts' },
+          { action: 'Read foo.ts' },
+          { action: 'Edit foo.ts' },
+          { action: 'Read foo.ts' },
+          { action: 'Edit foo.ts' },
+        ],
+      });
+
+      const observation = await learner.generateReflection(trajectory, makeRouting(), []);
+      expect(observation.execution.backtrackingOccurred).toBe(true);
+    });
+
+    it('should extract tools used', async () => {
+      const trajectory = makeTrajectory({
+        steps: [
+          { action: 'Read src/index.ts' },
+          { action: 'tool_use: Bash echo hello' },
+          { action: 'Edit src/index.ts' },
+        ],
+      });
+
+      const observation = await learner.generateReflection(trajectory, makeRouting(), []);
+      expect(observation.execution.toolsUsed).toContain('Read');
+      expect(observation.execution.toolsUsed).toContain('Bash');
+      expect(observation.execution.toolsUsed).toContain('Edit');
+    });
+
+    it('should detect decomposition usage from step thoughts', async () => {
+      const trajectory = makeTrajectory({
+        steps: [
+          { action: 'Read foo.ts', thought: 'first, I need to understand the code' },
+          { action: 'Edit foo.ts', thought: 'now fix the bug' },
+        ],
+      });
+
+      const observation = await learner.generateReflection(trajectory, makeRouting(), []);
+      expect(observation.execution.decompositionUsed).toBe(true);
+    });
+
+    it('should count refinement iterations', async () => {
+      const trajectory = makeTrajectory({
+        steps: [
+          { action: 'Edit foo.ts', observation: 'error: syntax error' },
+          { action: 'Edit foo.ts', thought: 'try fixing the syntax' },
+          { action: 'Bash npm test', observation: 'test failed' },
+          { action: 'Edit foo.ts', thought: 'fix the test failure' },
+        ],
+      });
+
+      const observation = await learner.generateReflection(trajectory, makeRouting(), []);
+      expect(observation.execution.refinementIterations).toBeGreaterThanOrEqual(1);
+    });
+  });
+
+  describe('assessOutcomeQuality (via generateReflection)', () => {
+    it('should rate excellent when effort is much less than estimate', async () => {
+      const trajectory = makeTrajectory({ success: true, llmCalls: 3 });
+      const routing = makeRouting({ estimatedBudget: 10 });
+      const obs = await learner.generateReflection(trajectory, routing, []);
+      // 3/10 = 0.3 <= 0.5 → excellent
+      expect(obs.outcome.quality).toBe('excellent');
+    });
+
+    it('should rate good when effort is within estimate', async () => {
+      const trajectory = makeTrajectory({ success: true, llmCalls: 8 });
+      const routing = makeRouting({ estimatedBudget: 10 });
+      const obs = await learner.generateReflection(trajectory, routing, []);
+      // 8/10 = 0.8 <= 1.0 → good
+      expect(obs.outcome.quality).toBe('good');
+    });
+
+    it('should rate acceptable when effort moderately overruns', async () => {
+      const trajectory = makeTrajectory({ success: true, llmCalls: 15 });
+      const routing = makeRouting({ estimatedBudget: 10 });
+      const obs = await learner.generateReflection(trajectory, routing, []);
+      // 15/10 = 1.5 <= 2.0 → acceptable
+      expect(obs.outcome.quality).toBe('acceptable');
+    });
+
+    it('should rate poor when effort vastly overruns', async () => {
+      const trajectory = makeTrajectory({ success: true, llmCalls: 25 });
+      const routing = makeRouting({ estimatedBudget: 10 });
+      const obs = await learner.generateReflection(trajectory, routing, []);
+      // 25/10 = 2.5 > 2.0 → poor
+      expect(obs.outcome.quality).toBe('poor');
+    });
+
+    it('should rate poor for failed trajectories regardless of effort', async () => {
+      const trajectory = makeTrajectory({ success: false, llmCalls: 3 });
+      const routing = makeRouting({ estimatedBudget: 10 });
+      const obs = await learner.generateReflection(trajectory, routing, []);
+      expect(obs.outcome.quality).toBe('poor');
+    });
+  });
+
+  describe('assessRetrievalQuality (via generateReflection)', () => {
+    it('should be neutral when no playbooks used', async () => {
+      const trajectory = makeTrajectory({ success: true });
+      const obs = await learner.generateReflection(trajectory, makeRouting(), []);
+      expect(obs.memoryUsage.retrievalQuality).toBe('neutral');
+    });
+
+    it('should be helpful when guidance followed and succeeded', async () => {
+      const trajectory = makeTrajectory({
+        success: true,
+        steps: [
+          { action: 'read the file carefully' },
+          { action: 'edit the function body' },
+        ],
+      });
+      const playbooks = [makePlaybookMatch('fix', ['read the file', 'edit the function'])];
+      const obs = await learner.generateReflection(trajectory, makeRouting(), playbooks);
+      expect(obs.memoryUsage.retrievalQuality).toBe('helpful');
+    });
+
+    it('should be misleading when guidance followed but failed', async () => {
+      const trajectory = makeTrajectory({
+        success: false,
+        steps: [
+          { action: 'read the file carefully' },
+          { action: 'edit the function body' },
+        ],
+      });
+      const playbooks = [makePlaybookMatch('fix', ['read the file', 'edit the function'])];
+      const obs = await learner.generateReflection(trajectory, makeRouting(), playbooks);
+      expect(obs.memoryUsage.retrievalQuality).toBe('misleading');
+    });
+  });
+
+  describe('learn', () => {
+    it('should return empty when too few observations', async () => {
+      const strategies = await learner.learn();
+      expect(strategies).toEqual([]);
+    });
+
+    it('should generate strategies after enough observations', async () => {
+      // Feed 10+ observations to reach threshold
+      for (let i = 0; i < 12; i++) {
+        const trajectory = makeTrajectory({ success: i % 3 !== 0 });
+        await learner.generateReflection(trajectory, makeRouting(), []);
+      }
+
+      const strategies = await learner.learn();
+      // Should have generated at least meta-strategies from MetaMemory
+      expect(strategies).toBeDefined();
+    });
+  });
+
+  describe('adjustRouting', () => {
+    it('should return base routing when no strategies exist', async () => {
+      const base = makeRouting();
+      const adjusted = await learner.adjustRouting(base, [], []);
+      expect(adjusted).toEqual(base);
+    });
+
+    it('should adjust confidence based on routing bias', async () => {
+      // Manually add a strategy
+      const { createMetaStrategy } = await import('../../src/types/meta.js');
+      await metaMemory.addStrategy(
+        createMetaStrategy({
+          name: 'boost-direct',
+          condition: { taskCharacteristics: [], memoryState: [] },
+          adjustment: {
+            routingBias: { direct: 0.15 },
+            retrievalModification: '',
+            executionHint: '',
+          },
+        })
+      );
+
+      const base = makeRouting({ confidence: 0.5 });
+      const adjusted = await learner.adjustRouting(base, [], []);
+      expect(adjusted.confidence).toBeCloseTo(0.65, 1);
+    });
+
+    it('should clamp adjusted confidence to [0.1, 0.95]', async () => {
+      const { createMetaStrategy } = await import('../../src/types/meta.js');
+      await metaMemory.addStrategy(
+        createMetaStrategy({
+          name: 'mega-boost',
+          condition: { taskCharacteristics: [], memoryState: [] },
+          adjustment: {
+            routingBias: { direct: 0.9 },
+            retrievalModification: '',
+            executionHint: '',
+          },
+        })
+      );
+
+      const base = makeRouting({ confidence: 0.9 });
+      const adjusted = await learner.adjustRouting(base, [], []);
+      expect(adjusted.confidence).toBeLessThanOrEqual(0.95);
+    });
+
+    it('should switch strategy when bias strongly favors alternative and base confidence low', async () => {
+      const { createMetaStrategy } = await import('../../src/types/meta.js');
+      await metaMemory.addStrategy(
+        createMetaStrategy({
+          name: 'prefer-explore',
+          condition: { taskCharacteristics: [], memoryState: [] },
+          adjustment: {
+            routingBias: { explore: 0.5 },
+            retrievalModification: '',
+            executionHint: 'try exploration',
+          },
+        })
+      );
+
+      const base = makeRouting({ strategy: 'direct', confidence: 0.3 });
+      const adjusted = await learner.adjustRouting(base, [], []);
+      expect(adjusted.strategy).toBe('explore');
+      expect(adjusted.reasoning).toContain('Meta-strategy');
+    });
+
+    it('should match strategies by task characteristics', async () => {
+      const { createMetaStrategy } = await import('../../src/types/meta.js');
+      await metaMemory.addStrategy(
+        createMetaStrategy({
+          name: 'multi-file-strategy',
+          condition: {
+            taskCharacteristics: ['multi-file change'],
+            memoryState: [],
+          },
+          adjustment: {
+            routingBias: { direct: 0.1 },
+            retrievalModification: '',
+            executionHint: 'use decomposition',
+          },
+        })
+      );
+
+      // Should match because "multi-file change" is included
+      const adjusted = await learner.adjustRouting(
+        makeRouting({ confidence: 0.5 }),
+        ['This involves a multi-file change'],
+        []
+      );
+      expect(adjusted.reasoning).toContain('decomposition');
+    });
+  });
+
+  describe('generateStrategiesFromAnalysis (via learn)', () => {
+    it('should generate avoid strategy for low success rate decisions', async () => {
+      // Create observations where "explore" has < 40% success rate
+      for (let i = 0; i < 10; i++) {
+        const trajectory = makeTrajectory({ success: i < 2 }); // 20% success
+        const routing = makeRouting({ strategy: 'explore' });
+        await learner.generateReflection(trajectory, routing, []);
+      }
+
+      const strategies = await learner.learn();
+      const avoidStrategy = strategies.find((s) => s.name.includes('avoid'));
+      // May or may not appear depending on MetaMemory's own strategy generation
+      if (avoidStrategy) {
+        expect(avoidStrategy.name).toContain('explore');
+      }
+    });
+  });
+
+  describe('strategy trigger interval', () => {
+    it('should auto-generate strategies at configured interval', async () => {
+      const intervalLearner = createMetaLearner(metaMemory, {
+        strategyGenerationInterval: 5,
+        minObservationsForStrategies: 3,
+      });
+
+      // Feed 5 observations to trigger auto-learn
+      for (let i = 0; i < 5; i++) {
+        await intervalLearner.generateReflection(
+          makeTrajectory({ success: i % 2 === 0 }),
+          makeRouting(),
+          []
+        );
+      }
+
+      // After 5 observations (interval=5), learn() should have been called
+      const stats = await metaMemory.getStats();
+      expect(stats.totalObservations).toBe(5);
+    });
+  });
+});