cognitive-core 0.2.4 → 0.3.1

package/tests/learning/meta-learner.test.ts

@@ -1,418 +0,0 @@
-import { describe, it, expect, beforeEach, afterEach } from 'vitest';
-import { MetaLearner, createMetaLearner } from '../../src/learning/meta-learner.js';
-import { MetaMemory, createMetaMemory } from '../../src/memory/meta.js';
-import { createSqlitePersistence } from '../../src/persistence/index.js';
-import { createTrajectory } from '../../src/types/trajectory.js';
-import { createTask } from '../../src/types/task.js';
-import { createStep } from '../../src/types/step.js';
-import { successOutcome, failureOutcome } from '../../src/types/outcome.js';
-import type { Trajectory } from '../../src/types/trajectory.js';
-import type { PlaybookMatch } from '../../src/memory/playbook.js';
-import type { RoutingDecision } from '../../src/learning/meta-learner.js';
-import { mkdtemp, rm } from 'node:fs/promises';
-import { join } from 'node:path';
-import { tmpdir } from 'node:os';
-
-function makeRouting(overrides?: Partial<RoutingDecision>): RoutingDecision {
-  return {
-    strategy: 'direct',
-    confidence: 0.8,
-    estimatedBudget: 10,
-    reasoning: 'direct match',
-    ...overrides,
-  };
-}
-
-function makeTrajectory(overrides?: {
-  success?: boolean;
-  steps?: Array<{ action: string; observation?: string; thought?: string }>;
-  llmCalls?: number;
-}): Trajectory {
-  const success = overrides?.success ?? true;
-  const rawSteps = overrides?.steps ?? [
-    { action: 'Read src/index.ts', observation: 'file contents' },
-    { action: 'Edit src/index.ts', observation: 'edited' },
-  ];
-  const steps = rawSteps.map((s) =>
-    createStep({
-      action: s.action,
-      observation: s.observation ?? '',
-      thought: s.thought,
-    })
-  );
-  return createTrajectory({
-    task: createTask({ domain: 'code', description: 'fix bug' }),
-    steps,
-    outcome: success ? successOutcome('fixed') : failureOutcome('failed'),
-    agentId: 'agent-1',
-    llmCalls: overrides?.llmCalls,
-  });
-}
-
-function makePlaybookMatch(name: string, tactics: string[] = []): PlaybookMatch {
-  return {
-    playbook: {
-      id: `pb-${name}`,
-      name,
-      domain: 'code',
-      trigger: { pattern: 'test', examples: [] },
-      guidance: { tactics, avoidances: [], context: '' },
-      confidence: 0.8,
-      successCount: 5,
-      failureCount: 1,
-      createdAt: new Date(),
-      updatedAt: new Date(),
-    },
-    score: 0.9,
-    source: 'text-similarity',
-  } as PlaybookMatch;
-}
-
-describe('MetaLearner', () => {
-  let tempDir: string;
-  let persistence: any;
-  let metaMemory: MetaMemory;
-  let learner: MetaLearner;
-
-  beforeEach(async () => {
-    tempDir = await mkdtemp(join(tmpdir(), 'meta-learner-test-'));
-    persistence = createSqlitePersistence({ baseDir: tempDir });
-    await persistence.init();
-    metaMemory = createMetaMemory(persistence);
-    await metaMemory.init();
-    learner = createMetaLearner(metaMemory);
-  });
-
-  afterEach(async () => {
-    persistence.close();
-    await rm(tempDir, { recursive: true, force: true });
-  });
-
-  describe('generateReflection', () => {
-    it('should create and store an observation for a successful trajectory', async () => {
-      const trajectory = makeTrajectory({ success: true });
-      const routing = makeRouting();
-      const playbooks = [makePlaybookMatch('fix-bugs', ['read file', 'edit file'])];
-
-      const observation = await learner.generateReflection(trajectory, routing, playbooks);
-
-      expect(observation).toBeDefined();
-      expect(observation.trajectoryId).toBe(trajectory.id);
-      expect(observation.routing.decision).toBe('direct');
-      expect(observation.routing.confidence).toBe(0.8);
-      expect(observation.outcome.success).toBe(true);
-      expect(observation.lessons.whatWorked.length).toBeGreaterThan(0);
-    });
-
-    it('should create observation for a failed trajectory', async () => {
-      const trajectory = makeTrajectory({
-        success: false,
-        steps: [
-          { action: 'Read src/broken.ts', observation: 'error: file not found' },
-          { action: 'Bash npm test', observation: 'error: tests failed' },
-        ],
-      });
-      const routing = makeRouting({ confidence: 0.9 });
-
-      const observation = await learner.generateReflection(trajectory, routing, []);
-
-      expect(observation.outcome.success).toBe(false);
-      expect(observation.outcome.quality).toBe('poor');
-      expect(observation.lessons.whatFailed.length).toBeGreaterThan(0);
-      expect(observation.lessons.suggestions.length).toBeGreaterThan(0);
-    });
-
-    it('should persist observation to MetaMemory', async () => {
-      const trajectory = makeTrajectory();
-      await learner.generateReflection(trajectory, makeRouting(), []);
-
-      const stored = await metaMemory.getByTrajectoryId(trajectory.id);
-      expect(stored).toBeDefined();
-      expect(stored!.trajectoryId).toBe(trajectory.id);
-    });
-
-    it('should detect backtracking when actions repeat', async () => {
-      const trajectory = makeTrajectory({
-        steps: [
-          { action: 'Read foo.ts' },
-          { action: 'Edit foo.ts' },
-          { action: 'Read foo.ts' },
-          { action: 'Edit foo.ts' },
-          { action: 'Read foo.ts' },
-          { action: 'Edit foo.ts' },
-        ],
-      });
-
-      const observation = await learner.generateReflection(trajectory, makeRouting(), []);
-      expect(observation.execution.backtrackingOccurred).toBe(true);
-    });
-
-    it('should extract tools used', async () => {
-      const trajectory = makeTrajectory({
-        steps: [
-          { action: 'Read src/index.ts' },
-          { action: 'tool_use: Bash echo hello' },
-          { action: 'Edit src/index.ts' },
-        ],
-      });
-
-      const observation = await learner.generateReflection(trajectory, makeRouting(), []);
-      expect(observation.execution.toolsUsed).toContain('Read');
-      expect(observation.execution.toolsUsed).toContain('Bash');
-      expect(observation.execution.toolsUsed).toContain('Edit');
-    });
-
-    it('should detect decomposition usage from step thoughts', async () => {
-      const trajectory = makeTrajectory({
-        steps: [
-          { action: 'Read foo.ts', thought: 'first, I need to understand the code' },
-          { action: 'Edit foo.ts', thought: 'now fix the bug' },
-        ],
-      });
-
-      const observation = await learner.generateReflection(trajectory, makeRouting(), []);
-      expect(observation.execution.decompositionUsed).toBe(true);
-    });
-
-    it('should count refinement iterations', async () => {
-      const trajectory = makeTrajectory({
-        steps: [
-          { action: 'Edit foo.ts', observation: 'error: syntax error' },
-          { action: 'Edit foo.ts', thought: 'try fixing the syntax' },
-          { action: 'Bash npm test', observation: 'test failed' },
-          { action: 'Edit foo.ts', thought: 'fix the test failure' },
-        ],
-      });
-
-      const observation = await learner.generateReflection(trajectory, makeRouting(), []);
-      expect(observation.execution.refinementIterations).toBeGreaterThanOrEqual(1);
-    });
-  });
-
-  describe('assessOutcomeQuality (via generateReflection)', () => {
-    it('should rate excellent when effort is much less than estimate', async () => {
-      const trajectory = makeTrajectory({ success: true, llmCalls: 3 });
-      const routing = makeRouting({ estimatedBudget: 10 });
-      const obs = await learner.generateReflection(trajectory, routing, []);
-      // 3/10 = 0.3 <= 0.5 → excellent
-      expect(obs.outcome.quality).toBe('excellent');
-    });
-
-    it('should rate good when effort is within estimate', async () => {
-      const trajectory = makeTrajectory({ success: true, llmCalls: 8 });
-      const routing = makeRouting({ estimatedBudget: 10 });
-      const obs = await learner.generateReflection(trajectory, routing, []);
-      // 8/10 = 0.8 <= 1.0 → good
-      expect(obs.outcome.quality).toBe('good');
-    });
-
-    it('should rate acceptable when effort moderately overruns', async () => {
-      const trajectory = makeTrajectory({ success: true, llmCalls: 15 });
-      const routing = makeRouting({ estimatedBudget: 10 });
-      const obs = await learner.generateReflection(trajectory, routing, []);
-      // 15/10 = 1.5 <= 2.0 → acceptable
-      expect(obs.outcome.quality).toBe('acceptable');
-    });
-
-    it('should rate poor when effort vastly overruns', async () => {
-      const trajectory = makeTrajectory({ success: true, llmCalls: 25 });
-      const routing = makeRouting({ estimatedBudget: 10 });
-      const obs = await learner.generateReflection(trajectory, routing, []);
-      // 25/10 = 2.5 > 2.0 → poor
-      expect(obs.outcome.quality).toBe('poor');
-    });
-
-    it('should rate poor for failed trajectories regardless of effort', async () => {
-      const trajectory = makeTrajectory({ success: false, llmCalls: 3 });
-      const routing = makeRouting({ estimatedBudget: 10 });
-      const obs = await learner.generateReflection(trajectory, routing, []);
-      expect(obs.outcome.quality).toBe('poor');
-    });
-  });
-
-  describe('assessRetrievalQuality (via generateReflection)', () => {
-    it('should be neutral when no playbooks used', async () => {
-      const trajectory = makeTrajectory({ success: true });
-      const obs = await learner.generateReflection(trajectory, makeRouting(), []);
-      expect(obs.memoryUsage.retrievalQuality).toBe('neutral');
-    });
-
-    it('should be helpful when guidance followed and succeeded', async () => {
-      const trajectory = makeTrajectory({
-        success: true,
-        steps: [
-          { action: 'read the file carefully' },
-          { action: 'edit the function body' },
-        ],
-      });
-      const playbooks = [makePlaybookMatch('fix', ['read the file', 'edit the function'])];
-      const obs = await learner.generateReflection(trajectory, makeRouting(), playbooks);
-      expect(obs.memoryUsage.retrievalQuality).toBe('helpful');
-    });
-
-    it('should be misleading when guidance followed but failed', async () => {
-      const trajectory = makeTrajectory({
-        success: false,
-        steps: [
-          { action: 'read the file carefully' },
-          { action: 'edit the function body' },
-        ],
-      });
-      const playbooks = [makePlaybookMatch('fix', ['read the file', 'edit the function'])];
-      const obs = await learner.generateReflection(trajectory, makeRouting(), playbooks);
-      expect(obs.memoryUsage.retrievalQuality).toBe('misleading');
-    });
-  });
-
-  describe('learn', () => {
-    it('should return empty when too few observations', async () => {
-      const strategies = await learner.learn();
-      expect(strategies).toEqual([]);
-    });
-
-    it('should generate strategies after enough observations', async () => {
-      // Feed 10+ observations to reach threshold
-      for (let i = 0; i < 12; i++) {
-        const trajectory = makeTrajectory({ success: i % 3 !== 0 });
-        await learner.generateReflection(trajectory, makeRouting(), []);
-      }
-
-      const strategies = await learner.learn();
-      // Should have generated at least meta-strategies from MetaMemory
-      expect(strategies).toBeDefined();
-    });
-  });
-
-  describe('adjustRouting', () => {
-    it('should return base routing when no strategies exist', async () => {
-      const base = makeRouting();
-      const adjusted = await learner.adjustRouting(base, [], []);
-      expect(adjusted).toEqual(base);
-    });
-
-    it('should adjust confidence based on routing bias', async () => {
-      // Manually add a strategy
-      const { createMetaStrategy } = await import('../../src/types/meta.js');
-      await metaMemory.addStrategy(
-        createMetaStrategy({
-          name: 'boost-direct',
-          condition: { taskCharacteristics: [], memoryState: [] },
-          adjustment: {
-            routingBias: { direct: 0.15 },
-            retrievalModification: '',
-            executionHint: '',
-          },
-        })
-      );
-
-      const base = makeRouting({ confidence: 0.5 });
-      const adjusted = await learner.adjustRouting(base, [], []);
-      expect(adjusted.confidence).toBeCloseTo(0.65, 1);
-    });
-
-    it('should clamp adjusted confidence to [0.1, 0.95]', async () => {
-      const { createMetaStrategy } = await import('../../src/types/meta.js');
-      await metaMemory.addStrategy(
-        createMetaStrategy({
-          name: 'mega-boost',
-          condition: { taskCharacteristics: [], memoryState: [] },
-          adjustment: {
-            routingBias: { direct: 0.9 },
-            retrievalModification: '',
-            executionHint: '',
-          },
-        })
-      );
-
-      const base = makeRouting({ confidence: 0.9 });
-      const adjusted = await learner.adjustRouting(base, [], []);
-      expect(adjusted.confidence).toBeLessThanOrEqual(0.95);
-    });
-
-    it('should switch strategy when bias strongly favors alternative and base confidence low', async () => {
-      const { createMetaStrategy } = await import('../../src/types/meta.js');
-      await metaMemory.addStrategy(
-        createMetaStrategy({
-          name: 'prefer-explore',
-          condition: { taskCharacteristics: [], memoryState: [] },
-          adjustment: {
-            routingBias: { explore: 0.5 },
-            retrievalModification: '',
-            executionHint: 'try exploration',
-          },
-        })
-      );
-
-      const base = makeRouting({ strategy: 'direct', confidence: 0.3 });
-      const adjusted = await learner.adjustRouting(base, [], []);
-      expect(adjusted.strategy).toBe('explore');
-      expect(adjusted.reasoning).toContain('Meta-strategy');
-    });
-
-    it('should match strategies by task characteristics', async () => {
-      const { createMetaStrategy } = await import('../../src/types/meta.js');
-      await metaMemory.addStrategy(
-        createMetaStrategy({
-          name: 'multi-file-strategy',
-          condition: {
-            taskCharacteristics: ['multi-file change'],
-            memoryState: [],
-          },
-          adjustment: {
-            routingBias: { direct: 0.1 },
-            retrievalModification: '',
-            executionHint: 'use decomposition',
-          },
-        })
-      );
-
-      // Should match because "multi-file change" is included
-      const adjusted = await learner.adjustRouting(
-        makeRouting({ confidence: 0.5 }),
-        ['This involves a multi-file change'],
-        []
-      );
-      expect(adjusted.reasoning).toContain('decomposition');
-    });
-  });
-
-  describe('generateStrategiesFromAnalysis (via learn)', () => {
-    it('should generate avoid strategy for low success rate decisions', async () => {
-      // Create observations where "explore" has < 40% success rate
-      for (let i = 0; i < 10; i++) {
-        const trajectory = makeTrajectory({ success: i < 2 }); // 20% success
-        const routing = makeRouting({ strategy: 'explore' });
-        await learner.generateReflection(trajectory, routing, []);
-      }
-
-      const strategies = await learner.learn();
-      const avoidStrategy = strategies.find((s) => s.name.includes('avoid'));
-      // May or may not appear depending on MetaMemory's own strategy generation
-      if (avoidStrategy) {
-        expect(avoidStrategy.name).toContain('explore');
-      }
-    });
-  });
-
-  describe('strategy trigger interval', () => {
-    it('should auto-generate strategies at configured interval', async () => {
-      const intervalLearner = createMetaLearner(metaMemory, {
-        strategyGenerationInterval: 5,
-        minObservationsForStrategies: 3,
-      });
-
-      // Feed 5 observations to trigger auto-learn
-      for (let i = 0; i < 5; i++) {
-        await intervalLearner.generateReflection(
-          makeTrajectory({ success: i % 2 === 0 }),
-          makeRouting(),
-          []
-        );
-      }
-
-      // After 5 observations (interval=5), learn() should have been called
-      const stats = await metaMemory.getStats();
-      expect(stats.totalObservations).toBe(5);
-    });
-  });
-});