cognitive-core 0.2.0 → 0.2.2

package/tests/memory/meta.test.ts

@@ -0,0 +1,399 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { MetaMemory, createMetaMemory } from '../../src/memory/meta.js';
+import { createSqlitePersistence } from '../../src/persistence/index.js';
+import { createMetaObservation, createMetaStrategy } from '../../src/types/meta.js';
+import type { MetaObservation, RoutingDecisionType } from '../../src/types/meta.js';
+import { mkdtemp, rm } from 'node:fs/promises';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+
+function makeObservation(overrides?: {
+  trajectoryId?: string;
+  decision?: RoutingDecisionType;
+  success?: boolean;
+  confidence?: number;
+  playbooksApplied?: string[];
+  retrievalQuality?: 'helpful' | 'neutral' | 'misleading';
+  timestamp?: Date;
+}): MetaObservation {
+  return createMetaObservation({
+    trajectoryId: overrides?.trajectoryId ?? `traj-${Math.random().toString(36).slice(2, 6)}`,
+    routing: {
+      decision: overrides?.decision ?? 'direct',
+      confidence: overrides?.confidence ?? 0.8,
+      reasoning: 'test routing',
+    },
+    memoryUsage: {
+      experiencesRetrieved: 3,
+      playbooksApplied: overrides?.playbooksApplied ?? [],
+      retrievalQuality: overrides?.retrievalQuality ?? 'neutral',
+    },
+    execution: {
+      decompositionUsed: false,
+      refinementIterations: 0,
+      backtrackingOccurred: false,
+      toolsUsed: ['Read', 'Edit'],
+    },
+    outcome: {
+      success: overrides?.success ?? true,
+      quality: overrides?.success === false ? 'poor' : 'good',
+      effortActual: 5,
+      effortEstimate: 8,
+    },
+    lessons: {
+      whatWorked: ['direct approach'],
+      whatFailed: [],
+      suggestions: [],
+    },
+    timestamp: overrides?.timestamp,
+  });
+}
+
+describe('MetaMemory', () => {
+  let tempDir: string;
+  let persistence: any;
+  let meta: MetaMemory;
+
+  beforeEach(async () => {
+    tempDir = await mkdtemp(join(tmpdir(), 'meta-mem-test-'));
+    persistence = createSqlitePersistence({ baseDir: tempDir });
+    await persistence.init();
+    meta = createMetaMemory(persistence);
+    await meta.init();
+  });
+
+  afterEach(async () => {
+    persistence.close();
+    await rm(tempDir, { recursive: true, force: true });
+  });
+
+  describe('recordObservation and retrieval', () => {
+    it('should store and retrieve observations', async () => {
+      const obs = makeObservation({ trajectoryId: 'traj-1' });
+      await meta.recordObservation(obs);
+
+      const all = await meta.getObservations();
+      expect(all).toHaveLength(1);
+      expect(all[0].trajectoryId).toBe('traj-1');
+    });
+
+    it('should retrieve by trajectory ID', async () => {
+      await meta.recordObservation(makeObservation({ trajectoryId: 'traj-A' }));
+      await meta.recordObservation(makeObservation({ trajectoryId: 'traj-B' }));
+
+      const result = await meta.getByTrajectoryId('traj-A');
+      expect(result).toBeDefined();
+      expect(result!.trajectoryId).toBe('traj-A');
+    });
+
+    it('should return undefined for non-existent trajectory ID', async () => {
+      const result = await meta.getByTrajectoryId('nonexistent');
+      expect(result).toBeUndefined();
+    });
+  });
+
+  describe('observation filtering', () => {
+    it('should filter by routing decision', async () => {
+      await meta.recordObservation(makeObservation({ decision: 'direct' }));
+      await meta.recordObservation(makeObservation({ decision: 'explore' }));
+      await meta.recordObservation(makeObservation({ decision: 'direct' }));
+
+      const results = await meta.getObservations({ routingDecision: 'direct' });
+      expect(results).toHaveLength(2);
+      expect(results.every((o) => o.routing.decision === 'direct')).toBe(true);
+    });
+
+    it('should filter by outcome', async () => {
+      await meta.recordObservation(makeObservation({ success: true }));
+      await meta.recordObservation(makeObservation({ success: false }));
+      await meta.recordObservation(makeObservation({ success: true }));
+
+      const successes = await meta.getObservations({ outcome: 'success' });
+      expect(successes).toHaveLength(2);
+
+      const failures = await meta.getObservations({ outcome: 'failure' });
+      expect(failures).toHaveLength(1);
+    });
+
+    it('should filter by date range', async () => {
+      const oldDate = new Date('2025-01-01');
+      const newDate = new Date('2026-03-01');
+      await meta.recordObservation(makeObservation({ timestamp: oldDate }));
+      await meta.recordObservation(makeObservation({ timestamp: newDate }));
+
+      const results = await meta.getObservations({ minDate: new Date('2026-01-01') });
+      expect(results).toHaveLength(1);
+    });
+
+    it('should limit results', async () => {
+      for (let i = 0; i < 5; i++) {
+        await meta.recordObservation(makeObservation());
+      }
+
+      const results = await meta.getObservations({ limit: 3 });
+      expect(results).toHaveLength(3);
+    });
+  });
+
+  describe('strategy management', () => {
+    it('should add and retrieve a strategy', async () => {
+      const strategy = createMetaStrategy({
+        name: 'test-strategy',
+        condition: { taskCharacteristics: ['complex'], memoryState: [] },
+        adjustment: {
+          routingBias: { explore: 0.2 },
+          retrievalModification: 'expand',
+          executionHint: 'decompose',
+        },
+      });
+
+      await meta.addStrategy(strategy);
+      const retrieved = await meta.getStrategy(strategy.id);
+
+      expect(retrieved).toBeDefined();
+      expect(retrieved!.name).toBe('test-strategy');
+      expect(retrieved!.condition.taskCharacteristics).toContain('complex');
+    });
+
+    it('should list all strategies', async () => {
+      for (let i = 0; i < 3; i++) {
+        await meta.addStrategy(
+          createMetaStrategy({
+            name: `strategy-${i}`,
+            condition: { taskCharacteristics: [], memoryState: [] },
+            adjustment: { routingBias: {}, retrievalModification: '', executionHint: '' },
+          })
+        );
+      }
+
+      const all = await meta.getAllStrategies();
+      expect(all).toHaveLength(3);
+    });
+
+    it('should return undefined for non-existent strategy', async () => {
+      const result = await meta.getStrategy('nonexistent');
+      expect(result).toBeUndefined();
+    });
+  });
+
+  describe('recordStrategyApplication', () => {
+    it('should update application count and success rate', async () => {
+      const strategy = createMetaStrategy({
+        name: 'test',
+        condition: { taskCharacteristics: [], memoryState: [] },
+        adjustment: { routingBias: {}, retrievalModification: '', executionHint: '' },
+      });
+      await meta.addStrategy(strategy);
+
+      await meta.recordStrategyApplication(strategy.id, true);
+      const updated = await meta.getStrategy(strategy.id);
+
+      expect(updated!.applicationCount).toBe(1);
+      expect(updated!.successRate).toBeGreaterThan(0);
+    });
+
+    it('should use exponential moving average for success rate', async () => {
+      const strategy = createMetaStrategy({
+        name: 'ema-test',
+        condition: { taskCharacteristics: [], memoryState: [] },
+        adjustment: { routingBias: {}, retrievalModification: '', executionHint: '' },
+      });
+      strategy.successRate = 0.5; // Set initial
+      await meta.addStrategy(strategy);
+
+      await meta.recordStrategyApplication(strategy.id, true);
+      const after = await meta.getStrategy(strategy.id);
+
+      // EMA: 0.5 * (1 - 0.1) + 1 * 0.1 = 0.55
+      expect(after!.successRate).toBeCloseTo(0.55, 2);
+    });
+
+    it('should no-op for nonexistent strategy', async () => {
+      // Should not throw
+      await meta.recordStrategyApplication('nonexistent', true);
+    });
+  });
+
+  describe('getRoutingAdjustments', () => {
+    it('should return empty when no strategies match', async () => {
+      await meta.addStrategy(
+        createMetaStrategy({
+          name: 'specific',
+          condition: {
+            taskCharacteristics: ['database migration'],
+            memoryState: ['no experiences'],
+          },
+          adjustment: { routingBias: {}, retrievalModification: '', executionHint: '' },
+        })
+      );
+
+      const results = await meta.getRoutingAdjustments({
+        taskCharacteristics: ['UI change'],
+        memoryState: ['plenty of experiences'],
+      });
+      expect(results).toHaveLength(0);
+    });
+
+    it('should match strategies with empty conditions (always applies)', async () => {
+      await meta.addStrategy(
+        createMetaStrategy({
+          name: 'universal',
+          condition: { taskCharacteristics: [], memoryState: [] },
+          adjustment: { routingBias: {}, retrievalModification: '', executionHint: 'always' },
+        })
+      );
+
+      const results = await meta.getRoutingAdjustments({
+        taskCharacteristics: ['anything'],
+        memoryState: ['anything'],
+      });
+      expect(results).toHaveLength(1);
+    });
+
+    it('should match via case-insensitive substring', async () => {
+      await meta.addStrategy(
+        createMetaStrategy({
+          name: 'multi-file',
+          condition: {
+            taskCharacteristics: ['multi-file'],
+            memoryState: [],
+          },
+          adjustment: { routingBias: {}, retrievalModification: '', executionHint: '' },
+        })
+      );
+
+      const results = await meta.getRoutingAdjustments({
+        taskCharacteristics: ['This is a Multi-File Change'],
+        memoryState: [],
+      });
+      expect(results).toHaveLength(1);
+    });
+  });
+
+  describe('analyzeRoutingEffectiveness', () => {
+    it('should aggregate success rates by routing decision', async () => {
+      // 3 successful direct, 2 failed direct
+      for (let i = 0; i < 5; i++) {
+        await meta.recordObservation(makeObservation({ decision: 'direct', success: i < 3 }));
+      }
+
+      const analysis = await meta.analyzeRoutingEffectiveness();
+      expect(analysis.byDecision.direct.total).toBeGreaterThanOrEqual(5);
+      expect(analysis.byDecision.direct.successful).toBeGreaterThanOrEqual(3);
+    });
+
+    it('should generate recommendations for low success rate', async () => {
+      // 5 failed explore decisions
+      for (let i = 0; i < 5; i++) {
+        await meta.recordObservation(makeObservation({ decision: 'explore', success: false }));
+      }
+
+      const analysis = await meta.analyzeRoutingEffectiveness();
+      expect(analysis.recommendations.length).toBeGreaterThan(0);
+      expect(analysis.recommendations[0]).toContain('explore');
+    });
+  });
+
+  describe('analyzeRetrievalQuality', () => {
+    it('should track helpful and misleading playbook patterns', async () => {
+      await meta.recordObservation(
+        makeObservation({ playbooksApplied: ['pb-1'], retrievalQuality: 'helpful' })
+      );
+      await meta.recordObservation(
+        makeObservation({ playbooksApplied: ['pb-1'], retrievalQuality: 'helpful' })
+      );
+      await meta.recordObservation(
+        makeObservation({ playbooksApplied: ['pb-2'], retrievalQuality: 'misleading' })
+      );
+
+      const analysis = await meta.analyzeRetrievalQuality();
+      expect(analysis.helpfulPatterns).toContain('pb-1');
+      expect(analysis.misleadingPatterns).toContain('pb-2');
+    });
+
+    it('should recommend refinement for consistently misleading playbooks', async () => {
+      for (let i = 0; i < 4; i++) {
+        await meta.recordObservation(
+          makeObservation({ playbooksApplied: ['pb-bad'], retrievalQuality: 'misleading' })
+        );
+      }
+
+      const analysis = await meta.analyzeRetrievalQuality();
+      expect(analysis.recommendations.length).toBeGreaterThan(0);
+      expect(analysis.recommendations[0]).toContain('pb-bad');
+    });
+  });
+
+  describe('generateMetaStrategies', () => {
+    it('should generate low-confidence-explore strategy when pattern detected', async () => {
+      // 5+ low-confidence observations with low success
+      for (let i = 0; i < 6; i++) {
+        await meta.recordObservation(
+          makeObservation({ confidence: 0.3, success: i === 0 }) // 1/6 success
+        );
+      }
+
+      const strategies = await meta.generateMetaStrategies();
+      const lowConf = strategies.find((s) => s.name === 'low-confidence-explore');
+      expect(lowConf).toBeDefined();
+      expect(lowConf!.adjustment.routingBias.explore).toBe(0.2);
+    });
+
+    it('should not duplicate existing strategies', async () => {
+      for (let i = 0; i < 6; i++) {
+        await meta.recordObservation(makeObservation({ confidence: 0.3, success: false }));
+      }
+
+      // Generate once
+      await meta.generateMetaStrategies();
+      // Generate again — should not duplicate
+      const second = await meta.generateMetaStrategies();
+      const lowConf = second.filter((s) => s.name === 'low-confidence-explore');
+      expect(lowConf).toHaveLength(0); // Already existed, not re-created
+    });
+  });
+
+  describe('getStats', () => {
+    it('should return zeroes for empty store', async () => {
+      const stats = await meta.getStats();
+      expect(stats.totalObservations).toBe(0);
+      expect(stats.totalStrategies).toBe(0);
+      expect(stats.averageSuccessRate).toBe(0);
+      expect(stats.recentTrend).toBe('stable');
+    });
+
+    it('should count observations by routing decision', async () => {
+      await meta.recordObservation(makeObservation({ decision: 'direct' }));
+      await meta.recordObservation(makeObservation({ decision: 'direct' }));
+      await meta.recordObservation(makeObservation({ decision: 'explore' }));
+
+      const stats = await meta.getStats();
+      expect(stats.totalObservations).toBe(3);
+      expect(stats.observationsByRouting.direct).toBe(2);
+      expect(stats.observationsByRouting.explore).toBe(1);
+    });
+
+    it('should compute average success rate', async () => {
+      await meta.recordObservation(makeObservation({ success: true }));
+      await meta.recordObservation(makeObservation({ success: true }));
+      await meta.recordObservation(makeObservation({ success: false }));
+
+      const stats = await meta.getStats();
+      expect(stats.averageSuccessRate).toBeCloseTo(2 / 3, 2);
+    });
+  });
+
+  describe('pruneOldObservations', () => {
+    it('should remove observations older than maxAge', async () => {
+      const oldDate = new Date(Date.now() - 60 * 24 * 60 * 60 * 1000); // 60 days ago
+      await meta.recordObservation(makeObservation({ timestamp: oldDate }));
+      await meta.recordObservation(makeObservation()); // now
+
+      const removed = await meta.pruneOldObservations(30 * 24 * 60 * 60 * 1000); // 30 day cutoff
+      expect(removed).toBeGreaterThanOrEqual(1);
+
+      const remaining = await meta.getObservations();
+      expect(remaining).toHaveLength(1);
+    });
+  });
+});