cognitive-core 0.0.2 → 0.1.2

package/tests/learning/effectiveness.test.ts

@@ -0,0 +1,542 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import {
+  LearningEffectivenessTracker,
+  createEffectivenessTracker,
+} from '../../src/learning/effectiveness.js';
+import { createTrajectory, createStep, createTask, successOutcome, failureOutcome } from '../../src/types/index.js';
+import type { PlaybookMatch } from '../../src/memory/playbook.js';
+import type { Playbook } from '../../src/types/playbook.js';
+import { createPlaybook } from '../../src/types/playbook.js';
+import { mkdtemp, rm } from 'node:fs/promises';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+
+describe('LearningEffectivenessTracker', () => {
+  let tempDir: string;
+  let tracker: LearningEffectivenessTracker;
+
+  beforeEach(async () => {
+    tempDir = await mkdtemp(join(tmpdir(), 'atlas-eff-test-'));
+    tracker = createEffectivenessTracker(tempDir);
+    await tracker.init();
+  });
+
+  afterEach(async () => {
+    await tracker.close();
+    await rm(tempDir, { recursive: true, force: true });
+  });
+
+  function makeTrajectory(success: boolean, stepCount: number, hasErrors = false) {
+    const steps = [];
+    for (let i = 0; i < stepCount; i++) {
+      steps.push(createStep({
+        thought: `Step ${i + 1}`,
+        action: `action-${i + 1}`,
+        observation: hasErrors && i === 1 ? 'error: something failed' : 'ok',
+      }));
+    }
+    return createTrajectory({
+      task: createTask({ domain: 'test', description: `Test task ${Date.now()}` }),
+      steps,
+      outcome: success ? successOutcome({ result: 'done' }) : failureOutcome('failed'),
+      agentId: 'test-agent',
+    });
+  }
+
+  function makePlaybook(name: string): Playbook {
+    return createPlaybook({
+      name,
+      applicability: {
+        situations: ['Test'],
+        triggers: [],
+        antiPatterns: [],
+        domains: ['test'],
+      },
+      guidance: { strategy: 'Test', tactics: [] },
+    });
+  }
+
+  function makePlaybookMatch(playbook: Playbook): PlaybookMatch {
+    return { playbook, score: 0.8, matchType: 'situation' };
+  }
+
+  describe('annotate', () => {
+    it('should create an annotation for a trajectory', async () => {
+      const trajectory = makeTrajectory(true, 3);
+      const pb = makePlaybook('test-playbook');
+      const matches = [makePlaybookMatch(pb)];
+
+      const annotation = await tracker.annotate(
+        trajectory,
+        matches,
+        ['exp-1'],
+        [pb.id],
+      );
+
+      expect(annotation.trajectoryId).toBe(trajectory.id);
+      expect(annotation.knowledgeSurfaced.playbookIds).toEqual([pb.id]);
+      expect(annotation.knowledgeSurfaced.playbookNames).toEqual(['test-playbook']);
+      expect(annotation.knowledgeSurfaced.experienceIds).toEqual(['exp-1']);
+      expect(annotation.knowledgeSurfaced.totalItems).toBe(2);
+      expect(annotation.knowledgeApplied.playbookIdsUsed).toEqual([pb.id]);
+      expect(annotation.knowledgeApplied.anyKnowledgeUsed).toBe(true);
+      expect(annotation.outcome.success).toBe(true);
+      expect(annotation.outcome.stepCount).toBe(3);
+    });
+
+    it('should count error recoveries', async () => {
+      const trajectory = makeTrajectory(true, 4, true);
+      const annotation = await tracker.annotate(trajectory, [], [], []);
+
+      // Step 1 (ok) -> Step 2 (error) -> Step 3 (ok) = 1 recovery
+      expect(annotation.outcome.errorRecoveries).toBe(1);
+    });
+
+    it('should count multiple error recoveries', async () => {
+      const steps = [
+        createStep({ thought: 'S1', action: 'a1', observation: 'ok' }),
+        createStep({ thought: 'S2', action: 'a2', observation: 'error: first fail' }),
+        createStep({ thought: 'S3', action: 'a3', observation: 'recovered' }),
+        createStep({ thought: 'S4', action: 'a4', observation: 'error: second fail' }),
+        createStep({ thought: 'S5', action: 'a5', observation: 'recovered again' }),
+      ];
+      const trajectory = createTrajectory({
+        task: createTask({ domain: 'test', description: 'multi-error' }),
+        steps,
+        outcome: successOutcome({ result: 'done' }),
+        agentId: 'test-agent',
+      });
+
+      const annotation = await tracker.annotate(trajectory, [], [], []);
+      expect(annotation.outcome.errorRecoveries).toBe(2);
+    });
+
+    it('should report zero error recoveries for clean trajectories', async () => {
+      const trajectory = makeTrajectory(true, 5, false);
+      const annotation = await tracker.annotate(trajectory, [], [], []);
+      expect(annotation.outcome.errorRecoveries).toBe(0);
+    });
+
+    it('should handle multiple playbooks surfaced in one task', async () => {
+      const pb1 = makePlaybook('pb-alpha');
+      const pb2 = makePlaybook('pb-beta');
+      const pb3 = makePlaybook('pb-gamma');
+      const matches = [makePlaybookMatch(pb1), makePlaybookMatch(pb2), makePlaybookMatch(pb3)];
+
+      const annotation = await tracker.annotate(
+        makeTrajectory(true, 3),
+        matches,
+        [],
+        [pb1.id, pb3.id],
+      );
+
+      expect(annotation.knowledgeSurfaced.playbookIds).toHaveLength(3);
+      expect(annotation.knowledgeSurfaced.playbookNames).toEqual(['pb-alpha', 'pb-beta', 'pb-gamma']);
+      expect(annotation.knowledgeSurfaced.totalItems).toBe(3);
+      expect(annotation.knowledgeApplied.playbookIdsUsed).toHaveLength(2);
+      expect(annotation.knowledgeApplied.anyKnowledgeUsed).toBe(true);
+    });
+
+    it('should handle experience-only guidance (no playbooks)', async () => {
+      const annotation = await tracker.annotate(
+        makeTrajectory(true, 3),
+        [],
+        ['exp-1', 'exp-2', 'exp-3'],
+        [],
+      );
+
+      expect(annotation.knowledgeSurfaced.playbookIds).toHaveLength(0);
+      expect(annotation.knowledgeSurfaced.experienceIds).toHaveLength(3);
+      expect(annotation.knowledgeSurfaced.totalItems).toBe(3);
+      expect(annotation.knowledgeApplied.anyKnowledgeUsed).toBe(false);
+      expect(annotation.outcome.domainHadPlaybooks).toBe(false);
+    });
+
+    it('should persist annotations', async () => {
+      const trajectory = makeTrajectory(true, 2);
+      await tracker.annotate(trajectory, [], [], []);
+
+      const count = await tracker.count();
+      expect(count).toBe(1);
+
+      const retrieved = await tracker.getByTrajectoryId(trajectory.id);
+      expect(retrieved).toBeDefined();
+      expect(retrieved!.trajectoryId).toBe(trajectory.id);
+    });
+
+    it('should set annotation id based on trajectory id', async () => {
+      const trajectory = makeTrajectory(true, 2);
+      const annotation = await tracker.annotate(trajectory, [], [], []);
+      expect(annotation.id).toBe(`ann-${trajectory.id}`);
+    });
+  });
+
+  describe('getAll and getByTrajectoryId', () => {
+    it('should return all annotations', async () => {
+      await tracker.annotate(makeTrajectory(true, 2), [], [], []);
+      await tracker.annotate(makeTrajectory(false, 3), [], [], []);
+      await tracker.annotate(makeTrajectory(true, 4), [], [], []);
+
+      const all = await tracker.getAll();
+      expect(all).toHaveLength(3);
+    });
+
+    it('should return undefined for non-existent trajectory id', async () => {
+      const result = await tracker.getByTrajectoryId('nonexistent-id');
+      expect(result).toBeUndefined();
+    });
+  });
+
+  describe('addReflection', () => {
+    it('should add a reflection to an existing annotation', async () => {
+      const trajectory = makeTrajectory(true, 2);
+      const annotation = await tracker.annotate(trajectory, [], [], []);
+
+      await tracker.addReflection(annotation.id, {
+        knowledgeRelevance: 'helpful',
+        notes: 'The playbook guidance was spot on',
+      });
+
+      const updated = await tracker.getByTrajectoryId(trajectory.id);
+      expect(updated!.reflection).toBeDefined();
+      expect(updated!.reflection!.knowledgeRelevance).toBe('helpful');
+      expect(updated!.reflection!.notes).toBe('The playbook guidance was spot on');
+    });
+
+    it('should no-op when annotation id does not exist', async () => {
+      await tracker.addReflection('nonexistent-ann', {
+        knowledgeRelevance: 'helpful',
+      });
+      expect(await tracker.count()).toBe(0);
+    });
+
+    it('should overwrite previous reflection', async () => {
+      const trajectory = makeTrajectory(true, 2);
+      const annotation = await tracker.annotate(trajectory, [], [], []);
+
+      await tracker.addReflection(annotation.id, {
+        knowledgeRelevance: 'helpful',
+      });
+      await tracker.addReflection(annotation.id, {
+        knowledgeRelevance: 'misleading',
+        notes: 'Actually it was wrong',
+      });
+
+      const updated = await tracker.getByTrajectoryId(trajectory.id);
+      expect(updated!.reflection!.knowledgeRelevance).toBe('misleading');
+      expect(updated!.reflection!.notes).toBe('Actually it was wrong');
+    });
+  });
+
+  describe('computeMetrics', () => {
+    it('should compute guided vs unguided success rates', async () => {
+      const pb = makePlaybook('guide-playbook');
+      const match = makePlaybookMatch(pb);
+
+      // 2 guided successes
+      await tracker.annotate(makeTrajectory(true, 3), [match], [], [pb.id]);
+      await tracker.annotate(makeTrajectory(true, 4), [match], [], [pb.id]);
+      // 1 guided failure
+      await tracker.annotate(makeTrajectory(false, 5), [match], [], []);
+
+      // 1 unguided success
+      await tracker.annotate(makeTrajectory(true, 6), [], [], []);
+      // 2 unguided failures
+      await tracker.annotate(makeTrajectory(false, 8), [], [], []);
+      await tracker.annotate(makeTrajectory(false, 7), [], [], []);
+
+      const metrics = await tracker.computeMetrics();
+
+      expect(metrics.totalTasks).toBe(6);
+      expect(metrics.guidedTasks).toBe(3);
+      expect(metrics.unguidedTasks).toBe(3);
+      expect(metrics.guidedSuccessRate).toBeCloseTo(2 / 3, 2);
+      expect(metrics.unguidedSuccessRate).toBeCloseTo(1 / 3, 2);
+      expect(metrics.successRateDelta).toBeCloseTo(1 / 3, 2);
+    });
+
+    it('should compute average step counts', async () => {
+      const pb = makePlaybook('step-playbook');
+      const match = makePlaybookMatch(pb);
+
+      // Guided: 3, 4 steps -> avg 3.5
+      await tracker.annotate(makeTrajectory(true, 3), [match], [], []);
+      await tracker.annotate(makeTrajectory(true, 4), [match], [], []);
+
+      // Unguided: 6, 8 steps -> avg 7
+      await tracker.annotate(makeTrajectory(true, 6), [], [], []);
+      await tracker.annotate(makeTrajectory(true, 8), [], [], []);
+
+      const metrics = await tracker.computeMetrics();
+
+      expect(metrics.guidedAvgSteps).toBeCloseTo(3.5, 1);
+      expect(metrics.unguidedAvgSteps).toBeCloseTo(7, 1);
+      expect(metrics.stepCountDelta).toBeCloseTo(-3.5, 1); // Negative = guidance reduces steps
+    });
+
+    it('should compute knowledge application rate', async () => {
+      const pb = makePlaybook('applied-playbook');
+      const match = makePlaybookMatch(pb);
+
+      // 2 guided, knowledge applied
+      await tracker.annotate(makeTrajectory(true, 3), [match], [], [pb.id]);
+      await tracker.annotate(makeTrajectory(true, 3), [match], [], [pb.id]);
+      // 1 guided, knowledge NOT applied
+      await tracker.annotate(makeTrajectory(false, 5), [match], [], []);
+
+      const metrics = await tracker.computeMetrics();
+
+      expect(metrics.knowledgeApplicationRate).toBeCloseTo(2 / 3, 2);
+      expect(metrics.appliedKnowledgeSuccessRate).toBe(1); // Both applied succeeded
+    });
+
+    it('should compute per-playbook effectiveness', async () => {
+      const pb1 = makePlaybook('good-playbook');
+      const pb2 = makePlaybook('bad-playbook');
+      const match1 = makePlaybookMatch(pb1);
+      const match2 = makePlaybookMatch(pb2);
+
+      // Good playbook: surfaced 3x, applied 3x, succeeded 2x
+      await tracker.annotate(makeTrajectory(true, 3), [match1], [], [pb1.id]);
+      await tracker.annotate(makeTrajectory(true, 4), [match1], [], [pb1.id]);
+      await tracker.annotate(makeTrajectory(false, 5), [match1], [], [pb1.id]);
+
+      // Bad playbook: surfaced 2x, applied 2x, succeeded 0x
+      await tracker.annotate(makeTrajectory(false, 6), [match2], [], [pb2.id]);
+      await tracker.annotate(makeTrajectory(false, 7), [match2], [], [pb2.id]);
+
+      const metrics = await tracker.computeMetrics();
+
+      expect(metrics.playbookEffectiveness).toHaveLength(2);
+
+      const goodPb = metrics.playbookEffectiveness.find(
+        (e) => e.playbookName === 'good-playbook'
+      );
+      expect(goodPb).toBeDefined();
+      expect(goodPb!.surfacedCount).toBe(3);
+      expect(goodPb!.appliedCount).toBe(3);
+      expect(goodPb!.appliedSuccessRate).toBeCloseTo(2 / 3, 2);
+
+      const badPb = metrics.playbookEffectiveness.find(
+        (e) => e.playbookName === 'bad-playbook'
+      );
+      expect(badPb).toBeDefined();
+      expect(badPb!.surfacedCount).toBe(2);
+      expect(badPb!.appliedSuccessRate).toBe(0);
+    });
+
+    it('should compute error recovery delta between guided and unguided', async () => {
+      const pb = makePlaybook('error-helper');
+      const match = makePlaybookMatch(pb);
+
+      // Guided: 1 error recovery each (avg 1)
+      await tracker.annotate(makeTrajectory(true, 4, true), [match], [], [pb.id]);
+      await tracker.annotate(makeTrajectory(true, 4, true), [match], [], [pb.id]);
+
+      // Unguided: no errors (avg 0)
+      await tracker.annotate(makeTrajectory(true, 3, false), [], [], []);
+      await tracker.annotate(makeTrajectory(true, 3, false), [], [], []);
+
+      const metrics = await tracker.computeMetrics();
+
+      expect(metrics.guidedAvgErrorRecoveries).toBe(1);
+      expect(metrics.unguidedAvgErrorRecoveries).toBe(0);
+    });
+
+    it('should compute per-playbook average step count', async () => {
+      const pb = makePlaybook('step-counter');
+      const match = makePlaybookMatch(pb);
+
+      // Applied with step counts 3, 5, 7 -> avg 5
+      await tracker.annotate(makeTrajectory(true, 3), [match], [], [pb.id]);
+      await tracker.annotate(makeTrajectory(true, 5), [match], [], [pb.id]);
+      await tracker.annotate(makeTrajectory(true, 7), [match], [], [pb.id]);
+
+      const metrics = await tracker.computeMetrics();
+      const pbMetrics = metrics.playbookEffectiveness.find(
+        (e) => e.playbookName === 'step-counter'
+      );
+
+      expect(pbMetrics).toBeDefined();
+      expect(pbMetrics!.appliedAvgSteps).toBe(5);
+    });
+
+    it('should track playbooks surfaced but not applied', async () => {
+      const pb = makePlaybook('ignored-playbook');
+      const match = makePlaybookMatch(pb);
+
+      // Surfaced 3x, applied 0x
+      await tracker.annotate(makeTrajectory(true, 3), [match], [], []);
+      await tracker.annotate(makeTrajectory(false, 4), [match], [], []);
+      await tracker.annotate(makeTrajectory(true, 5), [match], [], []);
+
+      const metrics = await tracker.computeMetrics();
+      const pbMetrics = metrics.playbookEffectiveness.find(
+        (e) => e.playbookName === 'ignored-playbook'
+      );
+
+      expect(pbMetrics).toBeDefined();
+      expect(pbMetrics!.surfacedCount).toBe(3);
+      expect(pbMetrics!.appliedCount).toBe(0);
+      expect(pbMetrics!.appliedSuccessRate).toBe(0);
+      expect(pbMetrics!.appliedAvgSteps).toBe(0);
+    });
+
+    it('should sort playbook effectiveness by surfaced count descending', async () => {
+      const pb1 = makePlaybook('popular');
+      const pb2 = makePlaybook('rare');
+      const match1 = makePlaybookMatch(pb1);
+      const match2 = makePlaybookMatch(pb2);
+
+      // Popular: surfaced 5x
+      for (let i = 0; i < 5; i++) {
+        await tracker.annotate(makeTrajectory(true, 3), [match1], [], []);
+      }
+      // Rare: surfaced 1x
+      await tracker.annotate(makeTrajectory(true, 3), [match2], [], []);
+
+      const metrics = await tracker.computeMetrics();
+      expect(metrics.playbookEffectiveness[0].playbookName).toBe('popular');
+      expect(metrics.playbookEffectiveness[1].playbookName).toBe('rare');
+    });
+
+    it('should filter by time window with since', async () => {
+      const now = Date.now();
+
+      await tracker.annotate(makeTrajectory(true, 3), [], [], []);
+
+      const metrics = await tracker.computeMetrics({
+        since: new Date(now + 60000), // In the future — should exclude everything
+      });
+
+      expect(metrics.totalTasks).toBe(0);
+    });
+
+    it('should filter by time window with until', async () => {
+      const past = new Date(Date.now() - 60000);
+
+      await tracker.annotate(makeTrajectory(true, 3), [], [], []);
+
+      const metrics = await tracker.computeMetrics({
+        until: past, // In the past — should exclude everything created now
+      });
+
+      expect(metrics.totalTasks).toBe(0);
+    });
+
+    it('should set window start and end in metrics', async () => {
+      const since = new Date('2025-01-01');
+      const until = new Date('2025-12-31');
+
+      const metrics = await tracker.computeMetrics({ since, until });
+
+      expect(metrics.windowStart).toEqual(since);
+      expect(metrics.windowEnd).toEqual(until);
+    });
+
+    it('should handle all-guided scenario', async () => {
+      const pb = makePlaybook('always-on');
+      const match = makePlaybookMatch(pb);
+
+      await tracker.annotate(makeTrajectory(true, 3), [match], [], [pb.id]);
+      await tracker.annotate(makeTrajectory(true, 4), [match], [], [pb.id]);
+
+      const metrics = await tracker.computeMetrics();
+
+      expect(metrics.guidedTasks).toBe(2);
+      expect(metrics.unguidedTasks).toBe(0);
+      expect(metrics.guidedSuccessRate).toBe(1);
+      expect(metrics.unguidedSuccessRate).toBe(0); // No data → 0
+    });
+
+    it('should handle all-unguided scenario', async () => {
+      await tracker.annotate(makeTrajectory(true, 3), [], [], []);
+      await tracker.annotate(makeTrajectory(false, 4), [], [], []);
+
+      const metrics = await tracker.computeMetrics();
+
+      expect(metrics.guidedTasks).toBe(0);
+      expect(metrics.unguidedTasks).toBe(2);
+      expect(metrics.guidedSuccessRate).toBe(0);
+      expect(metrics.unguidedSuccessRate).toBe(0.5);
+    });
+
+    it('should handle empty tracker gracefully', async () => {
+      const metrics = await tracker.computeMetrics();
+
+      expect(metrics.totalTasks).toBe(0);
+      expect(metrics.guidedSuccessRate).toBe(0);
+      expect(metrics.unguidedSuccessRate).toBe(0);
+      expect(metrics.successRateDelta).toBe(0);
+      expect(metrics.guidedAvgSteps).toBe(0);
+      expect(metrics.unguidedAvgSteps).toBe(0);
+      expect(metrics.knowledgeApplicationRate).toBe(0);
+      expect(metrics.appliedKnowledgeSuccessRate).toBe(0);
+      expect(metrics.playbookEffectiveness).toHaveLength(0);
+    });
+  });
+
+  describe('reflection-based relevance tracking', () => {
+    it('should compute relevance rate from reflections', async () => {
+      const pb = makePlaybook('reflected-playbook');
+      const match = makePlaybookMatch(pb);
+
+      const ann1 = await tracker.annotate(makeTrajectory(true, 3), [match], [], [pb.id]);
+      await tracker.addReflection(ann1.id, { knowledgeRelevance: 'helpful' });
+
+      const ann2 = await tracker.annotate(makeTrajectory(true, 4), [match], [], [pb.id]);
+      await tracker.addReflection(ann2.id, { knowledgeRelevance: 'irrelevant' });
+
+      const ann3 = await tracker.annotate(makeTrajectory(true, 5), [match], [], [pb.id]);
+      await tracker.addReflection(ann3.id, { knowledgeRelevance: 'helpful' });
+
+      const metrics = await tracker.computeMetrics();
+      const pbMetrics = metrics.playbookEffectiveness.find(
+        (e) => e.playbookName === 'reflected-playbook'
+      );
+
+      expect(pbMetrics).toBeDefined();
+      // 2 out of 3 reflections rated as helpful
+      expect(pbMetrics!.relevanceRate).toBeCloseTo(2 / 3, 2);
+    });
+
+    it('should not count misleading or not_assessed as relevant', async () => {
+      const pb = makePlaybook('mixed-feedback');
+      const match = makePlaybookMatch(pb);
+
+      const ann1 = await tracker.annotate(makeTrajectory(true, 3), [match], [], [pb.id]);
+      await tracker.addReflection(ann1.id, { knowledgeRelevance: 'misleading' });
+
+      const ann2 = await tracker.annotate(makeTrajectory(true, 3), [match], [], [pb.id]);
+      await tracker.addReflection(ann2.id, { knowledgeRelevance: 'not_assessed' });
+
+      const ann3 = await tracker.annotate(makeTrajectory(true, 3), [match], [], [pb.id]);
+      await tracker.addReflection(ann3.id, { knowledgeRelevance: 'helpful' });
+
+      const metrics = await tracker.computeMetrics();
+      const pbMetrics = metrics.playbookEffectiveness.find(
+        (e) => e.playbookName === 'mixed-feedback'
+      );
+
+      expect(pbMetrics).toBeDefined();
+      // Only 1 out of 3 is 'helpful'
+      expect(pbMetrics!.relevanceRate).toBeCloseTo(1 / 3, 2);
+    });
+
+    it('should report 0 relevance when no reflections exist', async () => {
+      const pb = makePlaybook('no-reflections');
+      const match = makePlaybookMatch(pb);
+
+      await tracker.annotate(makeTrajectory(true, 3), [match], [], [pb.id]);
+      await tracker.annotate(makeTrajectory(true, 4), [match], [], [pb.id]);
+
+      const metrics = await tracker.computeMetrics();
+      const pbMetrics = metrics.playbookEffectiveness.find(
+        (e) => e.playbookName === 'no-reflections'
+      );
+
+      expect(pbMetrics).toBeDefined();
+      expect(pbMetrics!.relevanceRate).toBe(0); // No assessments = 0 rate
+    });
+  });
+});