cognitive-core 0.2.4 → 0.3.1

package/tests/learning/effectiveness.test.ts

@@ -1,542 +0,0 @@
-import { describe, it, expect, beforeEach, afterEach } from 'vitest';
-import {
-  LearningEffectivenessTracker,
-  createEffectivenessTracker,
-} from '../../src/learning/effectiveness.js';
-import { createTrajectory, createStep, createTask, successOutcome, failureOutcome } from '../../src/types/index.js';
-import type { PlaybookMatch } from '../../src/memory/playbook.js';
-import type { Playbook } from '../../src/types/playbook.js';
-import { createPlaybook } from '../../src/types/playbook.js';
-import { mkdtemp, rm } from 'node:fs/promises';
-import { join } from 'node:path';
-import { tmpdir } from 'node:os';
-
-describe('LearningEffectivenessTracker', () => {
-  let tempDir: string;
-  let tracker: LearningEffectivenessTracker;
-
-  beforeEach(async () => {
-    tempDir = await mkdtemp(join(tmpdir(), 'atlas-eff-test-'));
-    tracker = createEffectivenessTracker(tempDir);
-    await tracker.init();
-  });
-
-  afterEach(async () => {
-    await tracker.close();
-    await rm(tempDir, { recursive: true, force: true });
-  });
-
-  function makeTrajectory(success: boolean, stepCount: number, hasErrors = false) {
-    const steps = [];
-    for (let i = 0; i < stepCount; i++) {
-      steps.push(createStep({
-        thought: `Step ${i + 1}`,
-        action: `action-${i + 1}`,
-        observation: hasErrors && i === 1 ? 'error: something failed' : 'ok',
-      }));
-    }
-    return createTrajectory({
-      task: createTask({ domain: 'test', description: `Test task ${Date.now()}` }),
-      steps,
-      outcome: success ? successOutcome({ result: 'done' }) : failureOutcome('failed'),
-      agentId: 'test-agent',
-    });
-  }
-
-  function makePlaybook(name: string): Playbook {
-    return createPlaybook({
-      name,
-      applicability: {
-        situations: ['Test'],
-        triggers: [],
-        antiPatterns: [],
-        domains: ['test'],
-      },
-      guidance: { strategy: 'Test', tactics: [] },
-    });
-  }
-
-  function makePlaybookMatch(playbook: Playbook): PlaybookMatch {
-    return { playbook, score: 0.8, matchType: 'situation' };
-  }
-
-  describe('annotate', () => {
-    it('should create an annotation for a trajectory', async () => {
-      const trajectory = makeTrajectory(true, 3);
-      const pb = makePlaybook('test-playbook');
-      const matches = [makePlaybookMatch(pb)];
-
-      const annotation = await tracker.annotate(
-        trajectory,
-        matches,
-        ['exp-1'],
-        [pb.id],
-      );
-
-      expect(annotation.trajectoryId).toBe(trajectory.id);
-      expect(annotation.knowledgeSurfaced.playbookIds).toEqual([pb.id]);
-      expect(annotation.knowledgeSurfaced.playbookNames).toEqual(['test-playbook']);
-      expect(annotation.knowledgeSurfaced.experienceIds).toEqual(['exp-1']);
-      expect(annotation.knowledgeSurfaced.totalItems).toBe(2);
-      expect(annotation.knowledgeApplied.playbookIdsUsed).toEqual([pb.id]);
-      expect(annotation.knowledgeApplied.anyKnowledgeUsed).toBe(true);
-      expect(annotation.outcome.success).toBe(true);
-      expect(annotation.outcome.stepCount).toBe(3);
-    });
-
-    it('should count error recoveries', async () => {
-      const trajectory = makeTrajectory(true, 4, true);
-      const annotation = await tracker.annotate(trajectory, [], [], []);
-
-      // Step 1 (ok) -> Step 2 (error) -> Step 3 (ok) = 1 recovery
-      expect(annotation.outcome.errorRecoveries).toBe(1);
-    });
-
-    it('should count multiple error recoveries', async () => {
-      const steps = [
-        createStep({ thought: 'S1', action: 'a1', observation: 'ok' }),
-        createStep({ thought: 'S2', action: 'a2', observation: 'error: first fail' }),
-        createStep({ thought: 'S3', action: 'a3', observation: 'recovered' }),
-        createStep({ thought: 'S4', action: 'a4', observation: 'error: second fail' }),
-        createStep({ thought: 'S5', action: 'a5', observation: 'recovered again' }),
-      ];
-      const trajectory = createTrajectory({
-        task: createTask({ domain: 'test', description: 'multi-error' }),
-        steps,
-        outcome: successOutcome({ result: 'done' }),
-        agentId: 'test-agent',
-      });
-
-      const annotation = await tracker.annotate(trajectory, [], [], []);
-      expect(annotation.outcome.errorRecoveries).toBe(2);
-    });
-
-    it('should report zero error recoveries for clean trajectories', async () => {
-      const trajectory = makeTrajectory(true, 5, false);
-      const annotation = await tracker.annotate(trajectory, [], [], []);
-      expect(annotation.outcome.errorRecoveries).toBe(0);
-    });
-
-    it('should handle multiple playbooks surfaced in one task', async () => {
-      const pb1 = makePlaybook('pb-alpha');
-      const pb2 = makePlaybook('pb-beta');
-      const pb3 = makePlaybook('pb-gamma');
-      const matches = [makePlaybookMatch(pb1), makePlaybookMatch(pb2), makePlaybookMatch(pb3)];
-
-      const annotation = await tracker.annotate(
-        makeTrajectory(true, 3),
-        matches,
-        [],
-        [pb1.id, pb3.id],
-      );
-
-      expect(annotation.knowledgeSurfaced.playbookIds).toHaveLength(3);
-      expect(annotation.knowledgeSurfaced.playbookNames).toEqual(['pb-alpha', 'pb-beta', 'pb-gamma']);
-      expect(annotation.knowledgeSurfaced.totalItems).toBe(3);
-      expect(annotation.knowledgeApplied.playbookIdsUsed).toHaveLength(2);
-      expect(annotation.knowledgeApplied.anyKnowledgeUsed).toBe(true);
-    });
-
-    it('should handle experience-only guidance (no playbooks)', async () => {
-      const annotation = await tracker.annotate(
-        makeTrajectory(true, 3),
-        [],
-        ['exp-1', 'exp-2', 'exp-3'],
-        [],
-      );
-
-      expect(annotation.knowledgeSurfaced.playbookIds).toHaveLength(0);
-      expect(annotation.knowledgeSurfaced.experienceIds).toHaveLength(3);
-      expect(annotation.knowledgeSurfaced.totalItems).toBe(3);
-      expect(annotation.knowledgeApplied.anyKnowledgeUsed).toBe(false);
-      expect(annotation.outcome.domainHadPlaybooks).toBe(false);
-    });
-
-    it('should persist annotations', async () => {
-      const trajectory = makeTrajectory(true, 2);
-      await tracker.annotate(trajectory, [], [], []);
-
-      const count = await tracker.count();
-      expect(count).toBe(1);
-
-      const retrieved = await tracker.getByTrajectoryId(trajectory.id);
-      expect(retrieved).toBeDefined();
-      expect(retrieved!.trajectoryId).toBe(trajectory.id);
-    });
-
-    it('should set annotation id based on trajectory id', async () => {
-      const trajectory = makeTrajectory(true, 2);
-      const annotation = await tracker.annotate(trajectory, [], [], []);
-      expect(annotation.id).toBe(`ann-${trajectory.id}`);
-    });
-  });
-
-  describe('getAll and getByTrajectoryId', () => {
-    it('should return all annotations', async () => {
-      await tracker.annotate(makeTrajectory(true, 2), [], [], []);
-      await tracker.annotate(makeTrajectory(false, 3), [], [], []);
-      await tracker.annotate(makeTrajectory(true, 4), [], [], []);
-
-      const all = await tracker.getAll();
-      expect(all).toHaveLength(3);
-    });
-
-    it('should return undefined for non-existent trajectory id', async () => {
-      const result = await tracker.getByTrajectoryId('nonexistent-id');
-      expect(result).toBeUndefined();
-    });
-  });
-
-  describe('addReflection', () => {
-    it('should add a reflection to an existing annotation', async () => {
-      const trajectory = makeTrajectory(true, 2);
-      const annotation = await tracker.annotate(trajectory, [], [], []);
-
-      await tracker.addReflection(annotation.id, {
-        knowledgeRelevance: 'helpful',
-        notes: 'The playbook guidance was spot on',
-      });
-
-      const updated = await tracker.getByTrajectoryId(trajectory.id);
-      expect(updated!.reflection).toBeDefined();
-      expect(updated!.reflection!.knowledgeRelevance).toBe('helpful');
-      expect(updated!.reflection!.notes).toBe('The playbook guidance was spot on');
-    });
-
-    it('should no-op when annotation id does not exist', async () => {
-      await tracker.addReflection('nonexistent-ann', {
-        knowledgeRelevance: 'helpful',
-      });
-      expect(await tracker.count()).toBe(0);
-    });
-
-    it('should overwrite previous reflection', async () => {
-      const trajectory = makeTrajectory(true, 2);
-      const annotation = await tracker.annotate(trajectory, [], [], []);
-
-      await tracker.addReflection(annotation.id, {
-        knowledgeRelevance: 'helpful',
-      });
-      await tracker.addReflection(annotation.id, {
-        knowledgeRelevance: 'misleading',
-        notes: 'Actually it was wrong',
-      });
-
-      const updated = await tracker.getByTrajectoryId(trajectory.id);
-      expect(updated!.reflection!.knowledgeRelevance).toBe('misleading');
-      expect(updated!.reflection!.notes).toBe('Actually it was wrong');
-    });
-  });
-
-  describe('computeMetrics', () => {
-    it('should compute guided vs unguided success rates', async () => {
-      const pb = makePlaybook('guide-playbook');
-      const match = makePlaybookMatch(pb);
-
-      // 2 guided successes
-      await tracker.annotate(makeTrajectory(true, 3), [match], [], [pb.id]);
-      await tracker.annotate(makeTrajectory(true, 4), [match], [], [pb.id]);
-      // 1 guided failure
-      await tracker.annotate(makeTrajectory(false, 5), [match], [], []);
-
-      // 1 unguided success
-      await tracker.annotate(makeTrajectory(true, 6), [], [], []);
-      // 2 unguided failures
-      await tracker.annotate(makeTrajectory(false, 8), [], [], []);
-      await tracker.annotate(makeTrajectory(false, 7), [], [], []);
-
-      const metrics = await tracker.computeMetrics();
-
-      expect(metrics.totalTasks).toBe(6);
-      expect(metrics.guidedTasks).toBe(3);
-      expect(metrics.unguidedTasks).toBe(3);
-      expect(metrics.guidedSuccessRate).toBeCloseTo(2 / 3, 2);
-      expect(metrics.unguidedSuccessRate).toBeCloseTo(1 / 3, 2);
-      expect(metrics.successRateDelta).toBeCloseTo(1 / 3, 2);
-    });
-
-    it('should compute average step counts', async () => {
-      const pb = makePlaybook('step-playbook');
-      const match = makePlaybookMatch(pb);
-
-      // Guided: 3, 4 steps -> avg 3.5
-      await tracker.annotate(makeTrajectory(true, 3), [match], [], []);
-      await tracker.annotate(makeTrajectory(true, 4), [match], [], []);
-
-      // Unguided: 6, 8 steps -> avg 7
-      await tracker.annotate(makeTrajectory(true, 6), [], [], []);
-      await tracker.annotate(makeTrajectory(true, 8), [], [], []);
-
-      const metrics = await tracker.computeMetrics();
-
-      expect(metrics.guidedAvgSteps).toBeCloseTo(3.5, 1);
-      expect(metrics.unguidedAvgSteps).toBeCloseTo(7, 1);
-      expect(metrics.stepCountDelta).toBeCloseTo(-3.5, 1); // Negative = guidance reduces steps
-    });
-
-    it('should compute knowledge application rate', async () => {
-      const pb = makePlaybook('applied-playbook');
-      const match = makePlaybookMatch(pb);
-
-      // 2 guided, knowledge applied
-      await tracker.annotate(makeTrajectory(true, 3), [match], [], [pb.id]);
-      await tracker.annotate(makeTrajectory(true, 3), [match], [], [pb.id]);
-      // 1 guided, knowledge NOT applied
-      await tracker.annotate(makeTrajectory(false, 5), [match], [], []);
-
-      const metrics = await tracker.computeMetrics();
-
-      expect(metrics.knowledgeApplicationRate).toBeCloseTo(2 / 3, 2);
-      expect(metrics.appliedKnowledgeSuccessRate).toBe(1); // Both applied succeeded
-    });
-
-    it('should compute per-playbook effectiveness', async () => {
-      const pb1 = makePlaybook('good-playbook');
-      const pb2 = makePlaybook('bad-playbook');
-      const match1 = makePlaybookMatch(pb1);
-      const match2 = makePlaybookMatch(pb2);
-
-      // Good playbook: surfaced 3x, applied 3x, succeeded 2x
-      await tracker.annotate(makeTrajectory(true, 3), [match1], [], [pb1.id]);
-      await tracker.annotate(makeTrajectory(true, 4), [match1], [], [pb1.id]);
-      await tracker.annotate(makeTrajectory(false, 5), [match1], [], [pb1.id]);
-
-      // Bad playbook: surfaced 2x, applied 2x, succeeded 0x
-      await tracker.annotate(makeTrajectory(false, 6), [match2], [], [pb2.id]);
-      await tracker.annotate(makeTrajectory(false, 7), [match2], [], [pb2.id]);
-
-      const metrics = await tracker.computeMetrics();
-
-      expect(metrics.playbookEffectiveness).toHaveLength(2);
-
-      const goodPb = metrics.playbookEffectiveness.find(
-        (e) => e.playbookName === 'good-playbook'
-      );
-      expect(goodPb).toBeDefined();
-      expect(goodPb!.surfacedCount).toBe(3);
-      expect(goodPb!.appliedCount).toBe(3);
-      expect(goodPb!.appliedSuccessRate).toBeCloseTo(2 / 3, 2);
-
-      const badPb = metrics.playbookEffectiveness.find(
-        (e) => e.playbookName === 'bad-playbook'
-      );
-      expect(badPb).toBeDefined();
-      expect(badPb!.surfacedCount).toBe(2);
-      expect(badPb!.appliedSuccessRate).toBe(0);
-    });
-
-    it('should compute error recovery delta between guided and unguided', async () => {
-      const pb = makePlaybook('error-helper');
-      const match = makePlaybookMatch(pb);
-
-      // Guided: 1 error recovery each (avg 1)
-      await tracker.annotate(makeTrajectory(true, 4, true), [match], [], [pb.id]);
-      await tracker.annotate(makeTrajectory(true, 4, true), [match], [], [pb.id]);
-
-      // Unguided: no errors (avg 0)
-      await tracker.annotate(makeTrajectory(true, 3, false), [], [], []);
-      await tracker.annotate(makeTrajectory(true, 3, false), [], [], []);
-
-      const metrics = await tracker.computeMetrics();
-
-      expect(metrics.guidedAvgErrorRecoveries).toBe(1);
-      expect(metrics.unguidedAvgErrorRecoveries).toBe(0);
-    });
-
-    it('should compute per-playbook average step count', async () => {
-      const pb = makePlaybook('step-counter');
-      const match = makePlaybookMatch(pb);
-
-      // Applied with step counts 3, 5, 7 -> avg 5
-      await tracker.annotate(makeTrajectory(true, 3), [match], [], [pb.id]);
-      await tracker.annotate(makeTrajectory(true, 5), [match], [], [pb.id]);
-      await tracker.annotate(makeTrajectory(true, 7), [match], [], [pb.id]);
-
-      const metrics = await tracker.computeMetrics();
-      const pbMetrics = metrics.playbookEffectiveness.find(
-        (e) => e.playbookName === 'step-counter'
-      );
-
-      expect(pbMetrics).toBeDefined();
-      expect(pbMetrics!.appliedAvgSteps).toBe(5);
-    });
-
-    it('should track playbooks surfaced but not applied', async () => {
-      const pb = makePlaybook('ignored-playbook');
-      const match = makePlaybookMatch(pb);
-
-      // Surfaced 3x, applied 0x
-      await tracker.annotate(makeTrajectory(true, 3), [match], [], []);
-      await tracker.annotate(makeTrajectory(false, 4), [match], [], []);
-      await tracker.annotate(makeTrajectory(true, 5), [match], [], []);
-
-      const metrics = await tracker.computeMetrics();
-      const pbMetrics = metrics.playbookEffectiveness.find(
-        (e) => e.playbookName === 'ignored-playbook'
-      );
-
-      expect(pbMetrics).toBeDefined();
-      expect(pbMetrics!.surfacedCount).toBe(3);
-      expect(pbMetrics!.appliedCount).toBe(0);
-      expect(pbMetrics!.appliedSuccessRate).toBe(0);
-      expect(pbMetrics!.appliedAvgSteps).toBe(0);
-    });
-
-    it('should sort playbook effectiveness by surfaced count descending', async () => {
-      const pb1 = makePlaybook('popular');
-      const pb2 = makePlaybook('rare');
-      const match1 = makePlaybookMatch(pb1);
-      const match2 = makePlaybookMatch(pb2);
-
-      // Popular: surfaced 5x
-      for (let i = 0; i < 5; i++) {
-        await tracker.annotate(makeTrajectory(true, 3), [match1], [], []);
-      }
-      // Rare: surfaced 1x
-      await tracker.annotate(makeTrajectory(true, 3), [match2], [], []);
-
-      const metrics = await tracker.computeMetrics();
-      expect(metrics.playbookEffectiveness[0].playbookName).toBe('popular');
-      expect(metrics.playbookEffectiveness[1].playbookName).toBe('rare');
-    });
-
-    it('should filter by time window with since', async () => {
-      const now = Date.now();
-
-      await tracker.annotate(makeTrajectory(true, 3), [], [], []);
-
-      const metrics = await tracker.computeMetrics({
-        since: new Date(now + 60000), // In the future — should exclude everything
-      });
-
-      expect(metrics.totalTasks).toBe(0);
-    });
-
-    it('should filter by time window with until', async () => {
-      const past = new Date(Date.now() - 60000);
-
-      await tracker.annotate(makeTrajectory(true, 3), [], [], []);
-
-      const metrics = await tracker.computeMetrics({
-        until: past, // In the past — should exclude everything created now
-      });
-
-      expect(metrics.totalTasks).toBe(0);
-    });
-
-    it('should set window start and end in metrics', async () => {
-      const since = new Date('2025-01-01');
-      const until = new Date('2025-12-31');
-
-      const metrics = await tracker.computeMetrics({ since, until });
-
-      expect(metrics.windowStart).toEqual(since);
-      expect(metrics.windowEnd).toEqual(until);
-    });
-
-    it('should handle all-guided scenario', async () => {
-      const pb = makePlaybook('always-on');
-      const match = makePlaybookMatch(pb);
-
-      await tracker.annotate(makeTrajectory(true, 3), [match], [], [pb.id]);
-      await tracker.annotate(makeTrajectory(true, 4), [match], [], [pb.id]);
-
-      const metrics = await tracker.computeMetrics();
-
-      expect(metrics.guidedTasks).toBe(2);
-      expect(metrics.unguidedTasks).toBe(0);
-      expect(metrics.guidedSuccessRate).toBe(1);
-      expect(metrics.unguidedSuccessRate).toBe(0); // No data → 0
-    });
-
-    it('should handle all-unguided scenario', async () => {
-      await tracker.annotate(makeTrajectory(true, 3), [], [], []);
-      await tracker.annotate(makeTrajectory(false, 4), [], [], []);
-
-      const metrics = await tracker.computeMetrics();
-
-      expect(metrics.guidedTasks).toBe(0);
-      expect(metrics.unguidedTasks).toBe(2);
-      expect(metrics.guidedSuccessRate).toBe(0);
-      expect(metrics.unguidedSuccessRate).toBe(0.5);
-    });
-
-    it('should handle empty tracker gracefully', async () => {
-      const metrics = await tracker.computeMetrics();
-
-      expect(metrics.totalTasks).toBe(0);
-      expect(metrics.guidedSuccessRate).toBe(0);
-      expect(metrics.unguidedSuccessRate).toBe(0);
-      expect(metrics.successRateDelta).toBe(0);
-      expect(metrics.guidedAvgSteps).toBe(0);
-      expect(metrics.unguidedAvgSteps).toBe(0);
-      expect(metrics.knowledgeApplicationRate).toBe(0);
-      expect(metrics.appliedKnowledgeSuccessRate).toBe(0);
-      expect(metrics.playbookEffectiveness).toHaveLength(0);
-    });
-  });
-
-  describe('reflection-based relevance tracking', () => {
-    it('should compute relevance rate from reflections', async () => {
-      const pb = makePlaybook('reflected-playbook');
-      const match = makePlaybookMatch(pb);
-
-      const ann1 = await tracker.annotate(makeTrajectory(true, 3), [match], [], [pb.id]);
-      await tracker.addReflection(ann1.id, { knowledgeRelevance: 'helpful' });
-
-      const ann2 = await tracker.annotate(makeTrajectory(true, 4), [match], [], [pb.id]);
-      await tracker.addReflection(ann2.id, { knowledgeRelevance: 'irrelevant' });
-
-      const ann3 = await tracker.annotate(makeTrajectory(true, 5), [match], [], [pb.id]);
-      await tracker.addReflection(ann3.id, { knowledgeRelevance: 'helpful' });
-
-      const metrics = await tracker.computeMetrics();
-      const pbMetrics = metrics.playbookEffectiveness.find(
-        (e) => e.playbookName === 'reflected-playbook'
-      );
-
-      expect(pbMetrics).toBeDefined();
-      // 2 out of 3 reflections rated as helpful
-      expect(pbMetrics!.relevanceRate).toBeCloseTo(2 / 3, 2);
-    });
-
-    it('should not count misleading or not_assessed as relevant', async () => {
-      const pb = makePlaybook('mixed-feedback');
-      const match = makePlaybookMatch(pb);
-
-      const ann1 = await tracker.annotate(makeTrajectory(true, 3), [match], [], [pb.id]);
-      await tracker.addReflection(ann1.id, { knowledgeRelevance: 'misleading' });
-
-      const ann2 = await tracker.annotate(makeTrajectory(true, 3), [match], [], [pb.id]);
-      await tracker.addReflection(ann2.id, { knowledgeRelevance: 'not_assessed' });
-
-      const ann3 = await tracker.annotate(makeTrajectory(true, 3), [match], [], [pb.id]);
-      await tracker.addReflection(ann3.id, { knowledgeRelevance: 'helpful' });
-
-      const metrics = await tracker.computeMetrics();
-      const pbMetrics = metrics.playbookEffectiveness.find(
-        (e) => e.playbookName === 'mixed-feedback'
-      );
-
-      expect(pbMetrics).toBeDefined();
-      // Only 1 out of 3 is 'helpful'
-      expect(pbMetrics!.relevanceRate).toBeCloseTo(1 / 3, 2);
-    });
-
-    it('should report 0 relevance when no reflections exist', async () => {
-      const pb = makePlaybook('no-reflections');
-      const match = makePlaybookMatch(pb);
-
-      await tracker.annotate(makeTrajectory(true, 3), [match], [], [pb.id]);
-      await tracker.annotate(makeTrajectory(true, 4), [match], [], [pb.id]);
-
-      const metrics = await tracker.computeMetrics();
-      const pbMetrics = metrics.playbookEffectiveness.find(
-        (e) => e.playbookName === 'no-reflections'
-      );
-
-      expect(pbMetrics).toBeDefined();
-      expect(pbMetrics!.relevanceRate).toBe(0); // No assessments = 0 rate
-    });
-  });
-});