cognitive-core 0.2.0 → 0.2.2

package/tests/workspace/efficacy-toolkit.test.ts

@@ -0,0 +1,404 @@
+import { describe, it, expect } from 'vitest';
+import {
+  computeNormalizedGain,
+  computeTemporalTrend,
+  computeDomainBreakdown,
+  buildEfficacyProfile,
+  buildPortfolioSnapshot,
+} from '../../src/workspace/efficacy-toolkit.js';
+import type { TaskAnnotation, PlaybookEffectivenessEntry } from '../../src/learning/effectiveness.js';
+import { createPlaybook } from '../../src/types/index.js';
+
+// ============================================================
+// Test Helpers
+// ============================================================
+
+function makeAnnotation(overrides: {
+  trajectoryId?: string;
+  playbookIds?: string[];
+  playbookNames?: string[];
+  appliedPlaybookIds?: string[];
+  success?: boolean;
+  stepCount?: number;
+  timestamp?: Date;
+}): TaskAnnotation {
+  return {
+    id: `ann-${Math.random().toString(36).slice(2, 6)}`,
+    trajectoryId: overrides.trajectoryId ?? `traj-${Math.random().toString(36).slice(2, 6)}`,
+    timestamp: overrides.timestamp ?? new Date(),
+    knowledgeSurfaced: {
+      playbookIds: overrides.playbookIds ?? [],
+      playbookNames: overrides.playbookNames ?? [],
+      experienceIds: [],
+      totalItems: (overrides.playbookIds ?? []).length,
+    },
+    knowledgeApplied: {
+      playbookIdsUsed: overrides.appliedPlaybookIds ?? [],
+      anyKnowledgeUsed: (overrides.appliedPlaybookIds ?? []).length > 0,
+    },
+    outcome: {
+      success: overrides.success ?? true,
+      stepCount: overrides.stepCount ?? 5,
+      errorRecoveries: 0,
+      domainHadPlaybooks: (overrides.playbookIds ?? []).length > 0,
+    },
+  };
+}
+
+function makePlaybook(overrides?: {
+  id?: string;
+  name?: string;
+  confidence?: number;
+  successCount?: number;
+  failureCount?: number;
+  domains?: string[];
+}) {
+  return createPlaybook({
+    id: overrides?.id ?? `pb-${Math.random().toString(36).slice(2, 6)}`,
+    name: overrides?.name ?? 'test-playbook',
+    confidence: overrides?.confidence ?? 0.7,
+    applicability: {
+      situations: ['debugging code'],
+      triggers: ['error'],
+      antiPatterns: [],
+      domains: overrides?.domains ?? ['typescript'],
+    },
+    guidance: {
+      strategy: 'Debug by reading error messages carefully',
+      tactics: ['Read the stack trace', 'Check the variable types'],
+    },
+    evolution: {
+      version: '1.0.0',
+      createdFrom: [],
+      failures: [],
+      refinements: [],
+      successCount: overrides?.successCount ?? 5,
+      failureCount: overrides?.failureCount ?? 2,
+    },
+  });
+}
+
+// ============================================================
+// computeNormalizedGain
+// ============================================================
+
+describe('computeNormalizedGain', () => {
+  it('should compute normalized gain correctly', () => {
+    // Raising from 50% to 75% = (0.75 - 0.5) / (1 - 0.5) = 0.5
+    expect(computeNormalizedGain(0.75, 0.5)).toBeCloseTo(0.5);
+  });
+
+  it('should handle perfect baseline (no room for improvement)', () => {
+    expect(computeNormalizedGain(1.0, 1.0)).toBe(0);
+  });
+
+  it('should handle zero baseline', () => {
+    // From 0% to 50% = (0.5 - 0) / (1 - 0) = 0.5
+    expect(computeNormalizedGain(0.5, 0)).toBeCloseTo(0.5);
+  });
+
+  it('should return negative for regression', () => {
+    // Guided is worse than unguided
+    expect(computeNormalizedGain(0.3, 0.5)).toBeLessThan(0);
+  });
+
+  it('should return 0 when both rates are zero', () => {
+    expect(computeNormalizedGain(0, 0)).toBe(0);
+  });
+
+  it('should return 1.0 for perfect guided on non-perfect baseline', () => {
+    expect(computeNormalizedGain(1.0, 0.5)).toBeCloseTo(1.0);
+  });
+});
+
+// ============================================================
+// computeTemporalTrend
+// ============================================================
+
+describe('computeTemporalTrend', () => {
+  it('should return stable for insufficient data', () => {
+    const trend = computeTemporalTrend([], 'pb-1');
+    expect(trend.direction).toBe('stable');
+    expect(trend.dataPoints).toBe(0);
+  });
+
+  it('should return stable for single data point', () => {
+    const ann = makeAnnotation({
+      appliedPlaybookIds: ['pb-1'],
+      success: true,
+    });
+    const trend = computeTemporalTrend([ann], 'pb-1');
+    expect(trend.direction).toBe('stable');
+    expect(trend.dataPoints).toBe(1);
+  });
+
+  it('should detect improving trend', () => {
+    const now = Date.now();
+    const annotations = Array.from({ length: 8 }, (_, i) =>
+      makeAnnotation({
+        appliedPlaybookIds: ['pb-1'],
+        // First 4 fail, last 4 succeed
+        success: i >= 4,
+        timestamp: new Date(now - (8 - i) * 86400000),
+      })
+    );
+
+    const trend = computeTemporalTrend(annotations, 'pb-1');
+    expect(trend.direction).toBe('improving');
+    expect(trend.slope).toBeGreaterThan(0);
+    expect(trend.dataPoints).toBe(8);
+  });
+
+  it('should detect decaying trend', () => {
+    const now = Date.now();
+    const annotations = Array.from({ length: 8 }, (_, i) =>
+      makeAnnotation({
+        appliedPlaybookIds: ['pb-1'],
+        // First 4 succeed, last 4 fail
+        success: i < 4,
+        timestamp: new Date(now - (8 - i) * 86400000),
+      })
+    );
+
+    const trend = computeTemporalTrend(annotations, 'pb-1');
+    expect(trend.direction).toBe('decaying');
+    expect(trend.slope).toBeLessThan(0);
+  });
+
+  it('should only include annotations for the target playbook', () => {
+    const ann1 = makeAnnotation({ appliedPlaybookIds: ['pb-1'], success: true });
+    const ann2 = makeAnnotation({ appliedPlaybookIds: ['pb-2'], success: false });
+
+    const trend = computeTemporalTrend([ann1, ann2], 'pb-1');
+    expect(trend.dataPoints).toBe(1);
+  });
+
+  it('should compute days since last use', () => {
+    const twoDaysAgo = new Date(Date.now() - 2 * 86400000);
+    const ann = makeAnnotation({
+      appliedPlaybookIds: ['pb-1'],
+      success: true,
+      timestamp: twoDaysAgo,
+    });
+
+    const trend = computeTemporalTrend([ann], 'pb-1');
+    expect(trend.daysSinceLastUse).toBeGreaterThan(1.5);
+    expect(trend.daysSinceLastUse).toBeLessThan(3);
+  });
+});
+
+// ============================================================
+// computeDomainBreakdown
+// ============================================================
+
+describe('computeDomainBreakdown', () => {
+  it('should compute per-domain success rates', () => {
+    const domainMap = new Map([
+      ['traj-1', 'typescript'],
+      ['traj-2', 'typescript'],
+      ['traj-3', 'python'],
+    ]);
+
+    const annotations = [
+      makeAnnotation({ trajectoryId: 'traj-1', appliedPlaybookIds: ['pb-1'], success: true }),
+      makeAnnotation({ trajectoryId: 'traj-2', appliedPlaybookIds: ['pb-1'], success: false }),
+      makeAnnotation({ trajectoryId: 'traj-3', appliedPlaybookIds: ['pb-1'], success: true }),
+    ];
+
+    const breakdown = computeDomainBreakdown(annotations, 'pb-1', domainMap);
+
+    expect(breakdown).toHaveLength(2);
+
+    const ts = breakdown.find(b => b.domain === 'typescript');
+    expect(ts).toBeDefined();
+    expect(ts!.appliedCount).toBe(2);
+    expect(ts!.successRate).toBeCloseTo(0.5);
+
+    const py = breakdown.find(b => b.domain === 'python');
+    expect(py).toBeDefined();
+    expect(py!.appliedCount).toBe(1);
+    expect(py!.successRate).toBe(1.0);
+  });
+
+  it('should skip annotations where playbook was not applied', () => {
+    const domainMap = new Map([['traj-1', 'typescript']]);
+    const annotations = [
+      makeAnnotation({ trajectoryId: 'traj-1', appliedPlaybookIds: ['pb-2'], success: true }),
+    ];
+
+    const breakdown = computeDomainBreakdown(annotations, 'pb-1', domainMap);
+    expect(breakdown).toHaveLength(0);
+  });
+
+  it('should sort by applied count descending', () => {
+    const domainMap = new Map([
+      ['traj-1', 'python'],
+      ['traj-2', 'typescript'],
+      ['traj-3', 'typescript'],
+    ]);
+
+    const annotations = [
+      makeAnnotation({ trajectoryId: 'traj-1', appliedPlaybookIds: ['pb-1'], success: true }),
+      makeAnnotation({ trajectoryId: 'traj-2', appliedPlaybookIds: ['pb-1'], success: true }),
+      makeAnnotation({ trajectoryId: 'traj-3', appliedPlaybookIds: ['pb-1'], success: true }),
+    ];
+
+    const breakdown = computeDomainBreakdown(annotations, 'pb-1', domainMap);
+    expect(breakdown[0].domain).toBe('typescript');
+    expect(breakdown[1].domain).toBe('python');
+  });
+});
+
+// ============================================================
+// buildEfficacyProfile
+// ============================================================
+
+describe('buildEfficacyProfile', () => {
+  it('should build a complete profile', () => {
+    const pb = makePlaybook({ id: 'pb-1', name: 'test-pb', successCount: 7, failureCount: 3 });
+    const eff: PlaybookEffectivenessEntry = {
+      playbookId: 'pb-1',
+      playbookName: 'test-pb',
+      surfacedCount: 10,
+      appliedCount: 8,
+      appliedSuccessRate: 0.75,
+      appliedAvgSteps: 5,
+      relevanceRate: 0.8,
+    };
+
+    const profile = buildEfficacyProfile(pb, [], eff, 0.5, new Map());
+
+    expect(profile.playbookId).toBe('pb-1');
+    expect(profile.successRate).toBeCloseTo(0.7);
+    expect(profile.normalizedGain).toBeCloseTo(0.5); // (0.75 - 0.5) / (1 - 0.5)
+    expect(profile.usage.adoptionRate).toBeCloseTo(0.8);
+    expect(profile.usage.appliedSuccessRate).toBe(0.75);
+    expect(profile.evolution.version).toBe('1.0.0');
+  });
+
+  it('should handle missing effectiveness entry', () => {
+    const pb = makePlaybook({ id: 'pb-1', successCount: 3, failureCount: 0 });
+    const profile = buildEfficacyProfile(pb, [], undefined, 0.5, new Map());
+
+    expect(profile.usage.surfacedCount).toBe(0);
+    expect(profile.usage.adoptionRate).toBe(0);
+    // Falls back to playbook success rate
+    expect(profile.normalizedGain).toBeCloseTo(
+      (1.0 - 0.5) / (1 - 0.5)
+    );
+  });
+});
+
+// ============================================================
+// buildPortfolioSnapshot
+// ============================================================
+
+describe('buildPortfolioSnapshot', () => {
+  it('should compute confidence bands', () => {
+    const playbooks = [
+      makePlaybook({ confidence: 0.9 }),
+      makePlaybook({ confidence: 0.8 }),
+      makePlaybook({ confidence: 0.5 }),
+      makePlaybook({ confidence: 0.3 }),
+    ];
+
+    const snapshot = buildPortfolioSnapshot(playbooks, [], [], 0.5, new Map());
+
+    expect(snapshot.confidenceBands.high).toBe(2);
+    expect(snapshot.confidenceBands.medium).toBe(1);
+    expect(snapshot.confidenceBands.low).toBe(1);
+    expect(snapshot.totalPlaybooks).toBe(4);
+  });
+
+  it('should detect stale playbooks', () => {
+    const now = Date.now();
+    const annotations = [
+      makeAnnotation({
+        appliedPlaybookIds: ['pb-1'],
+        success: true,
+        timestamp: new Date(now - 60 * 86400000), // 60 days ago
+      }),
+      // Need at least 2 data points for trend to report daysSinceLastUse properly
+      makeAnnotation({
+        appliedPlaybookIds: ['pb-1'],
+        success: true,
+        timestamp: new Date(now - 61 * 86400000),
+      }),
+    ];
+
+    const playbooks = [makePlaybook({ id: 'pb-1', name: 'stale-pb' })];
+    const snapshot = buildPortfolioSnapshot(playbooks, annotations, [], 0.5, new Map());
+
+    expect(snapshot.stale.length).toBeGreaterThanOrEqual(1);
+    expect(snapshot.stale[0].playbookId).toBe('pb-1');
+  });
+
+  it('should detect low adoption', () => {
+    const playbooks = [makePlaybook({ id: 'pb-1', name: 'ignored-pb' })];
+    const eff: PlaybookEffectivenessEntry[] = [{
+      playbookId: 'pb-1',
+      playbookName: 'ignored-pb',
+      surfacedCount: 20,
+      appliedCount: 2,
+      appliedSuccessRate: 0.5,
+      appliedAvgSteps: 5,
+      relevanceRate: 0.3,
+    }];
+
+    const snapshot = buildPortfolioSnapshot(playbooks, [], eff, 0.5, new Map());
+
+    expect(snapshot.lowAdoption).toHaveLength(1);
+    expect(snapshot.lowAdoption[0].playbookId).toBe('pb-1');
+  });
+
+  it('should detect redundant playbook pairs', () => {
+    const pb1 = createPlaybook({
+      id: 'pb-1',
+      name: 'debug-typescript-errors',
+      applicability: { situations: [], triggers: [], antiPatterns: [], domains: [] },
+      guidance: {
+        strategy: 'Read the error message and check the stack trace for type mismatches',
+        tactics: ['Read stack trace', 'Check types', 'Fix imports'],
+      },
+    });
+    const pb2 = createPlaybook({
+      id: 'pb-2',
+      name: 'fix-typescript-types',
+      applicability: { situations: [], triggers: [], antiPatterns: [], domains: [] },
+      guidance: {
+        strategy: 'Read the error message and check the stack trace for type mismatches',
+        tactics: ['Read stack trace', 'Check types', 'Fix type declarations'],
+      },
+    });
+
+    const snapshot = buildPortfolioSnapshot([pb1, pb2], [], [], 0.5, new Map());
+    expect(snapshot.redundant.length).toBeGreaterThanOrEqual(1);
+  });
+
+  it('should compute system-level normalized gain', () => {
+    const guided = Array.from({ length: 10 }, (_, i) =>
+      makeAnnotation({
+        playbookIds: ['pb-1'],
+        appliedPlaybookIds: ['pb-1'],
+        success: i < 8, // 80% success
+      })
+    );
+    const unguided = Array.from({ length: 10 }, (_, i) =>
+      makeAnnotation({
+        success: i < 5, // 50% success
+      })
+    );
+
+    const snapshot = buildPortfolioSnapshot(
+      [makePlaybook({ id: 'pb-1' })],
+      [...guided, ...unguided],
+      [],
+      0.5,
+      new Map()
+    );
+
+    expect(snapshot.systemMetrics.guidedSuccessRate).toBeCloseTo(0.8);
+    expect(snapshot.systemMetrics.unguidedSuccessRate).toBeCloseTo(0.5);
+    expect(snapshot.systemMetrics.normalizedGain).toBeCloseTo(0.6); // (0.8 - 0.5) / (1 - 0.5)
+  });
+});