cognitive-core 0.1.2 → 0.2.1

package/tests/workspace/templates/playbook-efficacy.test.ts

@@ -0,0 +1,377 @@
+import { describe, it, expect } from 'vitest';
+import { playbookEfficacyAuditTemplate } from '../../../src/workspace/templates/playbook-efficacy-audit.js';
+import { playbookDecayDetectionTemplate } from '../../../src/workspace/templates/playbook-decay-detection.js';
+import { playbookLifecycleReviewTemplate } from '../../../src/workspace/templates/playbook-lifecycle-review.js';
+import type { TaskAnnotation, PlaybookEffectivenessEntry } from '../../../src/learning/effectiveness.js';
+import { createPlaybook } from '../../../src/types/index.js';
+
+// ============================================================
+// Test Helpers
+// ============================================================
+
+function makeAnnotation(overrides: {
+  trajectoryId?: string;
+  playbookIds?: string[];
+  playbookNames?: string[];
+  appliedPlaybookIds?: string[];
+  success?: boolean;
+  stepCount?: number;
+  timestamp?: Date;
+}): TaskAnnotation {
+  return {
+    id: `ann-${Math.random().toString(36).slice(2, 6)}`,
+    trajectoryId: overrides.trajectoryId ?? `traj-${Math.random().toString(36).slice(2, 6)}`,
+    timestamp: overrides.timestamp ?? new Date(),
+    knowledgeSurfaced: {
+      playbookIds: overrides.playbookIds ?? [],
+      playbookNames: overrides.playbookNames ?? [],
+      experienceIds: [],
+      totalItems: (overrides.playbookIds ?? []).length,
+    },
+    knowledgeApplied: {
+      playbookIdsUsed: overrides.appliedPlaybookIds ?? [],
+      anyKnowledgeUsed: (overrides.appliedPlaybookIds ?? []).length > 0,
+    },
+    outcome: {
+      success: overrides.success ?? true,
+      stepCount: overrides.stepCount ?? 5,
+      errorRecoveries: 0,
+      domainHadPlaybooks: (overrides.playbookIds ?? []).length > 0,
+    },
+  };
+}
+
+function makePlaybook(overrides?: {
+  id?: string;
+  name?: string;
+  confidence?: number;
+  successCount?: number;
+  failureCount?: number;
+  failureRecords?: number;
+}) {
+  return createPlaybook({
+    id: overrides?.id ?? `pb-${Math.random().toString(36).slice(2, 6)}`,
+    name: overrides?.name ?? 'test-playbook',
+    confidence: overrides?.confidence ?? 0.7,
+    applicability: {
+      situations: ['debugging code'],
+      triggers: ['error'],
+      antiPatterns: [],
+      domains: ['typescript'],
+    },
+    guidance: {
+      strategy: 'Debug by reading error messages carefully',
+      tactics: ['Read the stack trace', 'Check variable types'],
+    },
+    evolution: {
+      version: '1.0.0',
+      createdFrom: [],
+      failures: Array.from(
+        { length: overrides?.failureRecords ?? 0 },
+        () => ({
+          trajectoryId: 'traj-x',
+          context: 'test',
+          failureMode: 'test failure',
+          timestamp: new Date(),
+        })
+      ),
+      refinements: [],
+      successCount: overrides?.successCount ?? 5,
+      failureCount: overrides?.failureCount ?? 2,
+    },
+  });
+}
+
+// ============================================================
+// playbookEfficacyAuditTemplate
+// ============================================================
+
+describe('playbookEfficacyAuditTemplate', () => {
+  it('should have correct metadata', () => {
+    expect(playbookEfficacyAuditTemplate.taskType).toBe('playbook-efficacy-audit');
+    expect(playbookEfficacyAuditTemplate.domain).toBe('meta-learning');
+  });
+
+  describe('assessComplexity', () => {
+    it('should return heuristic for low-usage playbooks', () => {
+      const pb = makePlaybook({ successCount: 1, failureCount: 0 });
+      const complexity = playbookEfficacyAuditTemplate.assessComplexity({
+        playbook: pb,
+        annotations: [],
+        playbookEffectiveness: undefined,
+        unguidedSuccessRate: 0.5,
+        trajectoryDomainMap: new Map(),
+      });
+      expect(complexity).toBe('heuristic');
+    });
+
+    it('should return standard for high-usage playbooks with failures', () => {
+      const pb = makePlaybook({ successCount: 15, failureCount: 8, failureRecords: 4 });
+      const complexity = playbookEfficacyAuditTemplate.assessComplexity({
+        playbook: pb,
+        annotations: [],
+        playbookEffectiveness: undefined,
+        unguidedSuccessRate: 0.5,
+        trajectoryDomainMap: new Map(),
+      });
+      expect(complexity).toBe('standard');
+    });
+
+    it('should return lightweight for moderate usage with some failures', () => {
+      const pb = makePlaybook({ successCount: 8, failureCount: 2, failureRecords: 1 });
+      const complexity = playbookEfficacyAuditTemplate.assessComplexity({
+        playbook: pb,
+        annotations: [],
+        playbookEffectiveness: undefined,
+        unguidedSuccessRate: 0.5,
+        trajectoryDomainMap: new Map(),
+      });
+      expect(complexity).toBe('lightweight');
+    });
+  });
+
+  describe('heuristicFallback', () => {
+    it('should return insufficient-data for low-usage playbooks', async () => {
+      const pb = makePlaybook({ successCount: 1, failureCount: 0 });
+      const result = await playbookEfficacyAuditTemplate.heuristicFallback!({
+        playbook: pb,
+        annotations: [],
+        playbookEffectiveness: undefined,
+        unguidedSuccessRate: 0.5,
+        trajectoryDomainMap: new Map(),
+      });
+
+      expect(result.assessment.health).toBe('insufficient-data');
+      expect(result.profile.totalUses).toBe(1);
+    });
+
+    it('should compute profile and health for adequate data', async () => {
+      const pb = makePlaybook({ successCount: 7, failureCount: 3 });
+      const result = await playbookEfficacyAuditTemplate.heuristicFallback!({
+        playbook: pb,
+        annotations: [],
+        playbookEffectiveness: undefined,
+        unguidedSuccessRate: 0.5,
+        trajectoryDomainMap: new Map(),
+      });
+
+      expect(result.assessment.health).toBe('healthy');
+      expect(result.profile.successRate).toBeCloseTo(0.7);
+      expect(result.profile.normalizedGain).toBeGreaterThan(0);
+    });
+
+    it('should flag underperforming playbooks', async () => {
+      const pb = makePlaybook({ successCount: 1, failureCount: 5 });
+      const result = await playbookEfficacyAuditTemplate.heuristicFallback!({
+        playbook: pb,
+        annotations: [],
+        playbookEffectiveness: undefined,
+        unguidedSuccessRate: 0.5,
+        trajectoryDomainMap: new Map(),
+      });
+
+      expect(result.assessment.health).toBe('underperforming');
+    });
+  });
+});
+
+// ============================================================
+// playbookDecayDetectionTemplate
+// ============================================================
+
+describe('playbookDecayDetectionTemplate', () => {
+  it('should have correct metadata', () => {
+    expect(playbookDecayDetectionTemplate.taskType).toBe('playbook-decay-detection');
+    expect(playbookDecayDetectionTemplate.domain).toBe('meta-learning');
+  });
+
+  describe('assessComplexity', () => {
+    it('should return heuristic for empty portfolio', () => {
+      const complexity = playbookDecayDetectionTemplate.assessComplexity({
+        playbooks: [],
+        annotations: [],
+        playbookEffectiveness: [],
+        unguidedSuccessRate: 0.5,
+      });
+      expect(complexity).toBe('heuristic');
+    });
+
+    it('should return heuristic for sparse annotations', () => {
+      const complexity = playbookDecayDetectionTemplate.assessComplexity({
+        playbooks: [makePlaybook()],
+        annotations: Array.from({ length: 5 }, () => makeAnnotation({})),
+        playbookEffectiveness: [],
+        unguidedSuccessRate: 0.5,
+      });
+      expect(complexity).toBe('heuristic');
+    });
+
+    it('should return standard for large portfolios', () => {
+      const playbooks = Array.from({ length: 25 }, () => makePlaybook());
+      const annotations = Array.from({ length: 50 }, () => makeAnnotation({}));
+      const complexity = playbookDecayDetectionTemplate.assessComplexity({
+        playbooks,
+        annotations,
+        playbookEffectiveness: [],
+        unguidedSuccessRate: 0.5,
+      });
+      expect(complexity).toBe('standard');
+    });
+  });
+
+  describe('heuristicFallback', () => {
+    it('should detect decaying playbooks', async () => {
+      const now = Date.now();
+      const pb = makePlaybook({ id: 'pb-1', name: 'decaying-pb' });
+      const annotations = Array.from({ length: 8 }, (_, i) =>
+        makeAnnotation({
+          appliedPlaybookIds: ['pb-1'],
+          success: i < 4, // First 4 succeed, last 4 fail
+          timestamp: new Date(now - (8 - i) * 86400000),
+        })
+      );
+
+      const result = await playbookDecayDetectionTemplate.heuristicFallback!({
+        playbooks: [pb],
+        annotations,
+        playbookEffectiveness: [],
+        unguidedSuccessRate: 0.5,
+      });
+
+      expect(result.decaying.length).toBeGreaterThanOrEqual(1);
+      expect(result.decaying[0].playbookId).toBe('pb-1');
+      expect(result.decaying[0].trend.direction).toBe('decaying');
+    });
+
+    it('should flag stale playbooks as at-risk', async () => {
+      const now = Date.now();
+      const pb = makePlaybook({ id: 'pb-1', name: 'stale-pb' });
+      const annotations = Array.from({ length: 4 }, (_, i) =>
+        makeAnnotation({
+          appliedPlaybookIds: ['pb-1'],
+          success: true,
+          timestamp: new Date(now - (60 + i) * 86400000), // 60+ days ago
+        })
+      );
+
+      const result = await playbookDecayDetectionTemplate.heuristicFallback!({
+        playbooks: [pb],
+        annotations,
+        playbookEffectiveness: [],
+        unguidedSuccessRate: 0.5,
+      });
+
+      // Could be in either decaying or atRisk depending on trend direction
+      const flagged = [...result.decaying, ...result.atRisk];
+      expect(flagged.some(s => s.playbookId === 'pb-1')).toBe(true);
+    });
+
+    it('should mark healthy playbooks', async () => {
+      const pb = makePlaybook({ id: 'pb-1' });
+      const result = await playbookDecayDetectionTemplate.heuristicFallback!({
+        playbooks: [pb],
+        annotations: [], // No annotations = insufficient data = healthy by default
+        playbookEffectiveness: [],
+        unguidedSuccessRate: 0.5,
+      });
+
+      expect(result.healthy).toContain('pb-1');
+    });
+  });
+});
+
+// ============================================================
+// playbookLifecycleReviewTemplate
+// ============================================================
+
+describe('playbookLifecycleReviewTemplate', () => {
+  it('should have correct metadata', () => {
+    expect(playbookLifecycleReviewTemplate.taskType).toBe('playbook-lifecycle-review');
+    expect(playbookLifecycleReviewTemplate.domain).toBe('meta-learning');
+  });
+
+  describe('assessComplexity', () => {
+    it('should return heuristic for empty portfolio', () => {
+      const complexity = playbookLifecycleReviewTemplate.assessComplexity({
+        playbooks: [],
+        annotations: [],
+        playbookEffectiveness: [],
+        unguidedSuccessRate: 0.5,
+        trajectoryDomainMap: new Map(),
+      });
+      expect(complexity).toBe('heuristic');
+    });
+
+    it('should return thorough for large portfolios', () => {
+      const playbooks = Array.from({ length: 35 }, () => makePlaybook());
+      const complexity = playbookLifecycleReviewTemplate.assessComplexity({
+        playbooks,
+        annotations: [],
+        playbookEffectiveness: [],
+        unguidedSuccessRate: 0.5,
+        trajectoryDomainMap: new Map(),
+      });
+      expect(complexity).toBe('thorough');
+    });
+  });
+
+  describe('heuristicFallback', () => {
+    it('should return snapshot with decaying recommendations', async () => {
+      const now = Date.now();
+      const pb = makePlaybook({ id: 'pb-1', name: 'old-pb' });
+      const annotations = Array.from({ length: 8 }, (_, i) =>
+        makeAnnotation({
+          appliedPlaybookIds: ['pb-1'],
+          success: i < 4,
+          timestamp: new Date(now - (8 - i) * 86400000),
+        })
+      );
+
+      const result = await playbookLifecycleReviewTemplate.heuristicFallback!({
+        playbooks: [pb],
+        annotations,
+        playbookEffectiveness: [],
+        unguidedSuccessRate: 0.5,
+        trajectoryDomainMap: new Map(),
+      });
+
+      expect(result.snapshot).toBeDefined();
+      expect(result.snapshot.totalPlaybooks).toBe(1);
+      // Decaying playbook should get an evolve recommendation
+      if (result.recommendations.length > 0) {
+        expect(result.recommendations[0].proposedState).toBe('evolve');
+      }
+    });
+
+    it('should return empty recommendations for empty portfolio', async () => {
+      const result = await playbookLifecycleReviewTemplate.heuristicFallback!({
+        playbooks: [],
+        annotations: [],
+        playbookEffectiveness: [],
+        unguidedSuccessRate: 0.5,
+        trajectoryDomainMap: new Map(),
+      });
+
+      expect(result.snapshot.totalPlaybooks).toBe(0);
+      expect(result.recommendations).toHaveLength(0);
+    });
+  });
+
+  describe('getComputeRequirements', () => {
+    it('should set higher timeout for thorough complexity', () => {
+      const reqs = playbookLifecycleReviewTemplate.getComputeRequirements!(
+        { playbooks: [], annotations: [], playbookEffectiveness: [], unguidedSuccessRate: 0, trajectoryDomainMap: new Map() },
+        'thorough'
+      );
+      expect(reqs.timeout).toBe(240_000);
+    });
+
+    it('should set standard timeout for lightweight', () => {
+      const reqs = playbookLifecycleReviewTemplate.getComputeRequirements!(
+        { playbooks: [], annotations: [], playbookEffectiveness: [], unguidedSuccessRate: 0, trajectoryDomainMap: new Map() },
+        'lightweight'
+      );
+      expect(reqs.timeout).toBe(120_000);
+    });
+  });
+});