cognitive-core 0.2.0 → 0.2.2

package/tests/workspace/templates/playbook-efficacy.test.ts

@@ -0,0 +1,377 @@
+import { describe, it, expect } from 'vitest';
+import { playbookEfficacyAuditTemplate } from '../../../src/workspace/templates/playbook-efficacy-audit.js';
+import { playbookDecayDetectionTemplate } from '../../../src/workspace/templates/playbook-decay-detection.js';
+import { playbookLifecycleReviewTemplate } from '../../../src/workspace/templates/playbook-lifecycle-review.js';
+import type { TaskAnnotation, PlaybookEffectivenessEntry } from '../../../src/learning/effectiveness.js';
+import { createPlaybook } from '../../../src/types/index.js';
+
+// ============================================================
+// Test Helpers
+// ============================================================
+
+function makeAnnotation(overrides: {
+  trajectoryId?: string;
+  playbookIds?: string[];
+  playbookNames?: string[];
+  appliedPlaybookIds?: string[];
+  success?: boolean;
+  stepCount?: number;
+  timestamp?: Date;
+}): TaskAnnotation {
+  return {
+    id: `ann-${Math.random().toString(36).slice(2, 6)}`,
+    trajectoryId: overrides.trajectoryId ?? `traj-${Math.random().toString(36).slice(2, 6)}`,
+    timestamp: overrides.timestamp ?? new Date(),
+    knowledgeSurfaced: {
+      playbookIds: overrides.playbookIds ?? [],
+      playbookNames: overrides.playbookNames ?? [],
+      experienceIds: [],
+      totalItems: (overrides.playbookIds ?? []).length,
+    },
+    knowledgeApplied: {
+      playbookIdsUsed: overrides.appliedPlaybookIds ?? [],
+      anyKnowledgeUsed: (overrides.appliedPlaybookIds ?? []).length > 0,
+    },
+    outcome: {
+      success: overrides.success ?? true,
+      stepCount: overrides.stepCount ?? 5,
+      errorRecoveries: 0,
+      domainHadPlaybooks: (overrides.playbookIds ?? []).length > 0,
+    },
+  };
+}
+
+function makePlaybook(overrides?: {
+  id?: string;
+  name?: string;
+  confidence?: number;
+  successCount?: number;
+  failureCount?: number;
+  failureRecords?: number;
+}) {
+  return createPlaybook({
+    id: overrides?.id ?? `pb-${Math.random().toString(36).slice(2, 6)}`,
+    name: overrides?.name ?? 'test-playbook',
+    confidence: overrides?.confidence ?? 0.7,
+    applicability: {
+      situations: ['debugging code'],
+      triggers: ['error'],
+      antiPatterns: [],
+      domains: ['typescript'],
+    },
+    guidance: {
+      strategy: 'Debug by reading error messages carefully',
+      tactics: ['Read the stack trace', 'Check variable types'],
+    },
+    evolution: {
+      version: '1.0.0',
+      createdFrom: [],
+      failures: Array.from(
+        { length: overrides?.failureRecords ?? 0 },
+        () => ({
+          trajectoryId: 'traj-x',
+          context: 'test',
+          failureMode: 'test failure',
+          timestamp: new Date(),
+        })
+      ),
+      refinements: [],
+      successCount: overrides?.successCount ?? 5,
+      failureCount: overrides?.failureCount ?? 2,
+    },
+  });
+}
+
+// ============================================================
+// playbookEfficacyAuditTemplate
+// ============================================================
+
+describe('playbookEfficacyAuditTemplate', () => {
+  it('should have correct metadata', () => {
+    expect(playbookEfficacyAuditTemplate.taskType).toBe('playbook-efficacy-audit');
+    expect(playbookEfficacyAuditTemplate.domain).toBe('meta-learning');
+  });
+
+  describe('assessComplexity', () => {
+    it('should return heuristic for low-usage playbooks', () => {
+      const pb = makePlaybook({ successCount: 1, failureCount: 0 });
+      const complexity = playbookEfficacyAuditTemplate.assessComplexity({
+        playbook: pb,
+        annotations: [],
+        playbookEffectiveness: undefined,
+        unguidedSuccessRate: 0.5,
+        trajectoryDomainMap: new Map(),
+      });
+      expect(complexity).toBe('heuristic');
+    });
+
+    it('should return standard for high-usage playbooks with failures', () => {
+      const pb = makePlaybook({ successCount: 15, failureCount: 8, failureRecords: 4 });
+      const complexity = playbookEfficacyAuditTemplate.assessComplexity({
+        playbook: pb,
+        annotations: [],
+        playbookEffectiveness: undefined,
+        unguidedSuccessRate: 0.5,
+        trajectoryDomainMap: new Map(),
+      });
+      expect(complexity).toBe('standard');
+    });
+
+    it('should return lightweight for moderate usage with some failures', () => {
+      const pb = makePlaybook({ successCount: 8, failureCount: 2, failureRecords: 1 });
+      const complexity = playbookEfficacyAuditTemplate.assessComplexity({
+        playbook: pb,
+        annotations: [],
+        playbookEffectiveness: undefined,
+        unguidedSuccessRate: 0.5,
+        trajectoryDomainMap: new Map(),
+      });
+      expect(complexity).toBe('lightweight');
+    });
+  });
+
+  describe('heuristicFallback', () => {
+    it('should return insufficient-data for low-usage playbooks', async () => {
+      const pb = makePlaybook({ successCount: 1, failureCount: 0 });
+      const result = await playbookEfficacyAuditTemplate.heuristicFallback!({
+        playbook: pb,
+        annotations: [],
+        playbookEffectiveness: undefined,
+        unguidedSuccessRate: 0.5,
+        trajectoryDomainMap: new Map(),
+      });
+
+      expect(result.assessment.health).toBe('insufficient-data');
+      expect(result.profile.totalUses).toBe(1);
+    });
+
+    it('should compute profile and health for adequate data', async () => {
+      const pb = makePlaybook({ successCount: 7, failureCount: 3 });
+      const result = await playbookEfficacyAuditTemplate.heuristicFallback!({
+        playbook: pb,
+        annotations: [],
+        playbookEffectiveness: undefined,
+        unguidedSuccessRate: 0.5,
+        trajectoryDomainMap: new Map(),
+      });
+
+      expect(result.assessment.health).toBe('healthy');
+      expect(result.profile.successRate).toBeCloseTo(0.7);
+      expect(result.profile.normalizedGain).toBeGreaterThan(0);
+    });
+
+    it('should flag underperforming playbooks', async () => {
+      const pb = makePlaybook({ successCount: 1, failureCount: 5 });
+      const result = await playbookEfficacyAuditTemplate.heuristicFallback!({
+        playbook: pb,
+        annotations: [],
+        playbookEffectiveness: undefined,
+        unguidedSuccessRate: 0.5,
+        trajectoryDomainMap: new Map(),
+      });
+
+      expect(result.assessment.health).toBe('underperforming');
+    });
+  });
+});
+
+// ============================================================
+// playbookDecayDetectionTemplate
+// ============================================================
+
+describe('playbookDecayDetectionTemplate', () => {
+  it('should have correct metadata', () => {
+    expect(playbookDecayDetectionTemplate.taskType).toBe('playbook-decay-detection');
+    expect(playbookDecayDetectionTemplate.domain).toBe('meta-learning');
+  });
+
+  describe('assessComplexity', () => {
+    it('should return heuristic for empty portfolio', () => {
+      const complexity = playbookDecayDetectionTemplate.assessComplexity({
+        playbooks: [],
+        annotations: [],
+        playbookEffectiveness: [],
+        unguidedSuccessRate: 0.5,
+      });
+      expect(complexity).toBe('heuristic');
+    });
+
+    it('should return heuristic for sparse annotations', () => {
+      const complexity = playbookDecayDetectionTemplate.assessComplexity({
+        playbooks: [makePlaybook()],
+        annotations: Array.from({ length: 5 }, () => makeAnnotation({})),
+        playbookEffectiveness: [],
+        unguidedSuccessRate: 0.5,
+      });
+      expect(complexity).toBe('heuristic');
+    });
+
+    it('should return standard for large portfolios', () => {
+      const playbooks = Array.from({ length: 25 }, () => makePlaybook());
+      const annotations = Array.from({ length: 50 }, () => makeAnnotation({}));
+      const complexity = playbookDecayDetectionTemplate.assessComplexity({
+        playbooks,
+        annotations,
+        playbookEffectiveness: [],
+        unguidedSuccessRate: 0.5,
+      });
+      expect(complexity).toBe('standard');
+    });
+  });
+
+  describe('heuristicFallback', () => {
+    it('should detect decaying playbooks', async () => {
+      const now = Date.now();
+      const pb = makePlaybook({ id: 'pb-1', name: 'decaying-pb' });
+      const annotations = Array.from({ length: 8 }, (_, i) =>
+        makeAnnotation({
+          appliedPlaybookIds: ['pb-1'],
+          success: i < 4, // First 4 succeed, last 4 fail
+          timestamp: new Date(now - (8 - i) * 86400000),
+        })
+      );
+
+      const result = await playbookDecayDetectionTemplate.heuristicFallback!({
+        playbooks: [pb],
+        annotations,
+        playbookEffectiveness: [],
+        unguidedSuccessRate: 0.5,
+      });
+
+      expect(result.decaying.length).toBeGreaterThanOrEqual(1);
+      expect(result.decaying[0].playbookId).toBe('pb-1');
+      expect(result.decaying[0].trend.direction).toBe('decaying');
+    });
+
+    it('should flag stale playbooks as at-risk', async () => {
+      const now = Date.now();
+      const pb = makePlaybook({ id: 'pb-1', name: 'stale-pb' });
+      const annotations = Array.from({ length: 4 }, (_, i) =>
+        makeAnnotation({
+          appliedPlaybookIds: ['pb-1'],
+          success: true,
+          timestamp: new Date(now - (60 + i) * 86400000), // 60+ days ago
+        })
+      );
+
+      const result = await playbookDecayDetectionTemplate.heuristicFallback!({
+        playbooks: [pb],
+        annotations,
+        playbookEffectiveness: [],
+        unguidedSuccessRate: 0.5,
+      });
+
+      // Could be in either decaying or atRisk depending on trend direction
+      const flagged = [...result.decaying, ...result.atRisk];
+      expect(flagged.some(s => s.playbookId === 'pb-1')).toBe(true);
+    });
+
+    it('should mark healthy playbooks', async () => {
+      const pb = makePlaybook({ id: 'pb-1' });
+      const result = await playbookDecayDetectionTemplate.heuristicFallback!({
+        playbooks: [pb],
+        annotations: [], // No annotations = insufficient data = healthy by default
+        playbookEffectiveness: [],
+        unguidedSuccessRate: 0.5,
+      });
+
+      expect(result.healthy).toContain('pb-1');
+    });
+  });
+});
+
+// ============================================================
+// playbookLifecycleReviewTemplate
+// ============================================================
+
+describe('playbookLifecycleReviewTemplate', () => {
+  it('should have correct metadata', () => {
+    expect(playbookLifecycleReviewTemplate.taskType).toBe('playbook-lifecycle-review');
+    expect(playbookLifecycleReviewTemplate.domain).toBe('meta-learning');
+  });
+
+  describe('assessComplexity', () => {
+    it('should return heuristic for empty portfolio', () => {
+      const complexity = playbookLifecycleReviewTemplate.assessComplexity({
+        playbooks: [],
+        annotations: [],
+        playbookEffectiveness: [],
+        unguidedSuccessRate: 0.5,
+        trajectoryDomainMap: new Map(),
+      });
+      expect(complexity).toBe('heuristic');
+    });
+
+    it('should return thorough for large portfolios', () => {
+      const playbooks = Array.from({ length: 35 }, () => makePlaybook());
+      const complexity = playbookLifecycleReviewTemplate.assessComplexity({
+        playbooks,
+        annotations: [],
+        playbookEffectiveness: [],
+        unguidedSuccessRate: 0.5,
+        trajectoryDomainMap: new Map(),
+      });
+      expect(complexity).toBe('thorough');
+    });
+  });
+
+  describe('heuristicFallback', () => {
+    it('should return snapshot with decaying recommendations', async () => {
+      const now = Date.now();
+      const pb = makePlaybook({ id: 'pb-1', name: 'old-pb' });
+      const annotations = Array.from({ length: 8 }, (_, i) =>
+        makeAnnotation({
+          appliedPlaybookIds: ['pb-1'],
+          success: i < 4,
+          timestamp: new Date(now - (8 - i) * 86400000),
+        })
+      );
+
+      const result = await playbookLifecycleReviewTemplate.heuristicFallback!({
+        playbooks: [pb],
+        annotations,
+        playbookEffectiveness: [],
+        unguidedSuccessRate: 0.5,
+        trajectoryDomainMap: new Map(),
+      });
+
+      expect(result.snapshot).toBeDefined();
+      expect(result.snapshot.totalPlaybooks).toBe(1);
+      // Decaying playbook should get an evolve recommendation
+      if (result.recommendations.length > 0) {
+        expect(result.recommendations[0].proposedState).toBe('evolve');
+      }
+    });
+
+    it('should return empty recommendations for empty portfolio', async () => {
+      const result = await playbookLifecycleReviewTemplate.heuristicFallback!({
+        playbooks: [],
+        annotations: [],
+        playbookEffectiveness: [],
+        unguidedSuccessRate: 0.5,
+        trajectoryDomainMap: new Map(),
+      });
+
+      expect(result.snapshot.totalPlaybooks).toBe(0);
+      expect(result.recommendations).toHaveLength(0);
+    });
+  });
+
+  describe('getComputeRequirements', () => {
+    it('should set higher timeout for thorough complexity', () => {
+      const reqs = playbookLifecycleReviewTemplate.getComputeRequirements!(
+        { playbooks: [], annotations: [], playbookEffectiveness: [], unguidedSuccessRate: 0, trajectoryDomainMap: new Map() },
+        'thorough'
+      );
+      expect(reqs.timeout).toBe(240_000);
+    });
+
+    it('should set standard timeout for lightweight', () => {
+      const reqs = playbookLifecycleReviewTemplate.getComputeRequirements!(
+        { playbooks: [], annotations: [], playbookEffectiveness: [], unguidedSuccessRate: 0, trajectoryDomainMap: new Map() },
+        'lightweight'
+      );
+      expect(reqs.timeout).toBe(120_000);
+    });
+  });
+});

package/.claude/settings.local.json

@@ -1,11 +0,0 @@
-{
-  "permissions": {
-    "allow": [
-      "Bash(npm install:*)",
-      "Bash(npm run build:*)",
-      "Bash(npm run test:run:*)",
-      "Bash(git add:*)",
-      "Bash(git commit:*)"
-    ]
-  }
-}

package/dist/learning/llm-extractor.d.ts

@@ -1,88 +0,0 @@
-/**
- * LLM-Based Playbook Extractor
- *
- * Uses language models to generate higher-quality playbooks
- * from trajectories. Complements the pattern-based extractor
- * with deeper semantic analysis.
- */
-import type { Playbook, Trajectory, Refinement } from '../types/index.js';
-import type { AnalysisResult } from './analyzer.js';
-import type { LLMProvider } from '../agents/types.js';
-/**
- * Configuration for LLM-based extraction
- */
-export interface LLMExtractorConfig {
-    /** Temperature for LLM calls (default: 0.2 for determinism) */
-    temperature: number;
-    /** Maximum tokens for response */
-    maxTokens: number;
-    /** Whether to extract from single trajectories (vs only batches) */
-    extractFromSingle: boolean;
-    /** Minimum complexity for LLM extraction */
-    minComplexity: number;
-}
-/**
- * Result from LLM extraction
- */
-export interface LLMExtractionResult {
-    playbooks: Playbook[];
-    refinements: Array<{
-        playbookId: string;
-        refinement: Refinement;
-    }>;
-    /** Raw LLM response for debugging */
-    rawResponse?: string;
-}
-/**
- * LLM-Based Playbook Extractor
- */
-export declare class LLMPlaybookExtractor {
-    private provider;
-    private config;
-    constructor(provider: LLMProvider, config?: Partial<LLMExtractorConfig>);
-    /**
-     * Extract playbooks from trajectories using LLM
-     */
-    extract(trajectories: Trajectory[], analyses: AnalysisResult[], existingPlaybooks?: Playbook[]): Promise<LLMExtractionResult>;
-    /**
-     * Extract a playbook from a single trajectory
-     */
-    private extractFromSingleTrajectory;
-    /**
-     * Extract playbooks from a batch of trajectories
-     */
-    private extractFromBatch;
-    /**
-     * Build prompt for single trajectory extraction
-     */
-    private buildSingleExtractionPrompt;
-    /**
-     * Build prompt for batch extraction
-     */
-    private buildBatchExtractionPrompt;
-    /**
-     * Parse LLM response into extraction result
-     */
-    private parseExtractionResponse;
-    /**
-     * Create a Playbook from LLM-extracted data
-     */
-    private createPlaybookFromLLMData;
-    /**
-     * Parse complexity string
-     */
-    private parseComplexity;
-    /**
-     * Check if two playbooks are similar
-     */
-    private isSimilarPlaybook;
-    /**
-     * Generate a refinement for an existing playbook based on a new trajectory
-     */
-    generateRefinement(playbook: Playbook, trajectory: Trajectory, context: string): Promise<Refinement>;
-}
-/**
- * Create an LLM-based playbook extractor
- */
-export declare function createLLMExtractor(provider: LLMProvider, config?: Partial<LLMExtractorConfig>): LLMPlaybookExtractor;
-//# sourceMappingURL=llm-extractor.d.ts.map

package/dist/learning/llm-extractor.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"llm-extractor.d.ts","sourceRoot":"","sources":["../../src/learning/llm-extractor.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EACV,QAAQ,EACR,UAAU,EACV,UAAU,EACX,MAAM,mBAAmB,CAAC;AAE3B,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AACpD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAEtD;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,+DAA+D;IAC/D,WAAW,EAAE,MAAM,CAAC;IACpB,kCAAkC;IAClC,SAAS,EAAE,MAAM,CAAC;IAClB,oEAAoE;IACpE,iBAAiB,EAAE,OAAO,CAAC;IAC3B,4CAA4C;IAC5C,aAAa,EAAE,MAAM,CAAC;CACvB;AASD;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,SAAS,EAAE,QAAQ,EAAE,CAAC;IACtB,WAAW,EAAE,KAAK,CAAC;QACjB,UAAU,EAAE,MAAM,CAAC;QACnB,UAAU,EAAE,UAAU,CAAC;KACxB,CAAC,CAAC;IACH,qCAAqC;IACrC,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,qBAAa,oBAAoB;IAC/B,OAAO,CAAC,QAAQ,CAAc;IAC9B,OAAO,CAAC,MAAM,CAAqB;gBAEvB,QAAQ,EAAE,WAAW,EAAE,MAAM,CAAC,EAAE,OAAO,CAAC,kBAAkB,CAAC;IAKvE;;OAEG;IACG,OAAO,CACX,YAAY,EAAE,UAAU,EAAE,EAC1B,QAAQ,EAAE,cAAc,EAAE,EAC1B,iBAAiB,GAAE,QAAQ,EAAO,GACjC,OAAO,CAAC,mBAAmB,CAAC;IAkE/B;;OAEG;YACW,2BAA2B;IAsBzC;;OAEG;YACW,gBAAgB;IAsB9B;;OAEG;IACH,OAAO,CAAC,2BAA2B;IAkEnC;;OAEG;IACH,OAAO,CAAC,0BAA0B;IAwDlC;;OAEG;IACH,OAAO,CAAC,uBAAuB;IA+D/B;;OAEG;IACH,OAAO,CAAC,yBAAyB;IAsCjC;;OAEG;IACH,OAAO,CAAC,eAAe;IASvB;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAYzB;;OAEG;IACG,kBAAkB,CACtB,QAAQ,EAAE,QAAQ,EAClB,UAAU,EAAE,UAAU,EACtB,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,UAAU,CAAC;CA0DvB;AAoBD;;GAEG;AACH,wBAAgB,kBAAkB,CAChC,QAAQ,EAAE,WAAW,EACrB,MAAM,CAAC,EAAE,OAAO,CAAC,kBAAkB,CAAC,GACnC,oBAAoB,CAEtB"}