cognitive-core 0.2.0 → 0.2.2

package/tests/memory/playbook-quality-validation.test.ts

@@ -0,0 +1,430 @@
+/**
+ * Playbook Quality Validation
+ *
+ * Validates that the compound-engineering seed pack playbooks are:
+ * 1. Structurally complete (all fields populated with meaningful content)
+ * 2. Internally consistent (cross-references resolve, domains cover expected areas)
+ * 3. Retrieval-effective (right playbooks surface for expected queries)
+ * 4. Non-redundant (no two playbooks are effectively duplicates)
+ *
+ * Modeled after CEP's "structural contract testing" approach — we test
+ * the shape and quality of the encoded knowledge, not prompt effectiveness.
+ */
+
+import { describe, it, expect, beforeAll, afterAll } from 'vitest';
+import { PlaybookLibrary, createPlaybookLibrary } from '../../src/memory/playbook.js';
+import { loadCuratedPlaybooks, BUNDLED_SEED_PACK_DIR } from '../../src/memory/curated-loader.js';
+import { createSqlitePersistence } from '../../src/persistence/index.js';
+import { textSimilarity } from '../../src/utils/similarity.js';
+import type { Playbook } from '../../src/types/playbook.js';
+import { mkdtemp, rm, readdir, readFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+
+// ============================================================================
+// Setup: load all playbooks once
+// ============================================================================
+
+let tempDir: string;
+let library: PlaybookLibrary;
+let persistence: ReturnType<typeof createSqlitePersistence>;
+let allPlaybooks: Playbook[];
+
+beforeAll(async () => {
+  tempDir = await mkdtemp(join(tmpdir(), 'pb-quality-'));
+  persistence = createSqlitePersistence({ baseDir: tempDir });
+  await persistence.init();
+  library = createPlaybookLibrary(persistence);
+  await library.init();
+  await loadCuratedPlaybooks(BUNDLED_SEED_PACK_DIR, library);
+  allPlaybooks = await library.getAll();
+});
+
+afterAll(async () => {
+  await library.close();
+  persistence.close();
+  await rm(tempDir, { recursive: true, force: true });
+});
+
+// ============================================================================
+// 1. Structural completeness
+// ============================================================================
+
+describe('Structural completeness', () => {
+  it('should have exactly 25 playbooks', () => {
+    expect(allPlaybooks.length).toBe(25);
+  });
+
+  it('every playbook should have a unique name', () => {
+    const names = allPlaybooks.map(p => p.name);
+    expect(new Set(names).size).toBe(names.length);
+  });
+
+  it('every playbook should have all required structural fields', () => {
+    for (const p of allPlaybooks) {
+      // Applicability
+      expect(p.applicability.situations.length, `${p.name}: needs situations`).toBeGreaterThanOrEqual(1);
+      expect(p.applicability.domains.length, `${p.name}: needs domains`).toBeGreaterThanOrEqual(1);
+      expect(p.applicability.triggers.length, `${p.name}: needs triggers`).toBeGreaterThanOrEqual(1);
+
+      // Guidance
+      expect(p.guidance.strategy.length, `${p.name}: strategy too short`).toBeGreaterThan(20);
+      expect(p.guidance.tactics.length, `${p.name}: needs ≥2 tactics`).toBeGreaterThanOrEqual(2);
+
+      // Verification
+      expect(p.verification.successIndicators.length, `${p.name}: needs success indicators`).toBeGreaterThanOrEqual(1);
+      expect(p.verification.failureIndicators.length, `${p.name}: needs failure indicators`).toBeGreaterThanOrEqual(1);
+
+      // Provenance
+      expect(p.provenance?.origin, `${p.name}: origin should be curated`).toBe('curated');
+      expect(p.provenance?.curatedBy, `${p.name}: curatedBy mismatch`).toBe('compound-engineering');
+
+      // Confidence
+      expect(p.confidence, `${p.name}: confidence too low`).toBeGreaterThanOrEqual(0.7);
+      expect(p.confidence, `${p.name}: confidence too high`).toBeLessThanOrEqual(0.95);
+    }
+  });
+
+  it('every situation should be a meaningful description (>10 chars)', () => {
+    for (const p of allPlaybooks) {
+      for (const situation of p.applicability.situations) {
+        expect(situation.length).toBeGreaterThan(10);
+      }
+    }
+  });
+
+  it('every tactic should be actionable (>15 chars)', () => {
+    for (const p of allPlaybooks) {
+      for (const tactic of p.guidance.tactics) {
+        expect(tactic.length).toBeGreaterThan(15);
+      }
+    }
+  });
+
+  it('every success/failure indicator should be specific (>10 chars)', () => {
+    for (const p of allPlaybooks) {
+      for (const indicator of p.verification.successIndicators) {
+        expect(indicator.length).toBeGreaterThan(10);
+      }
+      for (const indicator of p.verification.failureIndicators) {
+        expect(indicator.length).toBeGreaterThan(10);
+      }
+    }
+  });
+});
+
+// ============================================================================
+// 2. Content quality
+// ============================================================================
+
+describe('Content quality', () => {
+  it('strategies should not be generic boilerplate', () => {
+    // Phrases that indicate a strategy is too vague to be useful.
+    // Note: a strategy *critiquing* these phrases (e.g., "transform 'be careful'
+    // into concrete checklists") is fine — we check that the strategy isn't ONLY this.
+    for (const p of allPlaybooks) {
+      const strategy = p.guidance.strategy;
+      // Strategy should be substantial, not a single vague sentence
+      expect(strategy.length, `${p.name}: strategy too short to be actionable`).toBeGreaterThan(50);
+      // Should contain at least one concrete noun (not just adjectives/adverbs)
+      expect(strategy).toMatch(/[a-z]/i);
+    }
+  });
+
+  it('antiPatterns should exist for playbooks with complexity >= moderate', () => {
+    const complexPlaybooks = allPlaybooks.filter(
+      p => p.complexity === 'moderate' || p.complexity === 'complex',
+    );
+    // At least 80% should have anti-patterns
+    const withAntiPatterns = complexPlaybooks.filter(
+      p => p.applicability.antiPatterns.length > 0,
+    );
+    const ratio = withAntiPatterns.length / complexPlaybooks.length;
+    expect(ratio).toBeGreaterThanOrEqual(0.8);
+  });
+
+  it('review-orchestration should reference sub-reviewers', () => {
+    const orch = allPlaybooks.find(p => p.name === 'review-orchestration')!;
+    const allText = [orch.guidance.strategy, ...orch.guidance.tactics].join(' ').toLowerCase();
+
+    // Should mention key sub-reviewer domains
+    expect(allText).toContain('correctness');
+    expect(allText).toContain('testing');
+    expect(allText).toContain('security');
+    expect(allText).toContain('performance');
+  });
+
+  it('systematic-execution should reference plan-based workflow', () => {
+    const exec = allPlaybooks.find(p => p.name === 'systematic-execution')!;
+    const allText = [exec.guidance.strategy, ...exec.guidance.tactics].join(' ').toLowerCase();
+
+    expect(allText).toContain('plan');
+    expect(allText).toContain('test');
+    expect(allText).toContain('commit');
+  });
+
+  it('confidence-calibration should address uncertainty', () => {
+    const cal = allPlaybooks.find(p => p.name === 'confidence-calibration')!;
+    const allText = [
+      cal.guidance.strategy,
+      ...cal.guidance.tactics,
+      ...cal.applicability.situations,
+    ].join(' ').toLowerCase();
+
+    expect(allText).toMatch(/confiden|uncertain|calibrat/);
+  });
+});
+
+// ============================================================================
+// 3. Domain coverage
+// ============================================================================
+
+describe('Domain coverage', () => {
+  it('should cover the core CEP workflow phases', () => {
+    const allDomains = new Set(allPlaybooks.flatMap(p => p.applicability.domains));
+
+    // CEP's core loop: brainstorm → plan → execute → review → compound
+    expect(allDomains.has('brainstorming') || allDomains.has('requirements')).toBe(true);
+    expect(allDomains.has('planning') || allDomains.has('implementation-planning')).toBe(true);
+    expect(allDomains.has('execution') || allDomains.has('implementation')).toBe(true);
+    expect(allDomains.has('code-review') || allDomains.has('quality-assurance')).toBe(true);
+    expect(allDomains.has('knowledge-management') || allDomains.has('learning')).toBe(true);
+  });
+
+  it('should have review playbooks covering all specialized dimensions', () => {
+    const reviewPlaybooks = allPlaybooks.filter(
+      p => p.name.includes('review') || p.applicability.domains.includes('code-review'),
+    );
+
+    const reviewNames = reviewPlaybooks.map(p => p.name);
+
+    // CEP's review dimensions
+    expect(reviewNames).toContain('correctness-review');
+    expect(reviewNames).toContain('testing-review');
+    expect(reviewNames).toContain('maintainability-review');
+    expect(reviewNames).toContain('security-review');
+    expect(reviewNames).toContain('performance-review');
+    expect(reviewNames).toContain('review-orchestration');
+  });
+
+  it('should have planning playbooks covering analysis and feasibility', () => {
+    const planPlaybooks = allPlaybooks.filter(
+      p => p.name.includes('plan') || p.applicability.domains.includes('planning'),
+    );
+    expect(planPlaybooks.length).toBeGreaterThanOrEqual(2);
+  });
+
+  it('should have execution and debugging playbooks', () => {
+    const execNames = allPlaybooks.map(p => p.name);
+    expect(execNames).toContain('systematic-execution');
+    expect(execNames).toContain('bug-reproduction');
+    expect(execNames).toContain('error-recovery-patterns');
+  });
+});
+
+// ============================================================================
+// 4. Non-redundancy
+// ============================================================================
+
+describe('Non-redundancy', () => {
+  it('no two playbooks should have strategy similarity > 0.85', () => {
+    const redundantPairs: Array<{ a: string; b: string; sim: number }> = [];
+
+    for (let i = 0; i < allPlaybooks.length; i++) {
+      for (let j = i + 1; j < allPlaybooks.length; j++) {
+        const sim = textSimilarity(
+          allPlaybooks[i].guidance.strategy,
+          allPlaybooks[j].guidance.strategy,
+        );
+        if (sim > 0.85) {
+          redundantPairs.push({
+            a: allPlaybooks[i].name,
+            b: allPlaybooks[j].name,
+            sim,
+          });
+        }
+      }
+    }
+
+    expect(redundantPairs).toEqual([]);
+  });
+
+  it('no two playbooks should have identical situations', () => {
+    for (let i = 0; i < allPlaybooks.length; i++) {
+      for (let j = i + 1; j < allPlaybooks.length; j++) {
+        const sitsA = new Set(allPlaybooks[i].applicability.situations);
+        const sitsB = new Set(allPlaybooks[j].applicability.situations);
+        const overlap = [...sitsA].filter(s => sitsB.has(s));
+        expect(overlap).toEqual([]);
+      }
+    }
+  });
+});
+
+// ============================================================================
+// 5. Retrieval effectiveness
+// ============================================================================
+
+describe('Retrieval effectiveness', () => {
+  // For each expected query, the top result should contain the expected playbook
+  const retrievalTests: Array<{ query: string; expectedTop: string; desc: string }> = [
+    {
+      query: 'code review before PR',
+      expectedTop: 'review-orchestration',
+      desc: 'code review → review-orchestration',
+    },
+    {
+      query: 'check for security vulnerabilities in code',
+      expectedTop: 'security-review',
+      desc: 'security check → security-review',
+    },
+    {
+      query: 'optimize slow database queries',
+      expectedTop: 'performance-review',
+      desc: 'perf optimization → performance-review',
+    },
+    {
+      query: 'create implementation plan for feature',
+      expectedTop: 'implementation-planning',
+      desc: 'feature planning → implementation-planning',
+    },
+    {
+      query: 'reproduce a bug from issue report',
+      expectedTop: 'bug-reproduction',
+      desc: 'bug repro → bug-reproduction',
+    },
+    {
+      query: 'deploy and verify production release',
+      expectedTop: 'deployment-verification',
+      desc: 'deploy verify → deployment-verification',
+    },
+    {
+      query: 'gather requirements and brainstorm approach',
+      expectedTop: 'brainstorm-requirements',
+      desc: 'requirements → brainstorm-requirements',
+    },
+    {
+      query: 'document lessons learned from solved problem',
+      expectedTop: 'knowledge-compounding',
+      desc: 'learnings → knowledge-compounding',
+    },
+    {
+      query: 'handle errors and recover gracefully',
+      expectedTop: 'error-recovery-patterns',
+      desc: 'error recovery → error-recovery-patterns',
+    },
+    {
+      query: 'data migration safety and rollback',
+      expectedTop: 'data-migration-safety',
+      desc: 'data migration → data-migration-safety',
+    },
+  ];
+
+  it.each(retrievalTests)(
+    'should retrieve $desc',
+    async ({ query, expectedTop }) => {
+      const matches = await library.findMatching(query, { k: 5 });
+
+      expect(matches.length).toBeGreaterThan(0);
+
+      // Expected playbook should be in top 3 results
+      const top3Names = matches.slice(0, 3).map(m => m.playbook.name);
+      expect(top3Names).toContain(expectedTop);
+    },
+  );
+
+  it('review sub-types should rank higher than orchestrator for specific queries', async () => {
+    // When asking about "testing coverage gaps", testing-review should outrank review-orchestration
+    const matches = await library.findMatching('testing coverage gaps and missing tests', { k: 5 });
+    const names = matches.map(m => m.playbook.name);
+    const testingIdx = names.indexOf('testing-review');
+    const orchIdx = names.indexOf('review-orchestration');
+
+    if (testingIdx !== -1 && orchIdx !== -1) {
+      expect(testingIdx).toBeLessThan(orchIdx);
+    } else {
+      // At minimum, testing-review should be in results
+      expect(names).toContain('testing-review');
+    }
+  });
+
+  it('anti-patterns should exclude inappropriate playbooks', async () => {
+    // review-orchestration has anti-pattern: "Reviewing trivial formatting-only changes"
+    // Use an exact-enough query to trigger the 0.7 similarity threshold
+    const matches = await library.findMatching(
+      'Reviewing trivial formatting-only changes',
+      { k: 5, excludeAntiPatterns: true },
+    );
+
+    const names = matches.map(m => m.playbook.name);
+    // review-orchestration should NOT appear (anti-pattern match at ≥0.7 similarity)
+    expect(names).not.toContain('review-orchestration');
+  });
+});
+
+// ============================================================================
+// 6. JSON file integrity
+// ============================================================================
+
+describe('JSON file integrity', () => {
+  it('every .json file in seed pack should parse without error', async () => {
+    const files = await readdir(BUNDLED_SEED_PACK_DIR);
+    const jsonFiles = files.filter(f => f.endsWith('.json'));
+
+    expect(jsonFiles.length).toBe(25);
+
+    for (const file of jsonFiles) {
+      const content = await readFile(join(BUNDLED_SEED_PACK_DIR, file), 'utf-8');
+      expect(() => JSON.parse(content)).not.toThrow();
+    }
+  });
+
+  it('file names should match playbook names', async () => {
+    const files = await readdir(BUNDLED_SEED_PACK_DIR);
+    const jsonFiles = files.filter(f => f.endsWith('.json'));
+
+    for (const file of jsonFiles) {
+      const content = await readFile(join(BUNDLED_SEED_PACK_DIR, file), 'utf-8');
+      const parsed = JSON.parse(content);
+      const expectedName = file.replace('.json', '');
+      expect(parsed.name).toBe(expectedName);
+    }
+  });
+
+  it('all playbooks should have consistent curatedBy field', async () => {
+    const files = await readdir(BUNDLED_SEED_PACK_DIR);
+    const jsonFiles = files.filter(f => f.endsWith('.json'));
+
+    for (const file of jsonFiles) {
+      const content = await readFile(join(BUNDLED_SEED_PACK_DIR, file), 'utf-8');
+      const parsed = JSON.parse(content);
+      expect(parsed.curatedBy).toBe('compound-engineering');
+    }
+  });
+
+  it('confidence values should be within expected range', async () => {
+    const files = await readdir(BUNDLED_SEED_PACK_DIR);
+    const jsonFiles = files.filter(f => f.endsWith('.json'));
+
+    for (const file of jsonFiles) {
+      const content = await readFile(join(BUNDLED_SEED_PACK_DIR, file), 'utf-8');
+      const parsed = JSON.parse(content);
+      expect(parsed.confidence).toBeGreaterThanOrEqual(0.75);
+      expect(parsed.confidence).toBeLessThanOrEqual(0.90);
+    }
+  });
+
+  it('complexity values should be valid enum values', async () => {
+    const validComplexities = ['simple', 'moderate', 'complex'];
+    const files = await readdir(BUNDLED_SEED_PACK_DIR);
+    const jsonFiles = files.filter(f => f.endsWith('.json'));
+
+    for (const file of jsonFiles) {
+      const content = await readFile(join(BUNDLED_SEED_PACK_DIR, file), 'utf-8');
+      const parsed = JSON.parse(content);
+      if (parsed.complexity) {
+        expect(validComplexities).toContain(parsed.complexity);
+      }
+    }
+  });
+});