cognitive-core 0.0.2 → 0.1.2

package/tests/memory/provenance.test.ts

@@ -0,0 +1,639 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { PlaybookLibrary, createPlaybookLibrary } from '../../src/memory/playbook.js';
+import { loadCuratedPlaybooks, type CuratedPlaybookFile } from '../../src/memory/curated-loader.js';
+import type { Playbook, PlaybookProvenance } from '../../src/types/playbook.js';
+import { createPlaybook } from '../../src/types/playbook.js';
+import { mkdtemp, rm, mkdir, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+
+describe('Playbook Provenance', () => {
+  let tempDir: string;
+  let library: PlaybookLibrary;
+
+  beforeEach(async () => {
+    tempDir = await mkdtemp(join(tmpdir(), 'atlas-prov-test-'));
+    library = createPlaybookLibrary(tempDir);
+    await library.init();
+  });
+
+  afterEach(async () => {
+    await library.close();
+    await rm(tempDir, { recursive: true, force: true });
+  });
+
+  function createSamplePlaybook(overrides?: Partial<Playbook>): Playbook {
+    return createPlaybook({
+      name: `playbook-${Date.now()}-${Math.random().toString(36).slice(2, 6)}`,
+      applicability: {
+        situations: ['Test situation'],
+        triggers: [],
+        antiPatterns: [],
+        domains: ['test'],
+      },
+      guidance: {
+        strategy: 'Test strategy',
+        tactics: ['Step 1'],
+      },
+      ...overrides,
+    });
+  }
+
+  describe('createPlaybook default provenance', () => {
+    it('should default to extracted origin', () => {
+      const playbook = createPlaybook({
+        name: 'test-playbook',
+        applicability: {
+          situations: ['Test'],
+          triggers: [],
+          antiPatterns: [],
+          domains: ['test'],
+        },
+        guidance: {
+          strategy: 'Test',
+          tactics: [],
+        },
+      });
+
+      expect(playbook.provenance).toBeDefined();
+      expect(playbook.provenance.origin).toBe('extracted');
+      expect(playbook.provenance.recordedAt).toBeInstanceOf(Date);
+    });
+
+    it('should accept custom provenance', () => {
+      const provenance: PlaybookProvenance = {
+        origin: 'curated',
+        sourceFile: 'playbooks/test.json',
+        curatedBy: 'test-user',
+        recordedAt: new Date(),
+      };
+
+      const playbook = createPlaybook({
+        name: 'curated-playbook',
+        applicability: {
+          situations: ['Test'],
+          triggers: [],
+          antiPatterns: [],
+          domains: ['test'],
+        },
+        guidance: {
+          strategy: 'Test',
+          tactics: [],
+        },
+        provenance,
+      });
+
+      expect(playbook.provenance.origin).toBe('curated');
+      expect(playbook.provenance.sourceFile).toBe('playbooks/test.json');
+      expect(playbook.provenance.curatedBy).toBe('test-user');
+    });
+
+    it('should accept imported provenance with importedFrom', () => {
+      const playbook = createPlaybook({
+        name: 'imported-playbook',
+        applicability: {
+          situations: ['Test'],
+          triggers: [],
+          antiPatterns: [],
+          domains: ['test'],
+        },
+        guidance: { strategy: 'Test', tactics: [] },
+        provenance: {
+          origin: 'imported',
+          importedFrom: 'community-pack-v2',
+          recordedAt: new Date(),
+        },
+      });
+
+      expect(playbook.provenance.origin).toBe('imported');
+      expect(playbook.provenance.importedFrom).toBe('community-pack-v2');
+      expect(playbook.provenance.sourceFile).toBeUndefined();
+    });
+  });
+
+  describe('provenance preserved through evolution', () => {
+    it('should preserve provenance after recordSuccess', async () => {
+      const playbook = createSamplePlaybook({
+        name: 'curated-evolving',
+        provenance: { origin: 'curated', sourceFile: 'test.json', recordedAt: new Date() },
+      });
+      await library.add(playbook);
+
+      await library.recordSuccess(playbook.id, 'traj-1');
+
+      const updated = await library.get(playbook.id);
+      expect(updated!.provenance.origin).toBe('curated');
+      expect(updated!.provenance.sourceFile).toBe('test.json');
+    });
+
+    it('should preserve provenance after recordFailure', async () => {
+      const playbook = createSamplePlaybook({
+        name: 'imported-failing',
+        provenance: { origin: 'imported', importedFrom: 'dash', recordedAt: new Date() },
+      });
+      await library.add(playbook);
+
+      await library.recordFailure(playbook.id, 'traj-1', 'monorepo', 'symlink issue');
+
+      const updated = await library.get(playbook.id);
+      expect(updated!.provenance.origin).toBe('imported');
+      expect(updated!.provenance.importedFrom).toBe('dash');
+    });
+
+    it('should preserve provenance after addRefinement', async () => {
+      const playbook = createSamplePlaybook({
+        name: 'manual-refined',
+        provenance: { origin: 'manual', recordedAt: new Date() },
+      });
+      await library.add(playbook);
+
+      await library.addRefinement(playbook.id, 'ESM projects', 'Add .js extensions', 'failure');
+
+      const updated = await library.get(playbook.id);
+      expect(updated!.provenance.origin).toBe('manual');
+      expect(updated!.evolution.refinements).toHaveLength(1);
+    });
+
+    it('should preserve provenance after evolve', async () => {
+      const playbook = createSamplePlaybook({
+        name: 'curated-evolved',
+        provenance: { origin: 'curated', sourceFile: 'debug.json', curatedBy: 'alice', recordedAt: new Date() },
+      });
+      await library.add(playbook);
+
+      await library.evolve(playbook.id, { strategy: 'Updated strategy' });
+
+      const updated = await library.get(playbook.id);
+      expect(updated!.provenance.origin).toBe('curated');
+      expect(updated!.provenance.curatedBy).toBe('alice');
+      expect(updated!.guidance.strategy).toBe('Updated strategy');
+    });
+  });
+
+  describe('getByOrigin', () => {
+    it('should filter playbooks by origin', async () => {
+      await library.add(createSamplePlaybook({
+        name: 'extracted-1',
+        provenance: { origin: 'extracted', recordedAt: new Date() },
+      }));
+      await library.add(createSamplePlaybook({
+        name: 'curated-1',
+        provenance: { origin: 'curated', sourceFile: 'test.json', recordedAt: new Date() },
+      }));
+      await library.add(createSamplePlaybook({
+        name: 'curated-2',
+        provenance: { origin: 'curated', sourceFile: 'test2.json', recordedAt: new Date() },
+      }));
+      await library.add(createSamplePlaybook({
+        name: 'imported-1',
+        provenance: { origin: 'imported', importedFrom: 'dash', recordedAt: new Date() },
+      }));
+
+      const curated = await library.getByOrigin('curated');
+      expect(curated).toHaveLength(2);
+      expect(curated.every(p => p.provenance.origin === 'curated')).toBe(true);
+
+      const extracted = await library.getByOrigin('extracted');
+      expect(extracted).toHaveLength(1);
+
+      const imported = await library.getByOrigin('imported');
+      expect(imported).toHaveLength(1);
+      expect(imported[0].provenance.importedFrom).toBe('dash');
+    });
+
+    it('should return empty array when no playbooks match origin', async () => {
+      await library.add(createSamplePlaybook({
+        name: 'extracted-only',
+        provenance: { origin: 'extracted', recordedAt: new Date() },
+      }));
+
+      const curated = await library.getByOrigin('curated');
+      expect(curated).toHaveLength(0);
+
+      const manual = await library.getByOrigin('manual');
+      expect(manual).toHaveLength(0);
+    });
+  });
+
+  describe('deleteByOrigin', () => {
+    it('should delete only playbooks with the specified origin', async () => {
+      await library.add(createSamplePlaybook({
+        name: 'extracted-1',
+        provenance: { origin: 'extracted', recordedAt: new Date() },
+      }));
+      await library.add(createSamplePlaybook({
+        name: 'curated-1',
+        provenance: { origin: 'curated', sourceFile: 'test.json', recordedAt: new Date() },
+      }));
+      await library.add(createSamplePlaybook({
+        name: 'curated-2',
+        provenance: { origin: 'curated', sourceFile: 'test2.json', recordedAt: new Date() },
+      }));
+
+      const deleted = await library.deleteByOrigin('curated');
+      expect(deleted).toBe(2);
+
+      const remaining = await library.getAll();
+      expect(remaining).toHaveLength(1);
+      expect(remaining[0].provenance.origin).toBe('extracted');
+    });
+
+    it('should return 0 when no playbooks match the origin', async () => {
+      await library.add(createSamplePlaybook({
+        name: 'extracted-1',
+        provenance: { origin: 'extracted', recordedAt: new Date() },
+      }));
+
+      const deleted = await library.deleteByOrigin('imported');
+      expect(deleted).toBe(0);
+    });
+
+    it('should leave other origins untouched when deleting one', async () => {
+      await library.add(createSamplePlaybook({
+        name: 'e1',
+        provenance: { origin: 'extracted', recordedAt: new Date() },
+      }));
+      await library.add(createSamplePlaybook({
+        name: 'c1',
+        provenance: { origin: 'curated', recordedAt: new Date() },
+      }));
+      await library.add(createSamplePlaybook({
+        name: 'i1',
+        provenance: { origin: 'imported', importedFrom: 'x', recordedAt: new Date() },
+      }));
+      await library.add(createSamplePlaybook({
+        name: 'm1',
+        provenance: { origin: 'manual', recordedAt: new Date() },
+      }));
+
+      await library.deleteByOrigin('extracted');
+
+      expect(await library.count()).toBe(3);
+      expect(await library.getByOrigin('curated')).toHaveLength(1);
+      expect(await library.getByOrigin('imported')).toHaveLength(1);
+      expect(await library.getByOrigin('manual')).toHaveLength(1);
+    });
+  });
+
+  describe('playbooks without provenance', () => {
+    it('should not appear in getByOrigin results for any origin', async () => {
+      // Manually construct a playbook without provenance (pre-existing data scenario)
+      const noProvPlaybook = createSamplePlaybook({ name: 'no-prov' });
+      delete (noProvPlaybook as Record<string, unknown>).provenance;
+      await library.add(noProvPlaybook);
+
+      const extracted = await library.getByOrigin('extracted');
+      const curated = await library.getByOrigin('curated');
+      const imported = await library.getByOrigin('imported');
+      const manual = await library.getByOrigin('manual');
+
+      expect(extracted).toHaveLength(0);
+      expect(curated).toHaveLength(0);
+      expect(imported).toHaveLength(0);
+      expect(manual).toHaveLength(0);
+    });
+
+    it('should count as extracted in getProvenanceSummary', async () => {
+      const noProvPlaybook = createSamplePlaybook({ name: 'no-prov-summary' });
+      delete (noProvPlaybook as Record<string, unknown>).provenance;
+      await library.add(noProvPlaybook);
+
+      const summary = await library.getProvenanceSummary();
+      // Falls back to 'extracted' when provenance is undefined
+      expect(summary.extracted).toBe(1);
+    });
+
+    it('should not be deleted by deleteByOrigin for any origin', async () => {
+      const noProvPlaybook = createSamplePlaybook({ name: 'no-prov-delete' });
+      delete (noProvPlaybook as Record<string, unknown>).provenance;
+      await library.add(noProvPlaybook);
+
+      await library.deleteByOrigin('extracted');
+      await library.deleteByOrigin('curated');
+
+      expect(await library.count()).toBe(1);
+    });
+
+    it('should coexist with playbooks that have provenance', async () => {
+      const noProvPlaybook = createSamplePlaybook({ name: 'no-prov-coexist' });
+      delete (noProvPlaybook as Record<string, unknown>).provenance;
+      await library.add(noProvPlaybook);
+
+      await library.add(createSamplePlaybook({
+        name: 'with-prov',
+        provenance: { origin: 'curated', sourceFile: 'test.json', recordedAt: new Date() },
+      }));
+
+      const all = await library.getAll();
+      expect(all).toHaveLength(2);
+
+      const curated = await library.getByOrigin('curated');
+      expect(curated).toHaveLength(1);
+      expect(curated[0].name).toBe('with-prov');
+    });
+  });
+
+  describe('getProvenanceSummary', () => {
+    it('should return counts by origin', async () => {
+      await library.add(createSamplePlaybook({
+        name: 'e1',
+        provenance: { origin: 'extracted', recordedAt: new Date() },
+      }));
+      await library.add(createSamplePlaybook({
+        name: 'e2',
+        provenance: { origin: 'extracted', recordedAt: new Date() },
+      }));
+      await library.add(createSamplePlaybook({
+        name: 'c1',
+        provenance: { origin: 'curated', recordedAt: new Date() },
+      }));
+      await library.add(createSamplePlaybook({
+        name: 'm1',
+        provenance: { origin: 'manual', recordedAt: new Date() },
+      }));
+
+      const summary = await library.getProvenanceSummary();
+      expect(summary.extracted).toBe(2);
+      expect(summary.curated).toBe(1);
+      expect(summary.manual).toBe(1);
+      expect(summary.imported).toBe(0);
+    });
+
+    it('should return all zeros for empty library', async () => {
+      const summary = await library.getProvenanceSummary();
+      expect(summary.extracted).toBe(0);
+      expect(summary.curated).toBe(0);
+      expect(summary.imported).toBe(0);
+      expect(summary.manual).toBe(0);
+    });
+  });
+});
+
+describe('Curated Playbook Loader', () => {
+  let tempDir: string;
+  let curatedDir: string;
+  let library: PlaybookLibrary;
+
+  beforeEach(async () => {
+    tempDir = await mkdtemp(join(tmpdir(), 'atlas-curated-test-'));
+    curatedDir = join(tempDir, 'curated');
+    await mkdir(curatedDir, { recursive: true });
+    library = createPlaybookLibrary(tempDir);
+    await library.init();
+  });
+
+  afterEach(async () => {
+    await library.close();
+    await rm(tempDir, { recursive: true, force: true });
+  });
+
+  async function writeCuratedFile(filename: string, content: CuratedPlaybookFile): Promise<void> {
+    await writeFile(join(curatedDir, filename), JSON.stringify(content, null, 2));
+  }
+
+  it('should load curated playbooks from JSON files', async () => {
+    await writeCuratedFile('fix-ts-errors.json', {
+      name: 'fix-typescript-errors',
+      applicability: {
+        situations: ['Debugging TypeScript type errors'],
+        triggers: ['TS2322', 'TS2307'],
+        domains: ['typescript'],
+      },
+      guidance: {
+        strategy: 'Read error, trace to source, fix type mismatch',
+        tactics: ['Read full error message', 'Check type declarations'],
+      },
+      curatedBy: 'test-author',
+    });
+
+    await writeCuratedFile('optimize-react.json', {
+      name: 'optimize-react-renders',
+      applicability: {
+        situations: ['React component re-renders too often'],
+        domains: ['react'],
+      },
+      guidance: {
+        strategy: 'Profile renders, memoize expensive computations',
+        tactics: ['Use React.memo', 'Use useMemo for expensive calculations'],
+      },
+    });
+
+    const result = await loadCuratedPlaybooks(curatedDir, library);
+
+    expect(result.loaded).toBe(2);
+    expect(result.skipped).toBe(0);
+    expect(result.errors).toHaveLength(0);
+
+    const all = await library.getAll();
+    expect(all).toHaveLength(2);
+
+    const tsPlaybook = await library.getByName('fix-typescript-errors');
+    expect(tsPlaybook).toBeDefined();
+    expect(tsPlaybook!.provenance.origin).toBe('curated');
+    expect(tsPlaybook!.provenance.curatedBy).toBe('test-author');
+    expect(tsPlaybook!.confidence).toBe(0.7); // Curated default
+  });
+
+  it('should set sourceFile on loaded curated playbooks', async () => {
+    await writeCuratedFile('my-playbook.json', {
+      name: 'sourced-playbook',
+      applicability: {
+        situations: ['Test'],
+        domains: ['test'],
+      },
+      guidance: {
+        strategy: 'Test strategy',
+        tactics: ['Step 1'],
+      },
+    });
+
+    await loadCuratedPlaybooks(curatedDir, library);
+
+    const playbook = await library.getByName('sourced-playbook');
+    expect(playbook).toBeDefined();
+    expect(playbook!.provenance.sourceFile).toBeDefined();
+    expect(playbook!.provenance.sourceFile).toContain('my-playbook.json');
+  });
+
+  it('should load optional fields from curated files', async () => {
+    await writeCuratedFile('full.json', {
+      name: 'full-playbook',
+      applicability: {
+        situations: ['Full test'],
+        triggers: ['trigger-1', 'trigger-2'],
+        antiPatterns: ['not-this'],
+        domains: ['test'],
+      },
+      guidance: {
+        strategy: 'Full strategy',
+        tactics: ['Tactic A', 'Tactic B'],
+        steps: ['Step 1', 'Step 2', 'Step 3'],
+        codeExample: 'const x = 1;',
+      },
+      verification: {
+        successIndicators: ['Tests pass'],
+        failureIndicators: ['Build fails'],
+        rollbackStrategy: 'git revert',
+      },
+      confidence: 0.95,
+      complexity: 'complex',
+      estimatedEffort: 10,
+      curatedBy: 'expert',
+    });
+
+    await loadCuratedPlaybooks(curatedDir, library);
+
+    const playbook = await library.getByName('full-playbook');
+    expect(playbook).toBeDefined();
+    expect(playbook!.applicability.triggers).toEqual(['trigger-1', 'trigger-2']);
+    expect(playbook!.applicability.antiPatterns).toEqual(['not-this']);
+    expect(playbook!.guidance.steps).toEqual(['Step 1', 'Step 2', 'Step 3']);
+    expect(playbook!.guidance.codeExample).toBe('const x = 1;');
+    expect(playbook!.verification.successIndicators).toEqual(['Tests pass']);
+    expect(playbook!.verification.failureIndicators).toEqual(['Build fails']);
+    expect(playbook!.verification.rollbackStrategy).toBe('git revert');
+    expect(playbook!.confidence).toBe(0.95);
+    expect(playbook!.complexity).toBe('complex');
+    expect(playbook!.estimatedEffort).toBe(10);
+  });
+
+  it('should skip already-existing playbooks by name', async () => {
+    await writeCuratedFile('existing.json', {
+      name: 'already-exists',
+      applicability: {
+        situations: ['Test'],
+        domains: ['test'],
+      },
+      guidance: {
+        strategy: 'Original strategy',
+        tactics: ['Original tactic'],
+      },
+    });
+
+    // Load once
+    await loadCuratedPlaybooks(curatedDir, library);
+    const firstLoad = await library.getByName('already-exists');
+    expect(firstLoad).toBeDefined();
+
+    // Load again — should skip
+    const result = await loadCuratedPlaybooks(curatedDir, library);
+    expect(result.skipped).toBe(1);
+    expect(result.loaded).toBe(0);
+  });
+
+  it('should recreate curated playbooks when recreate=true', async () => {
+    await writeCuratedFile('recreatable.json', {
+      name: 'recreatable-playbook',
+      applicability: {
+        situations: ['Test situation'],
+        domains: ['test'],
+      },
+      guidance: {
+        strategy: 'V1 strategy',
+        tactics: ['V1 tactic'],
+      },
+    });
+
+    // Load initial
+    await loadCuratedPlaybooks(curatedDir, library);
+
+    // Add an extracted playbook too
+    const extracted = createPlaybook({
+      name: 'extracted-playbook',
+      applicability: { situations: ['Extracted'], triggers: [], antiPatterns: [], domains: ['test'] },
+      guidance: { strategy: 'Extracted', tactics: [] },
+      provenance: { origin: 'extracted', recordedAt: new Date() },
+    });
+    await library.add(extracted);
+
+    // Update the curated file
+    await writeCuratedFile('recreatable.json', {
+      name: 'recreatable-playbook',
+      applicability: {
+        situations: ['Test situation - updated'],
+        domains: ['test'],
+      },
+      guidance: {
+        strategy: 'V2 strategy',
+        tactics: ['V2 tactic'],
+      },
+    });
+
+    // Recreate
+    const result = await loadCuratedPlaybooks(curatedDir, library, { recreate: true });
+    expect(result.recreated).toBe(true);
+    expect(result.loaded).toBe(1);
+
+    // Curated was rebuilt
+    const reloaded = await library.getByName('recreatable-playbook');
+    expect(reloaded).toBeDefined();
+    expect(reloaded!.guidance.strategy).toBe('V2 strategy');
+
+    // Extracted playbook is untouched
+    const still = await library.getByOrigin('extracted');
+    expect(still).toHaveLength(1);
+    expect(still[0].name).toBe('extracted-playbook');
+  });
+
+  it('should ignore non-JSON files', async () => {
+    await writeFile(join(curatedDir, 'readme.md'), '# Not a playbook');
+    await writeFile(join(curatedDir, 'notes.txt'), 'some notes');
+    await writeCuratedFile('real-playbook.json', {
+      name: 'real-one',
+      applicability: { situations: ['Test'], domains: ['test'] },
+      guidance: { strategy: 'Test', tactics: ['Do it'] },
+    });
+
+    const result = await loadCuratedPlaybooks(curatedDir, library);
+
+    expect(result.loaded).toBe(1);
+    expect(result.errors).toHaveLength(0);
+    expect(await library.count()).toBe(1);
+  });
+
+  it('should handle invalid JSON files gracefully', async () => {
+    await writeFile(join(curatedDir, 'invalid.json'), '{ invalid json }}}');
+
+    const result = await loadCuratedPlaybooks(curatedDir, library);
+    expect(result.errors).toHaveLength(1);
+    expect(result.errors[0].file).toBe('invalid.json');
+    expect(result.loaded).toBe(0);
+  });
+
+  it('should handle missing required fields', async () => {
+    await writeCuratedFile('incomplete.json', {
+      name: 'incomplete',
+    } as unknown as CuratedPlaybookFile);
+
+    const result = await loadCuratedPlaybooks(curatedDir, library);
+    expect(result.errors).toHaveLength(1);
+    expect(result.errors[0].error).toContain('Missing required fields');
+  });
+
+  it('should return empty result for non-existent directory', async () => {
+    const result = await loadCuratedPlaybooks('/nonexistent/path', library);
+    expect(result.loaded).toBe(0);
+    expect(result.skipped).toBe(0);
+    expect(result.errors).toHaveLength(0);
+  });
+
+  it('should load valid files even when some are invalid', async () => {
+    await writeCuratedFile('good.json', {
+      name: 'good-playbook',
+      applicability: { situations: ['Test'], domains: ['test'] },
+      guidance: { strategy: 'Good', tactics: ['Do it'] },
+    });
+    await writeFile(join(curatedDir, 'bad.json'), '{ broken }');
+    await writeCuratedFile('also-good.json', {
+      name: 'also-good-playbook',
+      applicability: { situations: ['Another test'], domains: ['test'] },
+      guidance: { strategy: 'Also good', tactics: ['Do it too'] },
+    });
+
+    const result = await loadCuratedPlaybooks(curatedDir, library);
+
+    expect(result.loaded).toBe(2);
+    expect(result.errors).toHaveLength(1);
+    expect(await library.count()).toBe(2);
+  });
+});