cognitive-core 0.0.1 → 0.1.0

package/tests/learning/pipeline.test.ts

@@ -0,0 +1,176 @@
+import { describe, it, expect, beforeEach, vi } from 'vitest';
+import { LearningPipeline, createLearningPipeline } from '../../src/learning/pipeline.js';
+import { MemorySystem } from '../../src/memory/system.js';
+import { createTrajectory, createStep, createTask, successOutcome, failureOutcome } from '../../src/types/index.js';
+import * as fs from 'node:fs/promises';
+import * as path from 'node:path';
+
+describe('LearningPipeline', () => {
+  let pipeline: LearningPipeline;
+  let memory: MemorySystem;
+  let testDir: string;
+
+  beforeEach(async () => {
+    testDir = path.join(process.cwd(), '.test-pipeline-' + Date.now());
+    await fs.mkdir(testDir, { recursive: true });
+    memory = new MemorySystem(testDir);
+    await memory.init();
+    pipeline = createLearningPipeline(memory, {
+      minTrajectories: 3,
+      minSuccessRate: 0.5,
+      maxExperiences: 100,
+    });
+  });
+
+  afterEach(async () => {
+    await memory.close();
+    await fs.rm(testDir, { recursive: true, force: true });
+  });
+
+  const createTestTrajectory = (success: boolean, description: string) => {
+    return createTrajectory({
+      task: createTask({ domain: 'test', description }),
+      steps: [
+        createStep({
+          thought: 'Analyzing task',
+          action: 'read file',
+          observation: 'file contents',
+        }),
+        createStep({
+          thought: 'Applying solution',
+          action: 'edit file',
+          observation: success ? 'success' : 'error: failed',
+        }),
+      ],
+      outcome: success
+        ? successOutcome({ result: 'completed' })
+        : failureOutcome('Task failed'),
+      agentId: 'test-agent',
+    });
+  };
+
+  describe('processTrajectory', () => {
+    it('should process and store trajectory', async () => {
+      const trajectory = createTestTrajectory(true, 'Test task 1');
+      const result = await pipeline.processTrajectory(trajectory);
+
+      expect(result.stored).toBe(true);
+      expect(result.trajectoryId).toBe(trajectory.id);
+      expect(result.analysis).toBeDefined();
+      expect(result.analysis.success).toBe(true);
+    });
+
+    it('should accumulate trajectories for batch learning', async () => {
+      expect(pipeline.getAccumulatedCount()).toBe(0);
+
+      await pipeline.processTrajectory(createTestTrajectory(true, 'Task 1'));
+      expect(pipeline.getAccumulatedCount()).toBe(1);
+
+      await pipeline.processTrajectory(createTestTrajectory(true, 'Task 2'));
+      expect(pipeline.getAccumulatedCount()).toBe(2);
+    });
+
+    it('should analyze trajectory for key steps', async () => {
+      const trajectory = createTestTrajectory(true, 'Test analysis');
+      const result = await pipeline.processTrajectory(trajectory);
+
+      expect(result.analysis.keySteps).toBeDefined();
+      expect(Array.isArray(result.analysis.keySteps)).toBe(true);
+      expect(result.analysis.stepAttribution).toBeDefined();
+      expect(result.analysis.stepAttribution.length).toBe(trajectory.steps.length);
+    });
+  });
+
+  describe('shouldRunBatch', () => {
+    it('should return false when below minTrajectories', () => {
+      expect(pipeline.shouldRunBatch()).toBe(false);
+    });
+
+    it('should return true when above minTrajectories with sufficient success rate', async () => {
+      await pipeline.processTrajectory(createTestTrajectory(true, 'Task 1'));
+      await pipeline.processTrajectory(createTestTrajectory(true, 'Task 2'));
+      expect(pipeline.shouldRunBatch()).toBe(false); // Still below 3
+
+      await pipeline.processTrajectory(createTestTrajectory(true, 'Task 3'));
+      expect(pipeline.shouldRunBatch()).toBe(true); // Now at 3
+    });
+
+    it('should return false when success rate is too low', async () => {
+      await pipeline.processTrajectory(createTestTrajectory(false, 'Task 1'));
+      await pipeline.processTrajectory(createTestTrajectory(false, 'Task 2'));
+      await pipeline.processTrajectory(createTestTrajectory(false, 'Task 3'));
+
+      // 0% success rate, below 50% threshold
+      expect(pipeline.shouldRunBatch()).toBe(false);
+    });
+  });
+
+  describe('runBatchLearning', () => {
+    it('should return empty result when no trajectories accumulated', async () => {
+      const result = await pipeline.runBatchLearning();
+
+      expect(result.trajectoriesProcessed).toBe(0);
+      expect(result.playbooksExtracted).toBe(0);
+      expect(result.experiencesPruned).toBe(0);
+      expect(result.successRate).toBe(0);
+    });
+
+    it('should process accumulated trajectories', async () => {
+      await pipeline.processTrajectory(createTestTrajectory(true, 'Task 1'));
+      await pipeline.processTrajectory(createTestTrajectory(true, 'Task 2'));
+      await pipeline.processTrajectory(createTestTrajectory(false, 'Task 3'));
+
+      const result = await pipeline.runBatchLearning();
+
+      expect(result.trajectoriesProcessed).toBe(3);
+      expect(result.successRate).toBeCloseTo(2/3, 2);
+    });
+
+    it('should clear accumulated after batch', async () => {
+      await pipeline.processTrajectory(createTestTrajectory(true, 'Task 1'));
+      await pipeline.processTrajectory(createTestTrajectory(true, 'Task 2'));
+
+      expect(pipeline.getAccumulatedCount()).toBe(2);
+      await pipeline.runBatchLearning();
+      expect(pipeline.getAccumulatedCount()).toBe(0);
+    });
+  });
+
+  describe('clearAccumulated', () => {
+    it('should clear all accumulated trajectories', async () => {
+      await pipeline.processTrajectory(createTestTrajectory(true, 'Task 1'));
+      await pipeline.processTrajectory(createTestTrajectory(true, 'Task 2'));
+
+      expect(pipeline.getAccumulatedCount()).toBe(2);
+      pipeline.clearAccumulated();
+      expect(pipeline.getAccumulatedCount()).toBe(0);
+    });
+  });
+
+  describe('configuration', () => {
+    it('should use configurable maxExperiences for pruning', async () => {
+      // The pipeline was created with maxExperiences: 100
+      // This test verifies the config is passed correctly
+      const configuredPipeline = createLearningPipeline(memory, {
+        maxExperiences: 50,
+      });
+
+      // Process a trajectory to ensure it uses the config
+      const trajectory = createTestTrajectory(true, 'Config test');
+      const result = await configuredPipeline.processTrajectory(trajectory);
+      expect(result.stored).toBe(true);
+    });
+
+    it('should update configuration', async () => {
+      pipeline.updateConfig({ minTrajectories: 5 });
+
+      // Add 3 trajectories (was enough before update)
+      await pipeline.processTrajectory(createTestTrajectory(true, 'Task 1'));
+      await pipeline.processTrajectory(createTestTrajectory(true, 'Task 2'));
+      await pipeline.processTrajectory(createTestTrajectory(true, 'Task 3'));
+
+      // Should not run batch yet (need 5 now)
+      expect(pipeline.shouldRunBatch()).toBe(false);
+    });
+  });
+});

package/tests/learning/playbook-extractor-provenance.test.ts

@@ -0,0 +1,114 @@
+import { describe, it, expect } from 'vitest';
+import { PlaybookExtractor, createPlaybookExtractor } from '../../src/learning/playbook-extractor.js';
+import { createPlaybook } from '../../src/types/playbook.js';
+import { createTrajectory, createStep, createTask, successOutcome } from '../../src/types/index.js';
+import type { AnalysisResult } from '../../src/learning/analyzer.js';
+
+describe('PlaybookExtractor provenance', () => {
+  function makeTrajectory(description: string, domain: string, actions: string[]) {
+    return createTrajectory({
+      task: createTask({ domain, description }),
+      steps: actions.map((action) =>
+        createStep({
+          thought: `Doing: ${action}`,
+          action,
+          observation: 'done',
+        })
+      ),
+      outcome: successOutcome({ result: 'completed' }),
+      agentId: 'test-agent',
+    });
+  }
+
+  function makeAnalysis(trajectory: ReturnType<typeof makeTrajectory>): AnalysisResult {
+    return {
+      success: true,
+      keySteps: trajectory.steps.map((_, i) => i),
+      stepAttribution: trajectory.steps.map(() => 0.8),
+      errorPatterns: [],
+      abstractable: true,
+      trainingExamples: [],
+    };
+  }
+
+  it('should set origin to extracted on newly created playbooks', async () => {
+    const extractor = createPlaybookExtractor({ minTrajectories: 2 });
+
+    // Create 2 similar trajectories so the pattern meets frequency threshold
+    const t1 = makeTrajectory('Fix build error', 'code', ['read file', 'edit file']);
+    const t2 = makeTrajectory('Fix build error again', 'code', ['read file', 'edit file']);
+
+    const result = await extractor.extract(
+      [t1, t2],
+      [makeAnalysis(t1), makeAnalysis(t2)],
+      [],
+    );
+
+    // Should extract at least one playbook
+    expect(result.new.length).toBeGreaterThanOrEqual(1);
+
+    for (const playbook of result.new) {
+      expect(playbook.provenance).toBeDefined();
+      expect(playbook.provenance.origin).toBe('extracted');
+      expect(playbook.provenance.recordedAt).toBeInstanceOf(Date);
+      // Extracted playbooks should not have sourceFile or curatedBy
+      expect(playbook.provenance.sourceFile).toBeUndefined();
+      expect(playbook.provenance.curatedBy).toBeUndefined();
+    }
+  });
+
+  it('should preserve provenance from primary playbook when merging', async () => {
+    const extractor = createPlaybookExtractor({
+      minTrajectories: 1,
+      mergeThreshold: 0.3, // Low threshold to force merging
+    });
+
+    // Two similar trajectories to get one playbook from consolidation
+    const t1 = makeTrajectory('Debug failing test', 'test', ['search', 'run', 'test']);
+    const t2 = makeTrajectory('Debug failing test', 'test', ['search', 'run', 'test']);
+
+    const result = await extractor.extract(
+      [t1, t2],
+      [makeAnalysis(t1), makeAnalysis(t2)],
+      [],
+    );
+
+    // All playbooks should have extracted provenance
+    for (const playbook of result.new) {
+      expect(playbook.provenance.origin).toBe('extracted');
+    }
+  });
+
+  it('should generate updates for existing playbooks without changing provenance', async () => {
+    const extractor = createPlaybookExtractor({ minTrajectories: 2 });
+
+    const existingPlaybook = createPlaybook({
+      name: 'existing-debug',
+      applicability: {
+        situations: ['read file'],
+        triggers: [],
+        antiPatterns: [],
+        domains: ['code'],
+      },
+      guidance: {
+        strategy: 'Read then fix',
+        tactics: ['read', 'fix'],
+      },
+      provenance: { origin: 'curated', sourceFile: 'debug.json', recordedAt: new Date() },
+    });
+
+    const t1 = makeTrajectory('Read file to debug', 'code', ['read file', 'edit file']);
+    const t2 = makeTrajectory('Read file to debug', 'code', ['read file', 'edit file']);
+
+    const result = await extractor.extract(
+      [t1, t2],
+      [makeAnalysis(t1), makeAnalysis(t2)],
+      [existingPlaybook],
+    );
+
+    // Updates to existing playbooks should reference the existing playbook
+    // The existing playbook's provenance should remain unchanged
+    expect(existingPlaybook.provenance.origin).toBe('curated');
+    expect(existingPlaybook.provenance.sourceFile).toBe('debug.json');
+  });
+});

package/tests/learning/usage-inference.test.ts

@@ -0,0 +1,254 @@
+import { describe, it, expect } from 'vitest';
+import {
+  PlaybookUsageInference,
+  createUsageInference,
+} from '../../src/learning/usage-inference.js';
+import type { Playbook } from '../../src/types/playbook.js';
+import type { Trajectory } from '../../src/types/trajectory.js';
+
+describe('PlaybookUsageInference', () => {
+  let inference: PlaybookUsageInference;
+
+  beforeEach(() => {
+    inference = createUsageInference({
+      minMatchConfidence: 0.3,
+      tacticMatchThreshold: 0.5,
+    });
+  });
+
+  function createSamplePlaybook(overrides?: Partial<Playbook>): Playbook {
+    return {
+      id: `playbook-${Date.now()}-${Math.random().toString(36).slice(2)}`,
+      name: 'debug-error',
+      applicability: {
+        situations: ['Debugging runtime errors'],
+        triggers: ['Error:', 'Exception'],
+        antiPatterns: [],
+        domains: ['code'],
+      },
+      guidance: {
+        strategy: 'Identify the error, trace the cause, fix it',
+        tactics: [
+          'Read the error message carefully',
+          'Check the stack trace',
+          'Identify the root cause',
+          'Apply the fix',
+          'Verify the fix works',
+        ],
+      },
+      verification: {
+        successIndicators: ['Error resolved', 'Tests pass'],
+        failureIndicators: ['Error persists', 'New errors'],
+      },
+      evolution: {
+        version: '1.0.0',
+        createdFrom: [],
+        failures: [],
+        refinements: [],
+        successCount: 0,
+        failureCount: 0,
+      },
+      confidence: 0.8,
+      complexity: 'moderate',
+      estimatedEffort: 3,
+      createdAt: new Date(),
+      updatedAt: new Date(),
+      ...overrides,
+    };
+  }
+
+  function createSampleTrajectory(steps: Array<{
+    thought?: string;
+    action: string;
+    observation?: string;
+  }>, overrides?: Partial<Trajectory>): Trajectory {
+    return {
+      id: `traj-${Date.now()}`,
+      task: {
+        id: 'task-1',
+        description: 'Fix the error in the code',
+        domain: 'code',
+        context: {},
+        metadata: {},
+        createdAt: new Date(),
+      },
+      steps: steps.map((s, i) => ({
+        index: i,
+        thought: s.thought,
+        action: s.action,
+        observation: s.observation,
+        timestamp: new Date(),
+      })),
+      outcome: { success: true, quality: 'good' },
+      metadata: {},
+      startedAt: new Date(),
+      completedAt: new Date(),
+      ...overrides,
+    };
+  }
+
+  describe('inferUsage', () => {
+    it('should detect playbook usage when tactics match', async () => {
+      const playbook = createSamplePlaybook({
+        guidance: {
+          strategy: 'Debug by reading errors',
+          tactics: [
+            'Read the error message',
+            'Check the stack trace',
+            'Fix the issue',
+          ],
+        },
+      });
+
+      const trajectory = createSampleTrajectory([
+        {
+          thought: 'Let me read the error message carefully',
+          action: 'cat error.log',
+          observation: 'TypeError: Cannot read property of undefined',
+        },
+        {
+          thought: 'I should check the stack trace',
+          action: 'cat stacktrace.txt',
+          observation: 'at line 42 in main.ts',
+        },
+        {
+          thought: 'Now I will fix the issue',
+          action: 'edit main.ts',
+          observation: 'File updated',
+        },
+      ]);
+
+      const results = await inference.inferUsage(trajectory, [playbook]);
+
+      expect(results).toHaveLength(1);
+      expect(results[0].wasUsed).toBe(true);
+      expect(results[0].usageConfidence).toBeGreaterThan(0);
+    });
+
+    it('should detect non-usage when tactics do not match', async () => {
+      const playbook = createSamplePlaybook({
+        guidance: {
+          strategy: 'Performance optimization',
+          tactics: [
+            'Profile the application',
+            'Identify bottlenecks',
+            'Optimize hot paths',
+          ],
+        },
+      });
+
+      // Trajectory that does something completely different
+      const trajectory = createSampleTrajectory([
+        {
+          thought: 'Adding a new feature',
+          action: 'create new_feature.ts',
+          observation: 'File created',
+        },
+        {
+          thought: 'Writing the implementation',
+          action: 'edit new_feature.ts',
+          observation: 'Code added',
+        },
+      ]);
+
+      const results = await inference.inferUsage(trajectory, [playbook]);
+
+      expect(results).toHaveLength(1);
+      expect(results[0].wasUsed).toBe(false);
+    });
+
+    it('should infer success based on trajectory outcome', async () => {
+      const playbook = createSamplePlaybook({
+        guidance: {
+          strategy: 'Debug and fix',
+          tactics: ['Read the error', 'Fix the issue', 'Verify the fix'],
+        },
+        verification: {
+          successIndicators: ['Tests pass', 'Build succeeds'],
+          failureIndicators: ['Tests fail', 'Build error'],
+        },
+      });
+
+      // Create trajectory with matching tactics and successful outcome
+      const successfulTrajectory = createSampleTrajectory([
+        {
+          thought: 'Let me read the error carefully',
+          action: 'cat error.log',
+          observation: 'Error found',
+        },
+        {
+          thought: 'Now I will fix the issue',
+          action: 'edit file.ts',
+          observation: 'Fixed',
+        },
+        {
+          thought: 'Let me verify the fix works',
+          action: 'npm test',
+          observation: 'All tests pass',
+        },
+      ], {
+        outcome: { success: true, quality: 'good' },
+      });
+
+      const results = await inference.inferUsage(successfulTrajectory, [playbook]);
+
+      // Should infer success from the trajectory outcome when playbook is used
+      // If wasUsed is true and trajectory succeeded, wasSuccessful should reflect that
+      expect(results).toHaveLength(1);
+      if (results[0].wasUsed) {
+        expect(results[0].wasSuccessful).toBe(true);
+      }
+    });
+
+    it('should handle multiple playbooks', async () => {
+      const playbook1 = createSamplePlaybook({
+        id: 'playbook-1',
+        name: 'error-fixing',
+        guidance: {
+          strategy: 'Fix errors',
+          tactics: ['Read error', 'Fix code'],
+        },
+      });
+
+      const playbook2 = createSamplePlaybook({
+        id: 'playbook-2',
+        name: 'testing',
+        guidance: {
+          strategy: 'Test code',
+          tactics: ['Run tests', 'Check coverage'],
+        },
+      });
+
+      const trajectory = createSampleTrajectory([
+        {
+          thought: 'Reading the error message',
+          action: 'cat error.log',
+          observation: 'Found error',
+        },
+        {
+          thought: 'Fixing the code',
+          action: 'edit file.ts',
+          observation: 'Fixed',
+        },
+      ]);
+
+      const results = await inference.inferUsage(trajectory, [playbook1, playbook2]);
+
+      expect(results).toHaveLength(2);
+      // playbook1 should have higher usage confidence
+      const pb1Result = results.find(r => r.playbookId === 'playbook-1');
+      const pb2Result = results.find(r => r.playbookId === 'playbook-2');
+      expect(pb1Result?.usageConfidence).toBeGreaterThan(pb2Result?.usageConfidence || 0);
+    });
+
+    it('should return empty array when no playbooks provided', async () => {
+      const trajectory = createSampleTrajectory([
+        { action: 'some action', observation: 'result' },
+      ]);
+
+      const results = await inference.inferUsage(trajectory, []);
+
+      expect(results).toHaveLength(0);
+    });
+  });
+});