cognitive-core 0.0.2 → 0.1.2

package/tests/learning/knowledge-extractor.test.ts

@@ -0,0 +1,491 @@
+import { describe, it, expect } from 'vitest';
+import {
+  KnowledgeExtractor,
+  createKnowledgeExtractor,
+} from '../../src/learning/knowledge-extractor.js';
+import type { AnalysisResult } from '../../src/learning/analyzer.js';
+import {
+  createTrajectory,
+  createTask,
+  createStep,
+  successOutcome,
+  failureOutcome,
+} from '../../src/types/index.js';
+import type { Trajectory } from '../../src/types/index.js';
+
+function makeTrajectory(overrides?: {
+  id?: string;
+  domain?: string;
+  success?: boolean;
+  steps?: Array<{ action: string; observation: string }>;
+  metadata?: Record<string, unknown>;
+}): Trajectory {
+  const rawSteps = overrides?.steps ?? [
+    { action: 'read file', observation: 'file contents loaded' },
+    { action: 'edit code', observation: 'code updated' },
+    { action: 'run tests', observation: 'tests passed' },
+  ];
+
+  const steps = rawSteps.map((s) =>
+    createStep({ action: s.action, observation: s.observation })
+  );
+
+  return createTrajectory({
+    id: overrides?.id ?? 'traj-test-1',
+    task: createTask({
+      domain: overrides?.domain ?? 'code',
+      description: 'test task',
+    }),
+    steps,
+    outcome: overrides?.success !== false
+      ? successOutcome('done')
+      : failureOutcome('failed'),
+    agentId: 'test-agent',
+    metadata: overrides?.metadata ?? {},
+  });
+}
+
+function makeAnalysis(overrides?: Partial<AnalysisResult>): AnalysisResult {
+  return {
+    success: true,
+    keySteps: [],
+    errorPatterns: [],
+    abstractable: false,
+    trainingExamples: [],
+    ...overrides,
+  };
+}
+
+describe('KnowledgeExtractor', () => {
+  let extractor: KnowledgeExtractor;
+
+  describe('factory', () => {
+    it('should create with createKnowledgeExtractor', () => {
+      const ex = createKnowledgeExtractor();
+      expect(ex).toBeInstanceOf(KnowledgeExtractor);
+    });
+
+    it('should accept custom config', () => {
+      const ex = createKnowledgeExtractor({ initialConfidence: 0.5 });
+      expect(ex).toBeInstanceOf(KnowledgeExtractor);
+    });
+  });
+
+  describe('extract (overall)', () => {
+    beforeEach(() => {
+      extractor = new KnowledgeExtractor();
+    });
+
+    it('should return ExtractedKnowledge shape', () => {
+      const trajectory = makeTrajectory();
+      const analysis = makeAnalysis();
+
+      const result = extractor.extract(trajectory, analysis);
+
+      expect(result).toHaveProperty('observations');
+      expect(result).toHaveProperty('entityUpdates');
+      expect(result).toHaveProperty('contradictions');
+      expect(result).toHaveProperty('links');
+      expect(result).toHaveProperty('playbookLinks');
+      expect(Array.isArray(result.observations)).toBe(true);
+      expect(result.contradictions).toEqual([]);
+    });
+
+    it('should extract nothing from simple trajectory with no patterns', () => {
+      const trajectory = makeTrajectory({
+        steps: [
+          { action: 'think about problem', observation: 'plan formed' },
+        ],
+      });
+      const analysis = makeAnalysis();
+
+      const result = extractor.extract(trajectory, analysis);
+      // No error patterns, no config files, no failure→recovery
+      expect(result.observations).toHaveLength(0);
+    });
+  });
+
+  // =========================================================================
+  // Strategy 1: Error Patterns
+  // =========================================================================
+
+  describe('Strategy 1: Error patterns', () => {
+    beforeEach(() => {
+      extractor = new KnowledgeExtractor();
+    });
+
+    it('should extract observations from error patterns with suggestedFix', () => {
+      const trajectory = makeTrajectory();
+      const analysis = makeAnalysis({
+        errorPatterns: [
+          {
+            type: 'syntax',
+            signature: 'SyntaxError: Unexpected token',
+            description: 'Missing semicolon',
+            count: 1,
+            suggestedFix: 'Add semicolon at end of line',
+            stepIndices: [0],
+          },
+        ],
+      });
+
+      const result = extractor.extract(trajectory, analysis);
+      expect(result.observations.length).toBeGreaterThanOrEqual(1);
+
+      const errorObs = result.observations.find((o) =>
+        o.frontmatter.tags.includes('error-pattern')
+      );
+      expect(errorObs).toBeDefined();
+      expect(errorObs!.body).toContain('SyntaxError');
+      expect(errorObs!.body).toContain('Add semicolon');
+    });
+
+    it('should skip error patterns without suggestedFix', () => {
+      const trajectory = makeTrajectory();
+      const analysis = makeAnalysis({
+        errorPatterns: [
+          {
+            type: 'runtime',
+            signature: 'Error',
+            description: 'Something went wrong',
+            count: 1,
+            stepIndices: [0],
+            // no suggestedFix
+          },
+        ],
+      });
+
+      const result = extractor.extract(trajectory, analysis);
+      const errorObs = result.observations.filter((o) =>
+        o.frontmatter.tags.includes('error-pattern')
+      );
+      expect(errorObs).toHaveLength(0);
+    });
+
+    it('should set domain from trajectory', () => {
+      const trajectory = makeTrajectory({ domain: 'web-dev' });
+      const analysis = makeAnalysis({
+        errorPatterns: [
+          {
+            type: 'build',
+            signature: 'webpack compilation error',
+            description: 'Module not found',
+            count: 1,
+            suggestedFix: 'Install the missing module',
+            stepIndices: [0],
+          },
+        ],
+      });
+
+      const result = extractor.extract(trajectory, analysis);
+      const obs = result.observations.find((o) =>
+        o.frontmatter.tags.includes('error-pattern')
+      );
+      expect(obs!.frontmatter.domain).toContain('web-dev');
+    });
+  });
+
+  // =========================================================================
+  // Strategy 2: Config Facts
+  // =========================================================================
+
+  describe('Strategy 2: Config facts', () => {
+    beforeEach(() => {
+      extractor = new KnowledgeExtractor();
+    });
+
+    it('should detect config file references in step actions', () => {
+      const trajectory = makeTrajectory({
+        steps: [
+          {
+            action: 'cat package.json',
+            observation: '{"name": "my-app", "version": "1.2.3"}',
+          },
+        ],
+      });
+
+      const result = extractor.extract(trajectory, makeAnalysis());
+      const configObs = result.observations.filter((o) =>
+        o.frontmatter.tags.includes('config')
+      );
+      expect(configObs.length).toBeGreaterThanOrEqual(1);
+      expect(configObs[0].body).toContain('package.json');
+    });
+
+    it('should extract version numbers', () => {
+      const trajectory = makeTrajectory({
+        steps: [
+          {
+            action: 'read tsconfig.json',
+            observation: 'typescript version: 5.3.2',
+          },
+        ],
+      });
+
+      const result = extractor.extract(trajectory, makeAnalysis());
+      const configObs = result.observations.find((o) =>
+        o.body.includes('tsconfig')
+      );
+      expect(configObs).toBeDefined();
+      expect(configObs!.body).toContain('5.3.2');
+    });
+
+    it('should not duplicate config observations for the same file', () => {
+      const trajectory = makeTrajectory({
+        steps: [
+          { action: 'read package.json', observation: 'contents 1' },
+          { action: 'update package.json', observation: 'contents 2' },
+        ],
+      });
+
+      const result = extractor.extract(trajectory, makeAnalysis());
+      const packageObs = result.observations.filter(
+        (o) =>
+          o.frontmatter.tags.includes('config') &&
+          o.body.includes('package.json')
+      );
+      expect(packageObs).toHaveLength(1);
+    });
+  });
+
+  // =========================================================================
+  // Strategy 3: Causal Chains
+  // =========================================================================
+
+  describe('Strategy 3: Causal chains', () => {
+    beforeEach(() => {
+      extractor = new KnowledgeExtractor();
+    });
+
+    it('should detect failure → recovery sequences', () => {
+      const trajectory = makeTrajectory({
+        steps: [
+          { action: 'npm install foo', observation: 'error: package not found' },
+          { action: 'npm install @org/foo', observation: 'installed successfully' },
+        ],
+      });
+
+      const result = extractor.extract(trajectory, makeAnalysis());
+      const causalObs = result.observations.filter((o) =>
+        o.frontmatter.tags.includes('causal-chain')
+      );
+      expect(causalObs.length).toBeGreaterThanOrEqual(1);
+      expect(causalObs[0].body).toContain('Failed action');
+      expect(causalObs[0].body).toContain('Recovery action');
+    });
+
+    it('should create causal links', () => {
+      const trajectory = makeTrajectory({
+        steps: [
+          { action: 'run build', observation: 'error: types not found' },
+          { action: 'install @types/node', observation: 'installed' },
+        ],
+      });
+
+      const result = extractor.extract(trajectory, makeAnalysis());
+      expect(result.links.length).toBeGreaterThanOrEqual(1);
+      expect(result.links[0].relation).toBe('led-to');
+      expect(result.links[0].layer).toBe('causal');
+    });
+
+    it('should not create causal chain when no recovery follows', () => {
+      const trajectory = makeTrajectory({
+        steps: [
+          { action: 'run build', observation: 'error: compilation failed' },
+          { action: 'run build again', observation: 'error: still failed' },
+        ],
+      });
+
+      const result = extractor.extract(trajectory, makeAnalysis());
+      const causalObs = result.observations.filter((o) =>
+        o.frontmatter.tags.includes('causal-chain')
+      );
+      expect(causalObs).toHaveLength(0);
+    });
+
+    it('should look ahead up to 3 steps for recovery', () => {
+      const trajectory = makeTrajectory({
+        steps: [
+          { action: 'run test', observation: 'error: assertion failed' },
+          { action: 'read logs', observation: 'error details in log' },
+          { action: 'fix assertion', observation: 'error in fix' },
+          { action: 'fix properly', observation: 'all tests pass' },
+        ],
+      });
+
+      const result = extractor.extract(trajectory, makeAnalysis());
+      // The first failure (step 0) should find recovery at step 3 (within 3-step lookahead)
+      const causalObs = result.observations.filter((o) =>
+        o.frontmatter.tags.includes('causal-chain')
+      );
+      // At least the first failure should find a recovery
+      expect(causalObs.length).toBeGreaterThanOrEqual(1);
+    });
+  });
+
+  // =========================================================================
+  // Strategy 4: Entity Identification
+  // =========================================================================
+
+  describe('Strategy 4: Entity identification', () => {
+    beforeEach(() => {
+      extractor = new KnowledgeExtractor();
+    });
+
+    it('should produce entity updates for known entities', () => {
+      const trajectory = makeTrajectory({
+        domain: 'web',
+        steps: [
+          {
+            action: 'configure vitest',
+            observation: 'error: vitest config invalid',
+          },
+          {
+            action: 'fix vitest config',
+            observation: 'vitest config fixed, tests pass',
+          },
+        ],
+      });
+      const analysis = makeAnalysis({
+        errorPatterns: [
+          {
+            type: 'config',
+            signature: 'vitest config error',
+            description: 'Invalid vitest configuration',
+            count: 1,
+            suggestedFix: 'Update vitest.config.ts',
+            stepIndices: [0],
+          },
+        ],
+      });
+
+      const result = extractor.extract(trajectory, analysis, {
+        existingEntities: ['vitest'],
+      });
+
+      expect(result.entityUpdates.length).toBeGreaterThanOrEqual(1);
+      expect(result.entityUpdates[0].entityId).toBe('vitest');
+      expect(result.entityUpdates[0].trajectoryId).toBe(trajectory.id);
+    });
+
+    it('should skip entity updates when no existing entities', () => {
+      const trajectory = makeTrajectory({
+        steps: [
+          { action: 'use prisma', observation: 'error: prisma not configured' },
+        ],
+      });
+      const analysis = makeAnalysis({
+        errorPatterns: [
+          {
+            type: 'config',
+            signature: 'prisma error',
+            description: 'Prisma not configured',
+            count: 1,
+            suggestedFix: 'Run prisma init',
+            stepIndices: [0],
+          },
+        ],
+      });
+
+      const result = extractor.extract(trajectory, analysis);
+      expect(result.entityUpdates).toHaveLength(0);
+    });
+
+    it('should not create updates for entities not mentioned in observations', () => {
+      const trajectory = makeTrajectory({
+        steps: [
+          { action: 'read readme', observation: 'readme content' },
+        ],
+      });
+
+      const result = extractor.extract(trajectory, makeAnalysis(), {
+        existingEntities: ['prisma', 'react'],
+      });
+
+      expect(result.entityUpdates).toHaveLength(0);
+    });
+  });
+
+  // =========================================================================
+  // Cross-Reference: Playbook Links
+  // =========================================================================
+
+  describe('Playbook links', () => {
+    beforeEach(() => {
+      extractor = new KnowledgeExtractor();
+    });
+
+    it('should create playbook links from injectedPlaybooks metadata', () => {
+      const trajectory = makeTrajectory({
+        metadata: { injectedPlaybooks: ['pb-auth', 'pb-testing'] },
+        steps: [
+          {
+            action: 'read package.json',
+            observation: '{"name": "app"}',
+          },
+        ],
+      });
+
+      const result = extractor.extract(trajectory, makeAnalysis());
+
+      // Should have playbook links for each observation × each playbook
+      if (result.observations.length > 0) {
+        expect(result.playbookLinks.length).toBe(
+          result.observations.length * 2
+        );
+        expect(result.playbookLinks[0].relation).toBe('supports');
+      }
+    });
+
+    it('should not create playbook links when no injectedPlaybooks', () => {
+      const trajectory = makeTrajectory({
+        steps: [
+          {
+            action: 'read package.json',
+            observation: '{"name": "app"}',
+          },
+        ],
+      });
+
+      const result = extractor.extract(trajectory, makeAnalysis());
+      expect(result.playbookLinks).toHaveLength(0);
+    });
+  });
+
+  // =========================================================================
+  // Entity extraction from text
+  // =========================================================================
+
+  describe('entity extraction from text', () => {
+    beforeEach(() => {
+      extractor = new KnowledgeExtractor();
+    });
+
+    it('should extract known tool names from error patterns', () => {
+      const trajectory = makeTrajectory({ domain: 'web' });
+      const analysis = makeAnalysis({
+        errorPatterns: [
+          {
+            type: 'build',
+            signature: 'TypeScript error TS2307',
+            description: 'Cannot find module in React component',
+            count: 1,
+            suggestedFix: 'Install missing types',
+            stepIndices: [0],
+          },
+        ],
+      });
+
+      const result = extractor.extract(trajectory, analysis);
+      const obs = result.observations.find((o) =>
+        o.frontmatter.tags.includes('error-pattern')
+      );
+      expect(obs).toBeDefined();
+      // Should detect typescript and/or react
+      const entities = obs!.frontmatter.entities;
+      expect(
+        entities.includes('typescript') || entities.includes('react')
+      ).toBe(true);
+    });
+  });
+});

package/tests/learning/pipeline.test.ts

@@ -0,0 +1,176 @@
+import { describe, it, expect, beforeEach, vi } from 'vitest';
+import { LearningPipeline, createLearningPipeline } from '../../src/learning/pipeline.js';
+import { MemorySystem } from '../../src/memory/system.js';
+import { createTrajectory, createStep, createTask, successOutcome, failureOutcome } from '../../src/types/index.js';
+import * as fs from 'node:fs/promises';
+import * as path from 'node:path';
+
+describe('LearningPipeline', () => {
+  let pipeline: LearningPipeline;
+  let memory: MemorySystem;
+  let testDir: string;
+
+  beforeEach(async () => {
+    testDir = path.join(process.cwd(), '.test-pipeline-' + Date.now());
+    await fs.mkdir(testDir, { recursive: true });
+    memory = new MemorySystem(testDir);
+    await memory.init();
+    pipeline = createLearningPipeline(memory, {
+      minTrajectories: 3,
+      minSuccessRate: 0.5,
+      maxExperiences: 100,
+    });
+  });
+
+  afterEach(async () => {
+    await memory.close();
+    await fs.rm(testDir, { recursive: true, force: true });
+  });
+
+  const createTestTrajectory = (success: boolean, description: string) => {
+    return createTrajectory({
+      task: createTask({ domain: 'test', description }),
+      steps: [
+        createStep({
+          thought: 'Analyzing task',
+          action: 'read file',
+          observation: 'file contents',
+        }),
+        createStep({
+          thought: 'Applying solution',
+          action: 'edit file',
+          observation: success ? 'success' : 'error: failed',
+        }),
+      ],
+      outcome: success
+        ? successOutcome({ result: 'completed' })
+        : failureOutcome('Task failed'),
+      agentId: 'test-agent',
+    });
+  };
+
+  describe('processTrajectory', () => {
+    it('should process and store trajectory', async () => {
+      const trajectory = createTestTrajectory(true, 'Test task 1');
+      const result = await pipeline.processTrajectory(trajectory);
+
+      expect(result.stored).toBe(true);
+      expect(result.trajectoryId).toBe(trajectory.id);
+      expect(result.analysis).toBeDefined();
+      expect(result.analysis.success).toBe(true);
+    });
+
+    it('should accumulate trajectories for batch learning', async () => {
+      expect(pipeline.getAccumulatedCount()).toBe(0);
+
+      await pipeline.processTrajectory(createTestTrajectory(true, 'Task 1'));
+      expect(pipeline.getAccumulatedCount()).toBe(1);
+
+      await pipeline.processTrajectory(createTestTrajectory(true, 'Task 2'));
+      expect(pipeline.getAccumulatedCount()).toBe(2);
+    });
+
+    it('should analyze trajectory for key steps', async () => {
+      const trajectory = createTestTrajectory(true, 'Test analysis');
+      const result = await pipeline.processTrajectory(trajectory);
+
+      expect(result.analysis.keySteps).toBeDefined();
+      expect(Array.isArray(result.analysis.keySteps)).toBe(true);
+      expect(result.analysis.stepAttribution).toBeDefined();
+      expect(result.analysis.stepAttribution.length).toBe(trajectory.steps.length);
+    });
+  });
+
+  describe('shouldRunBatch', () => {
+    it('should return false when below minTrajectories', () => {
+      expect(pipeline.shouldRunBatch()).toBe(false);
+    });
+
+    it('should return true when above minTrajectories with sufficient success rate', async () => {
+      await pipeline.processTrajectory(createTestTrajectory(true, 'Task 1'));
+      await pipeline.processTrajectory(createTestTrajectory(true, 'Task 2'));
+      expect(pipeline.shouldRunBatch()).toBe(false); // Still below 3
+
+      await pipeline.processTrajectory(createTestTrajectory(true, 'Task 3'));
+      expect(pipeline.shouldRunBatch()).toBe(true); // Now at 3
+    });
+
+    it('should return false when success rate is too low', async () => {
+      await pipeline.processTrajectory(createTestTrajectory(false, 'Task 1'));
+      await pipeline.processTrajectory(createTestTrajectory(false, 'Task 2'));
+      await pipeline.processTrajectory(createTestTrajectory(false, 'Task 3'));
+
+      // 0% success rate, below 50% threshold
+      expect(pipeline.shouldRunBatch()).toBe(false);
+    });
+  });
+
+  describe('runBatchLearning', () => {
+    it('should return empty result when no trajectories accumulated', async () => {
+      const result = await pipeline.runBatchLearning();
+
+      expect(result.trajectoriesProcessed).toBe(0);
+      expect(result.playbooksExtracted).toBe(0);
+      expect(result.experiencesPruned).toBe(0);
+      expect(result.successRate).toBe(0);
+    });
+
+    it('should process accumulated trajectories', async () => {
+      await pipeline.processTrajectory(createTestTrajectory(true, 'Task 1'));
+      await pipeline.processTrajectory(createTestTrajectory(true, 'Task 2'));
+      await pipeline.processTrajectory(createTestTrajectory(false, 'Task 3'));
+
+      const result = await pipeline.runBatchLearning();
+
+      expect(result.trajectoriesProcessed).toBe(3);
+      expect(result.successRate).toBeCloseTo(2/3, 2);
+    });
+
+    it('should clear accumulated after batch', async () => {
+      await pipeline.processTrajectory(createTestTrajectory(true, 'Task 1'));
+      await pipeline.processTrajectory(createTestTrajectory(true, 'Task 2'));
+
+      expect(pipeline.getAccumulatedCount()).toBe(2);
+      await pipeline.runBatchLearning();
+      expect(pipeline.getAccumulatedCount()).toBe(0);
+    });
+  });
+
+  describe('clearAccumulated', () => {
+    it('should clear all accumulated trajectories', async () => {
+      await pipeline.processTrajectory(createTestTrajectory(true, 'Task 1'));
+      await pipeline.processTrajectory(createTestTrajectory(true, 'Task 2'));
+
+      expect(pipeline.getAccumulatedCount()).toBe(2);
+      pipeline.clearAccumulated();
+      expect(pipeline.getAccumulatedCount()).toBe(0);
+    });
+  });
+
+  describe('configuration', () => {
+    it('should use configurable maxExperiences for pruning', async () => {
+      // The pipeline was created with maxExperiences: 100
+      // This test verifies the config is passed correctly
+      const configuredPipeline = createLearningPipeline(memory, {
+        maxExperiences: 50,
+      });
+
+      // Process a trajectory to ensure it uses the config
+      const trajectory = createTestTrajectory(true, 'Config test');
+      const result = await configuredPipeline.processTrajectory(trajectory);
+      expect(result.stored).toBe(true);
+    });
+
+    it('should update configuration', async () => {
+      pipeline.updateConfig({ minTrajectories: 5 });
+
+      // Add 3 trajectories (was enough before update)
+      await pipeline.processTrajectory(createTestTrajectory(true, 'Task 1'));
+      await pipeline.processTrajectory(createTestTrajectory(true, 'Task 2'));
+      await pipeline.processTrajectory(createTestTrajectory(true, 'Task 3'));
+
+      // Should not run batch yet (need 5 now)
+      expect(pipeline.shouldRunBatch()).toBe(false);
+    });
+  });
+});