cognitive-core 0.0.2 → 0.1.2

package/tests/memory/knowledge-bank-extraction.test.ts

@@ -0,0 +1,558 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { mkdtemp, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { KnowledgeBank } from '../../src/memory/knowledge-bank.js';
+import type { KnowledgeBankConfig } from '../../src/types/config.js';
+import {
+  createObservation,
+  createEntityNote,
+} from '../../src/types/knowledge.js';
+import type {
+  ExtractedKnowledge,
+  Contradiction,
+  KnowledgeNote,
+} from '../../src/types/knowledge.js';
+import { InMemoryTrajectorySource } from '../../src/learning/trajectory-sources/in-memory.js';
+import {
+  createTrajectory,
+  createTask,
+  createStep,
+  successOutcome,
+  failureOutcome,
+} from '../../src/types/index.js';
+import type { Trajectory } from '../../src/types/index.js';
+import type { AnalysisResult } from '../../src/learning/analyzer.js';
+
+function defaultConfig(): KnowledgeBankConfig {
+  return {
+    enabled: true,
+    memoryDir: 'memory',
+    extraction: {
+      enabled: true,
+      useLlmExtraction: false,
+      minAttributionScore: 0.3,
+    },
+    defrag: {
+      minInboxItems: 5,
+      autoConsolidate: true,
+      consolidationThreshold: 3,
+      autoRegenerateSummaries: true,
+      llmSummaryGeneration: false,
+      autoResolveContradictions: false,
+    },
+    confidence: {
+      extractedInitial: 0.3,
+      agentAuthoredInitial: 0.5,
+      reinforcementAlpha: 0.1,
+      contradictionPenalty: 0.3,
+      dailyDecayRate: 0.001,
+      staleThreshold: 0.2,
+      decayGracePeriodDays: 90,
+    },
+    graph: {
+      enabledLayers: ['semantic', 'entity', 'causal'],
+      inferTemporalEdges: true,
+      inferCausalEdges: true,
+      inferEntityEdges: true,
+      domainSplitThreshold: 500,
+      sqlitePromotionThreshold: 10000,
+    },
+    surfacing: {
+      includeDomainSummaries: true,
+      includeEntityNotes: true,
+      maxNotesPerTask: 5,
+      maxTokensForKnowledge: 2000,
+      includeEvidence: true,
+      includeLinks: false,
+    },
+    minimemAware: false,
+  };
+}
+
+function makeTrajectory(overrides?: {
+  id?: string;
+  domain?: string;
+  success?: boolean;
+  steps?: Array<{ action: string; observation: string }>;
+  metadata?: Record<string, unknown>;
+}): Trajectory {
+  const rawSteps = overrides?.steps ?? [
+    { action: 'read file', observation: 'file contents loaded' },
+    { action: 'edit code', observation: 'code updated' },
+    { action: 'run tests', observation: 'tests passed' },
+  ];
+
+  return createTrajectory({
+    id: overrides?.id ?? 'traj-1',
+    task: createTask({
+      domain: overrides?.domain ?? 'code',
+      description: 'test task',
+    }),
+    steps: rawSteps.map((s) =>
+      createStep({ action: s.action, observation: s.observation })
+    ),
+    outcome: overrides?.success !== false
+      ? successOutcome('done')
+      : failureOutcome('failed'),
+    agentId: 'test-agent',
+    metadata: overrides?.metadata ?? {},
+  });
+}
+
+describe('KnowledgeBank Extraction', () => {
+  let tmpDir: string;
+  let bank: KnowledgeBank;
+
+  beforeEach(async () => {
+    tmpDir = await mkdtemp(join(tmpdir(), 'kb-extract-test-'));
+    bank = new KnowledgeBank(tmpDir, defaultConfig());
+    await bank.init();
+  });
+
+  afterEach(async () => {
+    try {
+      await bank.close();
+    } catch {
+      // ignore
+    }
+    await rm(tmpDir, { recursive: true, force: true });
+  });
+
+  // =========================================================================
+  // applyExtraction
+  // =========================================================================
+
+  describe('applyExtraction', () => {
+    it('should add observations to disk and graph', async () => {
+      const obs = createObservation({
+        id: 'k-test-apply-1',
+        title: 'Test Observation',
+        body: 'Extracted knowledge.',
+        domain: ['testing'],
+        entities: ['vitest'],
+        tags: ['unit-test'],
+        confidence: 0.3,
+        source: { origin: 'extracted', trajectories: ['t-1'] },
+      });
+
+      const extraction: ExtractedKnowledge = {
+        observations: [obs],
+        entityUpdates: [],
+        contradictions: [],
+        links: [],
+        playbookLinks: [],
+      };
+
+      await bank.applyExtraction(extraction);
+
+      const note = await bank.getNote('k-test-apply-1');
+      expect(note).not.toBeNull();
+      expect(note!.body).toContain('Extracted knowledge');
+
+      // Should be in graph
+      const graphNode = bank.graph.getNode('k-test-apply-1');
+      expect(graphNode).toBeDefined();
+    });
+
+    it('should apply entity updates to existing entities', async () => {
+      // First create an entity
+      const entity = createEntityNote({
+        entityName: 'vitest',
+        body: 'A testing framework.',
+        domain: ['testing'],
+        source: { origin: 'extracted' },
+      });
+      await bank.addObservation(entity);
+
+      const extraction: ExtractedKnowledge = {
+        observations: [],
+        entityUpdates: [
+          {
+            entityId: 'vitest',
+            section: '## New Evidence\n\nVitest supports snapshot testing.',
+            trajectoryId: 'traj-1',
+          },
+        ],
+        contradictions: [],
+        links: [],
+        playbookLinks: [],
+      };
+
+      await bank.applyExtraction(extraction);
+
+      const updated = await bank.getNote('vitest');
+      expect(updated).not.toBeNull();
+      expect(updated!.body).toContain('snapshot testing');
+      expect(updated!.body).toContain('traj-1');
+    });
+
+    it('should skip entity updates for non-existent entities', async () => {
+      const extraction: ExtractedKnowledge = {
+        observations: [],
+        entityUpdates: [
+          {
+            entityId: 'non-existent',
+            section: '## Something',
+            trajectoryId: 'traj-1',
+          },
+        ],
+        contradictions: [],
+        links: [],
+        playbookLinks: [],
+      };
+
+      // Should not throw
+      await expect(bank.applyExtraction(extraction)).resolves.not.toThrow();
+    });
+
+    it('should add graph edges for new links', async () => {
+      const obs1 = createObservation({
+        id: 'k-link-a',
+        title: 'Obs A',
+        body: 'A',
+        domain: ['code'],
+        entities: [],
+        tags: [],
+        source: { origin: 'extracted' },
+      });
+      const obs2 = createObservation({
+        id: 'k-link-b',
+        title: 'Obs B',
+        body: 'B',
+        domain: ['code'],
+        entities: [],
+        tags: [],
+        source: { origin: 'extracted' },
+      });
+
+      const extraction: ExtractedKnowledge = {
+        observations: [obs1, obs2],
+        entityUpdates: [],
+        contradictions: [],
+        links: [
+          {
+            from: 'k-link-a',
+            to: 'k-link-b',
+            relation: 'led-to',
+            layer: 'causal',
+          },
+        ],
+        playbookLinks: [],
+      };
+
+      await bank.applyExtraction(extraction);
+
+      // Check the causal layer has the edge
+      const causalLayer = bank.graph.getLayer('causal');
+      expect(causalLayer).not.toBeNull();
+      const edges = causalLayer!.getEdges('k-link-a', 'outgoing');
+      expect(edges.length).toBeGreaterThanOrEqual(1);
+      expect(edges[0].relation).toBe('led-to');
+    });
+
+    it('should add playbook cross-system links', async () => {
+      const obs = createObservation({
+        id: 'k-pb-link',
+        title: 'Obs with PB link',
+        body: 'Body',
+        domain: ['code'],
+        entities: [],
+        tags: [],
+        source: { origin: 'extracted' },
+      });
+
+      const extraction: ExtractedKnowledge = {
+        observations: [obs],
+        entityUpdates: [],
+        contradictions: [],
+        links: [],
+        playbookLinks: [
+          {
+            noteId: 'k-pb-link',
+            playbookId: 'pb-auth-flow',
+            relation: 'supports',
+          },
+        ],
+      };
+
+      await bank.applyExtraction(extraction);
+
+      const crossLinks = bank.graph.crossSystemLinks('k-pb-link');
+      expect(crossLinks.length).toBeGreaterThanOrEqual(1);
+      expect(crossLinks[0].crossSystem).toBe('playbook');
+    });
+  });
+
+  // =========================================================================
+  // recordContradiction
+  // =========================================================================
+
+  describe('recordContradiction', () => {
+    it('should add new observation and contradicts edge', async () => {
+      // Pre-existing note
+      const existing = createObservation({
+        id: 'k-old-fact',
+        title: 'Old Fact',
+        body: 'React uses class components.',
+        domain: ['web'],
+        entities: ['react'],
+        tags: [],
+        source: { origin: 'extracted' },
+      });
+      await bank.addObservation(existing);
+
+      const newObs = createObservation({
+        id: 'k-new-fact',
+        title: 'New Fact',
+        body: 'React primarily uses function components.',
+        domain: ['web'],
+        entities: ['react'],
+        tags: [],
+        source: { origin: 'extracted' },
+      });
+
+      const contradiction: Contradiction = {
+        existingNoteId: 'k-old-fact',
+        newObservation: newObs,
+        conflictDescription: 'Class vs function components',
+        resolution: 'coexist',
+      };
+
+      await bank.recordContradiction(contradiction);
+
+      // New observation should be on disk
+      const note = await bank.getNote('k-new-fact');
+      expect(note).not.toBeNull();
+
+      // Contradicts edge should exist in entity layer
+      const entityLayer = bank.graph.getLayer('entity');
+      expect(entityLayer).not.toBeNull();
+      const edges = entityLayer!.getEdges('k-new-fact', 'outgoing');
+      const contradictEdge = edges.find((e) => e.relation === 'contradicts');
+      expect(contradictEdge).toBeDefined();
+      expect(contradictEdge!.to).toBe('k-old-fact');
+    });
+
+    it('should set supersedes when resolution is supersede', async () => {
+      const existing = createObservation({
+        id: 'k-superseded',
+        title: 'Old',
+        body: 'Old info.',
+        domain: ['code'],
+        entities: [],
+        tags: [],
+        source: { origin: 'extracted' },
+      });
+      await bank.addObservation(existing);
+
+      const newObs = createObservation({
+        id: 'k-superseder',
+        title: 'New',
+        body: 'Updated info.',
+        domain: ['code'],
+        entities: [],
+        tags: [],
+        source: { origin: 'extracted' },
+      });
+
+      await bank.recordContradiction({
+        existingNoteId: 'k-superseded',
+        newObservation: newObs,
+        conflictDescription: 'Updated information',
+        resolution: 'supersede',
+      });
+
+      const note = await bank.getNote('k-superseder');
+      expect(note).not.toBeNull();
+      expect(note!.frontmatter.supersedes).toBe('k-superseded');
+    });
+
+    it('should not duplicate the observation if already added', async () => {
+      const obs = createObservation({
+        id: 'k-already-added',
+        title: 'Already Added',
+        body: 'Body.',
+        domain: ['code'],
+        entities: [],
+        tags: [],
+        source: { origin: 'extracted' },
+      });
+
+      // Add it first
+      await bank.addObservation(obs);
+
+      // Now record contradiction referencing the same observation
+      await bank.recordContradiction({
+        existingNoteId: 'k-some-other',
+        newObservation: obs,
+        conflictDescription: 'conflict',
+        resolution: 'coexist',
+      });
+
+      // Should still be retrievable (not errored)
+      const note = await bank.getNote('k-already-added');
+      expect(note).not.toBeNull();
+    });
+  });
+
+  // =========================================================================
+  // extractFromTrajectory
+  // =========================================================================
+
+  describe('extractFromTrajectory', () => {
+    it('should extract knowledge using heuristic fallback', async () => {
+      const trajectory = makeTrajectory({
+        id: 'traj-extract-1',
+        steps: [
+          {
+            action: 'npm install prisma',
+            observation: 'error: package not found',
+          },
+          {
+            action: 'npm install @prisma/client',
+            observation: 'installed successfully',
+          },
+        ],
+      });
+
+      const result = await bank.extractFromTrajectory(trajectory);
+
+      expect(result).toHaveProperty('observations');
+      expect(result).toHaveProperty('entityUpdates');
+      expect(result).toHaveProperty('links');
+      // The causal chain (error→fix) should produce observations
+      expect(result.observations.length).toBeGreaterThanOrEqual(1);
+    });
+
+    it('should apply extracted knowledge to bank', async () => {
+      const trajectory = makeTrajectory({
+        id: 'traj-extract-2',
+        steps: [
+          { action: 'read package.json', observation: '{"name": "app", "version": "2.0.0"}' },
+          { action: 'run build', observation: 'error: TypeScript compilation failed' },
+          { action: 'fix tsconfig.json', observation: 'build succeeded' },
+        ],
+      });
+
+      const result = await bank.extractFromTrajectory(trajectory);
+
+      // Observations should now be on disk
+      const allNotes = await bank.getAllNotes();
+      expect(allNotes.length).toBeGreaterThanOrEqual(result.observations.length);
+    });
+
+    it('should use provided analysis when given', async () => {
+      const trajectory = makeTrajectory({ id: 'traj-extract-3' });
+      const analysis: AnalysisResult = {
+        success: true,
+        keySteps: [],
+        errorPatterns: [
+          {
+            type: 'build',
+            signature: 'TS2307',
+            description: 'Cannot find module',
+            count: 1,
+            suggestedFix: 'Install types',
+            stepIndices: [0],
+          },
+        ],
+        abstractable: false,
+        trainingExamples: [],
+      };
+
+      const result = await bank.extractFromTrajectory(trajectory, analysis);
+
+      // Should use the provided analysis (which has an error pattern)
+      const errorObs = result.observations.filter((o) =>
+        o.frontmatter.tags.includes('error-pattern')
+      );
+      expect(errorObs.length).toBeGreaterThanOrEqual(1);
+    });
+  });
+
+  // =========================================================================
+  // extractFrom (TrajectorySource)
+  // =========================================================================
+
+  describe('extractFrom', () => {
+    it('should extract from an InMemoryTrajectorySource', async () => {
+      const t1 = makeTrajectory({
+        id: 'traj-src-1',
+        steps: [
+          { action: 'read package.json', observation: '{"name": "my-app"}' },
+        ],
+      });
+      const t2 = makeTrajectory({
+        id: 'traj-src-2',
+        steps: [
+          { action: 'run vitest', observation: 'error: test suite failed' },
+          { action: 'fix test', observation: 'tests pass' },
+        ],
+      });
+
+      const source = new InMemoryTrajectorySource([
+        { trajectory: t1 },
+        { trajectory: t2 },
+      ]);
+
+      const results = await bank.extractFrom(source);
+      expect(results).toHaveLength(2);
+
+      // All observations should be on disk
+      const allNotes = await bank.getAllNotes();
+      const totalObs = results.reduce(
+        (sum, r) => sum + r.observations.length,
+        0
+      );
+      expect(allNotes.length).toBeGreaterThanOrEqual(totalObs);
+    });
+
+    it('should use provided analyzer for missing analyses', async () => {
+      const t = makeTrajectory({ id: 'traj-src-3' });
+      const source = new InMemoryTrajectorySource([{ trajectory: t }]);
+
+      // Pass an analyzer
+      const { createAnalyzer } = await import(
+        '../../src/learning/analyzer.js'
+      );
+      const analyzer = createAnalyzer('simple');
+
+      const results = await bank.extractFrom(source, { analyzer });
+      expect(results).toHaveLength(1);
+    });
+
+    it('should use pre-computed analyses from source', async () => {
+      const t = makeTrajectory({ id: 'traj-src-4' });
+      const analysis: AnalysisResult = {
+        success: true,
+        keySteps: [],
+        errorPatterns: [
+          {
+            type: 'runtime',
+            signature: 'RangeError',
+            description: 'Stack overflow',
+            count: 1,
+            suggestedFix: 'Add recursion limit',
+            stepIndices: [0],
+          },
+        ],
+        abstractable: false,
+        trainingExamples: [],
+      };
+
+      const source = new InMemoryTrajectorySource([
+        { trajectory: t, analysis },
+      ]);
+
+      const results = await bank.extractFrom(source);
+      expect(results).toHaveLength(1);
+
+      // Should have the error pattern observation
+      const errorObs = results[0].observations.filter((o) =>
+        o.frontmatter.tags.includes('error-pattern')
+      );
+      expect(errorObs.length).toBeGreaterThanOrEqual(1);
+    });
+  });
+});