cognitive-core 0.2.5 → 0.3.1

package/tests/gap-fixes.test.ts

@@ -1,1116 +0,0 @@
-/**
- * Tests for the 5 gap fixes:
- *   Gap 1: LLMPlaybookExtractor deletion
- *   Gap 2: Atlas.solve() template delegation with fallback
- *   Gap 3: solution-evaluation template codebasePath / getResources
- *   Gap 4: MetaLearner.recordFromReflection bridge
- *   Gap 5: SolutionEvaluator & RefinementLoop setTaskRunner delegation
- */
-
-import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
-import { mkdtemp, rm } from 'node:fs/promises';
-import { join } from 'node:path';
-import { tmpdir } from 'node:os';
-
-// Types
-import { createTask } from '../src/types/task.js';
-import { createTrajectory } from '../src/types/trajectory.js';
-import { createStep } from '../src/types/step.js';
-import { successOutcome, failureOutcome } from '../src/types/outcome.js';
-import type { Trajectory } from '../src/types/trajectory.js';
-import type { Task } from '../src/types/task.js';
-import type { Playbook } from '../src/types/index.js';
-
-// Memory
-import { createMemorySystem, type MemorySystem } from '../src/memory/system.js';
-import { MetaMemory } from '../src/memory/meta.js';
-import { createSqlitePersistence } from '../src/persistence/index.js';
-
-// Learning
-import { MetaLearner, createMetaLearner } from '../src/learning/meta-learner.js';
-import type { MetaReflectionOutput } from '../src/workspace/templates/meta-reflection.js';
-import type { RoutingDecision } from '../src/learning/meta-learner.js';
-
-// Search
-import { SolutionEvaluator, createSolutionEvaluator } from '../src/search/evaluator.js';
-import { RefinementLoop, createRefinementLoop } from '../src/search/refinement-loop.js';
-import { createEvaluationResult, type EvaluationResult } from '../src/search/refinement-types.js';
-
-// Runtime
-import { AgentManager, createAgentManager } from '../src/runtime/manager.js';
-import { MockBackend, createMockBackend } from '../src/runtime/backends/mock.js';
-
-// Workspace / templates
-import type { AgenticTaskRunner } from '../src/workspace/runner.js';
-import { solutionEvaluationTemplate } from '../src/workspace/templates/solution-evaluation.js';
-import type { SolutionEvaluationInput } from '../src/workspace/templates/solution-evaluation.js';
-import { refinementAnalysisTemplate, type RefinementPlan } from '../src/workspace/templates/refinement-analysis.js';
-import { metaReflectionTemplate } from '../src/workspace/templates/meta-reflection.js';
-import { usageInferenceTemplate } from '../src/workspace/templates/usage-inference.js';
-
-// Atlas
-import { Atlas, createAtlasWithAgents } from '../src/atlas.js';
-
-// ============================================================
-// Helpers
-// ============================================================
-
-const makeTask = (overrides: Partial<Task> = {}) =>
-  createTask({
-    domain: 'code',
-    description: 'Fix the authentication bug',
-    ...overrides,
-  });
-
-const makeSuccessTrajectory = (overrides: Partial<Trajectory> = {}): Trajectory =>
-  createTrajectory({
-    task: makeTask(),
-    steps: [
-      createStep({ thought: 'analyze', action: 'read_file', observation: 'found code' }),
-      createStep({ thought: 'fix', action: 'write_file', observation: 'written' }),
-      createStep({ action: 'run_tests', observation: 'all tests pass' }),
-    ],
-    outcome: successOutcome('Bug fixed'),
-    agentId: 'test-agent',
-    wallTimeSeconds: 30,
-    llmCalls: 5,
-    ...overrides,
-  });
-
-const makeFailTrajectory = (overrides: Partial<Trajectory> = {}): Trajectory =>
-  createTrajectory({
-    task: makeTask(),
-    steps: [
-      createStep({ action: 'attempt', observation: 'error occurred' }),
-    ],
-    outcome: failureOutcome('Could not fix bug'),
-    agentId: 'test-agent',
-    wallTimeSeconds: 10,
-    llmCalls: 2,
-    ...overrides,
-  });
-
-const makeRouting = (overrides: Partial<RoutingDecision> = {}): RoutingDecision => ({
-  strategy: 'direct',
-  confidence: 0.8,
-  estimatedBudget: 10,
-  reasoning: 'High similarity to known patterns',
-  ...overrides,
-});
-
-const makePlaybook = (id: string, name: string): Playbook => ({
-  id,
-  name,
-  applicability: {
-    situations: ['Fix a bug'],
-    triggers: ['authentication'],
-    antiPatterns: [],
-    domains: ['code'],
-  },
-  guidance: {
-    strategy: 'Check token validation',
-    tactics: ['Verify token expiry', 'Check session storage'],
-    steps: ['Read code', 'Find bug', 'Fix', 'Test'],
-  },
-  verification: {
-    successIndicators: ['Tests pass'],
-    failureIndicators: ['Auth still fails'],
-  },
-  evolution: {
-    version: '1.0.0',
-    createdFrom: [],
-    failures: [],
-    refinements: [],
-    successCount: 5,
-    failureCount: 0,
-  },
-  confidence: 0.85,
-  complexity: 'moderate',
-  domain: 'code',
-  createdAt: new Date(),
-  updatedAt: new Date(),
-});
-
-/**
- * Wraps an array of values as a sequence for createMockTaskRunner.
- * Successive calls consume values in order; the last one repeats.
- */
-function seq(...values: unknown[]): { __seq: true; values: unknown[] } {
-  return { __seq: true, values };
-}
-
-function isSeq(v: unknown): v is { __seq: true; values: unknown[] } {
-  return typeof v === 'object' && v !== null && '__seq' in v && (v as Record<string, unknown>).__seq === true;
-}
-
-/**
- * Creates a mock AgenticTaskRunner that records calls and returns
- * configurable outputs per template taskType.
- *
- * Outputs can be:
- *   - A single value → returned every time
- *   - An Error → thrown every time
- *   - seq(v1, v2, ...) → values are consumed in order (last one repeats)
- */
-function createMockTaskRunner(
-  outputs: Record<string, unknown> = {}
-): AgenticTaskRunner & { calls: Array<{ taskType: string; input: unknown }> } {
-  const calls: Array<{ taskType: string; input: unknown }> = [];
-  const callCounts: Record<string, number> = {};
-
-  return {
-    calls,
-    run: vi.fn(async (template: { taskType: string }, input: unknown) => {
-      calls.push({ taskType: template.taskType, input });
-
-      const raw = outputs[template.taskType];
-      if (raw === undefined) {
-        throw new Error(`No mock output for template ${template.taskType}`);
-      }
-
-      let output: unknown;
-      if (isSeq(raw)) {
-        const idx = callCounts[template.taskType] ?? 0;
-        callCounts[template.taskType] = idx + 1;
-        output = raw.values[Math.min(idx, raw.values.length - 1)];
-      } else {
-        output = raw;
-      }
-
-      if (output instanceof Error) throw output;
-
-      return {
-        output,
-        trajectory: makeSuccessTrajectory(),
-        workspace: { id: 'mock-ws', path: '/tmp/mock-ws' },
-      };
-    }),
-  } as unknown as AgenticTaskRunner & { calls: Array<{ taskType: string; input: unknown }> };
-}
-
-// ============================================================
-// Gap 1: LLMPlaybookExtractor deletion
-// ============================================================
-
-describe('Gap 1: LLMPlaybookExtractor deletion', () => {
-  it('should not export LLMPlaybookExtractor from learning/index', async () => {
-    const learningExports = await import('../src/learning/index.js');
-    expect('LLMPlaybookExtractor' in learningExports).toBe(false);
-    expect('createLLMExtractor' in learningExports).toBe(false);
-  });
-
-  it('should not export LLMPlaybookExtractor from root index', async () => {
-    const rootExports = await import('../src/index.js');
-    expect('LLMPlaybookExtractor' in rootExports).toBe(false);
-    expect('createLLMExtractor' in rootExports).toBe(false);
-  });
-
-  it('should still export other learning components', async () => {
-    const learningExports = await import('../src/learning/index.js');
-    expect('PlaybookExtractor' in learningExports).toBe(true);
-    expect('MetaLearner' in learningExports).toBe(true);
-    expect('PlaybookUsageInference' in learningExports).toBe(true);
-    expect('UnifiedLearningPipeline' in learningExports).toBe(true);
-    expect('LearningEffectivenessTracker' in learningExports).toBe(true);
-  });
-});
-
-// ============================================================
-// Gap 3: solution-evaluation template codebasePath / getResources
-// ============================================================
-
-describe('Gap 3: solution-evaluation template codebasePath', () => {
-  const trajectory = makeSuccessTrajectory();
-  const task = makeTask();
-
-  it('should return empty resources when codebasePath is not set', () => {
-    const resources = solutionEvaluationTemplate.getResources({ trajectory, task });
-    expect(resources).toEqual([]);
-  });
-
-  it('should return a symlink resource when codebasePath is set', () => {
-    const resources = solutionEvaluationTemplate.getResources({
-      trajectory,
-      task,
-      codebasePath: '/home/user/my-project',
-    });
-
-    expect(resources).toHaveLength(1);
-    expect(resources[0]).toEqual({
-      path: 'codebase',
-      type: 'symlink',
-      source: '/home/user/my-project',
-      description: 'Codebase for solution verification',
-    });
-  });
-
-  it('should include codebase instruction in prompt when codebasePath is set', () => {
-    const prompt = solutionEvaluationTemplate.buildTaskPrompt({
-      trajectory,
-      task,
-      codebasePath: '/home/user/my-project',
-    });
-
-    expect(prompt).toContain('resources/codebase/');
-  });
-
-  it('should NOT include codebase instruction in prompt when codebasePath is absent', () => {
-    const prompt = solutionEvaluationTemplate.buildTaskPrompt({
-      trajectory,
-      task,
-    });
-
-    expect(prompt).not.toContain('resources/codebase/');
-  });
-
-  it('should still use heuristic complexity when task has verification', () => {
-    const taskWithVerification = makeTask({
-      verification: { type: 'exact_match', expected: 'pass' },
-    });
-
-    const complexity = solutionEvaluationTemplate.assessComplexity({
-      trajectory,
-      task: taskWithVerification,
-    });
-    expect(complexity).toBe('heuristic');
-  });
-});
-
-// ============================================================
-// Gap 4: MetaLearner.recordFromReflection
-// ============================================================
-
-describe('Gap 4: MetaLearner.recordFromReflection', () => {
-  let tempDir: string;
-  let metaMemory: MetaMemory;
-  let metaLearner: MetaLearner;
-  let persistence: any;
-
-  beforeEach(async () => {
-    tempDir = await mkdtemp(join(tmpdir(), 'meta-learner-test-'));
-    persistence = createSqlitePersistence({ baseDir: tempDir });
-    await persistence.init();
-    metaMemory = new MetaMemory(persistence);
-    await metaMemory.init();
-    metaLearner = createMetaLearner(metaMemory);
-  });
-
-  afterEach(async () => {
-    persistence.close();
-    await rm(tempDir, { recursive: true, force: true });
-  });
-
-  it('should create a MetaObservation from MetaReflectionOutput', async () => {
-    const reflection: MetaReflectionOutput = {
-      retrievalQuality: 'helpful',
-      outcomeQuality: 'good',
-      whatWorked: ['Direct routing was effective'],
-      whatFailed: [],
-      suggestions: ['Consider caching results'],
-      decompositionUsed: true,
-      backtrackingOccurred: false,
-      refinementIterations: 1,
-    };
-
-    const routing = makeRouting();
-    const observation = await metaLearner.recordFromReflection(
-      reflection,
-      'traj-123',
-      routing,
-      ['playbook-1', 'playbook-2'],
-      5,
-    );
-
-    expect(observation.trajectoryId).toBe('traj-123');
-    expect(observation.id).toBeDefined();
-    expect(observation.timestamp).toBeInstanceOf(Date);
-  });
-
-  it('should map routing fields correctly', async () => {
-    const routing = makeRouting({
-      strategy: 'explore',
-      confidence: 0.6,
-      reasoning: 'Unfamiliar task',
-    });
-
-    const observation = await metaLearner.recordFromReflection(
-      {
-        retrievalQuality: 'neutral',
-        outcomeQuality: 'acceptable',
-        whatWorked: [],
-        whatFailed: ['Exploration was slow'],
-        suggestions: [],
-        decompositionUsed: false,
-        backtrackingOccurred: true,
-        refinementIterations: 2,
-      },
-      'traj-456',
-      routing,
-      [],
-      8,
-    );
-
-    expect(observation.routing.decision).toBe('explore');
-    expect(observation.routing.confidence).toBe(0.6);
-    expect(observation.routing.reasoning).toBe('Unfamiliar task');
-  });
-
-  it('should map memory usage fields correctly', async () => {
-    const observation = await metaLearner.recordFromReflection(
-      {
-        retrievalQuality: 'misleading',
-        outcomeQuality: 'poor',
-        whatWorked: [],
-        whatFailed: ['Playbook was wrong'],
-        suggestions: ['Update playbook'],
-        decompositionUsed: false,
-        backtrackingOccurred: false,
-        refinementIterations: 0,
-      },
-      'traj-789',
-      makeRouting(),
-      ['pb-1'],
-      3,
-    );
-
-    expect(observation.memoryUsage.retrievalQuality).toBe('misleading');
-    expect(observation.memoryUsage.playbooksApplied).toEqual(['pb-1']);
-    expect(observation.memoryUsage.experiencesRetrieved).toBe(0);
-  });
-
-  it('should map execution fields correctly', async () => {
-    const observation = await metaLearner.recordFromReflection(
-      {
-        retrievalQuality: 'helpful',
-        outcomeQuality: 'excellent',
-        whatWorked: ['Decomposition was key'],
-        whatFailed: [],
-        suggestions: [],
-        decompositionUsed: true,
-        backtrackingOccurred: true,
-        refinementIterations: 3,
-      },
-      'traj-exec',
-      makeRouting(),
-      [],
-      4,
-    );
-
-    expect(observation.execution.decompositionUsed).toBe(true);
-    expect(observation.execution.backtrackingOccurred).toBe(true);
-    expect(observation.execution.refinementIterations).toBe(3);
-    expect(observation.execution.toolsUsed).toEqual([]);
-  });
-
-  it('should map outcome fields correctly', async () => {
-    const routing = makeRouting({ estimatedBudget: 10 });
-
-    const observation = await metaLearner.recordFromReflection(
-      {
-        retrievalQuality: 'neutral',
-        outcomeQuality: 'good',
-        whatWorked: ['Worked'],
-        whatFailed: [],
-        suggestions: [],
-        decompositionUsed: false,
-        backtrackingOccurred: false,
-        refinementIterations: 0,
-      },
-      'traj-outcome',
-      routing,
-      [],
-      7,
-    );
-
-    expect(observation.outcome.success).toBe(true); // outcomeQuality != 'poor'
-    expect(observation.outcome.quality).toBe('good');
-    expect(observation.outcome.effortActual).toBe(7);
-    expect(observation.outcome.effortEstimate).toBe(10);
-  });
-
-  it('should set success=false when outcomeQuality is poor', async () => {
-    const observation = await metaLearner.recordFromReflection(
-      {
-        retrievalQuality: 'neutral',
-        outcomeQuality: 'poor',
-        whatWorked: [],
-        whatFailed: ['Everything'],
-        suggestions: ['Retry'],
-        decompositionUsed: false,
-        backtrackingOccurred: false,
-        refinementIterations: 0,
-      },
-      'traj-poor',
-      makeRouting(),
-      [],
-      2,
-    );
-
-    expect(observation.outcome.success).toBe(false);
-    expect(observation.outcome.quality).toBe('poor');
-  });
-
-  it('should map lessons correctly', async () => {
-    const observation = await metaLearner.recordFromReflection(
-      {
-        retrievalQuality: 'neutral',
-        outcomeQuality: 'acceptable',
-        whatWorked: ['Step-by-step approach', 'Error handling'],
-        whatFailed: ['Initial attempt was too broad'],
-        suggestions: ['Start with narrower scope', 'Read docs first'],
-        decompositionUsed: false,
-        backtrackingOccurred: false,
-        refinementIterations: 0,
-      },
-      'traj-lessons',
-      makeRouting(),
-      [],
-      5,
-    );
-
-    expect(observation.lessons.whatWorked).toEqual([
-      'Step-by-step approach',
-      'Error handling',
-    ]);
-    expect(observation.lessons.whatFailed).toEqual([
-      'Initial attempt was too broad',
-    ]);
-    expect(observation.lessons.suggestions).toEqual([
-      'Start with narrower scope',
-      'Read docs first',
-    ]);
-  });
-
-  it('should persist the observation to MetaMemory', async () => {
-    await metaLearner.recordFromReflection(
-      {
-        retrievalQuality: 'helpful',
-        outcomeQuality: 'good',
-        whatWorked: ['Worked'],
-        whatFailed: [],
-        suggestions: [],
-        decompositionUsed: false,
-        backtrackingOccurred: false,
-        refinementIterations: 0,
-      },
-      'traj-persist',
-      makeRouting(),
-      [],
-      4,
-    );
-
-    const stats = await metaMemory.getStats();
-    expect(stats.totalObservations).toBeGreaterThanOrEqual(1);
-  });
-
-  it('should trigger learn() at strategyGenerationInterval', async () => {
-    const learnSpy = vi.spyOn(metaLearner, 'learn');
-
-    // Default interval is 10; create learner with interval=2 for testing
-    const fastLearner = createMetaLearner(metaMemory, {
-      strategyGenerationInterval: 2,
-    });
-
-    const reflection: MetaReflectionOutput = {
-      retrievalQuality: 'neutral',
-      outcomeQuality: 'good',
-      whatWorked: [],
-      whatFailed: [],
-      suggestions: [],
-      decompositionUsed: false,
-      backtrackingOccurred: false,
-      refinementIterations: 0,
-    };
-
-    const fastLearnSpy = vi.spyOn(fastLearner, 'learn');
-
-    await fastLearner.recordFromReflection(reflection, 't1', makeRouting(), [], 1);
-    expect(fastLearnSpy).not.toHaveBeenCalled();
-
-    await fastLearner.recordFromReflection(reflection, 't2', makeRouting(), [], 1);
-    expect(fastLearnSpy).toHaveBeenCalledTimes(1);
-
-    learnSpy.mockRestore();
-    fastLearnSpy.mockRestore();
-  });
-});
-
-// ============================================================
-// Gap 5a: SolutionEvaluator.setTaskRunner delegation
-// ============================================================
-
-describe('Gap 5a: SolutionEvaluator.setTaskRunner', () => {
-  let tempDir: string;
-  let memory: MemorySystem;
-  let agentManager: AgentManager;
-  let persistence: any;
-
-  beforeEach(async () => {
-    tempDir = await mkdtemp(join(tmpdir(), 'evaluator-test-'));
-    persistence = createSqlitePersistence({ baseDir: tempDir });
-    await persistence.init();
-    memory = createMemorySystem(persistence, tempDir);
-    await memory.init();
-    agentManager = createAgentManager(memory);
-    const mockBackend = createMockBackend({ success: true, result: 'ok', durationMs: 10 });
-    agentManager.registerBackend(mockBackend);
-  });
-
-  afterEach(async () => {
-    await memory.close();
-    persistence.close();
-    await rm(tempDir, { recursive: true, force: true });
-  });
-
-  it('should use heuristic when no taskRunner is set', async () => {
-    const evaluator = createSolutionEvaluator(null);
-    const result = await evaluator.evaluate(makeSuccessTrajectory(), makeTask());
-
-    expect(result.method).toBe('heuristic');
-    expect(result.score).toBeGreaterThan(0);
-  });
-
-  it('should delegate to workspace template when taskRunner is set', async () => {
-    const mockEvalResult = createEvaluationResult({
-      quality: 'excellent',
-      score: 0.95,
-      acceptable: true,
-      issues: [],
-      method: 'agent',
-    });
-
-    const runner = createMockTaskRunner({
-      'solution-evaluation': mockEvalResult,
-    });
-
-    const evaluator = createSolutionEvaluator(null);
-    evaluator.setTaskRunner(runner);
-
-    const result = await evaluator.evaluate(makeSuccessTrajectory(), makeTask());
-
-    expect(result.quality).toBe('excellent');
-    expect(result.score).toBe(0.95);
-    expect(runner.calls).toHaveLength(1);
-    expect(runner.calls[0].taskType).toBe('solution-evaluation');
-  });
-
-  it('should pass trajectory and task to the template', async () => {
-    const trajectory = makeSuccessTrajectory();
-    const task = makeTask({ description: 'Specific task desc' });
-
-    const runner = createMockTaskRunner({
-      'solution-evaluation': createEvaluationResult({
-        quality: 'good',
-        score: 0.8,
-        acceptable: true,
-        issues: [],
-        method: 'agent',
-      }),
-    });
-
-    const evaluator = createSolutionEvaluator(null);
-    evaluator.setTaskRunner(runner);
-    await evaluator.evaluate(trajectory, task);
-
-    const input = runner.calls[0].input as SolutionEvaluationInput;
-    expect(input.trajectory).toBe(trajectory);
-    expect(input.task).toBe(task);
-  });
-
-  it('should fall back to heuristic when taskRunner.run throws', async () => {
-    const runner = createMockTaskRunner({
-      'solution-evaluation': new Error('Workspace failed'),
-    });
-
-    const evaluator = createSolutionEvaluator(null);
-    evaluator.setTaskRunner(runner);
-
-    // suppress console.warn from fallback
-    const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
-
-    const result = await evaluator.evaluate(makeSuccessTrajectory(), makeTask());
-
-    expect(result.method).toBe('heuristic');
-    expect(runner.calls).toHaveLength(1);
-
-    warnSpy.mockRestore();
-  });
-
-  it('should prefer verification over workspace template', async () => {
-    const runner = createMockTaskRunner({
-      'solution-evaluation': createEvaluationResult({
-        quality: 'good', score: 0.8, acceptable: true, issues: [], method: 'agent',
-      }),
-    });
-
-    const evaluator = createSolutionEvaluator(null);
-    evaluator.setTaskRunner(runner);
-    evaluator.registerVerifier('code', async () => ({
-      passed: true,
-      confidence: 0.95,
-      details: 'All good',
-    }));
-
-    const result = await evaluator.evaluate(
-      makeSuccessTrajectory(),
-      makeTask({ domain: 'code' }),
-    );
-
-    // Verification took precedence — template never called
-    expect(result.method).toBe('verification');
-    expect(runner.calls).toHaveLength(0);
-  });
-
-  it('should fall through to workspace when verification confidence is low', async () => {
-    const runner = createMockTaskRunner({
-      'solution-evaluation': createEvaluationResult({
-        quality: 'good', score: 0.8, acceptable: true, issues: [], method: 'agent',
-      }),
-    });
-
-    const evaluator = createSolutionEvaluator(null);
-    evaluator.setTaskRunner(runner);
-    evaluator.registerVerifier('code', async () => ({
-      passed: true,
-      confidence: 0.3, // Below default 0.8 threshold
-    }));
-
-    const result = await evaluator.evaluate(
-      makeSuccessTrajectory(),
-      makeTask({ domain: 'code' }),
-    );
-
-    // Low-confidence verification → fell through to workspace
-    expect(result.method).toBe('agent');
-    expect(runner.calls).toHaveLength(1);
-  });
-});
-
-// ============================================================
-// Gap 5b: RefinementLoop.setTaskRunner delegation
-// ============================================================
-
-describe('Gap 5b: RefinementLoop.setTaskRunner', () => {
-  let tempDir: string;
-  let memory: MemorySystem;
-  let agentManager: AgentManager;
-  let mockBackend: MockBackend;
-  let persistence: any;
-
-  beforeEach(async () => {
-    tempDir = await mkdtemp(join(tmpdir(), 'refine-runner-test-'));
-    persistence = createSqlitePersistence({ baseDir: tempDir });
-    await persistence.init();
-    memory = createMemorySystem(persistence, tempDir);
-    await memory.init();
-    agentManager = createAgentManager(memory);
-    mockBackend = createMockBackend({
-      success: true,
-      result: 'Refined',
-      durationMs: 50,
-    });
-    agentManager.registerBackend(mockBackend);
-  });
-
-  afterEach(async () => {
-    await memory.close();
-    persistence.close();
-    await rm(tempDir, { recursive: true, force: true });
-  });
-
-  it('should propagate taskRunner to evaluator', () => {
-    const loop = createRefinementLoop(agentManager, memory);
-    const runner = createMockTaskRunner({});
-
-    loop.setTaskRunner(runner);
-
-    // The evaluator should also have the runner — verify indirectly:
-    // evaluator.setTaskRunner was called by RefinementLoop.setTaskRunner
-    const evaluator = loop.getEvaluator();
-    expect(evaluator).toBeDefined();
-    // We can verify the runner is set by checking that evaluator.evaluate
-    // tries the template path (which will throw since we gave no output)
-  });
-
-  it('should use refinement plan from template when runner is set', async () => {
-    const plan: RefinementPlan = {
-      rootCauses: ['Token not refreshed', 'Session timeout ignored'],
-      strategy: 'Implement token refresh with retry logic',
-      steps: ['Add refresh interceptor', 'Handle timeout gracefully'],
-      avoidPatterns: ['Hardcoded token values'],
-    };
-
-    // First solution-evaluation call (initial) → poor → triggers refinement
-    // Second call (after retry) → excellent → stops loop
-    const poorEval = createEvaluationResult({
-      quality: 'poor', score: 0.2, acceptable: false,
-      issues: [{ type: 'error', description: 'Auth failed', severity: 'critical' }],
-      method: 'agent',
-    });
-    const goodEval = createEvaluationResult({
-      quality: 'excellent', score: 0.95, acceptable: true, issues: [], method: 'agent',
-    });
-
-    const runner = createMockTaskRunner({
-      'refinement-analysis': plan,
-      'solution-evaluation': seq(poorEval, goodEval),
-    });
-
-    const loop = createRefinementLoop(agentManager, memory, {
-      maxIterations: 2,
-      minAcceptableScore: 0.7,
-    });
-    loop.setTaskRunner(runner);
-
-    const failTraj = makeFailTrajectory();
-    const spawnConfig = {
-      agentType: 'mock',
-      task: makeTask(),
-      timeout: 60000,
-    };
-
-    const result = await loop.refine(failTraj, spawnConfig);
-
-    // refinement-analysis should have been called for the retry
-    const refinementCalls = runner.calls.filter(c => c.taskType === 'refinement-analysis');
-    expect(refinementCalls.length).toBeGreaterThanOrEqual(1);
-    expect(result.iterations).toBeGreaterThanOrEqual(1);
-  });
-
-  it('should fall back to heuristic refinement when template fails', async () => {
-    const poorEval = createEvaluationResult({
-      quality: 'poor', score: 0.2, acceptable: false,
-      issues: [{ type: 'error', description: 'Failed', severity: 'critical' }],
-      method: 'agent',
-    });
-    const goodEval = createEvaluationResult({
-      quality: 'excellent', score: 0.95, acceptable: true, issues: [], method: 'agent',
-    });
-
-    const runner = createMockTaskRunner({
-      'refinement-analysis': new Error('Template failure'),
-      'solution-evaluation': seq(poorEval, goodEval),
-    });
-
-    const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
-
-    const loop = createRefinementLoop(agentManager, memory, {
-      maxIterations: 2,
-      minAcceptableScore: 0.7,
-    });
-    loop.setTaskRunner(runner);
-
-    const failTraj = makeFailTrajectory();
-    const spawnConfig = {
-      agentType: 'mock',
-      task: makeTask(),
-      timeout: 60000,
-    };
-
-    // Should not throw — falls back to heuristic prompt
-    const result = await loop.refine(failTraj, spawnConfig);
-    expect(result).toBeDefined();
-    expect(result.iterations).toBeGreaterThanOrEqual(1);
-
-    warnSpy.mockRestore();
-  });
-
-  it('should format RefinementPlan into prompt additions for the retry agent', async () => {
-    const plan: RefinementPlan = {
-      rootCauses: ['Stale cache', 'Race condition'],
-      strategy: 'Use lock-based approach with cache invalidation',
-      steps: ['Acquire lock', 'Invalidate cache', 'Retry operation'],
-      avoidPatterns: ['Polling without backoff'],
-    };
-
-    const poorEval = createEvaluationResult({
-      quality: 'poor', score: 0.2, acceptable: false,
-      issues: [{ type: 'error', description: 'Cache stale', severity: 'critical' }],
-      method: 'agent',
-    });
-    const goodEval = createEvaluationResult({
-      quality: 'excellent', score: 0.95, acceptable: true, issues: [], method: 'agent',
-    });
-
-    const runner = createMockTaskRunner({
-      'refinement-analysis': plan,
-      'solution-evaluation': seq(poorEval, goodEval),
-    });
-
-    const loop = createRefinementLoop(agentManager, memory, {
-      maxIterations: 2,
-      minAcceptableScore: 0.7,
-    });
-    loop.setTaskRunner(runner);
-
-    // Spy on AgentManager.spawn to capture the refined prompt
-    const spawnSpy = vi.spyOn(agentManager, 'spawn');
-
-    const failTraj = makeFailTrajectory();
-    await loop.refine(failTraj, {
-      agentType: 'mock',
-      task: makeTask(),
-      timeout: 60000,
-    });
-
-    // Find a spawn call that includes refinement content
-    const spawnCalls = spawnSpy.mock.calls;
-    const refinementCall = spawnCalls.find(call => {
-      const config = call[0];
-      return config.systemPromptAdditions?.includes('Root Cause Analysis');
-    });
-
-    expect(refinementCall).toBeDefined();
-    const additions = refinementCall![0].systemPromptAdditions!;
-    expect(additions).toContain('Stale cache');
-    expect(additions).toContain('Race condition');
-    expect(additions).toContain('Use lock-based approach');
-    expect(additions).toContain('Acquire lock');
-    expect(additions).toContain('DO NOT: Polling without backoff');
-
-    spawnSpy.mockRestore();
-  });
-});
-
-// ============================================================
-// Gap 2: Atlas.solve() template delegation and fallback
-// ============================================================
-
-describe('Gap 2: Atlas.solve() template delegation', () => {
-  let tempDir: string;
-  let atlas: Atlas;
-  let mockBackend: MockBackend;
-
-  beforeEach(async () => {
-    tempDir = await mkdtemp(join(tmpdir(), 'atlas-delegation-test-'));
-    mockBackend = createMockBackend({
-      success: true,
-      result: 'Task completed',
-      durationMs: 50,
-      toolCalls: [
-        { name: 'read_file', input: { path: 'src/main.ts' }, output: 'code' },
-      ],
-    });
-
-    atlas = createAtlasWithAgents([mockBackend], {
-      storage: { baseDir: tempDir },
-      learning: { minTrajectories: 100 }, // high threshold to avoid batch triggers
-      execution: { defaultAgentType: 'mock', maxExecutionTime: 60 },
-    });
-    await atlas.init();
-  });
-
-  afterEach(async () => {
-    await atlas.close();
-    await rm(tempDir, { recursive: true, force: true });
-  });
-
-  it('should call reflectOnExecution after solve', async () => {
-    // The metaLearner should have an observation after solve
-    const task = makeTask();
-    const result = await atlas.solve(task);
-
-    expect(result.trajectory).toBeDefined();
-    expect(result.trajectory.outcome.success).toBe(true);
-
-    // Meta learner should have been invoked (either via template fallback or heuristic)
-    const metaLearner = atlas.getMetaLearner();
-    expect(metaLearner).toBeDefined();
-  });
-
-  it('should handle solve when no playbooks exist (no usage inference needed)', async () => {
-    const task = makeTask({ description: 'A unique task with no playbooks' });
-    const result = await atlas.solve(task);
-
-    expect(result.trajectory).toBeDefined();
-    // No injected playbooks → no usage inference
-    expect(result.injectedPlaybooks).toEqual([]);
-  });
-
-  it('should infer playbook usage via template when playbooks are injected', async () => {
-    // Add a playbook to memory so it gets injected
-    const mem = atlas.getMemory();
-    await mem.playbooks.add(makePlaybook('pb-auth', 'Auth Bug Fix'));
-
-    // Solve a matching task
-    const task = makeTask({
-      description: 'Fix the authentication bug in login',
-      domain: 'code',
-    });
-    const result = await atlas.solve(task);
-
-    // Playbook was injected (may or may not get usage depending on heuristic)
-    // The key check: solve completes without error when playbooks are present
-    expect(result.trajectory).toBeDefined();
-    expect(result.trajectory.outcome.success).toBe(true);
-  });
-
-  it('should handle failed agent execution gracefully', async () => {
-    const failBackend = createMockBackend({
-      success: false,
-      result: undefined,
-      durationMs: 50,
-      error: 'Agent crashed',
-    });
-
-    const failAtlas = createAtlasWithAgents([failBackend], {
-      storage: { baseDir: tempDir + '-fail' },
-      learning: { minTrajectories: 100 },
-      execution: { defaultAgentType: 'mock', maxExecutionTime: 60 },
-    });
-    await failAtlas.init();
-
-    // Suppress warnings from template fallbacks in failed execution
-    const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
-    const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {});
-
-    const task = makeTask();
-    const result = await failAtlas.solve(task);
-
-    expect(result.trajectory).toBeDefined();
-    expect(result.trajectory.outcome.success).toBe(false);
-
-    await failAtlas.close();
-    await rm(tempDir + '-fail', { recursive: true, force: true });
-
-    warnSpy.mockRestore();
-    errorSpy.mockRestore();
-  });
-});
-
-// ============================================================
-// Template-level tests: meta-reflection heuristic correctness
-// ============================================================
-
-describe('Meta-reflection template heuristic', () => {
-  it('should return heuristic complexity for successful high-confidence tasks', () => {
-    const complexity = metaReflectionTemplate.assessComplexity({
-      trajectory: makeSuccessTrajectory(),
-      routingDecision: { strategy: 'direct', confidence: 0.9, estimatedBudget: 5, reasoning: 'ok' },
-      playbooksUsed: [],
-    });
-    expect(complexity).toBe('heuristic');
-  });
-
-  it('should return lightweight for failed tasks', () => {
-    const complexity = metaReflectionTemplate.assessComplexity({
-      trajectory: makeFailTrajectory(),
-      routingDecision: { strategy: 'explore', confidence: 0.4, estimatedBudget: 10, reasoning: 'uncertain' },
-      playbooksUsed: [],
-    });
-    expect(complexity).toBe('lightweight');
-  });
-
-  it('should compute correct heuristic outcomeQuality based on effort ratio', async () => {
-    const heuristic = metaReflectionTemplate.heuristicFallback!;
-
-    // effortRatio = 5/10 = 0.5 → 'excellent'
-    const excellent = await heuristic({
-      trajectory: makeSuccessTrajectory({ llmCalls: 5 }),
-      routingDecision: { strategy: 'direct', confidence: 0.9, estimatedBudget: 10, reasoning: 'ok' },
-      playbooksUsed: [],
-    });
-    expect(excellent.outcomeQuality).toBe('excellent');
-
-    // effortRatio = 10/10 = 1.0 → 'good'
-    const good = await heuristic({
-      trajectory: makeSuccessTrajectory({ llmCalls: 10 }),
-      routingDecision: { strategy: 'direct', confidence: 0.9, estimatedBudget: 10, reasoning: 'ok' },
-      playbooksUsed: [],
-    });
-    expect(good.outcomeQuality).toBe('good');
-
-    // effortRatio = 15/10 = 1.5 → 'acceptable'
-    const acceptable = await heuristic({
-      trajectory: makeSuccessTrajectory({ llmCalls: 15 }),
-      routingDecision: { strategy: 'direct', confidence: 0.9, estimatedBudget: 10, reasoning: 'ok' },
-      playbooksUsed: [],
-    });
-    expect(acceptable.outcomeQuality).toBe('acceptable');
-
-    // Failed trajectory → 'poor'
-    const poor = await heuristic({
-      trajectory: makeFailTrajectory(),
-      routingDecision: { strategy: 'explore', confidence: 0.5, estimatedBudget: 10, reasoning: 'guess' },
-      playbooksUsed: [],
-    });
-    expect(poor.outcomeQuality).toBe('poor');
-  });
-
-  it('should report helpful retrieval when playbooks used and successful', async () => {
-    const heuristic = metaReflectionTemplate.heuristicFallback!;
-    const result = await heuristic({
-      trajectory: makeSuccessTrajectory(),
-      routingDecision: { strategy: 'direct', confidence: 0.9, estimatedBudget: 10, reasoning: 'ok' },
-      playbooksUsed: [{
-        playbook: makePlaybook('pb-1', 'Auth Fix'),
-        score: 0.9,
-        matchType: 'situation',
-      }],
-    });
-
-    expect(result.retrievalQuality).toBe('helpful');
-  });
-
-  it('should report neutral retrieval when no playbooks used', async () => {
-    const heuristic = metaReflectionTemplate.heuristicFallback!;
-    const result = await heuristic({
-      trajectory: makeSuccessTrajectory(),
-      routingDecision: { strategy: 'direct', confidence: 0.9, estimatedBudget: 10, reasoning: 'ok' },
-      playbooksUsed: [],
-    });
-
-    expect(result.retrievalQuality).toBe('neutral');
-  });
-});
-
-// ============================================================
-// Template-level tests: refinement-analysis
-// ============================================================
-
-describe('Refinement-analysis template complexity', () => {
-  const baseInput = {
-    trajectory: makeFailTrajectory(),
-    task: makeTask(),
-    evaluation: createEvaluationResult({
-      quality: 'poor', score: 0.3, acceptable: false, issues: [], method: 'heuristic',
-    }),
-    previousAttempts: 1,
-  };
-
-  it('should return lightweight for first failure', () => {
-    expect(refinementAnalysisTemplate.assessComplexity(baseInput)).toBe('lightweight');
-  });
-
-  it('should return standard for repeated failures', () => {
-    expect(
-      refinementAnalysisTemplate.assessComplexity({ ...baseInput, previousAttempts: 2 })
-    ).toBe('standard');
-  });
-
-  it('should have no heuristic fallback', () => {
-    expect(refinementAnalysisTemplate.heuristicFallback).toBeUndefined();
-  });
-});
-
-// ============================================================
-// Template-level tests: usage-inference complexity
-// ============================================================
-
-describe('Usage-inference template complexity', () => {
-  it('should return heuristic when no playbooks injected', () => {
-    const complexity = usageInferenceTemplate.assessComplexity({
-      trajectory: makeSuccessTrajectory(),
-      injectedPlaybooks: [],
-    });
-    expect(complexity).toBe('heuristic');
-  });
-
-  it('should return heuristic for few playbooks and short trajectory', () => {
-    const complexity = usageInferenceTemplate.assessComplexity({
-      trajectory: makeSuccessTrajectory(), // 3 steps
-      injectedPlaybooks: [makePlaybook('p1', 'P1')],
-    });
-    expect(complexity).toBe('heuristic');
-  });
-});