cognitive-core 0.1.0 → 0.2.0

package/tests/gap-fixes.test.ts

@@ -0,0 +1,1116 @@
+/**
+ * Tests for the 5 gap fixes:
+ *   Gap 1: LLMPlaybookExtractor deletion
+ *   Gap 2: Atlas.solve() template delegation with fallback
+ *   Gap 3: solution-evaluation template codebasePath / getResources
+ *   Gap 4: MetaLearner.recordFromReflection bridge
+ *   Gap 5: SolutionEvaluator & RefinementLoop setTaskRunner delegation
+ */
+
+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
+import { mkdtemp, rm } from 'node:fs/promises';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+
+// Types
+import { createTask } from '../src/types/task.js';
+import { createTrajectory } from '../src/types/trajectory.js';
+import { createStep } from '../src/types/step.js';
+import { successOutcome, failureOutcome } from '../src/types/outcome.js';
+import type { Trajectory } from '../src/types/trajectory.js';
+import type { Task } from '../src/types/task.js';
+import type { Playbook } from '../src/types/index.js';
+
+// Memory
+import { createMemorySystem, type MemorySystem } from '../src/memory/system.js';
+import { MetaMemory } from '../src/memory/meta.js';
+import { createSqlitePersistence } from '../src/persistence/index.js';
+
+// Learning
+import { MetaLearner, createMetaLearner } from '../src/learning/meta-learner.js';
+import type { MetaReflectionOutput } from '../src/workspace/templates/meta-reflection.js';
+import type { RoutingDecision } from '../src/learning/meta-learner.js';
+
+// Search
+import { SolutionEvaluator, createSolutionEvaluator } from '../src/search/evaluator.js';
+import { RefinementLoop, createRefinementLoop } from '../src/search/refinement-loop.js';
+import { createEvaluationResult, type EvaluationResult } from '../src/search/refinement-types.js';
+
+// Runtime
+import { AgentManager, createAgentManager } from '../src/runtime/manager.js';
+import { MockBackend, createMockBackend } from '../src/runtime/backends/mock.js';
+
+// Workspace / templates
+import type { AgenticTaskRunner } from '../src/workspace/runner.js';
+import { solutionEvaluationTemplate } from '../src/workspace/templates/solution-evaluation.js';
+import type { SolutionEvaluationInput } from '../src/workspace/templates/solution-evaluation.js';
+import { refinementAnalysisTemplate, type RefinementPlan } from '../src/workspace/templates/refinement-analysis.js';
+import { metaReflectionTemplate } from '../src/workspace/templates/meta-reflection.js';
+import { usageInferenceTemplate } from '../src/workspace/templates/usage-inference.js';
+
+// Atlas
+import { Atlas, createAtlasWithAgents } from '../src/atlas.js';
+
+// ============================================================
+// Helpers
+// ============================================================
+
+const makeTask = (overrides: Partial<Task> = {}) =>
+  createTask({
+    domain: 'code',
+    description: 'Fix the authentication bug',
+    ...overrides,
+  });
+
+const makeSuccessTrajectory = (overrides: Partial<Trajectory> = {}): Trajectory =>
+  createTrajectory({
+    task: makeTask(),
+    steps: [
+      createStep({ thought: 'analyze', action: 'read_file', observation: 'found code' }),
+      createStep({ thought: 'fix', action: 'write_file', observation: 'written' }),
+      createStep({ action: 'run_tests', observation: 'all tests pass' }),
+    ],
+    outcome: successOutcome('Bug fixed'),
+    agentId: 'test-agent',
+    wallTimeSeconds: 30,
+    llmCalls: 5,
+    ...overrides,
+  });
+
+const makeFailTrajectory = (overrides: Partial<Trajectory> = {}): Trajectory =>
+  createTrajectory({
+    task: makeTask(),
+    steps: [
+      createStep({ action: 'attempt', observation: 'error occurred' }),
+    ],
+    outcome: failureOutcome('Could not fix bug'),
+    agentId: 'test-agent',
+    wallTimeSeconds: 10,
+    llmCalls: 2,
+    ...overrides,
+  });
+
+const makeRouting = (overrides: Partial<RoutingDecision> = {}): RoutingDecision => ({
+  strategy: 'direct',
+  confidence: 0.8,
+  estimatedBudget: 10,
+  reasoning: 'High similarity to known patterns',
+  ...overrides,
+});
+
+const makePlaybook = (id: string, name: string): Playbook => ({
+  id,
+  name,
+  applicability: {
+    situations: ['Fix a bug'],
+    triggers: ['authentication'],
+    antiPatterns: [],
+    domains: ['code'],
+  },
+  guidance: {
+    strategy: 'Check token validation',
+    tactics: ['Verify token expiry', 'Check session storage'],
+    steps: ['Read code', 'Find bug', 'Fix', 'Test'],
+  },
+  verification: {
+    successIndicators: ['Tests pass'],
+    failureIndicators: ['Auth still fails'],
+  },
+  evolution: {
+    version: '1.0.0',
+    createdFrom: [],
+    failures: [],
+    refinements: [],
+    successCount: 5,
+    failureCount: 0,
+  },
+  confidence: 0.85,
+  complexity: 'moderate',
+  domain: 'code',
+  createdAt: new Date(),
+  updatedAt: new Date(),
+});
+
+/**
+ * Wraps an array of values as a sequence for createMockTaskRunner.
+ * Successive calls consume values in order; the last one repeats.
+ */
+function seq(...values: unknown[]): { __seq: true; values: unknown[] } {
+  return { __seq: true, values };
+}
+
+function isSeq(v: unknown): v is { __seq: true; values: unknown[] } {
+  return typeof v === 'object' && v !== null && '__seq' in v && (v as Record<string, unknown>).__seq === true;
+}
+
+/**
+ * Creates a mock AgenticTaskRunner that records calls and returns
+ * configurable outputs per template taskType.
+ *
+ * Outputs can be:
+ *   - A single value → returned every time
+ *   - An Error → thrown every time
+ *   - seq(v1, v2, ...) → values are consumed in order (last one repeats)
+ */
+function createMockTaskRunner(
+  outputs: Record<string, unknown> = {}
+): AgenticTaskRunner & { calls: Array<{ taskType: string; input: unknown }> } {
+  const calls: Array<{ taskType: string; input: unknown }> = [];
+  const callCounts: Record<string, number> = {};
+
+  return {
+    calls,
+    run: vi.fn(async (template: { taskType: string }, input: unknown) => {
+      calls.push({ taskType: template.taskType, input });
+
+      const raw = outputs[template.taskType];
+      if (raw === undefined) {
+        throw new Error(`No mock output for template ${template.taskType}`);
+      }
+
+      let output: unknown;
+      if (isSeq(raw)) {
+        const idx = callCounts[template.taskType] ?? 0;
+        callCounts[template.taskType] = idx + 1;
+        output = raw.values[Math.min(idx, raw.values.length - 1)];
+      } else {
+        output = raw;
+      }
+
+      if (output instanceof Error) throw output;
+
+      return {
+        output,
+        trajectory: makeSuccessTrajectory(),
+        workspace: { id: 'mock-ws', path: '/tmp/mock-ws' },
+      };
+    }),
+  } as unknown as AgenticTaskRunner & { calls: Array<{ taskType: string; input: unknown }> };
+}
+
+// ============================================================
+// Gap 1: LLMPlaybookExtractor deletion
+// ============================================================
+
+describe('Gap 1: LLMPlaybookExtractor deletion', () => {
+  it('should not export LLMPlaybookExtractor from learning/index', async () => {
+    const learningExports = await import('../src/learning/index.js');
+    expect('LLMPlaybookExtractor' in learningExports).toBe(false);
+    expect('createLLMExtractor' in learningExports).toBe(false);
+  });
+
+  it('should not export LLMPlaybookExtractor from root index', async () => {
+    const rootExports = await import('../src/index.js');
+    expect('LLMPlaybookExtractor' in rootExports).toBe(false);
+    expect('createLLMExtractor' in rootExports).toBe(false);
+  });
+
+  it('should still export other learning components', async () => {
+    const learningExports = await import('../src/learning/index.js');
+    expect('PlaybookExtractor' in learningExports).toBe(true);
+    expect('MetaLearner' in learningExports).toBe(true);
+    expect('PlaybookUsageInference' in learningExports).toBe(true);
+    expect('UnifiedLearningPipeline' in learningExports).toBe(true);
+    expect('LearningEffectivenessTracker' in learningExports).toBe(true);
+  });
+});
+
+// ============================================================
+// Gap 3: solution-evaluation template codebasePath / getResources
+// ============================================================
+
+describe('Gap 3: solution-evaluation template codebasePath', () => {
+  const trajectory = makeSuccessTrajectory();
+  const task = makeTask();
+
+  it('should return empty resources when codebasePath is not set', () => {
+    const resources = solutionEvaluationTemplate.getResources({ trajectory, task });
+    expect(resources).toEqual([]);
+  });
+
+  it('should return a symlink resource when codebasePath is set', () => {
+    const resources = solutionEvaluationTemplate.getResources({
+      trajectory,
+      task,
+      codebasePath: '/home/user/my-project',
+    });
+
+    expect(resources).toHaveLength(1);
+    expect(resources[0]).toEqual({
+      path: 'codebase',
+      type: 'symlink',
+      source: '/home/user/my-project',
+      description: 'Codebase for solution verification',
+    });
+  });
+
+  it('should include codebase instruction in prompt when codebasePath is set', () => {
+    const prompt = solutionEvaluationTemplate.buildTaskPrompt({
+      trajectory,
+      task,
+      codebasePath: '/home/user/my-project',
+    });
+
+    expect(prompt).toContain('resources/codebase/');
+  });
+
+  it('should NOT include codebase instruction in prompt when codebasePath is absent', () => {
+    const prompt = solutionEvaluationTemplate.buildTaskPrompt({
+      trajectory,
+      task,
+    });
+
+    expect(prompt).not.toContain('resources/codebase/');
+  });
+
+  it('should still use heuristic complexity when task has verification', () => {
+    const taskWithVerification = makeTask({
+      verification: { type: 'exact_match', expected: 'pass' },
+    });
+
+    const complexity = solutionEvaluationTemplate.assessComplexity({
+      trajectory,
+      task: taskWithVerification,
+    });
+    expect(complexity).toBe('heuristic');
+  });
+});
+
+// ============================================================
+// Gap 4: MetaLearner.recordFromReflection
+// ============================================================
+
+describe('Gap 4: MetaLearner.recordFromReflection', () => {
+  let tempDir: string;
+  let metaMemory: MetaMemory;
+  let metaLearner: MetaLearner;
+  let persistence: any;
+
+  beforeEach(async () => {
+    tempDir = await mkdtemp(join(tmpdir(), 'meta-learner-test-'));
+    persistence = createSqlitePersistence({ baseDir: tempDir });
+    await persistence.init();
+    metaMemory = new MetaMemory(persistence);
+    await metaMemory.init();
+    metaLearner = createMetaLearner(metaMemory);
+  });
+
+  afterEach(async () => {
+    persistence.close();
+    await rm(tempDir, { recursive: true, force: true });
+  });
+
+  it('should create a MetaObservation from MetaReflectionOutput', async () => {
+    const reflection: MetaReflectionOutput = {
+      retrievalQuality: 'helpful',
+      outcomeQuality: 'good',
+      whatWorked: ['Direct routing was effective'],
+      whatFailed: [],
+      suggestions: ['Consider caching results'],
+      decompositionUsed: true,
+      backtrackingOccurred: false,
+      refinementIterations: 1,
+    };
+
+    const routing = makeRouting();
+    const observation = await metaLearner.recordFromReflection(
+      reflection,
+      'traj-123',
+      routing,
+      ['playbook-1', 'playbook-2'],
+      5,
+    );
+
+    expect(observation.trajectoryId).toBe('traj-123');
+    expect(observation.id).toBeDefined();
+    expect(observation.timestamp).toBeInstanceOf(Date);
+  });
+
+  it('should map routing fields correctly', async () => {
+    const routing = makeRouting({
+      strategy: 'explore',
+      confidence: 0.6,
+      reasoning: 'Unfamiliar task',
+    });
+
+    const observation = await metaLearner.recordFromReflection(
+      {
+        retrievalQuality: 'neutral',
+        outcomeQuality: 'acceptable',
+        whatWorked: [],
+        whatFailed: ['Exploration was slow'],
+        suggestions: [],
+        decompositionUsed: false,
+        backtrackingOccurred: true,
+        refinementIterations: 2,
+      },
+      'traj-456',
+      routing,
+      [],
+      8,
+    );
+
+    expect(observation.routing.decision).toBe('explore');
+    expect(observation.routing.confidence).toBe(0.6);
+    expect(observation.routing.reasoning).toBe('Unfamiliar task');
+  });
+
+  it('should map memory usage fields correctly', async () => {
+    const observation = await metaLearner.recordFromReflection(
+      {
+        retrievalQuality: 'misleading',
+        outcomeQuality: 'poor',
+        whatWorked: [],
+        whatFailed: ['Playbook was wrong'],
+        suggestions: ['Update playbook'],
+        decompositionUsed: false,
+        backtrackingOccurred: false,
+        refinementIterations: 0,
+      },
+      'traj-789',
+      makeRouting(),
+      ['pb-1'],
+      3,
+    );
+
+    expect(observation.memoryUsage.retrievalQuality).toBe('misleading');
+    expect(observation.memoryUsage.playbooksApplied).toEqual(['pb-1']);
+    expect(observation.memoryUsage.experiencesRetrieved).toBe(0);
+  });
+
+  it('should map execution fields correctly', async () => {
+    const observation = await metaLearner.recordFromReflection(
+      {
+        retrievalQuality: 'helpful',
+        outcomeQuality: 'excellent',
+        whatWorked: ['Decomposition was key'],
+        whatFailed: [],
+        suggestions: [],
+        decompositionUsed: true,
+        backtrackingOccurred: true,
+        refinementIterations: 3,
+      },
+      'traj-exec',
+      makeRouting(),
+      [],
+      4,
+    );
+
+    expect(observation.execution.decompositionUsed).toBe(true);
+    expect(observation.execution.backtrackingOccurred).toBe(true);
+    expect(observation.execution.refinementIterations).toBe(3);
+    expect(observation.execution.toolsUsed).toEqual([]);
+  });
+
+  it('should map outcome fields correctly', async () => {
+    const routing = makeRouting({ estimatedBudget: 10 });
+
+    const observation = await metaLearner.recordFromReflection(
+      {
+        retrievalQuality: 'neutral',
+        outcomeQuality: 'good',
+        whatWorked: ['Worked'],
+        whatFailed: [],
+        suggestions: [],
+        decompositionUsed: false,
+        backtrackingOccurred: false,
+        refinementIterations: 0,
+      },
+      'traj-outcome',
+      routing,
+      [],
+      7,
+    );
+
+    expect(observation.outcome.success).toBe(true); // outcomeQuality != 'poor'
+    expect(observation.outcome.quality).toBe('good');
+    expect(observation.outcome.effortActual).toBe(7);
+    expect(observation.outcome.effortEstimate).toBe(10);
+  });
+
+  it('should set success=false when outcomeQuality is poor', async () => {
+    const observation = await metaLearner.recordFromReflection(
+      {
+        retrievalQuality: 'neutral',
+        outcomeQuality: 'poor',
+        whatWorked: [],
+        whatFailed: ['Everything'],
+        suggestions: ['Retry'],
+        decompositionUsed: false,
+        backtrackingOccurred: false,
+        refinementIterations: 0,
+      },
+      'traj-poor',
+      makeRouting(),
+      [],
+      2,
+    );
+
+    expect(observation.outcome.success).toBe(false);
+    expect(observation.outcome.quality).toBe('poor');
+  });
+
+  it('should map lessons correctly', async () => {
+    const observation = await metaLearner.recordFromReflection(
+      {
+        retrievalQuality: 'neutral',
+        outcomeQuality: 'acceptable',
+        whatWorked: ['Step-by-step approach', 'Error handling'],
+        whatFailed: ['Initial attempt was too broad'],
+        suggestions: ['Start with narrower scope', 'Read docs first'],
+        decompositionUsed: false,
+        backtrackingOccurred: false,
+        refinementIterations: 0,
+      },
+      'traj-lessons',
+      makeRouting(),
+      [],
+      5,
+    );
+
+    expect(observation.lessons.whatWorked).toEqual([
+      'Step-by-step approach',
+      'Error handling',
+    ]);
+    expect(observation.lessons.whatFailed).toEqual([
+      'Initial attempt was too broad',
+    ]);
+    expect(observation.lessons.suggestions).toEqual([
+      'Start with narrower scope',
+      'Read docs first',
+    ]);
+  });
+
+  it('should persist the observation to MetaMemory', async () => {
+    await metaLearner.recordFromReflection(
+      {
+        retrievalQuality: 'helpful',
+        outcomeQuality: 'good',
+        whatWorked: ['Worked'],
+        whatFailed: [],
+        suggestions: [],
+        decompositionUsed: false,
+        backtrackingOccurred: false,
+        refinementIterations: 0,
+      },
+      'traj-persist',
+      makeRouting(),
+      [],
+      4,
+    );
+
+    const stats = await metaMemory.getStats();
+    expect(stats.totalObservations).toBeGreaterThanOrEqual(1);
+  });
+
+  it('should trigger learn() at strategyGenerationInterval', async () => {
+    const learnSpy = vi.spyOn(metaLearner, 'learn');
+
+    // Default interval is 10; create learner with interval=2 for testing
+    const fastLearner = createMetaLearner(metaMemory, {
+      strategyGenerationInterval: 2,
+    });
+
+    const reflection: MetaReflectionOutput = {
+      retrievalQuality: 'neutral',
+      outcomeQuality: 'good',
+      whatWorked: [],
+      whatFailed: [],
+      suggestions: [],
+      decompositionUsed: false,
+      backtrackingOccurred: false,
+      refinementIterations: 0,
+    };
+
+    const fastLearnSpy = vi.spyOn(fastLearner, 'learn');
+
+    await fastLearner.recordFromReflection(reflection, 't1', makeRouting(), [], 1);
+    expect(fastLearnSpy).not.toHaveBeenCalled();
+
+    await fastLearner.recordFromReflection(reflection, 't2', makeRouting(), [], 1);
+    expect(fastLearnSpy).toHaveBeenCalledTimes(1);
+
+    learnSpy.mockRestore();
+    fastLearnSpy.mockRestore();
+  });
+});
+
+// ============================================================
+// Gap 5a: SolutionEvaluator.setTaskRunner delegation
+// ============================================================
+
+describe('Gap 5a: SolutionEvaluator.setTaskRunner', () => {
+  let tempDir: string;
+  let memory: MemorySystem;
+  let agentManager: AgentManager;
+  let persistence: any;
+
+  beforeEach(async () => {
+    tempDir = await mkdtemp(join(tmpdir(), 'evaluator-test-'));
+    persistence = createSqlitePersistence({ baseDir: tempDir });
+    await persistence.init();
+    memory = createMemorySystem(persistence, tempDir);
+    await memory.init();
+    agentManager = createAgentManager(memory);
+    const mockBackend = createMockBackend({ success: true, result: 'ok', durationMs: 10 });
+    agentManager.registerBackend(mockBackend);
+  });
+
+  afterEach(async () => {
+    await memory.close();
+    persistence.close();
+    await rm(tempDir, { recursive: true, force: true });
+  });
+
+  it('should use heuristic when no taskRunner is set', async () => {
+    const evaluator = createSolutionEvaluator(null);
+    const result = await evaluator.evaluate(makeSuccessTrajectory(), makeTask());
+
+    expect(result.method).toBe('heuristic');
+    expect(result.score).toBeGreaterThan(0);
+  });
+
+  it('should delegate to workspace template when taskRunner is set', async () => {
+    const mockEvalResult = createEvaluationResult({
+      quality: 'excellent',
+      score: 0.95,
+      acceptable: true,
+      issues: [],
+      method: 'agent',
+    });
+
+    const runner = createMockTaskRunner({
+      'solution-evaluation': mockEvalResult,
+    });
+
+    const evaluator = createSolutionEvaluator(null);
+    evaluator.setTaskRunner(runner);
+
+    const result = await evaluator.evaluate(makeSuccessTrajectory(), makeTask());
+
+    expect(result.quality).toBe('excellent');
+    expect(result.score).toBe(0.95);
+    expect(runner.calls).toHaveLength(1);
+    expect(runner.calls[0].taskType).toBe('solution-evaluation');
+  });
+
+  it('should pass trajectory and task to the template', async () => {
+    const trajectory = makeSuccessTrajectory();
+    const task = makeTask({ description: 'Specific task desc' });
+
+    const runner = createMockTaskRunner({
+      'solution-evaluation': createEvaluationResult({
+        quality: 'good',
+        score: 0.8,
+        acceptable: true,
+        issues: [],
+        method: 'agent',
+      }),
+    });
+
+    const evaluator = createSolutionEvaluator(null);
+    evaluator.setTaskRunner(runner);
+    await evaluator.evaluate(trajectory, task);
+
+    const input = runner.calls[0].input as SolutionEvaluationInput;
+    expect(input.trajectory).toBe(trajectory);
+    expect(input.task).toBe(task);
+  });
+
+  it('should fall back to heuristic when taskRunner.run throws', async () => {
+    const runner = createMockTaskRunner({
+      'solution-evaluation': new Error('Workspace failed'),
+    });
+
+    const evaluator = createSolutionEvaluator(null);
+    evaluator.setTaskRunner(runner);
+
+    // suppress console.warn from fallback
+    const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
+
+    const result = await evaluator.evaluate(makeSuccessTrajectory(), makeTask());
+
+    expect(result.method).toBe('heuristic');
+    expect(runner.calls).toHaveLength(1);
+
+    warnSpy.mockRestore();
+  });
+
+  it('should prefer verification over workspace template', async () => {
+    const runner = createMockTaskRunner({
+      'solution-evaluation': createEvaluationResult({
+        quality: 'good', score: 0.8, acceptable: true, issues: [], method: 'agent',
+      }),
+    });
+
+    const evaluator = createSolutionEvaluator(null);
+    evaluator.setTaskRunner(runner);
+    evaluator.registerVerifier('code', async () => ({
+      passed: true,
+      confidence: 0.95,
+      details: 'All good',
+    }));
+
+    const result = await evaluator.evaluate(
+      makeSuccessTrajectory(),
+      makeTask({ domain: 'code' }),
+    );
+
+    // Verification took precedence — template never called
+    expect(result.method).toBe('verification');
+    expect(runner.calls).toHaveLength(0);
+  });
+
+  it('should fall through to workspace when verification confidence is low', async () => {
+    const runner = createMockTaskRunner({
+      'solution-evaluation': createEvaluationResult({
+        quality: 'good', score: 0.8, acceptable: true, issues: [], method: 'agent',
+      }),
+    });
+
+    const evaluator = createSolutionEvaluator(null);
+    evaluator.setTaskRunner(runner);
+    evaluator.registerVerifier('code', async () => ({
+      passed: true,
+      confidence: 0.3, // Below default 0.8 threshold
+    }));
+
+    const result = await evaluator.evaluate(
+      makeSuccessTrajectory(),
+      makeTask({ domain: 'code' }),
+    );
+
+    // Low-confidence verification → fell through to workspace
+    expect(result.method).toBe('agent');
+    expect(runner.calls).toHaveLength(1);
+  });
+});
+
+// ============================================================
+// Gap 5b: RefinementLoop.setTaskRunner delegation
+// ============================================================
+
+describe('Gap 5b: RefinementLoop.setTaskRunner', () => {
+  let tempDir: string;
+  let memory: MemorySystem;
+  let agentManager: AgentManager;
+  let mockBackend: MockBackend;
+  let persistence: any;
+
+  beforeEach(async () => {
+    tempDir = await mkdtemp(join(tmpdir(), 'refine-runner-test-'));
+    persistence = createSqlitePersistence({ baseDir: tempDir });
+    await persistence.init();
+    memory = createMemorySystem(persistence, tempDir);
+    await memory.init();
+    agentManager = createAgentManager(memory);
+    mockBackend = createMockBackend({
+      success: true,
+      result: 'Refined',
+      durationMs: 50,
+    });
+    agentManager.registerBackend(mockBackend);
+  });
+
+  afterEach(async () => {
+    await memory.close();
+    persistence.close();
+    await rm(tempDir, { recursive: true, force: true });
+  });
+
+  it('should propagate taskRunner to evaluator', () => {
+    const loop = createRefinementLoop(agentManager, memory);
+    const runner = createMockTaskRunner({});
+
+    loop.setTaskRunner(runner);
+
+    // The evaluator should also have the runner — verify indirectly:
+    // evaluator.setTaskRunner was called by RefinementLoop.setTaskRunner
+    const evaluator = loop.getEvaluator();
+    expect(evaluator).toBeDefined();
+    // We can verify the runner is set by checking that evaluator.evaluate
+    // tries the template path (which will throw since we gave no output)
+  });
+
+  it('should use refinement plan from template when runner is set', async () => {
+    const plan: RefinementPlan = {
+      rootCauses: ['Token not refreshed', 'Session timeout ignored'],
+      strategy: 'Implement token refresh with retry logic',
+      steps: ['Add refresh interceptor', 'Handle timeout gracefully'],
+      avoidPatterns: ['Hardcoded token values'],
+    };
+
+    // First solution-evaluation call (initial) → poor → triggers refinement
+    // Second call (after retry) → excellent → stops loop
+    const poorEval = createEvaluationResult({
+      quality: 'poor', score: 0.2, acceptable: false,
+      issues: [{ type: 'error', description: 'Auth failed', severity: 'critical' }],
+      method: 'agent',
+    });
+    const goodEval = createEvaluationResult({
+      quality: 'excellent', score: 0.95, acceptable: true, issues: [], method: 'agent',
+    });
+
+    const runner = createMockTaskRunner({
+      'refinement-analysis': plan,
+      'solution-evaluation': seq(poorEval, goodEval),
+    });
+
+    const loop = createRefinementLoop(agentManager, memory, {
+      maxIterations: 2,
+      minAcceptableScore: 0.7,
+    });
+    loop.setTaskRunner(runner);
+
+    const failTraj = makeFailTrajectory();
+    const spawnConfig = {
+      agentType: 'mock',
+      task: makeTask(),
+      timeout: 60000,
+    };
+
+    const result = await loop.refine(failTraj, spawnConfig);
+
+    // refinement-analysis should have been called for the retry
+    const refinementCalls = runner.calls.filter(c => c.taskType === 'refinement-analysis');
+    expect(refinementCalls.length).toBeGreaterThanOrEqual(1);
+    expect(result.iterations).toBeGreaterThanOrEqual(1);
+  });
+
+  it('should fall back to heuristic refinement when template fails', async () => {
+    const poorEval = createEvaluationResult({
+      quality: 'poor', score: 0.2, acceptable: false,
+      issues: [{ type: 'error', description: 'Failed', severity: 'critical' }],
+      method: 'agent',
+    });
+    const goodEval = createEvaluationResult({
+      quality: 'excellent', score: 0.95, acceptable: true, issues: [], method: 'agent',
+    });
+
+    const runner = createMockTaskRunner({
+      'refinement-analysis': new Error('Template failure'),
+      'solution-evaluation': seq(poorEval, goodEval),
+    });
+
+    const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
+
+    const loop = createRefinementLoop(agentManager, memory, {
+      maxIterations: 2,
+      minAcceptableScore: 0.7,
+    });
+    loop.setTaskRunner(runner);
+
+    const failTraj = makeFailTrajectory();
+    const spawnConfig = {
+      agentType: 'mock',
+      task: makeTask(),
+      timeout: 60000,
+    };
+
+    // Should not throw — falls back to heuristic prompt
+    const result = await loop.refine(failTraj, spawnConfig);
+    expect(result).toBeDefined();
+    expect(result.iterations).toBeGreaterThanOrEqual(1);
+
+    warnSpy.mockRestore();
+  });
+
+  it('should format RefinementPlan into prompt additions for the retry agent', async () => {
+    const plan: RefinementPlan = {
+      rootCauses: ['Stale cache', 'Race condition'],
+      strategy: 'Use lock-based approach with cache invalidation',
+      steps: ['Acquire lock', 'Invalidate cache', 'Retry operation'],
+      avoidPatterns: ['Polling without backoff'],
+    };
+
+    const poorEval = createEvaluationResult({
+      quality: 'poor', score: 0.2, acceptable: false,
+      issues: [{ type: 'error', description: 'Cache stale', severity: 'critical' }],
+      method: 'agent',
+    });
+    const goodEval = createEvaluationResult({
+      quality: 'excellent', score: 0.95, acceptable: true, issues: [], method: 'agent',
+    });
+
+    const runner = createMockTaskRunner({
+      'refinement-analysis': plan,
+      'solution-evaluation': seq(poorEval, goodEval),
+    });
+
+    const loop = createRefinementLoop(agentManager, memory, {
+      maxIterations: 2,
+      minAcceptableScore: 0.7,
+    });
+    loop.setTaskRunner(runner);
+
+    // Spy on AgentManager.spawn to capture the refined prompt
+    const spawnSpy = vi.spyOn(agentManager, 'spawn');
+
+    const failTraj = makeFailTrajectory();
+    await loop.refine(failTraj, {
+      agentType: 'mock',
+      task: makeTask(),
+      timeout: 60000,
+    });
+
+    // Find a spawn call that includes refinement content
+    const spawnCalls = spawnSpy.mock.calls;
+    const refinementCall = spawnCalls.find(call => {
+      const config = call[0];
+      return config.systemPromptAdditions?.includes('Root Cause Analysis');
+    });
+
+    expect(refinementCall).toBeDefined();
+    const additions = refinementCall![0].systemPromptAdditions!;
+    expect(additions).toContain('Stale cache');
+    expect(additions).toContain('Race condition');
+    expect(additions).toContain('Use lock-based approach');
+    expect(additions).toContain('Acquire lock');
+    expect(additions).toContain('DO NOT: Polling without backoff');
+
+    spawnSpy.mockRestore();
+  });
+});
+
+// ============================================================
+// Gap 2: Atlas.solve() template delegation and fallback
+// ============================================================
+
+describe('Gap 2: Atlas.solve() template delegation', () => {
+  let tempDir: string;
+  let atlas: Atlas;
+  let mockBackend: MockBackend;
+
+  beforeEach(async () => {
+    tempDir = await mkdtemp(join(tmpdir(), 'atlas-delegation-test-'));
+    mockBackend = createMockBackend({
+      success: true,
+      result: 'Task completed',
+      durationMs: 50,
+      toolCalls: [
+        { name: 'read_file', input: { path: 'src/main.ts' }, output: 'code' },
+      ],
+    });
+
+    atlas = createAtlasWithAgents([mockBackend], {
+      storage: { baseDir: tempDir },
+      learning: { minTrajectories: 100 }, // high threshold to avoid batch triggers
+      execution: { defaultAgentType: 'mock', maxExecutionTime: 60 },
+    });
+    await atlas.init();
+  });
+
+  afterEach(async () => {
+    await atlas.close();
+    await rm(tempDir, { recursive: true, force: true });
+  });
+
+  it('should call reflectOnExecution after solve', async () => {
+    // The metaLearner should have an observation after solve
+    const task = makeTask();
+    const result = await atlas.solve(task);
+
+    expect(result.trajectory).toBeDefined();
+    expect(result.trajectory.outcome.success).toBe(true);
+
+    // Meta learner should have been invoked (either via template fallback or heuristic)
+    const metaLearner = atlas.getMetaLearner();
+    expect(metaLearner).toBeDefined();
+  });
+
+  it('should handle solve when no playbooks exist (no usage inference needed)', async () => {
+    const task = makeTask({ description: 'A unique task with no playbooks' });
+    const result = await atlas.solve(task);
+
+    expect(result.trajectory).toBeDefined();
+    // No injected playbooks → no usage inference
+    expect(result.injectedPlaybooks).toEqual([]);
+  });
+
+  it('should infer playbook usage via template when playbooks are injected', async () => {
+    // Add a playbook to memory so it gets injected
+    const mem = atlas.getMemory();
+    await mem.playbooks.add(makePlaybook('pb-auth', 'Auth Bug Fix'));
+
+    // Solve a matching task
+    const task = makeTask({
+      description: 'Fix the authentication bug in login',
+      domain: 'code',
+    });
+    const result = await atlas.solve(task);
+
+    // Playbook was injected (may or may not get usage depending on heuristic)
+    // The key check: solve completes without error when playbooks are present
+    expect(result.trajectory).toBeDefined();
+    expect(result.trajectory.outcome.success).toBe(true);
+  });
+
+  it('should handle failed agent execution gracefully', async () => {
+    const failBackend = createMockBackend({
+      success: false,
+      result: undefined,
+      durationMs: 50,
+      error: 'Agent crashed',
+    });
+
+    const failAtlas = createAtlasWithAgents([failBackend], {
+      storage: { baseDir: tempDir + '-fail' },
+      learning: { minTrajectories: 100 },
+      execution: { defaultAgentType: 'mock', maxExecutionTime: 60 },
+    });
+    await failAtlas.init();
+
+    // Suppress warnings from template fallbacks in failed execution
+    const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
+    const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {});
+
+    const task = makeTask();
+    const result = await failAtlas.solve(task);
+
+    expect(result.trajectory).toBeDefined();
+    expect(result.trajectory.outcome.success).toBe(false);
+
+    await failAtlas.close();
+    await rm(tempDir + '-fail', { recursive: true, force: true });
+
+    warnSpy.mockRestore();
+    errorSpy.mockRestore();
+  });
+});
+
+// ============================================================
+// Template-level tests: meta-reflection heuristic correctness
+// ============================================================
+
+describe('Meta-reflection template heuristic', () => {
+  it('should return heuristic complexity for successful high-confidence tasks', () => {
+    const complexity = metaReflectionTemplate.assessComplexity({
+      trajectory: makeSuccessTrajectory(),
+      routingDecision: { strategy: 'direct', confidence: 0.9, estimatedBudget: 5, reasoning: 'ok' },
+      playbooksUsed: [],
+    });
+    expect(complexity).toBe('heuristic');
+  });
+
+  it('should return lightweight for failed tasks', () => {
+    const complexity = metaReflectionTemplate.assessComplexity({
+      trajectory: makeFailTrajectory(),
+      routingDecision: { strategy: 'explore', confidence: 0.4, estimatedBudget: 10, reasoning: 'uncertain' },
+      playbooksUsed: [],
+    });
+    expect(complexity).toBe('lightweight');
+  });
+
+  it('should compute correct heuristic outcomeQuality based on effort ratio', async () => {
+    const heuristic = metaReflectionTemplate.heuristicFallback!;
+
+    // effortRatio = 5/10 = 0.5 → 'excellent'
+    const excellent = await heuristic({
+      trajectory: makeSuccessTrajectory({ llmCalls: 5 }),
+      routingDecision: { strategy: 'direct', confidence: 0.9, estimatedBudget: 10, reasoning: 'ok' },
+      playbooksUsed: [],
+    });
+    expect(excellent.outcomeQuality).toBe('excellent');
+
+    // effortRatio = 10/10 = 1.0 → 'good'
+    const good = await heuristic({
+      trajectory: makeSuccessTrajectory({ llmCalls: 10 }),
+      routingDecision: { strategy: 'direct', confidence: 0.9, estimatedBudget: 10, reasoning: 'ok' },
+      playbooksUsed: [],
+    });
+    expect(good.outcomeQuality).toBe('good');
+
+    // effortRatio = 15/10 = 1.5 → 'acceptable'
+    const acceptable = await heuristic({
+      trajectory: makeSuccessTrajectory({ llmCalls: 15 }),
+      routingDecision: { strategy: 'direct', confidence: 0.9, estimatedBudget: 10, reasoning: 'ok' },
+      playbooksUsed: [],
+    });
+    expect(acceptable.outcomeQuality).toBe('acceptable');
+
+    // Failed trajectory → 'poor'
+    const poor = await heuristic({
+      trajectory: makeFailTrajectory(),
+      routingDecision: { strategy: 'explore', confidence: 0.5, estimatedBudget: 10, reasoning: 'guess' },
+      playbooksUsed: [],
+    });
+    expect(poor.outcomeQuality).toBe('poor');
+  });
+
+  it('should report helpful retrieval when playbooks used and successful', async () => {
+    const heuristic = metaReflectionTemplate.heuristicFallback!;
+    const result = await heuristic({
+      trajectory: makeSuccessTrajectory(),
+      routingDecision: { strategy: 'direct', confidence: 0.9, estimatedBudget: 10, reasoning: 'ok' },
+      playbooksUsed: [{
+        playbook: makePlaybook('pb-1', 'Auth Fix'),
+        score: 0.9,
+        matchType: 'situation',
+      }],
+    });
+
+    expect(result.retrievalQuality).toBe('helpful');
+  });
+
+  it('should report neutral retrieval when no playbooks used', async () => {
+    const heuristic = metaReflectionTemplate.heuristicFallback!;
+    const result = await heuristic({
+      trajectory: makeSuccessTrajectory(),
+      routingDecision: { strategy: 'direct', confidence: 0.9, estimatedBudget: 10, reasoning: 'ok' },
+      playbooksUsed: [],
+    });
+
+    expect(result.retrievalQuality).toBe('neutral');
+  });
+});
+
+// ============================================================
+// Template-level tests: refinement-analysis
+// ============================================================
+
+describe('Refinement-analysis template complexity', () => {
+  const baseInput = {
+    trajectory: makeFailTrajectory(),
+    task: makeTask(),
+    evaluation: createEvaluationResult({
+      quality: 'poor', score: 0.3, acceptable: false, issues: [], method: 'heuristic',
+    }),
+    previousAttempts: 1,
+  };
+
+  it('should return lightweight for first failure', () => {
+    expect(refinementAnalysisTemplate.assessComplexity(baseInput)).toBe('lightweight');
+  });
+
+  it('should return standard for repeated failures', () => {
+    expect(
+      refinementAnalysisTemplate.assessComplexity({ ...baseInput, previousAttempts: 2 })
+    ).toBe('standard');
+  });
+
+  it('should have no heuristic fallback', () => {
+    expect(refinementAnalysisTemplate.heuristicFallback).toBeUndefined();
+  });
+});
+
+// ============================================================
+// Template-level tests: usage-inference complexity
+// ============================================================
+
+describe('Usage-inference template complexity', () => {
+  it('should return heuristic when no playbooks injected', () => {
+    const complexity = usageInferenceTemplate.assessComplexity({
+      trajectory: makeSuccessTrajectory(),
+      injectedPlaybooks: [],
+    });
+    expect(complexity).toBe('heuristic');
+  });
+
+  it('should return heuristic for few playbooks and short trajectory', () => {
+    const complexity = usageInferenceTemplate.assessComplexity({
+      trajectory: makeSuccessTrajectory(), // 3 steps
+      injectedPlaybooks: [makePlaybook('p1', 'P1')],
+    });
+    expect(complexity).toBe('heuristic');
+  });
+});