agentic-orchestrator 0.1.2 → 0.1.4

package/apps/control-plane/test/performance-analytics.spec.ts

@@ -0,0 +1,338 @@
+import { describe, it, expect, beforeEach } from 'vitest';
+import { AopKernel } from '../src/index.js';
+import { PerformanceAnalyticsService } from '../src/application/services/performance-analytics-service.js';
+import { makeTempRepo } from './helpers.js';
+import path from 'node:path';
+import fs from 'node:fs/promises';
+
+let repoRoot: string;
+
+const ORCH_CTX = { actor_type: 'orchestrator', actor_id: 'test' };
+
+beforeEach(async () => {
+  repoRoot = await makeTempRepo(process.cwd());
+});
+
+describe('N5: Agent Performance Analytics', () => {
+  describe('PerformanceAnalyticsService', () => {
+    it('GIVEN no prior outcomes WHEN getAnalytics THEN returns empty snapshot', async () => {
+      const analyticsDir = path.join(repoRoot, '.aop', 'analytics');
+      await fs.mkdir(analyticsDir, { recursive: true });
+      const service = new PerformanceAnalyticsService({
+        analyticsPath: () => path.join(analyticsDir, 'performance.json'),
+        getPolicySnapshot: () => ({})
+      });
+
+      const snapshot = await service.getAnalytics();
+      expect(snapshot.outcomes).toEqual([]);
+      expect(snapshot.aggregates).toEqual([]);
+      expect(snapshot.generated_at).toBeTruthy();
+    });
+
+    it('GIVEN outcome recorded WHEN getAnalytics THEN outcome appears in list', async () => {
+      const analyticsDir = path.join(repoRoot, '.aop', 'analytics');
+      await fs.mkdir(analyticsDir, { recursive: true });
+      const service = new PerformanceAnalyticsService({
+        analyticsPath: () => path.join(analyticsDir, 'performance.json'),
+        getPolicySnapshot: () => ({})
+      });
+
+      await service.recordOutcome({
+        feature_id: 'feat-1',
+        provider: 'claude',
+        model: 'opus',
+        status: 'merged',
+        gate_pass: true,
+        retry_count: 0,
+        duration_ms: 5000,
+        cost_usd: 0.50,
+        recorded_at: new Date().toISOString()
+      });
+
+      const snapshot = await service.getAnalytics();
+      expect(snapshot.outcomes).toHaveLength(1);
+      expect(snapshot.outcomes[0].feature_id).toBe('feat-1');
+      expect(snapshot.outcomes[0].provider).toBe('claude');
+    });
+
+    it('GIVEN multiple outcomes for same provider/model WHEN getAnalytics THEN aggregates computed correctly', async () => {
+      const analyticsDir = path.join(repoRoot, '.aop', 'analytics');
+      await fs.mkdir(analyticsDir, { recursive: true });
+      const service = new PerformanceAnalyticsService({
+        analyticsPath: () => path.join(analyticsDir, 'performance.json'),
+        getPolicySnapshot: () => ({})
+      });
+
+      await service.recordOutcome({
+        feature_id: 'feat-a',
+        provider: 'claude',
+        model: 'opus',
+        status: 'merged',
+        gate_pass: true,
+        retry_count: 1,
+        duration_ms: 4000,
+        cost_usd: 0.40,
+        recorded_at: new Date().toISOString()
+      });
+
+      await service.recordOutcome({
+        feature_id: 'feat-b',
+        provider: 'claude',
+        model: 'opus',
+        status: 'merged',
+        gate_pass: true,
+        retry_count: 3,
+        duration_ms: 6000,
+        cost_usd: 0.60,
+        recorded_at: new Date().toISOString()
+      });
+
+      const snapshot = await service.getAnalytics();
+      expect(snapshot.aggregates).toHaveLength(1);
+      const agg = snapshot.aggregates[0];
+      expect(agg.provider).toBe('claude');
+      expect(agg.model).toBe('opus');
+      expect(agg.total_features).toBe(2);
+      expect(agg.success_count).toBe(2);
+      expect(agg.success_rate).toBe(1.0);
+      expect(agg.avg_retry_count).toBe(2);
+      expect(agg.avg_duration_ms).toBe(5000);
+      expect(agg.avg_cost_usd).toBeCloseTo(0.50);
+    });
+
+    it('GIVEN mixed providers WHEN getProviderStats with filter THEN only matching stats returned', async () => {
+      const analyticsDir = path.join(repoRoot, '.aop', 'analytics');
+      await fs.mkdir(analyticsDir, { recursive: true });
+      const service = new PerformanceAnalyticsService({
+        analyticsPath: () => path.join(analyticsDir, 'performance.json'),
+        getPolicySnapshot: () => ({})
+      });
+
+      await service.recordOutcome({
+        feature_id: 'feat-c1',
+        provider: 'claude',
+        model: 'opus',
+        status: 'merged',
+        gate_pass: true,
+        retry_count: 0,
+        duration_ms: 3000,
+        cost_usd: 0.30,
+        recorded_at: new Date().toISOString()
+      });
+
+      await service.recordOutcome({
+        feature_id: 'feat-g1',
+        provider: 'gemini',
+        model: 'pro',
+        status: 'merged',
+        gate_pass: true,
+        retry_count: 1,
+        duration_ms: 7000,
+        cost_usd: 0.70,
+        recorded_at: new Date().toISOString()
+      });
+
+      const claudeStats = await service.getProviderStats('claude');
+      expect(claudeStats).toHaveLength(1);
+      expect(claudeStats[0].provider).toBe('claude');
+
+      const geminiStats = await service.getProviderStats('gemini');
+      expect(geminiStats).toHaveLength(1);
+      expect(geminiStats[0].provider).toBe('gemini');
+    });
+
+    it('GIVEN feature merged WHEN recordOutcome THEN success_rate updates', async () => {
+      const analyticsDir = path.join(repoRoot, '.aop', 'analytics');
+      await fs.mkdir(analyticsDir, { recursive: true });
+      const service = new PerformanceAnalyticsService({
+        analyticsPath: () => path.join(analyticsDir, 'performance.json'),
+        getPolicySnapshot: () => ({})
+      });
+
+      // Record a failure first
+      await service.recordOutcome({
+        feature_id: 'feat-fail',
+        provider: 'claude',
+        model: 'sonnet',
+        status: 'failed',
+        gate_pass: false,
+        retry_count: 3,
+        duration_ms: 10000,
+        cost_usd: 1.00,
+        recorded_at: new Date().toISOString()
+      });
+
+      // Then a success
+      await service.recordOutcome({
+        feature_id: 'feat-pass',
+        provider: 'claude',
+        model: 'sonnet',
+        status: 'merged',
+        gate_pass: true,
+        retry_count: 0,
+        duration_ms: 5000,
+        cost_usd: 0.50,
+        recorded_at: new Date().toISOString()
+      });
+
+      const snapshot = await service.getAnalytics();
+      const agg = snapshot.aggregates.find((a) => a.model === 'sonnet');
+      expect(agg).toBeDefined();
+      expect(agg?.success_rate).toBe(0.5);
+      expect(agg?.total_features).toBe(2);
+      expect(agg?.success_count).toBe(1);
+    });
+
+    it('GIVEN feature failed WHEN recordOutcome THEN failure reflected in aggregates', async () => {
+      const analyticsDir = path.join(repoRoot, '.aop', 'analytics');
+      await fs.mkdir(analyticsDir, { recursive: true });
+      const service = new PerformanceAnalyticsService({
+        analyticsPath: () => path.join(analyticsDir, 'performance.json'),
+        getPolicySnapshot: () => ({})
+      });
+
+      await service.recordOutcome({
+        feature_id: 'feat-only-fail',
+        provider: 'codex',
+        model: 'gpt-4',
+        status: 'failed',
+        gate_pass: false,
+        retry_count: 5,
+        duration_ms: 20000,
+        cost_usd: 2.00,
+        recorded_at: new Date().toISOString()
+      });
+
+      const snapshot = await service.getAnalytics();
+      const agg = snapshot.aggregates.find((a) => a.provider === 'codex');
+      expect(agg).toBeDefined();
+      expect(agg?.success_rate).toBe(0);
+      expect(agg?.success_count).toBe(0);
+      expect(agg?.total_features).toBe(1);
+    });
+
+    it('GIVEN_outcomes_with_multiple_models_WHEN_getProviderStats_filtered_by_provider_and_model_THEN_returns_only_matching_model', async () => {
+      const analyticsDir = path.join(repoRoot, '.aop', 'analytics');
+      await fs.mkdir(analyticsDir, { recursive: true });
+      const service = new PerformanceAnalyticsService({
+        analyticsPath: () => path.join(analyticsDir, 'performance.json'),
+        getPolicySnapshot: () => ({})
+      });
+
+      await service.recordOutcome({
+        feature_id: 'feat-m1',
+        provider: 'claude',
+        model: 'opus',
+        status: 'merged',
+        gate_pass: true,
+        retry_count: 0,
+        duration_ms: 3000,
+        cost_usd: 0.30,
+        recorded_at: new Date().toISOString()
+      });
+      await service.recordOutcome({
+        feature_id: 'feat-m2',
+        provider: 'claude',
+        model: 'sonnet',
+        status: 'merged',
+        gate_pass: true,
+        retry_count: 0,
+        duration_ms: 2000,
+        cost_usd: 0.10,
+        recorded_at: new Date().toISOString()
+      });
+
+      const stats = await service.getProviderStats('claude', 'opus');
+      expect(stats).toHaveLength(1);
+      expect(stats[0].model).toBe('opus');
+    });
+  });
+
+  describe('Kernel tool handlers', () => {
+    it('GIVEN orchestrator role WHEN invoke performance.record_outcome THEN outcome recorded', async () => {
+      const kernel = new AopKernel(repoRoot);
+      await kernel.ensureLoaded();
+
+      const result = await kernel.invoke('performance.record_outcome', {
+        feature_id: 'feat-kern-1',
+        provider: 'claude',
+        model: 'opus',
+        status: 'merged',
+        gate_pass: true,
+        retry_count: 0,
+        duration_ms: 3000,
+        cost_usd: 0.25
+      }, ORCH_CTX);
+
+      expect(result.ok).toBe(true);
+      expect(result.data.total_outcomes).toBe(1);
+    });
+
+    it('GIVEN orchestrator role WHEN invoke performance.get_analytics THEN analytics returned', async () => {
+      const kernel = new AopKernel(repoRoot);
+      await kernel.ensureLoaded();
+
+      // Record an outcome first
+      await kernel.invoke('performance.record_outcome', {
+        feature_id: 'feat-kern-2',
+        provider: 'gemini',
+        model: 'pro',
+        status: 'merged',
+        gate_pass: true,
+        retry_count: 1,
+        duration_ms: 8000,
+        cost_usd: 0.80
+      }, ORCH_CTX);
+
+      const result = await kernel.invoke('performance.get_analytics', {}, ORCH_CTX);
+
+      expect(result.ok).toBe(true);
+      expect(result.data.outcomes).toHaveLength(1);
+      expect(result.data.aggregates).toHaveLength(1);
+      expect(result.data.aggregates[0].provider).toBe('gemini');
+      expect(result.data.generated_at).toBeTruthy();
+    });
+  });
+});
+
+describe('PerformanceAnalyticsService branch coverage', () => {
+  it('GIVEN_outcomes_WHEN_getProviderStats_with_only_model_filter_THEN_filters_by_model', async () => {
+    const analyticsDir = path.join(repoRoot, '.aop', 'analytics');
+    await fs.mkdir(analyticsDir, { recursive: true });
+    const service = new PerformanceAnalyticsService({
+      analyticsPath: () => path.join(analyticsDir, 'perf-model-only.json'),
+      getPolicySnapshot: () => ({})
+    });
+
+    await service.recordOutcome({ feature_id: 'f1', provider: 'claude', model: 'sonnet', status: 'merged', gate_pass: true, retry_count: 0, duration_ms: 1000, cost_usd: 0.1, recorded_at: new Date().toISOString() });
+    await service.recordOutcome({ feature_id: 'f2', provider: 'openai', model: 'gpt4', status: 'merged', gate_pass: true, retry_count: 0, duration_ms: 2000, cost_usd: 0.2, recorded_at: new Date().toISOString() });
+
+    const filtered = await service.getProviderStats(undefined, 'sonnet');
+    expect(filtered).toHaveLength(1);
+    expect(filtered[0].model).toBe('sonnet');
+  });
+});
+
+describe('AopKernel performanceGetAnalytics with provider filter', () => {
+  it('GIVEN_outcomes_recorded_WHEN_invoke_get_analytics_with_provider_filter_THEN_returns_filtered_stats', async () => {
+    const kernel = new AopKernel(repoRoot);
+    await kernel.ensureLoaded();
+
+    await kernel.invoke('performance.record_outcome', {
+      feature_id: 'feat-filter-1',
+      provider: 'openai',
+      model: 'gpt4',
+      status: 'merged',
+      gate_pass: true,
+      retry_count: 0,
+      duration_ms: 2000,
+      cost_usd: 0.10
+    }, ORCH_CTX);
+
+    const result = await kernel.invoke('performance.get_analytics', { provider: 'openai' }, ORCH_CTX);
+
+    expect(result.ok).toBe(true);
+    expect(result.data.aggregates).toHaveLength(1);
+    expect(result.data.aggregates[0].provider).toBe('openai');
+  });
+});

package/apps/control-plane/test/planning-wave-executor.spec.ts

@@ -226,4 +226,172 @@ describe('PlanningWaveExecutor post-QA reconciliation', () => {
     expect(note.decision).toBe('no_update');
     expect(note.reasons).toEqual(['reconciled_no_gaps']);
   });
+
+  it('GIVEN_plan_null_WHEN_reconciling_THEN_logs_no_update_missing_plan', async () => {
+    const featureId = 'feature_no_plan';
+    const toolCaller = {
+      callTool: vi.fn(async (_role: string, toolName: string) => {
+        if (toolName === TOOLS.FEATURE_GET_CONTEXT) {
+          return {
+            ok: true,
+            data: {
+              feature_id: featureId,
+              spec: 'Feature with no plan yet.',
+              state: { front_matter: { status: STATUS.QA } },
+              plan: null,
+              qa_test_index: { summary: { pending: 0, failed: 0, running: 0 } },
+              latest_evidence: {}
+            }
+          };
+        }
+        if (toolName === TOOLS.FEATURE_LOG_APPEND) {
+          return { ok: true, data: {} };
+        }
+        return { ok: true, data: {} };
+      })
+    };
+
+    const executor = new PlanningWaveExecutor({
+      toolCaller: toolCaller as never,
+      planGenerator: { generateInitialPlan: vi.fn() }
+    });
+
+    await executor.runPostQaReconciliation([featureId], 1);
+
+    const planUpdateCalls = toolCaller.callTool.mock.calls.filter((call) => call[1] === TOOLS.PLAN_UPDATE);
+    expect(planUpdateCalls).toHaveLength(0);
+
+    const logCall = toolCaller.callTool.mock.calls.find((call) => call[1] === TOOLS.FEATURE_LOG_APPEND);
+    expect(logCall).toBeTruthy();
+    const logPayload = (logCall as unknown[] | undefined)?.[2] as { note?: string } | undefined;
+    const note = JSON.parse(String(logPayload?.note));
+    expect(note.decision).toBe('no_update');
+    expect(note.reasons).toContain('missing_plan');
+  });
+
+  it('GIVEN_plan_with_invalid_version_WHEN_reconciling_THEN_logs_no_update_existing_plan_version_invalid', async () => {
+    const featureId = 'feature_invalid_version';
+    const toolCaller = {
+      callTool: vi.fn(async (_role: string, toolName: string) => {
+        if (toolName === TOOLS.FEATURE_GET_CONTEXT) {
+          return {
+            ok: true,
+            data: {
+              feature_id: featureId,
+              spec: 'Feature with invalid plan version.',
+              state: { front_matter: { status: STATUS.QA } },
+              plan: {
+                feature_id: featureId,
+                plan_version: 'not_a_number',
+                acceptance_criteria: []
+              },
+              qa_test_index: { summary: { pending: 0, failed: 0, running: 0 } },
+              latest_evidence: {}
+            }
+          };
+        }
+        if (toolName === TOOLS.FEATURE_LOG_APPEND) {
+          return { ok: true, data: {} };
+        }
+        return { ok: true, data: {} };
+      })
+    };
+
+    const executor = new PlanningWaveExecutor({
+      toolCaller: toolCaller as never,
+      planGenerator: { generateInitialPlan: vi.fn() }
+    });
+
+    await executor.runPostQaReconciliation([featureId], 1);
+
+    const planUpdateCalls = toolCaller.callTool.mock.calls.filter((call) => call[1] === TOOLS.PLAN_UPDATE);
+    expect(planUpdateCalls).toHaveLength(0);
+
+    const logCall = toolCaller.callTool.mock.calls.find((call) => call[1] === TOOLS.FEATURE_LOG_APPEND);
+    expect(logCall).toBeTruthy();
+    const logPayload = (logCall as unknown[] | undefined)?.[2] as { note?: string } | undefined;
+    const note = JSON.parse(String(logPayload?.note));
+    expect(note.decision).toBe('no_update');
+    expect(note.reasons).toContain('existing_plan_version_invalid');
+  });
+
+  it('GIVEN_no_gate_evidence_overall_WHEN_reconciling_THEN_logs_null_latest_gate_overall', async () => {
+    const featureId = 'feature_no_evidence';
+    const edgeChecklist = [
+      'Edge-case: boundary values and size limits are covered.',
+      'Edge-case: negative-path input validation and error handling are covered.',
+      'Edge-case: dependency failure and retry behavior are covered.'
+    ];
+    const toolCaller = {
+      callTool: vi.fn(async (_role: string, toolName: string) => {
+        if (toolName === TOOLS.FEATURE_GET_CONTEXT) {
+          return {
+            ok: true,
+            data: {
+              feature_id: featureId,
+              spec: 'Simple feature with no gate evidence yet.',
+              state: { front_matter: { status: STATUS.READY_TO_MERGE } },
+              plan: {
+                ...basePlan(featureId),
+                risk: edgeChecklist
+              },
+              qa_test_index: { summary: { pending: 0, failed: 0, running: 0, passed: 1, waived: 0 } },
+              latest_evidence: {}
+            }
+          };
+        }
+        if (toolName === TOOLS.FEATURE_LOG_APPEND) {
+          return { ok: true, data: {} };
+        }
+        return { ok: true, data: {} };
+      })
+    };
+
+    const executor = new PlanningWaveExecutor({
+      toolCaller: toolCaller as never,
+      planGenerator: { generateInitialPlan: vi.fn() }
+    });
+
+    await executor.runPostQaReconciliation([featureId], 1);
+
+    const logCall = toolCaller.callTool.mock.calls.find((call) => call[1] === TOOLS.FEATURE_LOG_APPEND);
+    expect(logCall).toBeTruthy();
+    const logPayload = (logCall as unknown[] | undefined)?.[2] as { note?: string } | undefined;
+    const note = JSON.parse(String(logPayload?.note));
+    expect(note.latest_gate_overall).toBeNull();
+  });
+
+  it('GIVEN_non_string_status_WHEN_reconciling_THEN_skips_feature', async () => {
+    const featureId = 'feature_bad_status';
+    const toolCaller = {
+      callTool: vi.fn(async (_role: string, toolName: string) => {
+        if (toolName === TOOLS.FEATURE_GET_CONTEXT) {
+          return {
+            ok: true,
+            data: {
+              feature_id: featureId,
+              spec: 'Feature with non-string status.',
+              state: { front_matter: { status: 42 } },
+              plan: basePlan(featureId),
+              qa_test_index: { summary: {} },
+              latest_evidence: {}
+            }
+          };
+        }
+        return { ok: true, data: {} };
+      })
+    };
+
+    const executor = new PlanningWaveExecutor({
+      toolCaller: toolCaller as never,
+      planGenerator: { generateInitialPlan: vi.fn() }
+    });
+
+    await executor.runPostQaReconciliation([featureId], 1);
+
+    const planUpdateCalls = toolCaller.callTool.mock.calls.filter((call) => call[1] === TOOLS.PLAN_UPDATE);
+    const logCalls = toolCaller.callTool.mock.calls.filter((call) => call[1] === TOOLS.FEATURE_LOG_APPEND);
+    expect(planUpdateCalls).toHaveLength(0);
+    expect(logCalls).toHaveLength(0);
+  });
 });