@avi770/testteam 1.2.0

package/core/__tests__/orchestrator.test.ts

@@ -0,0 +1,519 @@
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+import { Orchestrator } from '../orchestrator';
+import type { ValidatedConfig } from '../config';
+import type { AgentResult } from '../types';
+import type { LicenseTier } from '../license';
+import type { BaseAgent } from '../../agents/base-agent';
+import * as fs from 'fs/promises';
+import * as path from 'path';
+import * as os from 'os';
+
+const stubConfig: ValidatedConfig = {
+  schemaVersion: 1,
+  name: 'test',
+  auth: { loginUrl: 'http://localhost' },
+  modules: [{ id: 'mod', route: '/mod', sidebarIcon: true }],
+  environments: {},
+  portals: [],
+  breakpoints: [],
+  integrations: [],
+  workers: [],
+  workflows: [],
+  accuracy: { enabled: false, decimalPrecision: 2, currencySymbol: '$', timezone: 'UTC' },
+  tenancy: { enabled: false, isolationField: 'firmId', testFirms: [] },
+  costBudget: { maxPerRun: 100, maxPerAgent: 10, alertThreshold: 0.8, onExceeded: 'abort' },
+  agentClassification: { blocking: [1, 2], advisory: [23] },
+  thresholds: {
+    unitCoverage: { lines: 80, functions: 80, branches: 80, statements: 80 },
+    lighthouse: { accessibility: 90, performance: 90, bestPractices: 90, seo: 90 },
+    apiResponseTime: 2000,
+    bundleSizeLimit: 500000,
+    maxConsoleErrors: 0,
+  },
+  disabledAgents: [],
+  disabledAgentSet: new Set<number>(),
+};
+
+function createMockAgent(agentId: number, result?: Partial<AgentResult>): BaseAgent {
+  const defaultResult: AgentResult = {
+    agentId,
+    agentName: `agent-${agentId}`,
+    phase: 'alpha',
+    status: 'passed',
+    startedAt: new Date().toISOString(),
+    completedAt: new Date().toISOString(),
+    durationMs: 10,
+    findings: [],
+    evidence: [],
+    summary: `agent-${agentId}: All checks passed`,
+    ...result,
+  };
+
+  return {
+    agentId,
+    agentName: `agent-${agentId}`,
+    run: vi.fn().mockResolvedValue(defaultResult),
+  } as unknown as BaseAgent;
+}
+
+describe('Orchestrator', () => {
+  describe('dryRun', () => {
+    it('returns the correct agent IDs in order for alpha', () => {
+      const orchestrator = new Orchestrator(stubConfig);
+      const plan = orchestrator.dryRun('alpha');
+
+      expect(plan.map(e => e.agentId)).toEqual([1, 2, 23, 3, 4, 5]);
+    });
+
+    it('returns the correct agent IDs for beta', () => {
+      const orchestrator = new Orchestrator(stubConfig);
+      const plan = orchestrator.dryRun('beta');
+
+      expect(plan.map(e => e.agentId)).toEqual([5, 6, 7, 8, 12, 13, 14, 15, 16, 18, 19, 21, 22]);
+    });
+
+    it('returns the correct agent IDs for uat', () => {
+      const orchestrator = new Orchestrator(stubConfig);
+      const plan = orchestrator.dryRun('uat');
+
+      expect(plan.map(e => e.agentId)).toEqual([5, 6, 12, 13, 14, 15, 18, 19, 20, 21, 22, 17]);
+    });
+
+    it('returns the correct agent IDs for release', () => {
+      const orchestrator = new Orchestrator(stubConfig);
+      const plan = orchestrator.dryRun('release');
+
+      expect(plan.map(e => e.agentId)).toEqual([10]);
+    });
+
+    it('marks disabled agents as disabled in the dispatch plan', () => {
+      const configWithDisabled: ValidatedConfig = {
+        ...stubConfig,
+        disabledAgents: [2, 23],
+        disabledAgentSet: new Set([2, 23]),
+      };
+      const orchestrator = new Orchestrator(configWithDisabled);
+      const plan = orchestrator.dryRun('alpha');
+
+      const agent2 = plan.find(e => e.agentId === 2);
+      const agent23 = plan.find(e => e.agentId === 23);
+      expect(agent2?.status).toBe('disabled');
+      expect(agent23?.status).toBe('disabled');
+    });
+
+    it('marks unregistered agents as unregistered', () => {
+      const orchestrator = new Orchestrator(stubConfig);
+      const plan = orchestrator.dryRun('alpha');
+
+      // No agents registered, so all should be unregistered
+      expect(plan.every(e => e.status === 'unregistered')).toBe(true);
+    });
+
+    it('marks registered agents as scheduled', () => {
+      const orchestrator = new Orchestrator(stubConfig);
+      orchestrator.registerAgent(1, createMockAgent(1));
+
+      const plan = orchestrator.dryRun('alpha');
+      const agent1 = plan.find(e => e.agentId === 1);
+      expect(agent1?.status).toBe('scheduled');
+    });
+  });
+
+  describe('runPhase', () => {
+    let tmpDir: string;
+
+    beforeEach(async () => {
+      tmpDir = path.join(os.tmpdir(), `tf-orch-test-${Date.now()}-${Math.random().toString(36).slice(2)}`);
+      await fs.mkdir(tmpDir, { recursive: true });
+    });
+
+    it('calls agent.run() for registered agents and writes results', async () => {
+      const orchestrator = new Orchestrator(stubConfig, tmpDir);
+      const mockAgent = createMockAgent(1);
+      orchestrator.registerAgent(1, mockAgent);
+
+      const result = await orchestrator.runPhase('alpha');
+
+      expect(mockAgent.run).toHaveBeenCalledOnce();
+      expect(result.agentResults[1]).toBeDefined();
+      expect(result.agentResults[1].status).toBe('passed');
+
+      // Verify file was written
+      const filePath = path.join(tmpDir, 'agent-results', '1.json');
+      const fileContent = await fs.readFile(filePath, 'utf-8');
+      const parsed = JSON.parse(fileContent) as AgentResult;
+      expect(parsed.agentId).toBe(1);
+    });
+
+    it('skips disabled agents', async () => {
+      const configWithDisabled: ValidatedConfig = {
+        ...stubConfig,
+        disabledAgents: [1],
+        disabledAgentSet: new Set([1]),
+      };
+      const orchestrator = new Orchestrator(configWithDisabled, tmpDir);
+      const mockAgent = createMockAgent(1);
+      orchestrator.registerAgent(1, mockAgent);
+
+      const result = await orchestrator.runPhase('alpha');
+
+      expect(mockAgent.run).not.toHaveBeenCalled();
+      expect(result.skippedAgents).toContain(1);
+    });
+
+    it('skips unregistered agents and includes them in skippedAgents', async () => {
+      const orchestrator = new Orchestrator(stubConfig, tmpDir);
+      // Register only agent 1, not 2, 23, 3, 4, 5
+      orchestrator.registerAgent(1, createMockAgent(1));
+
+      const result = await orchestrator.runPhase('alpha');
+
+      expect(result.skippedAgents).toEqual(expect.arrayContaining([2, 23, 3, 4, 5]));
+      expect(result.skippedAgents).not.toContain(1);
+    });
+
+    it('evaluates the phase gate after running agents', async () => {
+      const orchestrator = new Orchestrator(stubConfig, tmpDir);
+      // Blocking agents 1 and 2 both pass
+      orchestrator.registerAgent(1, createMockAgent(1));
+      orchestrator.registerAgent(2, createMockAgent(2));
+
+      const result = await orchestrator.runPhase('alpha');
+
+      expect(result.gate).toBeDefined();
+      expect(result.gate.phase).toBe('alpha');
+      expect(result.gate.passed).toBe(true);
+    });
+
+    it('gate fails when a blocking agent fails', async () => {
+      const orchestrator = new Orchestrator(stubConfig, tmpDir);
+      orchestrator.registerAgent(1, createMockAgent(1, { status: 'failed' }));
+      orchestrator.registerAgent(2, createMockAgent(2));
+
+      const result = await orchestrator.runPhase('alpha');
+
+      expect(result.gate.passed).toBe(false);
+      expect(result.gate.failedBlockingAgents).toContain(1);
+    });
+
+    it('throws when required health checks fail', async () => {
+      const orchestrator = new Orchestrator(stubConfig, tmpDir);
+      orchestrator.registerHealthChecks([{
+        name: 'api-reachable',
+        check: async () => false,
+        required: true,
+      }]);
+
+      await expect(orchestrator.runPhase('alpha')).rejects.toThrow('Health checks failed');
+    });
+
+    it('proceeds when optional health checks fail', async () => {
+      const orchestrator = new Orchestrator(stubConfig, tmpDir);
+      orchestrator.registerHealthChecks([{
+        name: 'optional-check',
+        check: async () => false,
+        required: false,
+      }]);
+
+      const result = await orchestrator.runPhase('alpha');
+      expect(result).toBeDefined();
+    });
+
+    it('tracks cost via CostTracker', async () => {
+      const orchestrator = new Orchestrator(stubConfig, tmpDir);
+      orchestrator.registerAgent(1, createMockAgent(1));
+
+      const result = await orchestrator.runPhase('alpha');
+
+      expect(result.costTracker).toBeDefined();
+      expect(result.costTracker.getTotalCost()).toBeGreaterThanOrEqual(0);
+    });
+  });
+
+  describe('license gating', () => {
+    let tmpDir: string;
+
+    beforeEach(async () => {
+      tmpDir = path.join(os.tmpdir(), `tf-orch-license-${Date.now()}-${Math.random().toString(36).slice(2)}`);
+      await fs.mkdir(tmpDir, { recursive: true });
+    });
+
+    function createOrchestratorWithTier(tier: LicenseTier): Orchestrator {
+      const orchestrator = new Orchestrator(stubConfig, tmpDir);
+      orchestrator._getLicenseTier = () => tier;
+      return orchestrator;
+    }
+
+    /** Register all agents from a phase dispatch list. */
+    function registerPhaseAgents(orchestrator: Orchestrator, agentIds: readonly number[]): void {
+      for (const id of agentIds) {
+        orchestrator.registerAgent(id, createMockAgent(id));
+      }
+    }
+
+    // ── Phase gating ─────────────────────────────────────────────────
+
+    it('free tier blocks beta phase', async () => {
+      const orchestrator = createOrchestratorWithTier('free');
+      registerPhaseAgents(orchestrator, [5, 6, 7, 8, 12, 13, 14, 15, 16, 18, 19, 21, 22]);
+
+      const result = await orchestrator.runPhase('beta');
+
+      expect(result.gate.passed).toBe(false);
+      expect(result.skippedAgents.length).toBeGreaterThan(0);
+      expect(result.agentResults).toEqual({});
+    });
+
+    it('free tier blocks uat phase', async () => {
+      const orchestrator = createOrchestratorWithTier('free');
+      registerPhaseAgents(orchestrator, [5, 6, 12, 13, 14, 15, 18, 19, 20, 21, 22, 17]);
+
+      const result = await orchestrator.runPhase('uat');
+
+      expect(result.gate.passed).toBe(false);
+      expect(result.agentResults).toEqual({});
+    });
+
+    it('free tier allows alpha phase', async () => {
+      const orchestrator = createOrchestratorWithTier('free');
+      orchestrator.registerAgent(1, createMockAgent(1));
+      orchestrator.registerAgent(2, createMockAgent(2));
+
+      const result = await orchestrator.runPhase('alpha');
+
+      expect(result.agentResults[1]).toBeDefined();
+      expect(result.agentResults[1].status).toBe('passed');
+    });
+
+    it('pro tier allows beta phase', async () => {
+      const orchestrator = createOrchestratorWithTier('pro');
+      // Register only free+pro agents from beta dispatch
+      orchestrator.registerAgent(5, createMockAgent(5));
+      orchestrator.registerAgent(6, createMockAgent(6));
+
+      const result = await orchestrator.runPhase('beta');
+
+      expect(result.agentResults[5]).toBeDefined();
+      expect(result.agentResults[5].status).toBe('passed');
+      expect(result.agentResults[6]).toBeDefined();
+    });
+
+    it('pro tier allows uat phase', async () => {
+      const orchestrator = createOrchestratorWithTier('pro');
+      orchestrator.registerAgent(5, createMockAgent(5));
+
+      const result = await orchestrator.runPhase('uat');
+
+      expect(result.agentResults[5]).toBeDefined();
+    });
+
+    it('enterprise tier allows all phases', async () => {
+      const orchestrator = createOrchestratorWithTier('enterprise');
+      orchestrator.registerAgent(10, createMockAgent(10));
+
+      const result = await orchestrator.runPhase('release');
+
+      expect(result.agentResults[10]).toBeDefined();
+      expect(result.agentResults[10].status).toBe('passed');
+    });
+
+    // ── Agent gating ─────────────────────────────────────────────────
+
+    it('free tier skips pro agents with info finding', async () => {
+      const orchestrator = createOrchestratorWithTier('free');
+      // Alpha phase: [1, 2, 23, 3, 4, 5]. Agent 23 is pro tier.
+      registerPhaseAgents(orchestrator, [1, 2, 23, 3, 4, 5]);
+
+      const result = await orchestrator.runPhase('alpha');
+
+      // Agent 23 (Migration Tester) should be skipped due to license
+      expect(result.skippedAgents).toContain(23);
+      expect(result.agentResults[23]).toBeDefined();
+      expect(result.agentResults[23].status).toBe('skipped');
+
+      // Should have an info-severity finding
+      const findings = result.agentResults[23].findings;
+      expect(findings).toHaveLength(1);
+      expect(findings[0].severity).toBe('info');
+      expect(findings[0].description).toContain('requires Pro tier');
+      expect(findings[0].description).toContain('current: free');
+    });
+
+    it('free tier runs free agents normally', async () => {
+      const orchestrator = createOrchestratorWithTier('free');
+      // Register only free-tier agents from alpha dispatch
+      orchestrator.registerAgent(1, createMockAgent(1));
+      orchestrator.registerAgent(2, createMockAgent(2));
+      orchestrator.registerAgent(3, createMockAgent(3));
+      orchestrator.registerAgent(4, createMockAgent(4));
+      orchestrator.registerAgent(5, createMockAgent(5));
+
+      const result = await orchestrator.runPhase('alpha');
+
+      expect(result.agentResults[1].status).toBe('passed');
+      expect(result.agentResults[2].status).toBe('passed');
+      expect(result.agentResults[3].status).toBe('passed');
+      expect(result.agentResults[4].status).toBe('passed');
+      expect(result.agentResults[5].status).toBe('passed');
+    });
+
+    it('pro tier skips enterprise agents with info finding', async () => {
+      const orchestrator = createOrchestratorWithTier('pro');
+      // Beta phase includes enterprise agents: 14, 16, 18, 19
+      registerPhaseAgents(orchestrator, [5, 6, 7, 8, 12, 13, 14, 15, 16, 18, 19, 21, 22]);
+
+      const result = await orchestrator.runPhase('beta');
+
+      // Enterprise agents 14, 16, 18, 19 should be skipped
+      expect(result.skippedAgents).toContain(14);
+      expect(result.skippedAgents).toContain(16);
+      expect(result.skippedAgents).toContain(18);
+      expect(result.skippedAgents).toContain(19);
+
+      // Verify skipped results have info findings
+      expect(result.agentResults[14].status).toBe('skipped');
+      expect(result.agentResults[14].findings[0].severity).toBe('info');
+      expect(result.agentResults[14].findings[0].description).toContain('Enterprise');
+
+      // Pro agents should run normally
+      expect(result.agentResults[6].status).toBe('passed');
+      expect(result.agentResults[7].status).toBe('passed');
+      expect(result.agentResults[12].status).toBe('passed');
+    });
+
+    it('pro tier allows all phases but gates enterprise agents', async () => {
+      const orchestrator = createOrchestratorWithTier('pro');
+      // UAT includes enterprise agents: 14, 18, 19, 20
+      registerPhaseAgents(orchestrator, [5, 6, 12, 13, 14, 15, 18, 19, 20, 21, 22, 17]);
+
+      const result = await orchestrator.runPhase('uat');
+
+      // Phase itself is allowed
+      expect(result.agentResults[5]).toBeDefined();
+      expect(result.agentResults[5].status).toBe('passed');
+
+      // Enterprise agents are gated
+      expect(result.agentResults[14].status).toBe('skipped');
+      expect(result.agentResults[18].status).toBe('skipped');
+      expect(result.agentResults[19].status).toBe('skipped');
+      expect(result.agentResults[20].status).toBe('skipped');
+    });
+
+    it('enterprise tier allows all agents', async () => {
+      const orchestrator = createOrchestratorWithTier('enterprise');
+      // Beta phase: all agents should run
+      registerPhaseAgents(orchestrator, [5, 6, 7, 8, 12, 13, 14, 15, 16, 18, 19, 21, 22]);
+
+      const result = await orchestrator.runPhase('beta');
+
+      // No agents should be license-gated
+      const skippedWithInfoFinding = Object.values(result.agentResults).filter(
+        r => r.status === 'skipped' && r.findings.some(f => f.severity === 'info'),
+      );
+      expect(skippedWithInfoFinding).toHaveLength(0);
+
+      // All agents should have run
+      expect(result.agentResults[14].status).toBe('passed');
+      expect(result.agentResults[16].status).toBe('passed');
+      expect(result.agentResults[18].status).toBe('passed');
+      expect(result.agentResults[19].status).toBe('passed');
+    });
+
+    // ── Cost budget enforcement ────────────────────────────────────
+
+    it('skips remaining agents when cost budget is exceeded with abort policy', async () => {
+      const abortConfig: ValidatedConfig = {
+        ...stubConfig,
+        costBudget: { maxPerRun: 0.0001, maxPerAgent: 10, alertThreshold: 0.8, onExceeded: 'abort' },
+      };
+      const orchestrator = new Orchestrator(abortConfig, tmpDir);
+      orchestrator._getLicenseTier = () => 'enterprise';
+
+      // Register all alpha agents
+      for (const id of [1, 2, 23, 3, 4, 5]) {
+        orchestrator.registerAgent(id, createMockAgent(id, { durationMs: 100000 }));
+      }
+
+      const result = await orchestrator.runPhase('alpha');
+
+      // Some agents should have been skipped due to budget
+      expect(result.skippedAgents.length).toBeGreaterThan(0);
+    });
+
+    it('continues with warning when cost budget is exceeded with alert-continue policy', async () => {
+      const warnConfig: ValidatedConfig = {
+        ...stubConfig,
+        costBudget: { maxPerRun: 0.0001, maxPerAgent: 10, alertThreshold: 0.8, onExceeded: 'alert-continue' },
+      };
+      const orchestrator = new Orchestrator(warnConfig, tmpDir);
+      orchestrator._getLicenseTier = () => 'enterprise';
+
+      const consoleSpy = vi.spyOn(console, 'warn').mockImplementation(() => undefined);
+
+      for (const id of [1, 2, 23, 3, 4, 5]) {
+        orchestrator.registerAgent(id, createMockAgent(id, { durationMs: 100000 }));
+      }
+
+      const result = await orchestrator.runPhase('alpha');
+
+      // All agents should have run despite budget exceeded
+      expect(result.agentResults[1]).toBeDefined();
+      consoleSpy.mockRestore();
+    });
+
+    it('runs parallel-safe agents concurrently in batches', async () => {
+      const orchestrator = new Orchestrator(stubConfig, tmpDir);
+      orchestrator._getLicenseTier = () => 'enterprise';
+
+      // Beta phase has parallel-safe agents: 6, 7, 8, 12, 13, etc.
+      for (const id of [5, 6, 7, 8, 12, 13, 14, 15, 16, 18, 19, 21, 22]) {
+        orchestrator.registerAgent(id, createMockAgent(id));
+      }
+
+      const result = await orchestrator.runPhase('beta');
+
+      // All parallel-safe agents should have results
+      expect(result.agentResults[6]).toBeDefined();
+      expect(result.agentResults[7]).toBeDefined();
+    });
+
+    it('getRunDir returns the run directory', () => {
+      const orchestrator = new Orchestrator(stubConfig, '/my/run/dir');
+      expect(orchestrator.getRunDir()).toBe('/my/run/dir');
+    });
+
+    it('fix loop does not dispatch when no findings exist', async () => {
+      const healerAgent = createMockAgent(9);
+      const orchestrator = new Orchestrator(stubConfig, tmpDir);
+      orchestrator._getLicenseTier = () => 'enterprise';
+      orchestrator.registerAgent(1, createMockAgent(1)); // no findings
+      orchestrator.registerAgent(9, healerAgent);
+
+      await orchestrator.runPhase('alpha');
+
+      expect(healerAgent.run).not.toHaveBeenCalled();
+    });
+
+    it('skipped agents produce findings with info severity and license description', async () => {
+      const orchestrator = createOrchestratorWithTier('free');
+      // Register agent 23 (pro) in alpha phase
+      orchestrator.registerAgent(1, createMockAgent(1));
+      orchestrator.registerAgent(23, createMockAgent(23));
+
+      const result = await orchestrator.runPhase('alpha');
+
+      const gatedResult = result.agentResults[23];
+      expect(gatedResult).toBeDefined();
+      expect(gatedResult.status).toBe('skipped');
+      expect(gatedResult.durationMs).toBe(0);
+      expect(gatedResult.findings).toHaveLength(1);
+
+      const finding = gatedResult.findings[0];
+      expect(finding.id).toBe('license-gate-23');
+      expect(finding.severity).toBe('info');
+      expect(finding.type).toBe('infra-issue');
+      expect(finding.module).toBe('license');
+      expect(finding.agentId).toBe(23);
+    });
+  });
+});

package/core/__tests__/phase-gate.test.ts

@@ -0,0 +1,43 @@
+import { describe, it, expect } from 'vitest';
+import type { AgentResult } from '../types';
+import { evaluateGate } from '../phase-gate';
+
+type PartialAgentResults = Record<number, Pick<AgentResult, 'status'>>;
+
+describe('Phase Gate', () => {
+  it('passes when all blocking agents pass', () => {
+    const result = evaluateGate('alpha', {
+      blocking: [1, 2, 4],
+      advisory: [12],
+    }, {
+      1: { status: 'passed' },
+      2: { status: 'passed' },
+      4: { status: 'passed' },
+      12: { status: 'failed' },
+    } as unknown as PartialAgentResults as Record<number, AgentResult>);
+    expect(result.passed).toBe(true);
+  });
+
+  it('fails when any blocking agent fails', () => {
+    const result = evaluateGate('alpha', {
+      blocking: [1, 2],
+      advisory: [],
+    }, {
+      1: { status: 'passed' },
+      2: { status: 'failed' },
+    } as unknown as PartialAgentResults as Record<number, AgentResult>);
+    expect(result.passed).toBe(false);
+    expect(result.failedBlockingAgents).toEqual([2]);
+  });
+
+  it('skips disabled agents', () => {
+    const result = evaluateGate('beta', {
+      blocking: [5, 6, 7],
+      advisory: [],
+    }, {
+      5: { status: 'passed' },
+      6: { status: 'passed' },
+    } as unknown as PartialAgentResults as Record<number, AgentResult>, new Set([7]));
+    expect(result.passed).toBe(true);
+  });
+});