@avi770/testteam 1.2.0

package/agents/__tests__/15-regression-sentinel.test.ts

@@ -0,0 +1,657 @@
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+import { join } from 'path';
+import type { ValidatedConfig } from '../../core/config';
+import type { Phase } from '../../core/types';
+
+// ---------------------------------------------------------------------------
+// Stub config
+// ---------------------------------------------------------------------------
+
+const stubConfig = {
+  schemaVersion: 1,
+  name: 'test',
+  auth: {
+    loginUrl: 'http://localhost/login',
+    credentials: { admin: { email: 'admin@test.com', password: 'secret' } },
+  },
+  modules: [
+    { id: 'dashboard', route: '/dashboard', sidebarIcon: true },
+    { id: 'clients', route: '/clients', sidebarIcon: true },
+  ],
+  environments: {
+    beta: { baseUrl: 'http://localhost:3000', seed: false },
+  },
+  portals: [],
+  breakpoints: [],
+  integrations: [],
+  workers: [],
+  workflows: [],
+  accuracy: { enabled: true, decimalPrecision: 2, currencySymbol: '$', timezone: 'UTC' },
+  tenancy: { enabled: false, isolationField: 'firmId', testFirms: [] },
+  costBudget: { maxPerRun: 100, maxPerAgent: 10, alertThreshold: 0.8, onExceeded: 'abort' as const },
+  agentClassification: { blocking: [], advisory: [] },
+  thresholds: {
+    unitCoverage: { lines: 80, functions: 80, branches: 80, statements: 80 },
+    lighthouse: { accessibility: 90, performance: 90, bestPractices: 90, seo: 90 },
+    apiResponseTime: 2000,
+    bundleSizeLimit: 500000,
+    maxConsoleErrors: 0,
+  },
+  disabledAgents: [],
+  disabledAgentSet: new Set<number>(),
+} satisfies ValidatedConfig;
+
+// ---------------------------------------------------------------------------
+// Mock node:fs/promises
+// ---------------------------------------------------------------------------
+
+const statMock = vi.fn();
+const readdirMock = vi.fn();
+const mkdirMock = vi.fn();
+const readFileMock = vi.fn();
+const writeFileMock = vi.fn();
+
+vi.mock('node:fs/promises', async (importOriginal) => {
+  const actual = await importOriginal<typeof import('node:fs/promises')>();
+  return {
+    ...actual,
+    stat: (...args: unknown[]) => statMock(...args),
+    readdir: (...args: unknown[]) => readdirMock(...args),
+    mkdir: (...args: unknown[]) => mkdirMock(...args),
+    readFile: (...args: unknown[]) => readFileMock(...args),
+    writeFile: (...args: unknown[]) => writeFileMock(...args),
+    copyFile: vi.fn().mockResolvedValue(undefined),
+  };
+});
+
+// ---------------------------------------------------------------------------
+// Mock screenshot helper
+// ---------------------------------------------------------------------------
+
+const compareScreenshotsMock = vi.fn();
+const updateBaselineMock = vi.fn();
+
+vi.mock('../../helpers/screenshot', () => ({
+  compareScreenshots: (...args: unknown[]) => compareScreenshotsMock(...args),
+  updateBaseline: (...args: unknown[]) => updateBaselineMock(...args),
+  captureScreenshot: vi.fn().mockResolvedValue('/tmp/test-run/screenshot.png'),
+}));
+
+// ---------------------------------------------------------------------------
+// Dynamic import AFTER mocks
+// ---------------------------------------------------------------------------
+
+const { RegressionSentinelAgent } = await import('../15-regression-sentinel');
+
+// Compute paths the same way the agent does so they match on all platforms
+const RUN_DIR = join('/tmp', 'test-runs', 'run-001');
+const BASELINES_DIR = join('/tmp', 'test-runs', 'baselines');
+const PARENT_DIR = join('/tmp', 'test-runs');
+
+function createAgent(phase: Phase = 'beta'): InstanceType<typeof RegressionSentinelAgent> {
+  return new RegressionSentinelAgent(stubConfig, phase, RUN_DIR);
+}
+
+// ---------------------------------------------------------------------------
+// Tests
+// ---------------------------------------------------------------------------
+
+describe('RegressionSentinelAgent', () => {
+  beforeEach(() => {
+    statMock.mockReset();
+    readdirMock.mockReset();
+    mkdirMock.mockReset();
+    readFileMock.mockReset();
+    writeFileMock.mockReset();
+    compareScreenshotsMock.mockReset();
+    updateBaselineMock.mockReset().mockResolvedValue(join(BASELINES_DIR, 'screenshot.png'));
+
+    // Default: parent dir exists, run dir exists, baselines dir does NOT yet exist (first run)
+    statMock.mockImplementation((p: string) => {
+      if (p === PARENT_DIR) return Promise.resolve({ isDirectory: () => true });
+      if (p === RUN_DIR) return Promise.resolve({ isDirectory: () => true });
+      if (p === BASELINES_DIR) return Promise.reject(new Error('ENOENT'));
+      return Promise.reject(new Error('ENOENT'));
+    });
+
+    readdirMock.mockImplementation((dir: string) => {
+      if (dir === RUN_DIR) return Promise.resolve(['smoke-dashboard.png', 'smoke-clients.png']);
+      return Promise.resolve([]);
+    });
+
+    mkdirMock.mockResolvedValue(undefined);
+    writeFileMock.mockResolvedValue(undefined);
+  });
+
+  it('has agentId 15 and agentName "Regression Sentinel"', () => {
+    const agent = createAgent();
+    expect(agent.agentId).toBe(15);
+    expect(agent.agentName).toBe('Regression Sentinel');
+  });
+
+  // -------------------------------------------------------------------------
+  // preFlight()
+  // -------------------------------------------------------------------------
+
+  describe('preFlight()', () => {
+    it('throws when parent directory does not exist', async () => {
+      statMock.mockImplementation((p: string) => {
+        if (p === PARENT_DIR) return Promise.reject(new Error('ENOENT')); // parent missing
+        return Promise.reject(new Error('ENOENT'));
+      });
+
+      const agent = createAgent();
+      const result = await agent.run();
+
+      expect(result.status).toBe('failed');
+      expect(result.findings[0].description).toContain('Parent directory does not exist');
+    });
+
+    it('passes when parent directory exists (baselines may not yet exist)', async () => {
+      const agent = createAgent();
+      const result = await agent.run();
+
+      expect(result.findings.some((f) => f.description.includes('Parent directory'))).toBe(false);
+    });
+  });
+
+  // -------------------------------------------------------------------------
+  // First-run baseline creation
+  // -------------------------------------------------------------------------
+
+  describe('first-run baseline creation', () => {
+    it('creates baselines directory and copies all screenshots on first run', async () => {
+      const agent = createAgent();
+      await agent.run();
+
+      expect(mkdirMock).toHaveBeenCalledWith(BASELINES_DIR, { recursive: true });
+      expect(updateBaselineMock).toHaveBeenCalledTimes(2); // one per screenshot
+    });
+
+    it('returns no findings on first run', async () => {
+      const agent = createAgent();
+      const result = await agent.run();
+
+      expect(result.findings).toHaveLength(0);
+    });
+
+    it('returns passed status on first run', async () => {
+      const agent = createAgent();
+      const result = await agent.run();
+
+      expect(result.status).toBe('passed');
+    });
+
+    it('adds evidence for each baseline created on first run', async () => {
+      updateBaselineMock.mockImplementation(async (_src: string, dir: string) => {
+        return `${dir}/baseline.png`;
+      });
+
+      const agent = createAgent();
+      const result = await agent.run();
+
+      const screenshotEvidence = result.evidence.filter((e) => e.type === 'screenshot');
+      expect(screenshotEvidence.length).toBeGreaterThanOrEqual(2);
+    });
+  });
+
+  // -------------------------------------------------------------------------
+  // First-run: baselines dir exists but contains only .gitkeep
+  // -------------------------------------------------------------------------
+
+  describe('first-run when baselines dir contains only .gitkeep', () => {
+    beforeEach(() => {
+      // Baselines dir EXISTS (committed as empty placeholder) but has only .gitkeep
+      statMock.mockImplementation((p: string) => {
+        if (p === PARENT_DIR) return Promise.resolve({ isDirectory: () => true });
+        if (p === RUN_DIR) return Promise.resolve({ isDirectory: () => true });
+        if (p === BASELINES_DIR) return Promise.resolve({ isDirectory: () => true }); // dir exists...
+        return Promise.reject(new Error('ENOENT'));
+      });
+
+      // ...but only contains .gitkeep
+      readdirMock.mockImplementation((dir: string) => {
+        if (dir === BASELINES_DIR) return Promise.resolve(['.gitkeep']);
+        return Promise.resolve(['smoke-dashboard.png', 'smoke-clients.png']);
+      });
+    });
+
+    it('treats .gitkeep-only baselines dir as first run and creates baselines', async () => {
+      const agent = createAgent();
+      await agent.run();
+
+      expect(mkdirMock).toHaveBeenCalledWith(BASELINES_DIR, { recursive: true });
+      expect(updateBaselineMock).toHaveBeenCalledTimes(2);
+    });
+
+    it('returns no findings when baselines dir has only .gitkeep', async () => {
+      const agent = createAgent();
+      const result = await agent.run();
+
+      expect(result.findings).toHaveLength(0);
+    });
+
+    it('returns passed status when baselines dir has only .gitkeep', async () => {
+      const agent = createAgent();
+      const result = await agent.run();
+
+      expect(result.status).toBe('passed');
+    });
+
+    it('logs the first-run message to console', async () => {
+      const consoleSpy = vi.spyOn(console, 'log').mockImplementation(() => undefined);
+
+      const agent = createAgent();
+      await agent.run();
+
+      expect(consoleSpy).toHaveBeenCalledWith(
+        expect.stringContaining('No baselines found — creating initial baselines from this run'),
+      );
+
+      consoleSpy.mockRestore();
+    });
+  });
+
+  // -------------------------------------------------------------------------
+  // Second-run: baselines dir has real files (not .gitkeep-only) → compare
+  // -------------------------------------------------------------------------
+
+  describe('second-run baseline comparison (baselines exist beyond .gitkeep)', () => {
+    beforeEach(() => {
+      statMock.mockImplementation((p: string) => {
+        if (p === PARENT_DIR) return Promise.resolve({ isDirectory: () => true });
+        if (p === RUN_DIR) return Promise.resolve({ isDirectory: () => true });
+        if (p === BASELINES_DIR) return Promise.resolve({ isDirectory: () => true });
+        if (p.startsWith(BASELINES_DIR)) return Promise.resolve({ isFile: () => true });
+        return Promise.reject(new Error('ENOENT'));
+      });
+
+      // Baselines dir has real PNG baselines — NOT treated as first run
+      readdirMock.mockImplementation((dir: string) => {
+        if (dir === BASELINES_DIR) return Promise.resolve(['smoke-dashboard.png', 'smoke-clients.png']);
+        if (dir === RUN_DIR) return Promise.resolve(['smoke-dashboard.png', 'smoke-clients.png']);
+        return Promise.resolve([]);
+      });
+
+      compareScreenshotsMock.mockResolvedValue({ match: true, diffPercentage: 0 });
+    });
+
+    it('does NOT create new baselines when real baseline files exist', async () => {
+      const agent = createAgent();
+      await agent.run();
+
+      expect(mkdirMock).not.toHaveBeenCalled();
+      expect(updateBaselineMock).not.toHaveBeenCalled();
+    });
+
+    it('compares screenshots against existing baselines', async () => {
+      const agent = createAgent();
+      await agent.run();
+
+      expect(compareScreenshotsMock).toHaveBeenCalledTimes(2);
+    });
+  });
+
+  // -------------------------------------------------------------------------
+  // Subsequent run: matching screenshots → no findings
+  // -------------------------------------------------------------------------
+
+  describe('matching screenshots → no findings', () => {
+    beforeEach(() => {
+      // Baselines now exist
+      statMock.mockImplementation((p: string) => {
+        if (p === PARENT_DIR) return Promise.resolve({ isDirectory: () => true });
+        if (p === RUN_DIR) return Promise.resolve({ isDirectory: () => true });
+        if (p === BASELINES_DIR) return Promise.resolve({ isDirectory: () => true });
+        if (p.startsWith(BASELINES_DIR)) return Promise.resolve({ isFile: () => true });
+        return Promise.reject(new Error('ENOENT'));
+      });
+
+      // Baselines dir has real PNG files — NOT treated as first run
+      readdirMock.mockImplementation((dir: string) => {
+        if (dir === BASELINES_DIR) return Promise.resolve(['smoke-dashboard.png', 'smoke-clients.png']);
+        if (dir === RUN_DIR) return Promise.resolve(['smoke-dashboard.png', 'smoke-clients.png']);
+        return Promise.resolve([]);
+      });
+
+      compareScreenshotsMock.mockResolvedValue({ match: true, diffPercentage: 0 });
+    });
+
+    it('returns no findings when all screenshots match baselines', async () => {
+      const agent = createAgent();
+      const result = await agent.run();
+
+      const regressionFindings = result.findings.filter((f) => f.id.includes('visual-regression'));
+      expect(regressionFindings).toHaveLength(0);
+    });
+
+    it('returns passed status when all screenshots match', async () => {
+      const agent = createAgent();
+      const result = await agent.run();
+
+      expect(result.status).toBe('passed');
+    });
+  });
+
+  // -------------------------------------------------------------------------
+  // Subsequent run: different screenshots → findings
+  // -------------------------------------------------------------------------
+
+  describe('visual regression detection', () => {
+    beforeEach(() => {
+      statMock.mockImplementation((p: string) => {
+        if (p === PARENT_DIR) return Promise.resolve({ isDirectory: () => true });
+        if (p === RUN_DIR) return Promise.resolve({ isDirectory: () => true });
+        if (p === BASELINES_DIR) return Promise.resolve({ isDirectory: () => true });
+        if (p.startsWith(BASELINES_DIR)) return Promise.resolve({ isFile: () => true });
+        return Promise.reject(new Error('ENOENT'));
+      });
+
+      // Baselines dir has real PNG files — NOT treated as first run
+      readdirMock.mockImplementation((dir: string) => {
+        if (dir === BASELINES_DIR) return Promise.resolve(['smoke-dashboard.png', 'smoke-clients.png']);
+        if (dir === RUN_DIR) return Promise.resolve(['smoke-dashboard.png', 'smoke-clients.png']);
+        return Promise.resolve([]);
+      });
+    });
+
+    it('creates a medium-severity finding when screenshot diff exceeds threshold', async () => {
+      compareScreenshotsMock.mockResolvedValue({
+        match: false,
+        diffPercentage: 5.3,
+        diffPath: join(RUN_DIR, 'smoke-dashboard_diff.png'),
+      });
+
+      const agent = createAgent();
+      const result = await agent.run();
+
+      const regressionFinding = result.findings.find(
+        (f) => f.id.includes('visual-regression') && f.id.includes('dashboard'),
+      );
+      expect(regressionFinding).toBeDefined();
+      expect(regressionFinding!.severity).toBe('medium');
+      expect(regressionFinding!.type).toBe('code-bug-cosmetic');
+    });
+
+    it('includes the diff percentage in the finding description', async () => {
+      compareScreenshotsMock.mockResolvedValue({
+        match: false,
+        diffPercentage: 2.75,
+        diffPath: join(RUN_DIR, 'smoke-dashboard_diff.png'),
+      });
+
+      const agent = createAgent();
+      const result = await agent.run();
+
+      const regressionFinding = result.findings.find((f) => f.id.includes('visual-regression'));
+      expect(regressionFinding!.description).toContain('2.75');
+    });
+
+    it('adds diff image as evidence for each regression finding', async () => {
+      compareScreenshotsMock.mockResolvedValue({
+        match: false,
+        diffPercentage: 3.0,
+        diffPath: join(RUN_DIR, 'smoke-dashboard_diff.png'),
+      });
+
+      const agent = createAgent();
+      const result = await agent.run();
+
+      const diffEvidence = result.evidence.filter((e) =>
+        e.path.includes('_diff.png'),
+      );
+      expect(diffEvidence.length).toBeGreaterThan(0);
+    });
+
+    it('creates a finding for each differing screenshot', async () => {
+      compareScreenshotsMock.mockResolvedValue({
+        match: false,
+        diffPercentage: 1.5,
+      });
+
+      const agent = createAgent();
+      const result = await agent.run();
+
+      const regressionFindings = result.findings.filter((f) => f.id.includes('visual-regression'));
+      expect(regressionFindings).toHaveLength(2); // one per screenshot
+    });
+
+    it('does NOT create findings for screenshots that match', async () => {
+      // First screenshot matches, second does not
+      compareScreenshotsMock
+        .mockResolvedValueOnce({ match: true, diffPercentage: 0 })
+        .mockResolvedValueOnce({ match: false, diffPercentage: 2.0 });
+
+      const agent = createAgent();
+      const result = await agent.run();
+
+      const regressionFindings = result.findings.filter((f) => f.id.includes('visual-regression'));
+      expect(regressionFindings).toHaveLength(1);
+    });
+  });
+
+  // -------------------------------------------------------------------------
+  // API schema comparison
+  // -------------------------------------------------------------------------
+
+  describe('API schema comparison', () => {
+    beforeEach(() => {
+      statMock.mockImplementation((p: string) => {
+        if (p === PARENT_DIR) return Promise.resolve({ isDirectory: () => true });
+        if (p === RUN_DIR) return Promise.resolve({ isDirectory: () => true });
+        if (p === BASELINES_DIR) return Promise.resolve({ isDirectory: () => true });
+        if (p.startsWith(BASELINES_DIR) && p.endsWith('.png')) return Promise.resolve({ isFile: () => true });
+        return Promise.reject(new Error('ENOENT'));
+      });
+
+      // Baselines dir has real PNG files — NOT treated as first run
+      readdirMock.mockImplementation((dir: string) => {
+        if (dir === BASELINES_DIR) return Promise.resolve(['smoke-dashboard.png', 'smoke-clients.png']);
+        if (dir === RUN_DIR) return Promise.resolve(['smoke-dashboard.png', 'smoke-clients.png']);
+        return Promise.resolve([]);
+      });
+
+      // All screenshots match
+      compareScreenshotsMock.mockResolvedValue({ match: true, diffPercentage: 0 });
+    });
+
+    it('creates a schema baseline when current snapshot exists but no baseline schema', async () => {
+      statMock.mockImplementation((p: string) => {
+        if (p === PARENT_DIR) return Promise.resolve({ isDirectory: () => true });
+        if (p === RUN_DIR) return Promise.resolve({ isDirectory: () => true });
+        if (p === BASELINES_DIR) return Promise.resolve({ isDirectory: () => true });
+        if (p.startsWith(BASELINES_DIR) && p.endsWith('.png')) return Promise.resolve({ isFile: () => true });
+        if (p === join(RUN_DIR, 'api-schema-snapshot.json')) return Promise.resolve({ isFile: () => true }); // current schema exists
+        if (p === join(BASELINES_DIR, 'api-schema-snapshot.json')) return Promise.reject(new Error('ENOENT')); // baseline does not
+        return Promise.reject(new Error('ENOENT'));
+      });
+
+      readFileMock.mockResolvedValue(JSON.stringify({ '/api/clients': { id: 'string' } }));
+
+      const agent = createAgent();
+      await agent.run();
+
+      expect(writeFileMock).toHaveBeenCalledWith(
+        join(BASELINES_DIR, 'api-schema-snapshot.json'),
+        expect.anything(),
+      );
+    });
+
+    it('creates a finding when an endpoint is removed from schema', async () => {
+      statMock.mockImplementation((p: string) => {
+        if (p === PARENT_DIR) return Promise.resolve({ isDirectory: () => true });
+        if (p === RUN_DIR) return Promise.resolve({ isDirectory: () => true });
+        if (p === BASELINES_DIR) return Promise.resolve({ isDirectory: () => true });
+        if (p.startsWith(BASELINES_DIR) && p.endsWith('.png')) return Promise.resolve({ isFile: () => true });
+        if (p === join(RUN_DIR, 'api-schema-snapshot.json')) return Promise.resolve({ isFile: () => true });
+        if (p === join(BASELINES_DIR, 'api-schema-snapshot.json')) return Promise.resolve({ isFile: () => true });
+        return Promise.reject(new Error('ENOENT'));
+      });
+
+      const currentSchema = { '/api/clients': { id: 'string' } };
+      const baselineSchema = { '/api/clients': { id: 'string' }, '/api/invoices': { id: 'string' } }; // invoices removed!
+
+      readFileMock
+        .mockResolvedValueOnce(JSON.stringify(currentSchema))
+        .mockResolvedValueOnce(JSON.stringify(baselineSchema));
+
+      const agent = createAgent();
+      const result = await agent.run();
+
+      const schemaFinding = result.findings.find((f) => f.id.includes('schema-removed'));
+      expect(schemaFinding).toBeDefined();
+      expect(schemaFinding!.severity).toBe('high');
+      expect(schemaFinding!.description).toContain('/api/invoices');
+    });
+
+    it('creates a finding when an endpoint schema changes', async () => {
+      statMock.mockImplementation((p: string) => {
+        if (p === PARENT_DIR) return Promise.resolve({ isDirectory: () => true });
+        if (p === RUN_DIR) return Promise.resolve({ isDirectory: () => true });
+        if (p === BASELINES_DIR) return Promise.resolve({ isDirectory: () => true });
+        if (p.startsWith(BASELINES_DIR) && p.endsWith('.png')) return Promise.resolve({ isFile: () => true });
+        if (p === join(RUN_DIR, 'api-schema-snapshot.json')) return Promise.resolve({ isFile: () => true });
+        if (p === join(BASELINES_DIR, 'api-schema-snapshot.json')) return Promise.resolve({ isFile: () => true });
+        return Promise.reject(new Error('ENOENT'));
+      });
+
+      const currentSchema = { '/api/clients': { id: 'string', name: 'string', status: 'string' } }; // added status
+      const baselineSchema = { '/api/clients': { id: 'string', name: 'string' } };
+
+      readFileMock
+        .mockResolvedValueOnce(JSON.stringify(currentSchema))
+        .mockResolvedValueOnce(JSON.stringify(baselineSchema));
+
+      const agent = createAgent();
+      const result = await agent.run();
+
+      const schemaFinding = result.findings.find((f) => f.id.includes('schema-changed'));
+      expect(schemaFinding).toBeDefined();
+      expect(schemaFinding!.severity).toBe('medium');
+    });
+
+    it('creates a finding when schema JSON is malformed', async () => {
+      statMock.mockImplementation((p: string) => {
+        if (p === PARENT_DIR) return Promise.resolve({ isDirectory: () => true });
+        if (p === RUN_DIR) return Promise.resolve({ isDirectory: () => true });
+        if (p === BASELINES_DIR) return Promise.resolve({ isDirectory: () => true });
+        if (p.startsWith(BASELINES_DIR) && p.endsWith('.png')) return Promise.resolve({ isFile: () => true });
+        if (p === join(RUN_DIR, 'api-schema-snapshot.json')) return Promise.resolve({ isFile: () => true });
+        if (p === join(BASELINES_DIR, 'api-schema-snapshot.json')) return Promise.resolve({ isFile: () => true });
+        return Promise.reject(new Error('ENOENT'));
+      });
+
+      readFileMock.mockResolvedValue('not valid json');
+
+      const agent = createAgent();
+      const result = await agent.run();
+
+      const parseFinding = result.findings.find(f => f.id.includes('schema-parse-error'));
+      expect(parseFinding).toBeDefined();
+      expect(parseFinding!.severity).toBe('low');
+    });
+
+    it('creates a new baseline for a screenshot with no prior baseline', async () => {
+      statMock.mockImplementation((p: string) => {
+        if (p === PARENT_DIR) return Promise.resolve({ isDirectory: () => true });
+        if (p === RUN_DIR) return Promise.resolve({ isDirectory: () => true });
+        if (p === BASELINES_DIR) return Promise.resolve({ isDirectory: () => true });
+        // baseline for dashboard exists, but NOT for clients
+        if (p === join(BASELINES_DIR, 'smoke-dashboard.png')) return Promise.resolve({ isFile: () => true });
+        if (p === join(BASELINES_DIR, 'smoke-clients.png')) return Promise.reject(new Error('ENOENT'));
+        return Promise.reject(new Error('ENOENT'));
+      });
+
+      readdirMock.mockImplementation((dir: string) => {
+        if (dir === BASELINES_DIR) return Promise.resolve(['smoke-dashboard.png']);
+        if (dir === RUN_DIR) return Promise.resolve(['smoke-dashboard.png', 'smoke-clients.png']);
+        return Promise.resolve([]);
+      });
+
+      compareScreenshotsMock.mockResolvedValue({ match: true, diffPercentage: 0 });
+
+      const agent = createAgent();
+      await agent.run();
+
+      // updateBaseline called for the new screenshot (clients)
+      expect(updateBaselineMock).toHaveBeenCalledTimes(1);
+    });
+
+    it('creates a finding when compareScreenshots throws', async () => {
+      statMock.mockImplementation((p: string) => {
+        if (p === PARENT_DIR) return Promise.resolve({ isDirectory: () => true });
+        if (p === RUN_DIR) return Promise.resolve({ isDirectory: () => true });
+        if (p === BASELINES_DIR) return Promise.resolve({ isDirectory: () => true });
+        if (p.startsWith(BASELINES_DIR)) return Promise.resolve({ isFile: () => true });
+        return Promise.reject(new Error('ENOENT'));
+      });
+
+      readdirMock.mockImplementation((dir: string) => {
+        if (dir === BASELINES_DIR) return Promise.resolve(['smoke-dashboard.png']);
+        if (dir === RUN_DIR) return Promise.resolve(['smoke-dashboard.png']);
+        return Promise.resolve([]);
+      });
+
+      compareScreenshotsMock.mockRejectedValue(new Error('Image decode error'));
+
+      const agent = createAgent();
+      const result = await agent.run();
+
+      const compareErrorFinding = result.findings.find(f => f.id.includes('compare-error'));
+      expect(compareErrorFinding).toBeDefined();
+      expect(compareErrorFinding!.severity).toBe('medium');
+    });
+
+    it('creates a critical finding when run directory does not exist', async () => {
+      statMock.mockImplementation((p: string) => {
+        if (p === PARENT_DIR) return Promise.resolve({ isDirectory: () => true });
+        if (p === RUN_DIR) return Promise.reject(new Error('ENOENT'));
+        return Promise.reject(new Error('ENOENT'));
+      });
+
+      const agent = createAgent();
+      const result = await agent.run();
+
+      const noRunDirFinding = result.findings.find(f => f.id === '15-no-run-dir');
+      expect(noRunDirFinding).toBeDefined();
+      expect(noRunDirFinding!.severity).toBe('critical');
+    });
+
+    it('handles readdir failure for run directory gracefully', async () => {
+      statMock.mockImplementation((p: string) => {
+        if (p === PARENT_DIR) return Promise.resolve({ isDirectory: () => true });
+        if (p === RUN_DIR) return Promise.resolve({ isDirectory: () => true });
+        return Promise.reject(new Error('ENOENT'));
+      });
+
+      readdirMock.mockRejectedValue(new Error('Permission denied'));
+
+      const agent = createAgent();
+      const result = await agent.run();
+
+      // Should still complete (empty screenshot list, first run)
+      expect(result.status).toBe('passed');
+    });
+
+    it('returns no schema findings when schemas are identical', async () => {
+      statMock.mockImplementation((p: string) => {
+        if (p === PARENT_DIR) return Promise.resolve({ isDirectory: () => true });
+        if (p === RUN_DIR) return Promise.resolve({ isDirectory: () => true });
+        if (p === BASELINES_DIR) return Promise.resolve({ isDirectory: () => true });
+        if (p.startsWith(BASELINES_DIR) && p.endsWith('.png')) return Promise.resolve({ isFile: () => true });
+        if (p === join(RUN_DIR, 'api-schema-snapshot.json')) return Promise.resolve({ isFile: () => true });
+        if (p === join(BASELINES_DIR, 'api-schema-snapshot.json')) return Promise.resolve({ isFile: () => true });
+        return Promise.reject(new Error('ENOENT'));
+      });
+
+      const schema = JSON.stringify({ '/api/clients': { id: 'string' } });
+      readFileMock.mockResolvedValue(schema);
+
+      const agent = createAgent();
+      const result = await agent.run();
+
+      const schemaFindings = result.findings.filter((f) => f.id.includes('schema'));
+      expect(schemaFindings).toHaveLength(0);
+    });
+  });
+});