@avi770/testteam 2.0.0 → 3.2.0

package/agents/15-regression-sentinel.ts

@@ -1,307 +0,0 @@
-import { mkdir, readdir, readFile, writeFile, stat } from 'node:fs/promises';
-import { join } from 'path';
-import type { Finding, Phase } from '../core/types';
-import type { ValidatedConfig } from '../core/config';
-import { BaseAgent } from './base-agent';
-import { compareScreenshots, updateBaseline } from '../helpers/screenshot';
-
-/** Pixel-diff threshold for screenshot comparison (percentage). */
-const SCREENSHOT_DIFF_THRESHOLD = 0.1;
-
-/** Directory name for baseline screenshots inside runDir's parent. */
-const BASELINES_DIR_NAME = 'baselines';
-
-/** Shape of a stored API schema snapshot. */
-interface SchemaSnapshot {
-  [endpoint: string]: unknown;
-}
-
-async function fileExists(p: string): Promise<boolean> {
-  try { await stat(p); return true; } catch { return false; }
-}
-
-export class RegressionSentinelAgent extends BaseAgent {
-  readonly agentId = 15;
-  readonly agentName = 'Regression Sentinel';
-
-  /** Absolute path to the baselines directory (sibling of runDir). */
-  private readonly baselinesDir: string;
-
-  constructor(config: ValidatedConfig, phase: Phase, runDir: string) {
-    super(config, phase, runDir);
-    // Place baselines as a sibling directory to runDir
-    const parentDir = join(runDir, '..');
-    this.baselinesDir = join(parentDir, BASELINES_DIR_NAME);
-  }
-
-  protected async preFlight(): Promise<void> {
-    // Ensure the parent directory of runDir exists (so baselinesDir can be created)
-    const parentDir = join(this.runDir, '..');
-    const parentExists = await fileExists(parentDir);
-    if (!parentExists) {
-      throw new Error(
-        `Parent directory does not exist: ${parentDir}. Run directory path appears invalid.`,
-      );
-    }
-
-    // baselinesDir will be created on first run — pre-flight only checks the parent
-  }
-
-  protected async execute(): Promise<Finding[]> {
-    const findings: Finding[] = [];
-
-    // Ensure the run directory has screenshots to compare
-    const runDirExists = await fileExists(this.runDir);
-    if (!runDirExists) {
-      findings.push({
-        id: '15-no-run-dir',
-        type: 'infra-issue',
-        severity: 'critical',
-        agentId: this.agentId,
-        module: 'regression-sentinel',
-        description: `Run directory does not exist: ${this.runDir}`,
-      });
-      return findings;
-    }
-
-    let allFiles: string[];
-    try {
-      allFiles = await readdir(this.runDir);
-    } catch {
-      allFiles = [];
-    }
-
-    const screenshotFiles = allFiles.filter(
-      (f) => f.endsWith('.png') && !f.endsWith('_diff.png'),
-    );
-
-    const isFirstRun = await this.isFirstRun();
-
-    if (isFirstRun) {
-      // First run: create baselines from all current screenshots
-      this.logFirstRun(screenshotFiles.length);
-      await mkdir(this.baselinesDir, { recursive: true });
-
-      for (const screenshotFile of screenshotFiles) {
-        const currentPath = join(this.runDir, screenshotFile);
-        const baselinePath = await updateBaseline(currentPath, this.baselinesDir);
-
-        this.addEvidence({
-          type: 'screenshot',
-          path: baselinePath,
-          description: `Baseline created for ${screenshotFile}`,
-        });
-      }
-
-      // Store note as evidence
-      const notePath = join(this.baselinesDir, 'first-run-note.json');
-      await writeFile(
-        notePath,
-        JSON.stringify(
-          {
-            note: 'First run: baselines established from current screenshots',
-            screenshotCount: screenshotFiles.length,
-            createdAt: new Date().toISOString(),
-          },
-          null,
-          2,
-        ),
-      );
-      this.addEvidence({
-        type: 'log',
-        path: notePath,
-        description: `First-run baseline note: ${screenshotFiles.length} baseline(s) created`,
-      });
-
-      // First run always passes — return no findings
-      return findings;
-    }
-
-    // Subsequent runs: compare screenshots against baselines
-    for (const screenshotFile of screenshotFiles) {
-      const currentPath = join(this.runDir, screenshotFile);
-      const baselinePath = join(this.baselinesDir, screenshotFile);
-
-      const baselineExists = await fileExists(baselinePath);
-      if (!baselineExists) {
-        // New screenshot with no baseline yet — establish it
-        await updateBaseline(currentPath, this.baselinesDir);
-        this.addEvidence({
-          type: 'screenshot',
-          path: join(this.baselinesDir, screenshotFile),
-          description: `New baseline created for ${screenshotFile} (no prior baseline existed)`,
-        });
-        continue;
-      }
-
-      let compareResult;
-      try {
-        compareResult = await compareScreenshots(currentPath, baselinePath, SCREENSHOT_DIFF_THRESHOLD);
-      } catch (err) {
-        findings.push({
-          id: `15-compare-error-${screenshotFile.replace(/\.png$/, '')}`,
-          type: 'infra-issue',
-          severity: 'medium',
-          agentId: this.agentId,
-          module: 'regression-sentinel',
-          description: `Failed to compare screenshot "${screenshotFile}": ${err instanceof Error ? err.message : String(err)}`,
-        });
-        continue;
-      }
-
-      if (!compareResult.match) {
-        const diffPercent = compareResult.diffPercentage.toFixed(2);
-        const safeName = screenshotFile.replace(/\.png$/, '');
-
-        const evidencePath = compareResult.diffPath ?? currentPath;
-        this.addEvidence({
-          type: 'screenshot',
-          path: evidencePath,
-          description: `Visual diff for "${screenshotFile}": ${diffPercent}% pixels changed`,
-        });
-
-        findings.push({
-          id: `15-visual-regression-${safeName}`,
-          type: 'code-bug-cosmetic',
-          severity: 'medium',
-          agentId: this.agentId,
-          module: 'regression-sentinel',
-          description:
-            `Visual regression detected in "${screenshotFile}": ${diffPercent}% pixel difference (threshold: ${SCREENSHOT_DIFF_THRESHOLD}%)`,
-          screenshot: evidencePath,
-          suggestedFix:
-            'Review the visual diff image. If the change is intentional, update the baseline by deleting the old baseline file.',
-        });
-      }
-    }
-
-    // API schema comparison
-    const schemaFindings = await this.compareApiSchemas();
-    findings.push(...schemaFindings);
-
-    return findings;
-  }
-
-  /**
-   * Determines whether this is a first run.
-   *
-   * A first run is defined as:
-   *  - The baselines directory does not exist at all, OR
-   *  - The baselines directory exists but contains only a `.gitkeep` file
-   *    (i.e. it was committed as an empty placeholder with no real baselines).
-   */
-  private async isFirstRun(): Promise<boolean> {
-    const baseDirExists = await fileExists(this.baselinesDir);
-    if (!baseDirExists) {
-      return true;
-    }
-
-    let entries: string[];
-    try {
-      entries = await readdir(this.baselinesDir);
-    } catch {
-      return true;
-    }
-
-    const meaningfulEntries = entries.filter((e) => e !== '.gitkeep');
-    return meaningfulEntries.length === 0;
-  }
-
-  /** Emits a structured console log so callers can observe the first-run event. */
-  private logFirstRun(screenshotCount: number): void {
-    console.log(
-      `[RegressionSentinel] No baselines found — creating initial baselines from this run ` +
-        `(${screenshotCount} screenshot(s) found in ${this.runDir})`,
-    );
-  }
-
-  private async compareApiSchemas(): Promise<Finding[]> {
-    const findings: Finding[] = [];
-
-    const currentSchemaPath = join(this.runDir, 'api-schema-snapshot.json');
-    const baselineSchemaPath = join(this.baselinesDir, 'api-schema-snapshot.json');
-
-    const currentExists = await fileExists(currentSchemaPath);
-    if (!currentExists) {
-      // No schema snapshot in this run — nothing to compare
-      return findings;
-    }
-
-    const baselineExists = await fileExists(baselineSchemaPath);
-    if (!baselineExists) {
-      // First time seeing schema snapshot — create baseline
-      const schemaContent = await readFile(currentSchemaPath);
-      await writeFile(baselineSchemaPath, schemaContent);
-      this.addEvidence({
-        type: 'snapshot',
-        path: baselineSchemaPath,
-        description: 'API schema snapshot baseline created',
-      });
-      return findings;
-    }
-
-    try {
-      const currentSchema = JSON.parse(
-        await readFile(currentSchemaPath, 'utf-8'),
-      ) as SchemaSnapshot;
-      const baselineSchema = JSON.parse(
-        await readFile(baselineSchemaPath, 'utf-8'),
-      ) as SchemaSnapshot;
-
-      const currentKeys = Object.keys(currentSchema).sort();
-      const baselineKeys = Object.keys(baselineSchema).sort();
-
-      // Check for removed endpoints
-      for (const key of baselineKeys) {
-        if (!(key in currentSchema)) {
-          findings.push({
-            id: `15-schema-removed-${key.replace(/[^a-zA-Z0-9_-]/g, '_')}`,
-            type: 'code-bug-logic',
-            severity: 'high',
-            agentId: this.agentId,
-            module: 'regression-sentinel',
-            description: `API endpoint "${key}" was present in baseline but is missing in current run — possible breaking change`,
-            suggestedFix: 'Verify endpoint removal is intentional and update API consumers accordingly',
-          });
-        }
-      }
-
-      // Check for shape changes in existing endpoints
-      for (const key of currentKeys) {
-        if (key in baselineSchema) {
-          const currentShape = JSON.stringify(currentSchema[key]);
-          const baselineShape = JSON.stringify(baselineSchema[key]);
-          if (currentShape !== baselineShape) {
-            findings.push({
-              id: `15-schema-changed-${key.replace(/[^a-zA-Z0-9_-]/g, '_')}`,
-              type: 'code-bug-logic',
-              severity: 'medium',
-              agentId: this.agentId,
-              module: 'regression-sentinel',
-              description: `API schema for endpoint "${key}" changed from baseline`,
-              suggestedFix:
-                'Verify schema change is intentional, update clients, and refresh the baseline snapshot',
-            });
-          }
-        }
-      }
-
-      this.addEvidence({
-        type: 'snapshot',
-        path: currentSchemaPath,
-        description: `API schema snapshot compared against baseline (${currentKeys.length} endpoints)`,
-      });
-    } catch (err) {
-      findings.push({
-        id: '15-schema-parse-error',
-        type: 'infra-issue',
-        severity: 'low',
-        agentId: this.agentId,
-        module: 'regression-sentinel',
-        description: `Failed to parse API schema snapshots for comparison: ${err instanceof Error ? err.message : String(err)}`,
-      });
-    }
-
-    return findings;
-  }
-}

package/agents/16-chaos-agent.ts

@@ -1,228 +0,0 @@
-import type { Browser, BrowserContext, Page } from 'playwright';
-import type { Finding } from '../core/types';
-import { BaseAgent } from './base-agent';
-import { resolveEnvironment } from '../helpers/env-resolver';
-
-export class ChaosAgent extends BaseAgent {
-  readonly agentId = 16;
-  readonly agentName = 'Chaos Agent';
-
-  protected async preFlight(): Promise<void> {
-    if (this.phase !== 'beta' && this.phase !== 'uat') {
-      throw new Error(`ChaosAgent requires phase "beta" or "uat" but got "${this.phase}"`);
-    }
-
-    await resolveEnvironment(this.config, this.phase);
-  }
-
-  protected async execute(): Promise<Finding[]> {
-    const findings: Finding[] = [];
-
-    const envKey = this.phase === 'uat' ? 'uat' : 'beta';
-    const env = this.config.environments[envKey] ?? Object.values(this.config.environments)[0];
-    if (!env) {
-      findings.push({
-        id: `${this.agentId}-no-env`,
-        type: 'infra-issue',
-        severity: 'critical',
-        agentId: this.agentId,
-        module: 'chaos',
-        description: 'No environment configured — cannot run chaos scenarios',
-      });
-      return findings;
-    }
-
-    const isUat = this.phase === 'uat';
-
-    const loginUrl = this.config.auth.loginUrl ?? `${env.baseUrl}/login`;
-    const credentials =
-      this.config.auth.credentials?.['admin'] ??
-      Object.values(this.config.auth.credentials ?? {})[0];
-
-    const { chromium } = await import('playwright');
-
-    // -----------------------------------------------------------------------
-    // Scenario 1: Token expiry — expired JWT should redirect to login
-    // -----------------------------------------------------------------------
-    try {
-      let browser: Browser | null = null;
-      let context: BrowserContext | null = null;
-      let page: Page | null = null;
-      try {
-        browser = await chromium.launch({ headless: true });
-        context = await browser.newContext();
-        page = await context.newPage();
-
-        // Inject an expired JWT into localStorage before navigating
-        await page.goto(env.baseUrl);
-        await page.evaluate(() => {
-          // Minimal expired JWT: header.payload.signature — exp in the past
-          const expiredJwt =
-            'eyJhbGciOiJIUzI1NiJ9.' +
-            btoa(JSON.stringify({ exp: 1, sub: 'test' })).replace(/=/g, '') +
-            '.invalid';
-          localStorage.setItem('access_token', expiredJwt);
-          localStorage.setItem('token', expiredJwt);
-          sessionStorage.setItem('access_token', expiredJwt);
-        });
-
-        // Navigate to a protected module — should redirect to login
-        const firstModule = this.config.modules[0];
-        const protectedUrl = firstModule
-          ? `${env.baseUrl}${firstModule.route}`
-          : `${env.baseUrl}/dashboard`;
-
-        await page.goto(protectedUrl, { waitUntil: 'networkidle' });
-
-        const finalUrl = page.url();
-        const redirectedToLogin =
-          finalUrl.includes('/login') ||
-          finalUrl.includes('/auth') ||
-          finalUrl.includes('/signin');
-
-        if (!redirectedToLogin) {
-          findings.push({
-            id: `${this.agentId}-token-expiry`,
-            type: 'code-bug-security',
-            severity: 'critical',
-            agentId: this.agentId,
-            module: 'auth',
-            description:
-              `Token expiry scenario: app did not redirect to login after expired JWT was set. ` +
-              `Final URL: ${finalUrl}`,
-            suggestedFix: 'Validate JWT expiry on every protected route and redirect to login when expired.',
-          });
-        }
-      } finally {
-        if (page) await page.close().catch(() => undefined);
-        if (context) await context.close().catch(() => undefined);
-        if (browser) await browser.close().catch(() => undefined);
-      }
-    } catch {
-      findings.push({
-        id: `${this.agentId}-token-expiry-skipped`,
-        type: 'infra-issue',
-        severity: 'medium',
-        agentId: this.agentId,
-        module: 'chaos',
-        description: 'Chaos test requires a running app with auth — skipped',
-      });
-      return findings;
-    }
-
-    // -----------------------------------------------------------------------
-    // Scenario 2: Network throttle — abort selected API routes and verify no crash
-    // -----------------------------------------------------------------------
-    {
-      let browser: Browser | null = null;
-      let context: BrowserContext | null = null;
-      let page: Page | null = null;
-      const uncaughtErrors: string[] = [];
-      try {
-        browser = await chromium.launch({ headless: true });
-        context = await browser.newContext();
-        page = await context.newPage();
-
-        // Abort all XHR/fetch requests to /api/* to simulate network degradation
-        await page.route('**/api/**', (route) => {
-          route.abort('failed').catch(() => undefined);
-        });
-
-        page.on('pageerror', (err) => {
-          uncaughtErrors.push(err.message);
-        });
-
-        // Login without API interception (need to reach login first)
-        if (credentials) {
-          try {
-            await page.goto(loginUrl, { waitUntil: 'networkidle', timeout: 15_000 });
-          } catch {
-            // Tolerable under route abort
-          }
-        }
-
-        const firstModule = this.config.modules[0];
-        const targetUrl = firstModule
-          ? `${env.baseUrl}${firstModule.route}`
-          : `${env.baseUrl}/dashboard`;
-
-        try {
-          await page.goto(targetUrl, { waitUntil: 'domcontentloaded', timeout: 15_000 });
-        } catch {
-          // Expected — the page may time out when API routes are aborted
-        }
-
-        // Wait a moment for any unhandled errors to surface
-        await new Promise<void>((resolve) => setTimeout(resolve, 1_000));
-
-        if (uncaughtErrors.length > 0) {
-          findings.push({
-            id: `${this.agentId}-network-throttle`,
-            type: 'code-bug-logic',
-            severity: 'high',
-            agentId: this.agentId,
-            module: firstModule?.id ?? 'app',
-            description:
-              `Network throttle scenario: ${uncaughtErrors.length} uncaught JS error(s) when API routes were aborted. ` +
-              `First error: ${uncaughtErrors[0]}`,
-            suggestedFix:
-              'Add error boundaries and graceful loading states for all API-dependent components.',
-          });
-        }
-      } finally {
-        if (page) await page.close().catch(() => undefined);
-        if (context) await context.close().catch(() => undefined);
-        if (browser) await browser.close().catch(() => undefined);
-      }
-    }
-
-    // -----------------------------------------------------------------------
-    // Scenario 3: Large data — verify paginated list endpoints don't crash
-    // Skipped in UAT mode (destructive: seeds large data)
-    // -----------------------------------------------------------------------
-    if (!isUat) {
-      try {
-        const listPaths = ['/api/clients', '/api/invoices', '/api/transactions'];
-        for (const path of listPaths) {
-          const url = `${env.baseUrl}${path}?page=1&limit=1000`;
-          let response: Response | null = null;
-          try {
-            response = await fetch(url, {
-              method: 'GET',
-              signal: AbortSignal.timeout(15_000),
-            });
-          } catch {
-            // Endpoint may not exist — skip silently
-            continue;
-          }
-
-          if (response.status === 500 || response.status === 502 || response.status === 503) {
-            findings.push({
-              id: `${this.agentId}-large-data-${path.replace(/\//g, '-')}`,
-              type: 'code-bug-logic',
-              severity: 'high',
-              agentId: this.agentId,
-              module: 'api',
-              description:
-                `Large data scenario: ${path} returned ${response.status} when requesting 1 000 records. ` +
-                `The endpoint may lack server-side pagination or timeout protection.`,
-              suggestedFix:
-                'Enforce server-side pagination limits and add query timeout guards.',
-            });
-          }
-        }
-      } catch (error) {
-        findings.push({
-          id: `${this.agentId}-large-data-error`,
-          type: 'infra-issue',
-          severity: 'medium',
-          agentId: this.agentId,
-          module: 'api',
-          description: `Large data scenario failed unexpectedly: ${error instanceof Error ? error.message : String(error)}`,
-        });
-      }
-    }
-
-    return findings;
-  }
-}