@bugzy-ai/bugzy 1.16.0 → 1.18.0

package/templates/playwright/reporters/__tests__/bugzy-reporter-failure-classification.test.ts

@@ -0,0 +1,299 @@
+import { test, expect } from '@playwright/test';
+import { classifyFailures } from '../bugzy-reporter';
+import * as fs from 'fs';
+import * as path from 'path';
+import * as os from 'os';
+
+function makeManifest(overrides: Partial<{
+  timestamp: string;
+  testCases: Array<{
+    id: string;
+    name: string;
+    totalExecutions: number;
+    finalStatus: string;
+    executions: Array<{
+      number: number;
+      status: string;
+      duration: number;
+      videoFile: string | null;
+      hasTrace: boolean;
+      hasScreenshots: boolean;
+      error: string | null;
+    }>;
+  }>;
+}> = {}) {
+  const testCases = overrides.testCases ?? [];
+  const totalExecutions = testCases.reduce((sum, tc) => sum + tc.executions.length, 0);
+  const passed = testCases.filter(tc => tc.finalStatus === 'passed').length;
+  const failed = testCases.length - passed;
+
+  return {
+    bugzyExecutionId: 'local-test',
+    timestamp: overrides.timestamp ?? '20260216-120000',
+    startTime: '2026-02-16T12:00:00.000Z',
+    endTime: '2026-02-16T12:01:00.000Z',
+    status: failed > 0 ? 'failed' : 'passed',
+    stats: { totalTests: testCases.length, passed, failed, totalExecutions },
+    testCases,
+  };
+}
+
+function makeTestCase(id: string, finalStatus: string, error?: string) {
+  return {
+    id,
+    name: id.replace(/^TC-\d+-/, '').replace(/-/g, ' '),
+    totalExecutions: 1,
+    finalStatus,
+    executions: [{
+      number: 1,
+      status: finalStatus,
+      duration: 1000,
+      videoFile: null,
+      hasTrace: false,
+      hasScreenshots: false,
+      error: error ?? null,
+    }],
+  };
+}
+
+function setupTestRunsDir(manifests: Array<{ timestamp: string; manifest: any }>) {
+  const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'bugzy-test-'));
+  const testRunsRoot = path.join(tmpDir, 'test-runs');
+  fs.mkdirSync(testRunsRoot, { recursive: true });
+
+  for (const { timestamp, manifest } of manifests) {
+    const runDir = path.join(testRunsRoot, timestamp);
+    fs.mkdirSync(runDir, { recursive: true });
+    fs.writeFileSync(
+      path.join(runDir, 'manifest.json'),
+      JSON.stringify(manifest, null, 2)
+    );
+  }
+
+  return testRunsRoot;
+}
+
+test.describe('classifyFailures', () => {
+  test('returns empty arrays when no failures', () => {
+    const manifest = makeManifest({
+      testCases: [makeTestCase('TC-001-login', 'passed')],
+    });
+
+    const result = classifyFailures(manifest, '/nonexistent');
+
+    expect(result.newFailures).toHaveLength(0);
+    expect(result.knownFailures).toHaveLength(0);
+  });
+
+  test('all failures are new when no previous runs exist', () => {
+    const manifest = makeManifest({
+      timestamp: '20260216-120000',
+      testCases: [
+        makeTestCase('TC-001-login', 'failed', 'timeout'),
+        makeTestCase('TC-002-checkout', 'failed', 'element not found'),
+      ],
+    });
+
+    const testRunsRoot = setupTestRunsDir([]);
+
+    const result = classifyFailures(manifest, testRunsRoot);
+
+    expect(result.newFailures).toHaveLength(2);
+    expect(result.knownFailures).toHaveLength(0);
+    expect(result.newFailures[0].id).toBe('TC-001-login');
+    expect(result.newFailures[0].error).toBe('timeout');
+    expect(result.newFailures[1].id).toBe('TC-002-checkout');
+  });
+
+  test('failure is new when test passed in recent run', () => {
+    const previousManifest = makeManifest({
+      timestamp: '20260215-120000',
+      testCases: [
+        makeTestCase('TC-001-login', 'passed'),
+        makeTestCase('TC-002-checkout', 'passed'),
+      ],
+    });
+
+    const testRunsRoot = setupTestRunsDir([
+      { timestamp: '20260215-120000', manifest: previousManifest },
+    ]);
+
+    const currentManifest = makeManifest({
+      timestamp: '20260216-120000',
+      testCases: [
+        makeTestCase('TC-001-login', 'failed', 'timeout'),
+      ],
+    });
+
+    const result = classifyFailures(currentManifest, testRunsRoot);
+
+    expect(result.newFailures).toHaveLength(1);
+    expect(result.knownFailures).toHaveLength(0);
+    expect(result.newFailures[0].id).toBe('TC-001-login');
+    expect(result.newFailures[0].lastPassedRun).toBe('20260215-120000');
+  });
+
+  test('failure is known when test failed in all previous runs', () => {
+    const prev1 = makeManifest({
+      timestamp: '20260215-120000',
+      testCases: [makeTestCase('TC-001-login', 'failed', 'timeout')],
+    });
+    const prev2 = makeManifest({
+      timestamp: '20260214-120000',
+      testCases: [makeTestCase('TC-001-login', 'failed', 'timeout')],
+    });
+
+    const testRunsRoot = setupTestRunsDir([
+      { timestamp: '20260215-120000', manifest: prev1 },
+      { timestamp: '20260214-120000', manifest: prev2 },
+    ]);
+
+    const currentManifest = makeManifest({
+      timestamp: '20260216-120000',
+      testCases: [makeTestCase('TC-001-login', 'failed', 'timeout')],
+    });
+
+    const result = classifyFailures(currentManifest, testRunsRoot);
+
+    expect(result.newFailures).toHaveLength(0);
+    expect(result.knownFailures).toHaveLength(1);
+    expect(result.knownFailures[0].id).toBe('TC-001-login');
+  });
+
+  test('mixed new and known failures', () => {
+    const previousManifest = makeManifest({
+      timestamp: '20260215-120000',
+      testCases: [
+        makeTestCase('TC-001-login', 'passed'),
+        makeTestCase('TC-002-checkout', 'failed', 'always broken'),
+      ],
+    });
+
+    const testRunsRoot = setupTestRunsDir([
+      { timestamp: '20260215-120000', manifest: previousManifest },
+    ]);
+
+    const currentManifest = makeManifest({
+      timestamp: '20260216-120000',
+      testCases: [
+        makeTestCase('TC-001-login', 'failed', 'new regression'),
+        makeTestCase('TC-002-checkout', 'failed', 'still broken'),
+      ],
+    });
+
+    const result = classifyFailures(currentManifest, testRunsRoot);
+
+    expect(result.newFailures).toHaveLength(1);
+    expect(result.newFailures[0].id).toBe('TC-001-login');
+    expect(result.newFailures[0].lastPassedRun).toBe('20260215-120000');
+
+    expect(result.knownFailures).toHaveLength(1);
+    expect(result.knownFailures[0].id).toBe('TC-002-checkout');
+  });
+
+  test('new test not in history is treated as new failure', () => {
+    const previousManifest = makeManifest({
+      timestamp: '20260215-120000',
+      testCases: [makeTestCase('TC-001-login', 'passed')],
+    });
+
+    const testRunsRoot = setupTestRunsDir([
+      { timestamp: '20260215-120000', manifest: previousManifest },
+    ]);
+
+    const currentManifest = makeManifest({
+      timestamp: '20260216-120000',
+      testCases: [
+        makeTestCase('TC-003-new-feature', 'failed', 'new test fails'),
+      ],
+    });
+
+    const result = classifyFailures(currentManifest, testRunsRoot);
+
+    expect(result.newFailures).toHaveLength(1);
+    expect(result.newFailures[0].id).toBe('TC-003-new-feature');
+    expect(result.newFailures[0].lastPassedRun).toBeNull();
+  });
+
+  test('respects BUGZY_FAILURE_LOOKBACK env var', () => {
+    // Set lookback to 1
+    const origEnv = process.env.BUGZY_FAILURE_LOOKBACK;
+    process.env.BUGZY_FAILURE_LOOKBACK = '1';
+
+    try {
+      // Run 1: test passed
+      // Run 2: test failed
+      // Run 3 (current): test failed
+      // With lookback=1, only run 2 is checked (most recent)
+      const run1 = makeManifest({
+        timestamp: '20260213-120000',
+        testCases: [makeTestCase('TC-001-login', 'passed')],
+      });
+      const run2 = makeManifest({
+        timestamp: '20260214-120000',
+        testCases: [makeTestCase('TC-001-login', 'failed', 'broken')],
+      });
+
+      const testRunsRoot = setupTestRunsDir([
+        { timestamp: '20260213-120000', manifest: run1 },
+        { timestamp: '20260214-120000', manifest: run2 },
+      ]);
+
+      const currentManifest = makeManifest({
+        timestamp: '20260215-120000',
+        testCases: [makeTestCase('TC-001-login', 'failed', 'still broken')],
+      });
+
+      const result = classifyFailures(currentManifest, testRunsRoot);
+
+      // With lookback=1, only sees run2 where test failed → known failure
+      expect(result.knownFailures).toHaveLength(1);
+      expect(result.newFailures).toHaveLength(0);
+    } finally {
+      if (origEnv !== undefined) {
+        process.env.BUGZY_FAILURE_LOOKBACK = origEnv;
+      } else {
+        delete process.env.BUGZY_FAILURE_LOOKBACK;
+      }
+    }
+  });
+
+  test('handles timedOut status as failure', () => {
+    const previousManifest = makeManifest({
+      timestamp: '20260215-120000',
+      testCases: [makeTestCase('TC-001-login', 'passed')],
+    });
+
+    const testRunsRoot = setupTestRunsDir([
+      { timestamp: '20260215-120000', manifest: previousManifest },
+    ]);
+
+    const currentManifest = makeManifest({
+      timestamp: '20260216-120000',
+      testCases: [makeTestCase('TC-001-login', 'timedOut', 'Test timeout')],
+    });
+
+    const result = classifyFailures(currentManifest, testRunsRoot);
+
+    expect(result.newFailures).toHaveLength(1);
+    expect(result.newFailures[0].id).toBe('TC-001-login');
+  });
+
+  test('skips current run timestamp when reading previous manifests', () => {
+    // Only the current run exists - should be treated as first run
+    const currentManifest = makeManifest({
+      timestamp: '20260216-120000',
+      testCases: [makeTestCase('TC-001-login', 'failed', 'error')],
+    });
+
+    const testRunsRoot = setupTestRunsDir([
+      { timestamp: '20260216-120000', manifest: currentManifest },
+    ]);
+
+    const result = classifyFailures(currentManifest, testRunsRoot);
+
+    // First run - all failures are new
+    expect(result.newFailures).toHaveLength(1);
+    expect(result.knownFailures).toHaveLength(0);
+  });
+});

package/templates/playwright/reporters/bugzy-reporter.ts

@@ -48,6 +48,16 @@ interface ManifestTestCase {
   executions: ManifestExecution[];
 }
 
+/**
+ * Failure classification entry for new vs known failures
+ */
+interface FailureClassification {
+  id: string;
+  name: string;
+  error: string | null;
+  lastPassedRun: string | null;
+}
+
 /**
  * Manifest structure for test run sessions
  */
@@ -64,6 +74,132 @@ interface Manifest {
     totalExecutions: number;
   };
   testCases: ManifestTestCase[];
+  new_failures?: FailureClassification[];
+  known_failures?: FailureClassification[];
+}
+
+/**
+ * Classify failures as new or known by checking previous test run manifests.
+ *
+ * A failure is "new" if the test passed in any of the last N runs.
+ * A failure is "known" if the test failed in ALL of the last N runs (or no prior data exists for that specific test).
+ * If there are no previous runs at all (first run), all failures are treated as "new".
+ *
+ * @param currentManifest - The current run's manifest
+ * @param testRunsRoot - Path to the test-runs/ directory
+ * @returns Object with newFailures and knownFailures arrays
+ */
+export function classifyFailures(
+  currentManifest: Manifest,
+  testRunsRoot: string
+): { newFailures: FailureClassification[]; knownFailures: FailureClassification[] } {
+  const lookback = parseInt(process.env.BUGZY_FAILURE_LOOKBACK || '5', 10);
+  const newFailures: FailureClassification[] = [];
+  const knownFailures: FailureClassification[] = [];
+
+  // Get failed test cases from current manifest
+  const failedTests = currentManifest.testCases.filter(
+    tc => tc.finalStatus === 'failed' || tc.finalStatus === 'timedOut'
+  );
+
+  if (failedTests.length === 0) {
+    return { newFailures, knownFailures };
+  }
+
+  // Read previous manifests
+  const previousManifests: Manifest[] = [];
+  if (fs.existsSync(testRunsRoot)) {
+    const dirs = fs.readdirSync(testRunsRoot)
+      .filter(d => {
+        try {
+          return fs.statSync(path.join(testRunsRoot, d)).isDirectory();
+        } catch {
+          return false;
+        }
+      })
+      .sort()
+      .reverse(); // Latest first
+
+    for (const dir of dirs) {
+      // Skip current run
+      if (dir === currentManifest.timestamp) continue;
+
+      if (previousManifests.length >= lookback) break;
+
+      const manifestPath = path.join(testRunsRoot, dir, 'manifest.json');
+      if (fs.existsSync(manifestPath)) {
+        try {
+          const manifest: Manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf-8'));
+          previousManifests.push(manifest);
+        } catch {
+          // Skip invalid manifests
+        }
+      }
+    }
+  }
+
+  // If no previous runs exist, all failures are new (first run)
+  if (previousManifests.length === 0) {
+    for (const tc of failedTests) {
+      const lastExec = tc.executions[tc.executions.length - 1];
+      newFailures.push({
+        id: tc.id,
+        name: tc.name,
+        error: lastExec?.error || null,
+        lastPassedRun: null,
+      });
+    }
+    return { newFailures, knownFailures };
+  }
+
+  // For each failed test, check if it passed in any previous run
+  for (const tc of failedTests) {
+    const lastExec = tc.executions[tc.executions.length - 1];
+    let lastPassedRun: string | null = null;
+
+    for (const prevManifest of previousManifests) {
+      const prevTc = prevManifest.testCases.find(ptc => ptc.id === tc.id);
+      if (prevTc && (prevTc.finalStatus === 'passed')) {
+        lastPassedRun = prevManifest.timestamp;
+        break;
+      }
+    }
+
+    if (lastPassedRun) {
+      // Test passed recently, so this is a new failure
+      newFailures.push({
+        id: tc.id,
+        name: tc.name,
+        error: lastExec?.error || null,
+        lastPassedRun,
+      });
+    } else {
+      // Check if test exists in any previous run at all
+      const existsInPrevious = previousManifests.some(
+        pm => pm.testCases.some(ptc => ptc.id === tc.id)
+      );
+
+      if (!existsInPrevious) {
+        // New test that doesn't exist in history - treat as new failure
+        newFailures.push({
+          id: tc.id,
+          name: tc.name,
+          error: lastExec?.error || null,
+          lastPassedRun: null,
+        });
+      } else {
+        // Failed in all previous runs - known failure
+        knownFailures.push({
+          id: tc.id,
+          name: tc.name,
+          error: lastExec?.error || null,
+          lastPassedRun: null,
+        });
+      }
+    }
+  }
+
+  return { newFailures, knownFailures };
 }
 
 /**
@@ -144,7 +280,7 @@ export function mergeManifests(existing: Manifest | null, current: Manifest): Ma
   const hasFailure = mergedTestCases.some(tc => tc.finalStatus === 'failed' || tc.finalStatus === 'timedOut');
   const status = hasFailure ? 'failed' : current.status;
 
-  return {
+  const merged: Manifest = {
     bugzyExecutionId: current.bugzyExecutionId,
     timestamp: existing.timestamp, // Keep original session timestamp
     startTime,
@@ -158,6 +294,21 @@ export function mergeManifests(existing: Manifest | null, current: Manifest): Ma
     },
     testCases: mergedTestCases,
   };
+
+  // Preserve failure classification (current run's classification wins)
+  if (current.new_failures) {
+    merged.new_failures = current.new_failures;
+  } else if (existing.new_failures) {
+    merged.new_failures = existing.new_failures;
+  }
+
+  if (current.known_failures) {
+    merged.known_failures = current.known_failures;
+  } else if (existing.known_failures) {
+    merged.known_failures = existing.known_failures;
+  }
+
+  return merged;
 }
 
 /**
@@ -559,6 +710,26 @@ class BugzyReporter implements Reporter {
     // Merge with existing manifest data
     const merged = mergeManifests(existingManifest, currentManifest);
 
+    // Classify failures as new vs known
+    if (merged.stats.failed > 0) {
+      try {
+        const testRunsRoot = path.join(process.cwd(), 'test-runs');
+        const { newFailures, knownFailures } = classifyFailures(merged, testRunsRoot);
+        if (newFailures.length > 0) {
+          merged.new_failures = newFailures;
+        }
+        if (knownFailures.length > 0) {
+          merged.known_failures = knownFailures;
+        }
+
+        console.log(`\n🔍 Failure Classification:`);
+        console.log(`   New failures: ${newFailures.length}`);
+        console.log(`   Known failures: ${knownFailures.length}`);
+      } catch (err) {
+        console.warn(`⚠️ Could not classify failures: ${err}`);
+      }
+    }
+
     // Write atomically (temp file + rename)
     const tmpPath = manifestPath + '.tmp';
     fs.writeFileSync(tmpPath, JSON.stringify(merged, null, 2));