@bugzy-ai/bugzy 1.15.0 → 1.16.0

package/templates/playwright/reporters/__tests__/bugzy-reporter-manifest-merge.test.ts

@@ -0,0 +1,329 @@
+import { test, expect } from '@playwright/test';
+import { mergeManifests } from '../bugzy-reporter';
+
+function makeExecution(overrides: Partial<{
+  number: number;
+  status: string;
+  duration: number;
+  videoFile: string | null;
+  hasTrace: boolean;
+  hasScreenshots: boolean;
+  error: string | null;
+}> = {}) {
+  return {
+    number: 1,
+    status: 'passed',
+    duration: 1000,
+    videoFile: 'video.webm',
+    hasTrace: false,
+    hasScreenshots: false,
+    error: null,
+    ...overrides,
+  };
+}
+
+function makeTestCase(id: string, executions: ReturnType<typeof makeExecution>[], finalStatus?: string) {
+  const lastExec = executions[executions.length - 1];
+  return {
+    id,
+    name: id.replace(/^TC-\d+-/, '').replace(/-/g, ' '),
+    totalExecutions: executions.length,
+    finalStatus: finalStatus ?? lastExec.status,
+    executions,
+  };
+}
+
+function makeManifest(overrides: Partial<{
+  bugzyExecutionId: string;
+  timestamp: string;
+  startTime: string;
+  endTime: string;
+  status: string;
+  stats: { totalTests: number; passed: number; failed: number; totalExecutions: number };
+  testCases: ReturnType<typeof makeTestCase>[];
+}> = {}) {
+  const testCases = overrides.testCases ?? [];
+  const totalExecutions = testCases.reduce((sum, tc) => sum + tc.executions.length, 0);
+  const passed = testCases.filter(tc => tc.finalStatus === 'passed').length;
+  const failed = testCases.length - passed;
+
+  return {
+    bugzyExecutionId: 'local-20260127-060129',
+    timestamp: '20260127-060129',
+    startTime: '2026-01-27T06:01:29.000Z',
+    endTime: '2026-01-27T06:02:00.000Z',
+    status: 'passed',
+    stats: {
+      totalTests: testCases.length,
+      passed,
+      failed,
+      totalExecutions,
+      ...overrides.stats,
+    },
+    ...overrides,
+    testCases,
+  };
+}
+
+test.describe('mergeManifests', () => {
+  test('returns current manifest unchanged when existing is null', () => {
+    const current = makeManifest({
+      testCases: [makeTestCase('TC-001-login', [makeExecution()])],
+    });
+
+    const result = mergeManifests(null, current);
+
+    expect(result).toEqual(current);
+  });
+
+  test('merges test cases from both manifests', () => {
+    const existing = makeManifest({
+      testCases: [
+        makeTestCase('TC-001-login', [makeExecution({ number: 1 })]),
+      ],
+    });
+
+    const current = makeManifest({
+      startTime: '2026-01-27T06:05:00.000Z',
+      endTime: '2026-01-27T06:06:00.000Z',
+      testCases: [
+        makeTestCase('TC-002-checkout', [makeExecution({ number: 1 })]),
+      ],
+    });
+
+    const result = mergeManifests(existing, current);
+
+    expect(result.testCases).toHaveLength(2);
+    expect(result.testCases.map(tc => tc.id)).toContain('TC-001-login');
+    expect(result.testCases.map(tc => tc.id)).toContain('TC-002-checkout');
+    expect(result.stats.totalTests).toBe(2);
+    expect(result.stats.totalExecutions).toBe(2);
+  });
+
+  test('merges executions for the same test case across runs', () => {
+    const existing = makeManifest({
+      testCases: [
+        makeTestCase('TC-001-login', [
+          makeExecution({ number: 1, status: 'failed', error: 'timeout' }),
+        ], 'failed'),
+      ],
+    });
+
+    const current = makeManifest({
+      startTime: '2026-01-27T06:05:00.000Z',
+      endTime: '2026-01-27T06:06:00.000Z',
+      testCases: [
+        makeTestCase('TC-001-login', [
+          makeExecution({ number: 2, status: 'passed' }),
+        ]),
+      ],
+    });
+
+    const result = mergeManifests(existing, current);
+
+    expect(result.testCases).toHaveLength(1);
+    const tc = result.testCases[0];
+    expect(tc.executions).toHaveLength(2);
+    expect(tc.executions[0].number).toBe(1);
+    expect(tc.executions[0].status).toBe('failed');
+    expect(tc.executions[1].number).toBe(2);
+    expect(tc.executions[1].status).toBe('passed');
+    expect(tc.totalExecutions).toBe(2);
+    expect(tc.finalStatus).toBe('passed'); // Latest execution status
+  });
+
+  test('current run wins on execution number collision', () => {
+    const existing = makeManifest({
+      testCases: [
+        makeTestCase('TC-001-login', [
+          makeExecution({ number: 3, status: 'failed', duration: 500 }),
+        ], 'failed'),
+      ],
+    });
+
+    const current = makeManifest({
+      startTime: '2026-01-27T06:05:00.000Z',
+      endTime: '2026-01-27T06:06:00.000Z',
+      testCases: [
+        makeTestCase('TC-001-login', [
+          makeExecution({ number: 3, status: 'passed', duration: 1200 }),
+        ]),
+      ],
+    });
+
+    const result = mergeManifests(existing, current);
+
+    const tc = result.testCases[0];
+    expect(tc.executions).toHaveLength(1);
+    expect(tc.executions[0].status).toBe('passed');
+    expect(tc.executions[0].duration).toBe(1200);
+  });
+
+  test('preserves test cases that only exist in existing manifest', () => {
+    const existing = makeManifest({
+      testCases: [
+        makeTestCase('TC-001-login', [makeExecution({ number: 1 })]),
+        makeTestCase('TC-002-checkout', [makeExecution({ number: 1 })]),
+      ],
+    });
+
+    const current = makeManifest({
+      startTime: '2026-01-27T06:05:00.000Z',
+      endTime: '2026-01-27T06:06:00.000Z',
+      testCases: [
+        makeTestCase('TC-001-login', [makeExecution({ number: 2 })]),
+      ],
+    });
+
+    const result = mergeManifests(existing, current);
+
+    expect(result.testCases).toHaveLength(2);
+    const checkout = result.testCases.find(tc => tc.id === 'TC-002-checkout');
+    expect(checkout).toBeDefined();
+    expect(checkout!.executions).toHaveLength(1);
+    expect(checkout!.executions[0].number).toBe(1);
+  });
+
+  test('recalculates stats correctly from merged data', () => {
+    const existing = makeManifest({
+      testCases: [
+        makeTestCase('TC-001-login', [
+          makeExecution({ number: 1, status: 'failed' }),
+        ], 'failed'),
+        makeTestCase('TC-002-checkout', [
+          makeExecution({ number: 1, status: 'passed' }),
+        ]),
+      ],
+    });
+
+    const current = makeManifest({
+      startTime: '2026-01-27T06:05:00.000Z',
+      endTime: '2026-01-27T06:06:00.000Z',
+      testCases: [
+        makeTestCase('TC-001-login', [
+          makeExecution({ number: 2, status: 'passed' }),
+        ]),
+        makeTestCase('TC-003-profile', [
+          makeExecution({ number: 1, status: 'failed' }),
+        ], 'failed'),
+      ],
+    });
+
+    const result = mergeManifests(existing, current);
+
+    expect(result.stats.totalTests).toBe(3);
+    // TC-001: exec-1 (failed) + exec-2 (passed) = 2 execs, finalStatus=passed
+    // TC-002: exec-1 (passed) = 1 exec, finalStatus=passed
+    // TC-003: exec-1 (failed) = 1 exec, finalStatus=failed
+    expect(result.stats.totalExecutions).toBe(4);
+    expect(result.stats.passed).toBe(2); // TC-001 and TC-002
+    expect(result.stats.failed).toBe(1); // TC-003
+  });
+
+  test('uses earliest startTime and latest endTime', () => {
+    const existing = makeManifest({
+      startTime: '2026-01-27T06:01:00.000Z',
+      endTime: '2026-01-27T06:02:00.000Z',
+      testCases: [makeTestCase('TC-001-login', [makeExecution()])],
+    });
+
+    const current = makeManifest({
+      startTime: '2026-01-27T06:05:00.000Z',
+      endTime: '2026-01-27T06:06:00.000Z',
+      testCases: [makeTestCase('TC-001-login', [makeExecution({ number: 2 })])],
+    });
+
+    const result = mergeManifests(existing, current);
+
+    expect(result.startTime).toBe('2026-01-27T06:01:00.000Z');
+    expect(result.endTime).toBe('2026-01-27T06:06:00.000Z');
+  });
+
+  test('sets status to failed if any test case has failed finalStatus', () => {
+    const existing = makeManifest({
+      status: 'passed',
+      testCases: [
+        makeTestCase('TC-001-login', [makeExecution({ number: 1, status: 'passed' })]),
+      ],
+    });
+
+    const current = makeManifest({
+      status: 'passed',
+      startTime: '2026-01-27T06:05:00.000Z',
+      endTime: '2026-01-27T06:06:00.000Z',
+      testCases: [
+        makeTestCase('TC-002-checkout', [
+          makeExecution({ number: 1, status: 'failed' }),
+        ], 'failed'),
+      ],
+    });
+
+    const result = mergeManifests(existing, current);
+
+    expect(result.status).toBe('failed');
+  });
+
+  test('preserves original session timestamp from existing manifest', () => {
+    const existing = makeManifest({
+      timestamp: '20260127-060129',
+      testCases: [makeTestCase('TC-001-login', [makeExecution()])],
+    });
+
+    const current = makeManifest({
+      timestamp: '20260127-060500',
+      startTime: '2026-01-27T06:05:00.000Z',
+      endTime: '2026-01-27T06:06:00.000Z',
+      testCases: [makeTestCase('TC-001-login', [makeExecution({ number: 2 })])],
+    });
+
+    const result = mergeManifests(existing, current);
+
+    expect(result.timestamp).toBe('20260127-060129');
+  });
+
+  test('handles timedOut status as failure in merged status', () => {
+    const existing = makeManifest({
+      status: 'passed',
+      testCases: [
+        makeTestCase('TC-001-login', [
+          makeExecution({ number: 1, status: 'timedOut' }),
+        ], 'timedOut'),
+      ],
+    });
+
+    const current = makeManifest({
+      status: 'passed',
+      startTime: '2026-01-27T06:05:00.000Z',
+      endTime: '2026-01-27T06:06:00.000Z',
+      testCases: [
+        makeTestCase('TC-002-checkout', [makeExecution({ number: 1 })]),
+      ],
+    });
+
+    const result = mergeManifests(existing, current);
+
+    expect(result.status).toBe('failed');
+  });
+
+  test('does not mutate input manifests', () => {
+    const existingExec = makeExecution({ number: 1, status: 'failed' });
+    const existing = makeManifest({
+      testCases: [makeTestCase('TC-001-login', [existingExec], 'failed')],
+    });
+    const existingSnapshot = JSON.parse(JSON.stringify(existing));
+
+    const current = makeManifest({
+      startTime: '2026-01-27T06:05:00.000Z',
+      endTime: '2026-01-27T06:06:00.000Z',
+      testCases: [
+        makeTestCase('TC-001-login', [makeExecution({ number: 2, status: 'passed' })]),
+      ],
+    });
+    const currentSnapshot = JSON.parse(JSON.stringify(current));
+
+    mergeManifests(existing, current);
+
+    expect(existing).toEqual(existingSnapshot);
+    expect(current).toEqual(currentSnapshot);
+  });
+});

package/templates/playwright/reporters/__tests__/playwright.config.ts

@@ -0,0 +1,5 @@
+import { defineConfig } from '@playwright/test';
+
+export default defineConfig({
+  testDir: '.',
+});

package/templates/playwright/reporters/bugzy-reporter.ts

@@ -24,6 +24,142 @@ interface StepData {
   duration?: number;
 }
 
+/**
+ * Manifest execution entry
+ */
+interface ManifestExecution {
+  number: number;
+  status: string;
+  duration: number;
+  videoFile: string | null;
+  hasTrace: boolean;
+  hasScreenshots: boolean;
+  error: string | null;
+}
+
+/**
+ * Manifest test case entry
+ */
+interface ManifestTestCase {
+  id: string;
+  name: string;
+  totalExecutions: number;
+  finalStatus: string;
+  executions: ManifestExecution[];
+}
+
+/**
+ * Manifest structure for test run sessions
+ */
+interface Manifest {
+  bugzyExecutionId: string;
+  timestamp: string;
+  startTime: string;
+  endTime: string;
+  status: string;
+  stats: {
+    totalTests: number;
+    passed: number;
+    failed: number;
+    totalExecutions: number;
+  };
+  testCases: ManifestTestCase[];
+}
+
+/**
+ * Merge an existing manifest with the current run's manifest.
+ * If existing is null, returns current as-is.
+ * Deduplicates executions by number (current run wins on collision).
+ * Recalculates stats from the merged data.
+ */
+export function mergeManifests(existing: Manifest | null, current: Manifest): Manifest {
+  if (!existing) {
+    return current;
+  }
+
+  // Build map of test cases by id from existing manifest
+  const testCaseMap = new Map<string, ManifestTestCase>();
+  for (const tc of existing.testCases) {
+    testCaseMap.set(tc.id, { ...tc, executions: [...tc.executions] });
+  }
+
+  // Merge current run's test cases
+  for (const tc of current.testCases) {
+    const existingTc = testCaseMap.get(tc.id);
+    if (existingTc) {
+      // Merge executions: build a map keyed by execution number
+      const execMap = new Map<number, ManifestExecution>();
+      for (const exec of existingTc.executions) {
+        execMap.set(exec.number, exec);
+      }
+      // Current run's executions overwrite on collision
+      for (const exec of tc.executions) {
+        execMap.set(exec.number, exec);
+      }
+      // Sort by execution number
+      const mergedExecs = Array.from(execMap.values()).sort((a, b) => a.number - b.number);
+      const finalStatus = mergedExecs[mergedExecs.length - 1].status;
+
+      testCaseMap.set(tc.id, {
+        id: tc.id,
+        name: tc.name,
+        totalExecutions: mergedExecs.length,
+        finalStatus,
+        executions: mergedExecs,
+      });
+    } else {
+      // New test case from current run
+      testCaseMap.set(tc.id, { ...tc, executions: [...tc.executions] });
+    }
+  }
+
+  // Build merged test cases array
+  const mergedTestCases = Array.from(testCaseMap.values());
+
+  // Recalculate stats
+  let totalTests = 0;
+  let totalExecutions = 0;
+  let passedTests = 0;
+  let failedTests = 0;
+
+  for (const tc of mergedTestCases) {
+    totalTests++;
+    totalExecutions += tc.executions.length;
+    if (tc.finalStatus === 'passed') {
+      passedTests++;
+    } else {
+      failedTests++;
+    }
+  }
+
+  // Use earliest startTime, latest endTime
+  const startTime = new Date(existing.startTime) < new Date(current.startTime)
+    ? existing.startTime
+    : current.startTime;
+  const endTime = new Date(existing.endTime) > new Date(current.endTime)
+    ? existing.endTime
+    : current.endTime;
+
+  // Status: if any test case failed, overall is failed
+  const hasFailure = mergedTestCases.some(tc => tc.finalStatus === 'failed' || tc.finalStatus === 'timedOut');
+  const status = hasFailure ? 'failed' : current.status;
+
+  return {
+    bugzyExecutionId: current.bugzyExecutionId,
+    timestamp: existing.timestamp, // Keep original session timestamp
+    startTime,
+    endTime,
+    status,
+    stats: {
+      totalTests,
+      passed: passedTests,
+      failed: failedTests,
+      totalExecutions,
+    },
+    testCases: mergedTestCases,
+  };
+}
+
 /**
  * Bugzy Custom Playwright Reporter
  *
@@ -393,8 +529,8 @@ class BugzyReporter implements Reporter {
       });
     }
 
-    // Generate manifest.json
-    const manifest = {
+    // Build current run's manifest
+    const currentManifest: Manifest = {
       bugzyExecutionId: this.bugzyExecutionId,
       timestamp: this.timestamp,
       startTime: this.startTime.toISOString(),
@@ -409,14 +545,37 @@ class BugzyReporter implements Reporter {
       testCases,
     };
 
+    // Read existing manifest for merge (if session is being reused)
     const manifestPath = path.join(this.testRunDir, 'manifest.json');
-    fs.writeFileSync(manifestPath, JSON.stringify(manifest, null, 2));
+    let existingManifest: Manifest | null = null;
+    if (fs.existsSync(manifestPath)) {
+      try {
+        existingManifest = JSON.parse(fs.readFileSync(manifestPath, 'utf-8'));
+      } catch (err) {
+        console.warn(`⚠️ Could not parse existing manifest, will overwrite: ${err}`);
+      }
+    }
 
-    console.log(`\n📊 Test Run Summary:`);
+    // Merge with existing manifest data
+    const merged = mergeManifests(existingManifest, currentManifest);
+
+    // Write atomically (temp file + rename)
+    const tmpPath = manifestPath + '.tmp';
+    fs.writeFileSync(tmpPath, JSON.stringify(merged, null, 2));
+    fs.renameSync(tmpPath, manifestPath);
+
+    console.log(`\n📊 Test Run Summary (this run):`);
     console.log(`   Total tests: ${totalTests}`);
     console.log(`   Passed: ${passedTests}`);
     console.log(`   Failed: ${failedTests}`);
     console.log(`   Total executions: ${totalExecutions}`);
+
+    if (existingManifest) {
+      console.log(`\n🔗 Merged with previous session data:`);
+      console.log(`   Session total tests: ${merged.stats.totalTests}`);
+      console.log(`   Session total executions: ${merged.stats.totalExecutions}`);
+    }
+
     console.log(`   Manifest: ${manifestPath}\n`);
   }