@testivai/witness-playwright 0.1.0

package/src/cli/init.ts

@@ -0,0 +1,119 @@
+#!/usr/bin/env node
+
+import * as fs from 'fs-extra';
+import * as path from 'path';
+
+/**
+ * Basic CLI init script for TestivAI Playwright SDK
+ * Creates a testivai.config.ts file with sensible defaults
+ */
+
+const DEFAULT_CONFIG = `/**
+ * TestivAI Configuration File
+ * 
+ * This file configures how TestivAI analyzes visual differences in your tests.
+ * 
+ * Layout Settings:
+ * - sensitivity: 0-4 scale (0=strict/precise, 4=very lenient)
+ * - tolerance: Base pixel tolerance for layout differences
+ * 
+ * AI Settings:
+ * - sensitivity: 0-4 scale (0=conservative, 4=aggressive)
+ * - confidence: 0.0-1.0 scale (minimum confidence required for AI_BUG verdict)
+ * 
+ * Custom Configuration Examples:
+ * 
+ * // For strict testing (critical components):
+ * export default {
+ *   layout: { sensitivity: 0, tolerance: 0.5 },
+ *   ai: { sensitivity: 0, confidence: 0.9 }
+ * };
+ * 
+ * // For lenient testing (dynamic content):
+ * export default {
+ *   layout: { sensitivity: 4, tolerance: 3.0 },
+ *   ai: { sensitivity: 3, confidence: 0.6 }
+ * };
+ * 
+ * // Per-selector tolerances (advanced):
+ * export default {
+ *   layout: { 
+ *     sensitivity: 2, 
+ *     tolerance: 1.0,
+ *     selectorTolerances: {
+ *       '.carousel': 4.0,      // Carousel components shift
+ *       '.dropdown': 2.0,      // Dropdowns can vary
+ *       '.tooltip': 5.0,       // Tooltips are highly variable
+ *       '.submit-button': 0.5  // Critical buttons need precision
+ *     }
+ *   },
+ *   ai: { sensitivity: 2, confidence: 0.7 }
+ * };
+ * 
+ * // Environment-specific overrides:
+ * export default {
+ *   layout: { sensitivity: 2, tolerance: 1.0 },
+ *   ai: { sensitivity: 2, confidence: 0.7 },
+ *   environments: {
+ *     ci: {
+ *       layout: { sensitivity: 1, tolerance: 0.5 },  // Stricter in CI
+ *       ai: { sensitivity: 1, confidence: 0.8 }
+ *     },
+ *     development: {
+ *       layout: { sensitivity: 3, tolerance: 2.0 },  // More lenient locally
+ *       ai: { sensitivity: 3, confidence: 0.6 }
+ *     }
+ *   }
+ * };
+ */
+
+export default {
+  layout: {
+    sensitivity: 2,        // Balanced sensitivity (0-4 scale)
+    tolerance: 1.0,        // 1 pixel base tolerance
+  },
+  ai: {
+    sensitivity: 2,        // Balanced AI analysis (0-4 scale)
+    confidence: 0.7,       // 70% confidence required for AI_BUG
+  }
+};
+`;
+
+async function createConfigFile(): Promise<void> {
+  const configPath = path.join(process.cwd(), 'testivai.config.ts');
+  
+  // Check if config already exists
+  if (await fs.pathExists(configPath)) {
+    console.log('❌ TestivAI config already exists at:', configPath);
+    console.log('   Delete it first if you want to regenerate it.');
+    process.exit(1);
+  }
+  
+  try {
+    // Create the config file
+    await fs.writeFile(configPath, DEFAULT_CONFIG, 'utf8');
+    console.log('✅ TestivAI configuration created successfully!');
+    console.log('📁 Config file:', configPath);
+    console.log('');
+    console.log('📖 Next steps:');
+    console.log('   1. Review and customize the configuration in testivai.config.ts');
+    console.log('   2. Update your playwright.config.ts to use TestivAI reporter');
+    console.log('   3. Run your tests with: npx playwright test');
+    console.log('');
+    console.log('💡 Need help? Check the comments in testivai.config.ts for examples');
+    
+  } catch (error) {
+    console.error('❌ Failed to create configuration file:', error);
+    process.exit(1);
+  }
+}
+
+// Main execution
+if (require.main === module) {
+  createConfigFile().catch(error => {
+    console.error('❌ CLI init failed:', error);
+    process.exit(1);
+  });
+}
+
+export { createConfigFile };

package/src/config/loader.ts

@@ -0,0 +1,219 @@
+import * as fs from 'fs-extra';
+import * as path from 'path';
+import { TestivAIProjectConfig, TestivAIConfig, LayoutConfig, AIConfig } from '../types';
+
+/**
+ * Default configuration when no config file is found
+ */
+const DEFAULT_CONFIG: TestivAIProjectConfig = {
+  layout: {
+    sensitivity: 2,        // Balanced sensitivity (0-4 scale)
+    tolerance: 1.0,        // 1 pixel base tolerance
+  },
+  ai: {
+    sensitivity: 2,        // Balanced AI analysis (0-4 scale)
+    confidence: 0.7,       // 70% confidence required for AI_BUG
+  }
+};
+
+/**
+ * Load TestivAI configuration from file system
+ * Supports both .ts and .js config files
+ * 
+ * @returns Promise<TestivAIProjectConfig> The loaded configuration or defaults
+ */
+export async function loadConfig(): Promise<TestivAIProjectConfig> {
+  // Try TypeScript config first, then JavaScript
+  const tsConfigPath = path.join(process.cwd(), 'testivai.config.ts');
+  const jsConfigPath = path.join(process.cwd(), 'testivai.config.js');
+  
+  try {
+    let configPath: string;
+    let configModule: any;
+    
+    // Check for TypeScript config
+    if (await fs.pathExists(tsConfigPath)) {
+      configPath = tsConfigPath;
+    } else if (await fs.pathExists(jsConfigPath)) {
+      configPath = jsConfigPath;
+    } else {
+      console.log('⚠️  No testivai.config.ts or testivai.config.js found, using defaults');
+      return DEFAULT_CONFIG;
+    }
+    
+    // Load configuration based on file type
+    if (configPath.endsWith('.js')) {
+      // For .js files, use require to get CommonJS module
+      // Clear require cache to ensure fresh load
+      delete require.cache[require.resolve(configPath)];
+      configModule = require(configPath);
+    } else {
+      // For .ts files, use dynamic import (ES module)
+      configModule = await import(configPath);
+    }
+    
+    const config = configModule.default || configModule;
+    
+    // Validate and merge with defaults
+    return validateAndMergeConfig(config);
+    
+  } catch (error) {
+    console.warn('⚠️  Failed to load testivai config, using defaults:', error);
+    return DEFAULT_CONFIG;
+  }
+}
+
+/**
+ * Merge per-test configuration with project configuration
+ * 
+ * @param projectConfig The project-level configuration
+ * @param testConfig Optional per-test configuration overrides
+ * @returns TestivAIConfig The effective configuration for this test
+ */
+export function mergeTestConfig(
+  projectConfig: TestivAIProjectConfig, 
+  testConfig?: TestivAIConfig
+): TestivAIConfig {
+  if (!testConfig) {
+    return {
+      layout: projectConfig.layout,
+      ai: projectConfig.ai
+    };
+  }
+  
+  return {
+    layout: {
+      ...projectConfig.layout,
+      ...testConfig.layout
+    },
+    ai: {
+      ...projectConfig.ai,
+      ...testConfig.ai
+    },
+    selectors: testConfig.selectors
+  };
+}
+
+/**
+ * Detect current environment (CI, development, production)
+ * 
+ * @returns string The detected environment
+ */
+export function detectEnvironment(): string {
+  // Check for common CI environment variables
+  if (process.env.CI || process.env.CONTINUOUS_INTEGRATION) {
+    return 'ci';
+  }
+  
+  // Check for production indicators
+  if (process.env.NODE_ENV === 'production') {
+    return 'production';
+  }
+  
+  // Default to development
+  return 'development';
+}
+
+/**
+ * Apply environment-specific overrides to configuration
+ * 
+ * @param config The base configuration
+ * @returns TestivAIProjectConfig Configuration with environment overrides applied
+ */
+export function applyEnvironmentOverrides(config: TestivAIProjectConfig): TestivAIProjectConfig {
+  const environment = detectEnvironment();
+  
+  if (!config.environments) {
+    return config;
+  }
+  
+  // Type-safe environment override access
+  let envOverrides: { layout?: Partial<LayoutConfig>; ai?: Partial<AIConfig> } | undefined;
+  
+  switch (environment) {
+    case 'ci':
+      envOverrides = config.environments.ci;
+      break;
+    case 'development':
+      envOverrides = config.environments.development;
+      break;
+    case 'production':
+      envOverrides = config.environments.production;
+      break;
+  }
+  
+  if (!envOverrides) {
+    return config;
+  }
+  
+  return {
+    layout: {
+      ...config.layout,
+      ...envOverrides.layout
+    },
+    ai: {
+      ...config.ai,
+      ...envOverrides.ai
+    },
+    environments: config.environments // Keep the original environments config
+  };
+}
+
+/**
+ * Validate configuration values and merge with defaults
+ * 
+ * @param config The configuration to validate
+ * @returns TestivAIProjectConfig Validated configuration
+ */
+function validateAndMergeConfig(config: any): TestivAIProjectConfig {
+  // Guard against null/undefined config
+  if (!config) {
+    console.warn('⚠️  Config is null or undefined, using defaults');
+    return DEFAULT_CONFIG;
+  }
+  
+  // Basic validation
+  const validatedConfig: TestivAIProjectConfig = {
+    layout: {
+      sensitivity: validateRange(config.layout?.sensitivity ?? DEFAULT_CONFIG.layout.sensitivity, 0, 4, 'layout.sensitivity', DEFAULT_CONFIG.layout.sensitivity),
+      tolerance: validateRange(config.layout?.tolerance ?? DEFAULT_CONFIG.layout.tolerance, 0, 100, 'layout.tolerance', DEFAULT_CONFIG.layout.tolerance),
+      selectorTolerances: config.layout?.selectorTolerances,
+      useRelativeTolerance: config.layout?.useRelativeTolerance,
+      relativeTolerance: config.layout?.relativeTolerance
+    },
+    ai: {
+      sensitivity: validateRange(config.ai?.sensitivity ?? DEFAULT_CONFIG.ai.sensitivity, 0, 4, 'ai.sensitivity', DEFAULT_CONFIG.ai.sensitivity),
+      confidence: validateRange(config.ai?.confidence ?? DEFAULT_CONFIG.ai.confidence, 0, 1, 'ai.confidence', DEFAULT_CONFIG.ai.confidence),
+      enableReasoning: config.ai?.enableReasoning
+    },
+    environments: config.environments
+  };
+  
+  return applyEnvironmentOverrides(validatedConfig);
+}
+
+/**
+ * Validate that a number is within the expected range
+ * 
+ * @param value The value to validate
+ * @param min Minimum allowed value
+ * @param max Maximum allowed value
+ * @param field Field name for error messages
+ * @param defaultValue Default value to use when validation fails
+ * @returns number The validated value
+ */
+function validateRange(value: any, min: number, max: number, field: string, defaultValue: number): number {
+  const numValue = Number(value);
+  
+  if (isNaN(numValue)) {
+    console.warn(`⚠️  Invalid ${field}: ${value}, using default: ${defaultValue}`);
+    return defaultValue;
+  }
+  
+  if (numValue < min || numValue > max) {
+    console.warn(`⚠️  ${field} must be between ${min} and ${max}, got ${numValue}, using default: ${defaultValue}`);
+    return defaultValue;
+  }
+  
+  return numValue;
+}

package/src/index.ts

@@ -0,0 +1,9 @@
+import { snapshot } from './snapshot';
+
+export const testivai = {
+  witness: snapshot,
+};
+
+export * from './types';
+export * from './reporter';
+export * from './ci';

package/src/reporter-types.ts

@@ -0,0 +1,5 @@
+// This file isolates the Playwright-specific types for the reporter
+// to allow for clean mocking in the Jest test environment.
+import type { Reporter, FullConfig, Suite, FullResult } from '@playwright/test/reporter';
+
+export type { Reporter, FullConfig, Suite, FullResult };

package/src/reporter.ts

@@ -0,0 +1,148 @@
+import { Reporter, FullConfig, Suite, FullResult } from './reporter-types';
+import * as fs from 'fs-extra';
+import * as path from 'path';
+import simpleGit, { SimpleGit } from 'simple-git';
+import axios from 'axios';
+import { BatchPayload, BrowserInfo, GitInfo, SnapshotPayload } from './types';
+import { getCiRunId } from './ci';
+
+interface TestivaiReporterOptions {
+  apiUrl?: string;
+  apiKey?: string;
+}
+
+export class TestivAIPlaywrightReporter implements Reporter {
+  private options: TestivaiReporterOptions;
+  private gitInfo: GitInfo | null = null;
+  private browserInfo: BrowserInfo | null = null;
+  private runId: string | null = null;
+  private tempDir = path.join(process.cwd(), '.testivai', 'temp');
+
+  constructor(options: TestivaiReporterOptions = {}) {
+    this.options = {
+      apiUrl: options.apiUrl || process.env.TESTIVAI_API_URL,
+      apiKey: options.apiKey || process.env.TESTIVAI_API_KEY,
+    };
+  }
+
+  async onBegin(config: FullConfig, suite: Suite): Promise<void> {
+    if (!this.options.apiUrl || !this.options.apiKey) {
+      console.error('Testivai Reporter: API URL or API Key is not configured. Disabling reporter.');
+      this.options.apiUrl = undefined; // Disable reporter
+      return;
+    }
+
+    // 1. Clean temp directory
+    await fs.emptyDir(this.tempDir);
+
+    // 2. Capture Git metadata
+    try {
+      const git: SimpleGit = simpleGit();
+      const [branch, commit] = await Promise.all([
+        git.revparse(['--abbrev-ref', 'HEAD']),
+        git.revparse(['HEAD']),
+      ]);
+      this.gitInfo = { branch, commit };
+    } catch (error) {
+      console.error('Testivai Reporter: Could not get Git information.', error);
+      this.gitInfo = { branch: 'unknown', commit: 'unknown' };
+    }
+
+    // 3. Capture Browser info from the first project
+    const project = suite.suites[0]?.project();
+    if (project) {
+      this.browserInfo = {
+        name: project.use.browserName || 'unknown',
+        version: 'unknown', // Playwright does not easily expose browser version
+        viewportWidth: project.use.viewport?.width || 0,
+        viewportHeight: project.use.viewport?.height || 0,
+        userAgent: project.use.userAgent || 'unknown',
+        os: 'unknown',
+      };
+    }
+
+    // 4. Get CI Run ID
+    this.runId = getCiRunId();
+    if (this.runId) {
+      console.log(`Testivai Reporter: Detected CI environment. Run ID: ${this.runId}`);
+    }
+  }
+
+  async onEnd(result: FullResult): Promise<void> {
+    if (!this.options.apiUrl) {
+      return; // Reporter is disabled
+    }
+
+    console.log('Testivai Reporter: Test run finished. Preparing to upload evidence...');
+
+    try {
+      const snapshotFiles = await fs.readdir(this.tempDir);
+      const jsonFiles = snapshotFiles.filter(f => f.endsWith('.json'));
+
+      if (jsonFiles.length === 0) {
+        console.log('Testivai Reporter: No snapshots found to upload.');
+        return;
+      }
+
+      const snapshots: SnapshotPayload[] = [];
+      const filesToUpload: { filePath: string, contentType: string }[] = [];
+
+      for (const jsonFile of jsonFiles) {
+        const metadataPath = path.join(this.tempDir, jsonFile);
+        const metadata = await fs.readJson(metadataPath);
+
+        const domPath = metadata.files.dom;
+        const screenshotPath = metadata.files.screenshot;
+
+        const snapshotPayload: SnapshotPayload = {
+          ...metadata,
+          dom: { html: await fs.readFile(domPath, 'utf-8') },
+          layout: metadata.layout,
+          testivaiConfig: metadata.testivaiConfig
+        };
+        snapshots.push(snapshotPayload);
+
+        filesToUpload.push({ filePath: screenshotPath, contentType: 'image/png' });
+      }
+
+      const batchPayload: Omit<BatchPayload, 'batchId'> = {
+        git: this.gitInfo!,
+        browser: this.browserInfo!,
+        snapshots,
+        timestamp: Date.now(),
+        runId: this.runId,
+      };
+
+      // Start batch and get upload URLs
+      const startBatchResponse = await axios.post(`${this.options.apiUrl}/api/v1/ingest/start-batch`, batchPayload, {
+        headers: { 'Authorization': `Bearer ${this.options.apiKey}` },
+      });
+
+      const { batchId, uploadInstructions } = startBatchResponse.data;
+
+      // Upload files
+      const uploadPromises = filesToUpload.map((file, index) => {
+        const instruction = uploadInstructions[index];
+        return fs.readFile(file.filePath).then(buffer => 
+          axios.put(instruction.url, buffer, { headers: { 'Content-Type': file.contentType } })
+        );
+      });
+
+      await Promise.all(uploadPromises);
+
+      // Finalize batch
+      await axios.post(`${this.options.apiUrl}/api/v1/ingest/finish-batch/${batchId}`, {}, {
+        headers: { 'Authorization': `Bearer ${this.options.apiKey}` },
+      });
+
+      console.log(`Testivai Reporter: Successfully uploaded ${snapshots.length} snapshots with Batch ID: ${batchId}`);
+
+      // Clean up temp files
+      await fs.emptyDir(this.tempDir);
+      console.log('Testivai Reporter: Cleaned up temporary evidence files.');
+
+    } catch (error) {
+      console.error('Testivai Reporter: An error occurred during the onEnd hook:', error);
+    }
+  }
+}

package/src/snapshot.ts

@@ -0,0 +1,103 @@
+import { Page, TestInfo } from '@playwright/test';
+import * as fs from 'fs-extra';
+import * as path from 'path';
+import { URL } from 'url';
+import { SnapshotPayload, LayoutData, TestivAIConfig } from './types';
+import { loadConfig, mergeTestConfig } from './config/loader';
+
+/**
+ * Generates a safe filename from a URL.
+ * @param pageUrl The URL of the page.
+ * @returns A sanitized string suitable for a filename.
+ */
+function getSnapshotNameFromUrl(pageUrl: string): string {
+  // Handle data URIs, which are common in test environments
+  if (pageUrl.startsWith('data:')) {
+    return 'snapshot';
+  }
+
+  try {
+    const url = new URL(pageUrl);
+    const pathName = url.pathname.substring(1).replace(/\//g, '_'); // remove leading slash and replace others
+    return pathName || 'home';
+  } catch (error) {
+    // Fallback for invalid URLs
+    return 'snapshot';
+  }
+}
+
+/**
+ * Captures a snapshot of the page, including a screenshot, DOM, and layout data.
+ * The evidence is stored in a temporary directory for the reporter to process later.
+ *
+ * @param page The Playwright Page object.
+ * @param testInfo The Playwright TestInfo object, passed from the test.
+ * @param name An optional unique name for the snapshot. If not provided, a name is generated from the URL.
+ * @param config Optional TestivAI configuration for this snapshot (overrides project defaults).
+ */
+export async function snapshot(
+  page: Page,
+  testInfo: TestInfo,
+  name?: string,
+  config?: TestivAIConfig
+): Promise<void> {
+  // Load project configuration and merge with test-specific overrides
+  const projectConfig = await loadConfig();
+  const effectiveConfig = mergeTestConfig(projectConfig, config);
+  
+  const outputDir = path.join(process.cwd(), '.testivai', 'temp');
+  await fs.ensureDir(outputDir);
+
+  const snapshotName = name || getSnapshotNameFromUrl(page.url());
+  const timestamp = Date.now();
+  const safeName = snapshotName.replace(/[^a-z0-9]/gi, '_').toLowerCase();
+  const baseFilename = `${timestamp}_${safeName}`;
+
+  // 1. Capture full-page screenshot
+  const screenshotPath = path.join(outputDir, `${baseFilename}.png`);
+  await page.screenshot({ path: screenshotPath, fullPage: true });
+
+  // 2. Dump full-page DOM
+  const domPath = path.join(outputDir, `${baseFilename}.html`);
+  const htmlContent = await page.content();
+  await fs.writeFile(domPath, htmlContent);
+
+  // 3. Extract bounding boxes for requested selectors
+  const selectors = effectiveConfig.selectors ?? ['body'];
+  const layout: Record<string, LayoutData> = {};
+
+  for (const selector of selectors) {
+    const element = page.locator(selector).first();
+    const boundingBox = await element.boundingBox();
+    if (boundingBox) {
+      layout[selector] = {
+        ...boundingBox,
+        top: boundingBox.y,
+        left: boundingBox.x,
+        right: boundingBox.x + boundingBox.width,
+        bottom: boundingBox.y + boundingBox.height,
+      };
+    }
+  }
+
+  // 4. Save metadata with configuration
+  const metadataPath = path.join(outputDir, `${baseFilename}.json`);
+  const metadata: Partial<SnapshotPayload> = {
+    snapshotName,
+    testName: testInfo.title,
+    timestamp,
+    url: page.url(),
+    viewport: page.viewportSize() || undefined,
+  };
+
+  await fs.writeJson(metadataPath, {
+    ...metadata,
+    files: {
+      screenshot: screenshotPath,
+      dom: domPath,
+    },
+    layout,
+    // Store the effective configuration for the reporter
+    testivaiConfig: effectiveConfig
+  });
+}