@testivai/witness-playwright 1.0.0

package/src/ci.ts

@@ -0,0 +1,140 @@
+/**
+ * Utilities for detecting CI/CD environments and extracting a unique run identifier.
+ */
+
+/**
+ * CI environment information for integration feedback (e.g., GitHub commit statuses, PR comments).
+ */
+export interface CiInfo {
+  /** CI provider name */
+  provider: string;
+  /** Pull/Merge request number (if available) */
+  prNumber?: number;
+  /** URL to the CI run (if available) */
+  runUrl?: string;
+  /** CI build/run identifier */
+  buildId?: string;
+}
+
+/**
+ * Gets a unique identifier for the current CI run.
+ * This ID is used to group snapshots from parallel shards into a single batch.
+ *
+ * @returns A unique string identifier for the CI run, or null if not in a known CI environment.
+ */
+export function getCiRunId(): string | null {
+  // GitHub Actions
+  if (process.env.GITHUB_RUN_ID) {
+    return `github-${process.env.GITHUB_RUN_ID}`;
+  }
+  // GitLab CI
+  if (process.env.GITLAB_CI) {
+    return `gitlab-${process.env.CI_PIPELINE_ID}`;
+  }
+  // Jenkins
+  if (process.env.JENKINS_URL) {
+    return `jenkins-${process.env.BUILD_ID}`;
+  }
+  // CircleCI
+  if (process.env.CIRCLECI) {
+    return `circleci-${process.env.CIRCLE_WORKFLOW_ID}`;
+  }
+  // Travis CI
+  if (process.env.TRAVIS) {
+    return `travis-${process.env.TRAVIS_BUILD_ID}`;
+  }
+
+  return null;
+}
+
+/**
+ * Gets detailed CI environment information including PR number and run URL.
+ * Used for integration feedback (GitHub commit statuses, PR comments, etc.).
+ *
+ * @returns CiInfo object if in a known CI environment, or null otherwise.
+ */
+export function getCiInfo(): CiInfo | null {
+  // GitHub Actions
+  if (process.env.GITHUB_ACTIONS) {
+    const serverUrl = process.env.GITHUB_SERVER_URL || 'https://github.com';
+    const repo = process.env.GITHUB_REPOSITORY;
+    const runId = process.env.GITHUB_RUN_ID;
+
+    // PR number: available via GITHUB_EVENT_NUMBER in pull_request events,
+    // or from GITHUB_REF (refs/pull/<number>/merge)
+    let prNumber: number | undefined;
+    if (process.env.GITHUB_EVENT_NUMBER) {
+      prNumber = parseInt(process.env.GITHUB_EVENT_NUMBER, 10) || undefined;
+    } else if (process.env.GITHUB_REF?.startsWith('refs/pull/')) {
+      const match = process.env.GITHUB_REF.match(/^refs\/pull\/(\d+)\//);
+      if (match) {
+        prNumber = parseInt(match[1], 10);
+      }
+    }
+
+    return {
+      provider: 'github_actions',
+      prNumber,
+      runUrl: repo && runId ? `${serverUrl}/${repo}/actions/runs/${runId}` : undefined,
+      buildId: runId,
+    };
+  }
+
+  // GitLab CI
+  if (process.env.GITLAB_CI) {
+    const prNumber = process.env.CI_MERGE_REQUEST_IID
+      ? parseInt(process.env.CI_MERGE_REQUEST_IID, 10) || undefined
+      : undefined;
+
+    return {
+      provider: 'gitlab_ci',
+      prNumber,
+      runUrl: process.env.CI_PIPELINE_URL || undefined,
+      buildId: process.env.CI_PIPELINE_ID,
+    };
+  }
+
+  // CircleCI
+  if (process.env.CIRCLECI) {
+    const prNumber = process.env.CIRCLE_PULL_REQUEST
+      ? parseInt(process.env.CIRCLE_PULL_REQUEST.split('/').pop() || '', 10) || undefined
+      : undefined;
+
+    return {
+      provider: 'circleci',
+      prNumber,
+      runUrl: process.env.CIRCLE_BUILD_URL || undefined,
+      buildId: process.env.CIRCLE_WORKFLOW_ID,
+    };
+  }
+
+  // Jenkins
+  if (process.env.JENKINS_URL) {
+    const prNumber = process.env.CHANGE_ID
+      ? parseInt(process.env.CHANGE_ID, 10) || undefined
+      : undefined;
+
+    return {
+      provider: 'jenkins',
+      prNumber,
+      runUrl: process.env.BUILD_URL || undefined,
+      buildId: process.env.BUILD_ID,
+    };
+  }
+
+  // Travis CI
+  if (process.env.TRAVIS) {
+    const prNumber = process.env.TRAVIS_PULL_REQUEST && process.env.TRAVIS_PULL_REQUEST !== 'false'
+      ? parseInt(process.env.TRAVIS_PULL_REQUEST, 10) || undefined
+      : undefined;
+
+    return {
+      provider: 'travis_ci',
+      prNumber,
+      runUrl: process.env.TRAVIS_BUILD_WEB_URL || undefined,
+      buildId: process.env.TRAVIS_BUILD_ID,
+    };
+  }
+
+  return null;
+}

package/src/cli/index.ts

@@ -0,0 +1,49 @@
+#!/usr/bin/env node
+
+import { Command } from 'commander';
+import chalk from 'chalk';
+import { createConfigFile } from './init';
+
+const program = new Command();
+
+// Display banner
+const showBanner = () => {
+  console.log();
+  console.log(chalk.cyan.bold('  TestivAI'));
+  console.log(chalk.gray('  Catch Visual Bugs Automatically'));
+  console.log(chalk.gray('  AI that catches real bugs, ignores the noise.'));
+  console.log();
+};
+
+program
+  .name('testivai')
+  .description('TestivAI Playwright SDK CLI')
+  .version(require('../../package.json').version)
+  .hook('preAction', () => {
+    if (!process.argv.includes('--quiet') && !process.argv.includes('-q')) {
+      showBanner();
+    }
+  });
+
+// Global options
+program
+  .option('-q, --quiet', 'Suppress banner output (ideal for CI)');
+
+program
+  .command('init')
+  .description('Initialize TestivAI configuration file')
+  .action(async () => {
+    try {
+      await createConfigFile();
+    } catch (error) {
+      console.error('❌ Failed to initialize:', error);
+      process.exit(1);
+    }
+  });
+
+program.parse(process.argv);
+
+// Show help if no command provided
+if (!process.argv.slice(2).length) {
+  program.outputHelp();
+}

package/src/cli/init.ts

@@ -0,0 +1,131 @@
+#!/usr/bin/env node
+
+import * as fs from 'fs-extra';
+import * as path from 'path';
+
+/**
+ * Basic CLI init script for TestivAI Playwright SDK
+ * Creates a testivai.config.ts file with sensible defaults
+ */
+
+const DEFAULT_CONFIG = `/**
+ * TestivAI Configuration File
+ * 
+ * This file configures how TestivAI analyzes visual differences in your tests.
+ * 
+ * Layout Settings:
+ * - sensitivity: 0-4 scale (0=strict/precise, 4=very lenient)
+ * - tolerance: Base pixel tolerance for layout differences
+ * 
+ * AI Settings:
+ * - sensitivity: 0-4 scale (0=conservative, 4=aggressive)
+ * - confidence: 0.0-1.0 scale (minimum confidence required for AI_BUG verdict)
+ * 
+ * Custom Configuration Examples:
+ * 
+ * // For strict testing (critical components):
+ * export default {
+ *   layout: { sensitivity: 0, tolerance: 0.5 },
+ *   ai: { sensitivity: 0, confidence: 0.9 }
+ * };
+ * 
+ * // For lenient testing (dynamic content):
+ * export default {
+ *   layout: { sensitivity: 4, tolerance: 3.0 },
+ *   ai: { sensitivity: 3, confidence: 0.6 }
+ * };
+ * 
+ * // Per-selector tolerances (advanced):
+ * export default {
+ *   layout: { 
+ *     sensitivity: 2, 
+ *     tolerance: 1.0,
+ *     selectorTolerances: {
+ *       '.carousel': 4.0,      // Carousel components shift
+ *       '.dropdown': 2.0,      // Dropdowns can vary
+ *       '.tooltip': 5.0,       // Tooltips are highly variable
+ *       '.submit-button': 0.5  // Critical buttons need precision
+ *     }
+ *   },
+ *   ai: { sensitivity: 2, confidence: 0.7 }
+ * };
+ * 
+ * // Environment-specific overrides:
+ * export default {
+ *   layout: { sensitivity: 2, tolerance: 1.0 },
+ *   ai: { sensitivity: 2, confidence: 0.7 },
+ *   environments: {
+ *     ci: {
+ *       layout: { sensitivity: 1, tolerance: 0.5 },  // Stricter in CI
+ *       ai: { sensitivity: 1, confidence: 0.8 }
+ *     },
+ *     development: {
+ *       layout: { sensitivity: 3, tolerance: 2.0 },  // More lenient locally
+ *       ai: { sensitivity: 3, confidence: 0.6 }
+ *     }
+ *   }
+ * };
+ */
+
+export default {
+  // Your TestivAI API key - get this from the dashboard after creating a project
+  apiKey: process.env.TESTIVAI_API_KEY || '',
+  
+  // API endpoint - automatically points to production
+  apiUrl: process.env.TESTIVAI_API_URL || 'https://core-api.testiv.ai',
+  
+  layout: {
+    sensitivity: 2,        // Balanced sensitivity (0-4 scale)
+    tolerance: 1.0,        // 1 pixel base tolerance
+  },
+  ai: {
+    sensitivity: 2,        // Balanced AI analysis (0-4 scale)
+    confidence: 0.7,       // 70% confidence required for AI_BUG
+  }
+};
+`;
+
+async function createConfigFile(): Promise<void> {
+  const configPath = path.join(process.cwd(), 'testivai.config.ts');
+  
+  // Check if config already exists
+  if (await fs.pathExists(configPath)) {
+    console.log('❌ TestivAI config already exists at:', configPath);
+    console.log('   Delete it first if you want to regenerate it.');
+    process.exit(1);
+  }
+  
+  try {
+    // Create the config file
+    await fs.writeFile(configPath, DEFAULT_CONFIG, 'utf8');
+    console.log('✅ TestivAI configuration created successfully!');
+    console.log('📁 Config file:', configPath);
+    console.log('');
+    console.log('📖 Next steps:');
+    console.log('   1. Set up your API key:');
+    console.log('      TESTIVAI_API_KEY=tstvai-your-key-here');
+    console.log('');
+    console.log('   2. Update your playwright.config.ts to use TestivAI reporter:');
+    console.log('      reporter: [[\'@testivai/witness-playwright/reporter\']]');
+    console.log('');
+    console.log('   3. Review and customize testivai.config.ts');
+    console.log('   4. Run your tests: npx playwright test');
+    console.log('');
+    console.log('💡 Get your API key from: https://dashboard.testiv.ai');
+    console.log('💡 The SDK automatically connects to the production API - no URL configuration needed!');
+    
+  } catch (error) {
+    console.error('❌ Failed to create configuration file:', error);
+    process.exit(1);
+  }
+}
+
+// Main execution
+if (require.main === module) {
+  createConfigFile().catch(error => {
+    console.error('❌ CLI init failed:', error);
+    process.exit(1);
+  });
+}
+
+export { createConfigFile };

package/src/config/loader.ts

@@ -0,0 +1,238 @@
+import * as fs from 'fs-extra';
+import * as path from 'path';
+import { TestivAIProjectConfig, TestivAIConfig, LayoutConfig, AIConfig } from '../types';
+
+/**
+ * Default configuration when no config file is found
+ */
+const DEFAULT_CONFIG: TestivAIProjectConfig = {
+  layout: {
+    sensitivity: 2,        // Balanced sensitivity (0-4 scale)
+    tolerance: 1.0,        // 1 pixel base tolerance
+  },
+  ai: {
+    sensitivity: 2,        // Balanced AI analysis (0-4 scale)
+    confidence: 0.7,       // 70% confidence required for AI_BUG
+  }
+};
+
+/**
+ * Load TestivAI configuration from file system
+ * Supports both .ts and .js config files
+ * 
+ * @returns Promise<TestivAIProjectConfig> The loaded configuration or defaults
+ */
+export async function loadConfig(): Promise<TestivAIProjectConfig> {
+  // Try TypeScript config first, then JavaScript
+  const tsConfigPath = path.join(process.cwd(), 'testivai.config.ts');
+  const jsConfigPath = path.join(process.cwd(), 'testivai.config.js');
+  
+  try {
+    let configPath: string;
+    let configModule: any;
+    
+    // Check for TypeScript config
+    if (await fs.pathExists(tsConfigPath)) {
+      configPath = tsConfigPath;
+    } else if (await fs.pathExists(jsConfigPath)) {
+      configPath = jsConfigPath;
+    } else {
+      console.log('⚠️  No testivai.config.ts or testivai.config.js found, using defaults');
+      return DEFAULT_CONFIG;
+    }
+    
+    // Load configuration based on file type
+    if (configPath.endsWith('.js')) {
+      // For .js files, use require to get CommonJS module
+      // Clear require cache to ensure fresh load
+      delete require.cache[require.resolve(configPath)];
+      configModule = require(configPath);
+    } else {
+      // For .ts files, use dynamic import (ES module)
+      configModule = await import(configPath);
+    }
+    
+    const config = configModule.default || configModule;
+    
+    // Validate and merge with defaults
+    return validateAndMergeConfig(config);
+    
+  } catch (error) {
+    console.warn('⚠️  Failed to load testivai config, using defaults:', error);
+    return DEFAULT_CONFIG;
+  }
+}
+
+/**
+ * Merge per-test configuration with project configuration
+ * 
+ * @param projectConfig The project-level configuration
+ * @param testConfig Optional per-test configuration overrides
+ * @returns TestivAIConfig The effective configuration for this test
+ */
+export function mergeTestConfig(
+  projectConfig: TestivAIProjectConfig, 
+  testConfig?: TestivAIConfig
+): TestivAIConfig {
+  if (!testConfig) {
+    return {
+      layout: projectConfig.layout,
+      ai: projectConfig.ai,
+      // @renamed: dom → structure (IP protection)
+      structure: projectConfig.structure,
+      performanceMetrics: projectConfig.performanceMetrics
+    };
+  }
+  
+  return {
+    layout: {
+      ...projectConfig.layout,
+      ...testConfig.layout
+    },
+    ai: {
+      ...projectConfig.ai,
+      ...testConfig.ai
+    },
+    // @renamed: dom → structure (IP protection)
+    structure: {
+      ...projectConfig.structure,
+      ...testConfig.structure
+    },
+    performanceMetrics: {
+      ...projectConfig.performanceMetrics,
+      ...testConfig.performanceMetrics
+    },
+    selectors: testConfig.selectors,
+    useCDP: testConfig.useCDP
+  };
+}
+
+/**
+ * Detect current environment (CI, development, production)
+ * 
+ * @returns string The detected environment
+ */
+export function detectEnvironment(): string {
+  // Check for common CI environment variables
+  if (process.env.CI || process.env.CONTINUOUS_INTEGRATION) {
+    return 'ci';
+  }
+  
+  // Check for production indicators
+  if (process.env.NODE_ENV === 'production') {
+    return 'production';
+  }
+  
+  // Default to development
+  return 'development';
+}
+
+/**
+ * Apply environment-specific overrides to configuration
+ * 
+ * @param config The base configuration
+ * @returns TestivAIProjectConfig Configuration with environment overrides applied
+ */
+export function applyEnvironmentOverrides(config: TestivAIProjectConfig): TestivAIProjectConfig {
+  const environment = detectEnvironment();
+  
+  if (!config.environments) {
+    return config;
+  }
+  
+  // Type-safe environment override access
+  let envOverrides: { layout?: Partial<LayoutConfig>; ai?: Partial<AIConfig> } | undefined;
+  
+  switch (environment) {
+    case 'ci':
+      envOverrides = config.environments.ci;
+      break;
+    case 'development':
+      envOverrides = config.environments.development;
+      break;
+    case 'production':
+      envOverrides = config.environments.production;
+      break;
+  }
+  
+  if (!envOverrides) {
+    return config;
+  }
+  
+  return {
+    layout: {
+      ...config.layout,
+      ...envOverrides.layout
+    },
+    ai: {
+      ...config.ai,
+      ...envOverrides.ai
+    },
+    environments: config.environments // Keep the original environments config
+  };
+}
+
+/**
+ * Validate configuration values and merge with defaults
+ * 
+ * @param config The configuration to validate
+ * @returns TestivAIProjectConfig Validated configuration
+ */
+function validateAndMergeConfig(config: any): TestivAIProjectConfig {
+  // Guard against null/undefined config
+  if (!config) {
+    console.warn('⚠️  Config is null or undefined, using defaults');
+    return DEFAULT_CONFIG;
+  }
+  
+  // Basic validation
+  const validatedConfig: TestivAIProjectConfig = {
+    // API configuration with production default
+    apiKey: config.apiKey,
+    apiUrl: config.apiUrl || 'https://core-api.testiv.ai',
+    layout: {
+      sensitivity: validateRange(config.layout?.sensitivity ?? DEFAULT_CONFIG.layout.sensitivity, 0, 4, 'layout.sensitivity', DEFAULT_CONFIG.layout.sensitivity),
+      tolerance: validateRange(config.layout?.tolerance ?? DEFAULT_CONFIG.layout.tolerance, 0, 100, 'layout.tolerance', DEFAULT_CONFIG.layout.tolerance),
+      selectorTolerances: config.layout?.selectorTolerances,
+      useRelativeTolerance: config.layout?.useRelativeTolerance,
+      relativeTolerance: config.layout?.relativeTolerance
+    },
+    ai: {
+      sensitivity: validateRange(config.ai?.sensitivity ?? DEFAULT_CONFIG.ai.sensitivity, 0, 4, 'ai.sensitivity', DEFAULT_CONFIG.ai.sensitivity),
+      confidence: validateRange(config.ai?.confidence ?? DEFAULT_CONFIG.ai.confidence, 0, 1, 'ai.confidence', DEFAULT_CONFIG.ai.confidence),
+      enableReasoning: config.ai?.enableReasoning
+    },
+    // @renamed: dom → structure (IP protection)
+    structure: config.structure, // Pass through structure analysis configuration as-is
+    performanceMetrics: config.performanceMetrics, // Pass through performance metrics configuration as-is
+    environments: config.environments
+  };
+  
+  return applyEnvironmentOverrides(validatedConfig);
+}
+
+/**
+ * Validate that a number is within the expected range
+ * 
+ * @param value The value to validate
+ * @param min Minimum allowed value
+ * @param max Maximum allowed value
+ * @param field Field name for error messages
+ * @param defaultValue Default value to use when validation fails
+ * @returns number The validated value
+ */
+function validateRange(value: any, min: number, max: number, field: string, defaultValue: number): number {
+  const numValue = Number(value);
+  
+  if (isNaN(numValue)) {
+    console.warn(`⚠️  Invalid ${field}: ${value}, using default: ${defaultValue}`);
+    return defaultValue;
+  }
+  
+  if (numValue < min || numValue > max) {
+    console.warn(`⚠️  ${field} must be between ${min} and ${max}, got ${numValue}, using default: ${defaultValue}`);
+    return defaultValue;
+  }
+  
+  return numValue;
+}

package/src/index.ts

@@ -0,0 +1,14 @@
+import { snapshot } from './snapshot';
+import { getCiRunId } from './ci';
+
+export const testivai = {
+  witness: snapshot,
+  ci: getCiRunId,
+};
+
+// Re-export types for convenience
+// @renamed: DOMAnalysisConfig → StructureAnalysisConfig, DOMAnalysis → StructureAnalysis (IP protection)
+export type { TestivAIConfig, TestivAIProjectConfig, StructureAnalysisConfig, StructureAnalysis } from './types';
+
+// Structure analyzer is now handled on the backend
+// The types are kept for backwards compatibility

package/src/reporter-entry.ts

@@ -0,0 +1,6 @@
+// Entry point for Playwright reporter
+// Playwright expects: module.exports = ReporterClass
+import { TestivAIPlaywrightReporter } from './reporter';
+
+// This will compile to: module.exports = TestivAIPlaywrightReporter
+export = TestivAIPlaywrightReporter;

package/src/reporter-types.ts

@@ -0,0 +1,5 @@
+// This file isolates the Playwright-specific types for the reporter
+// to allow for clean mocking in the Jest test environment.
+import type { Reporter, FullConfig, Suite, FullResult } from '@playwright/test/reporter';
+
+export type { Reporter, FullConfig, Suite, FullResult };