@optimizely/ocp-cli 1.2.13 → 1.2.14

package/src/test/e2e/utils/output-validator.ts

@@ -0,0 +1,661 @@
+import { CLIExecutionResult } from './cli-executor';
+
+export enum OutputType {
+  JSON = 'json',
+  YAML = 'yaml',
+  TABLE = 'table',
+  PLAIN_TEXT = 'plain_text'
+}
+
+export interface ValidationResult {
+  isValid: boolean;
+  errors: string[];
+  warnings: string[];
+  outputType: OutputType;
+  metadata?: {
+    lineCount?: number;
+    columnCount?: number;
+    jsonKeys?: string[];
+    yamlKeys?: string[];
+    tableHeaders?: string[];
+    detectedPatterns?: string[];
+    exitCode?: number;
+    executionTime?: number;
+    timedOut?: boolean;
+  };
+}
+
+export interface SchemaValidationRule {
+  path: string;
+  type: 'string' | 'number' | 'boolean' | 'array' | 'object';
+  required?: boolean;
+  pattern?: RegExp;
+  minLength?: number;
+  maxLength?: number;
+  min?: number;
+  max?: number;
+  enum?: any[];
+}
+
+export interface TextPattern {
+  name: string;
+  pattern: RegExp;
+  required?: boolean;
+  description?: string;
+}
+
+export interface TableValidationOptions {
+  expectedColumns?: string[];
+  minRows?: number;
+  maxRows?: number;
+  columnSeparator?: string | RegExp;
+  headerRequired?: boolean;
+  allowEmptyRows?: boolean;
+}
+
+export interface JsonValidationOptions {
+  schema?: SchemaValidationRule[];
+  strictMode?: boolean;
+  allowAdditionalProperties?: boolean;
+}
+
+export interface YamlValidationOptions {
+  schema?: SchemaValidationRule[];
+  strictMode?: boolean;
+  allowMultipleDocuments?: boolean;
+}
+
+export interface TextValidationOptions {
+  patterns?: TextPattern[];
+  minLines?: number;
+  maxLines?: number;
+  encoding?: string;
+  allowEmptyOutput?: boolean;
+}
+
+export class OutputValidator {
+  /**
+   * Detect the type of output with improved accuracy
+   */
+  static detectOutputType(output: string): OutputType {
+    const trimmed = output.trim();
+    
+    if (!trimmed) {
+      return OutputType.PLAIN_TEXT;
+    }
+    
+    // Check for JSON first (most specific)
+    if (this.isValidJson(trimmed)) {
+      return OutputType.JSON;
+    }
+    
+    // Check for YAML (improved detection)
+    if (this.looksLikeYaml(trimmed)) {
+      return OutputType.YAML;
+    }
+    
+    // Check for table format (improved detection)
+    if (this.looksLikeTable(trimmed)) {
+      return OutputType.TABLE;
+    }
+    
+    return OutputType.PLAIN_TEXT;
+  }
+
+  /**
+   * Comprehensive output validation with detailed results
+   */
+  static validateOutput(
+    output: string, 
+    expectedType?: OutputType,
+    options?: JsonValidationOptions | YamlValidationOptions | TableValidationOptions | TextValidationOptions
+  ): ValidationResult {
+    const detectedType = this.detectOutputType(output);
+    
+    // Check if detected type matches expected type
+    if (expectedType && detectedType !== expectedType) {
+      return {
+        isValid: false,
+        errors: [`Expected ${expectedType} output, but detected ${detectedType}`],
+        warnings: [],
+        outputType: detectedType
+      };
+    }
+    
+    // Validate based on detected type
+    switch (detectedType) {
+      case OutputType.JSON:
+        return this.validateJson(output, options as JsonValidationOptions);
+      case OutputType.YAML:
+        return this.validateYaml(output, options as YamlValidationOptions);
+      case OutputType.TABLE:
+        return this.validateTable(output, options as TableValidationOptions);
+      case OutputType.PLAIN_TEXT:
+        return this.validatePlainText(output, options as TextValidationOptions);
+      default:
+        return {
+          isValid: false,
+          errors: [`Unknown output type: ${detectedType}`],
+          warnings: [],
+          outputType: detectedType
+        };
+    }
+  }
+
+  /**
+   * Enhanced JSON validation with schema support
+   */
+  static validateJson(output: string, options: JsonValidationOptions = {}): ValidationResult {
+    const errors: string[] = [];
+    const warnings: string[] = [];
+    const metadata: any = {};
+    
+    try {
+      const parsed = JSON.parse(output);
+      
+      // Extract JSON keys for metadata
+      if (typeof parsed === 'object' && parsed !== null) {
+        metadata.jsonKeys = this.extractJsonKeys(parsed);
+      }
+      
+      // Schema validation if provided
+      if (options.schema) {
+        const schemaErrors = this.validateJsonSchema(parsed, options.schema);
+        errors.push(...schemaErrors);
+      }
+      
+      // Check for additional properties if strict mode
+      if (options.strictMode && !options.allowAdditionalProperties && options.schema) {
+        const additionalProps = this.findAdditionalProperties(parsed, options.schema);
+        if (additionalProps.length > 0) {
+          warnings.push(`Additional properties found: ${additionalProps.join(', ')}`);
+        }
+      }
+      
+      return {
+        isValid: errors.length === 0,
+        errors,
+        warnings,
+        outputType: OutputType.JSON,
+        metadata
+      };
+    } catch (error) {
+      return {
+        isValid: false,
+        errors: [`Invalid JSON: ${error instanceof Error ? error.message : String(error)}`],
+        warnings: [],
+        outputType: OutputType.JSON
+      };
+    }
+  }
+
+  /**
+   * YAML validation with schema support
+   */
+  static validateYaml(output: string, options: YamlValidationOptions = {}): ValidationResult {
+    const errors: string[] = [];
+    const warnings: string[] = [];
+    const metadata: any = {};
+    
+    try {
+      // Basic YAML parsing (simplified - would need yaml library for full support)
+      const yamlKeys = this.extractYamlKeys(output);
+      metadata.yamlKeys = yamlKeys;
+      
+      // Check for multiple documents
+      const documentCount = (output.match(/^---/gm) || []).length;
+      if (documentCount > 1 && !options.allowMultipleDocuments) {
+        warnings.push(`Multiple YAML documents found (${documentCount})`);
+      }
+      
+      // Additional YAML-specific warnings could be added here
+      if (options.strictMode && yamlKeys.length === 0) {
+        warnings.push('No YAML keys detected in strict mode');
+      }
+      
+      // Basic structure validation
+      if (yamlKeys.length === 0) {
+        errors.push('No YAML keys found in output');
+      }
+      
+      return {
+        isValid: errors.length === 0,
+        errors,
+        warnings,
+        outputType: OutputType.YAML,
+        metadata
+      };
+    } catch (error) {
+      return {
+        isValid: false,
+        errors: [`Invalid YAML: ${error instanceof Error ? error.message : String(error)}`],
+        warnings: [],
+        outputType: OutputType.YAML
+      };
+    }
+  }
+
+  /**
+   * Enhanced table validation with comprehensive options
+   */
+  static validateTable(output: string, options: TableValidationOptions = {}): ValidationResult {
+    const errors: string[] = [];
+    const warnings: string[] = [];
+    const metadata: any = {};
+    
+    const lines = output.split('\n');
+    const nonEmptyLines = lines.filter(line => line.trim());
+    
+    metadata.lineCount = lines.length;
+    
+    if (nonEmptyLines.length === 0) {
+      errors.push('Empty table output');
+      return { 
+        isValid: false, 
+        errors, 
+        warnings, 
+        outputType: OutputType.TABLE,
+        metadata 
+      };
+    }
+    
+    // Detect table structure
+    const tableInfo = this.analyzeTableStructure(output, options.columnSeparator);
+    metadata.columnCount = tableInfo.columnCount;
+    metadata.tableHeaders = tableInfo.headers;
+    
+    // Validate row count
+    if (options.minRows && tableInfo.dataRows < options.minRows) {
+      errors.push(`Expected at least ${options.minRows} data rows, found ${tableInfo.dataRows}`);
+    }
+    if (options.maxRows && tableInfo.dataRows > options.maxRows) {
+      errors.push(`Expected at most ${options.maxRows} data rows, found ${tableInfo.dataRows}`);
+    }
+    
+    // Validate headers
+    if (options.headerRequired !== false && !tableInfo.hasHeader) {
+      errors.push('Table header is required but not found');
+    }
+    
+    // Check for expected columns
+    if (options.expectedColumns && tableInfo.headers) {
+      const missingColumns = options.expectedColumns.filter(col => 
+        !tableInfo.headers!.some(header => header.toLowerCase().includes(col.toLowerCase()))
+      );
+      if (missingColumns.length > 0) {
+        errors.push(`Missing expected columns: ${missingColumns.join(', ')}`);
+      }
+    }
+    
+    // Check for empty rows
+    if (!options.allowEmptyRows && tableInfo.emptyRows > 0) {
+      warnings.push(`Found ${tableInfo.emptyRows} empty rows in table`);
+    }
+    
+    return {
+      isValid: errors.length === 0,
+      errors,
+      warnings,
+      outputType: OutputType.TABLE,
+      metadata
+    };
+  }
+
+  /**
+   * Plain text validation with pattern matching
+   */
+  static validatePlainText(output: string, options: TextValidationOptions = {}): ValidationResult {
+    const errors: string[] = [];
+    const warnings: string[] = [];
+    const metadata: any = {};
+    
+    const lines = output.split('\n');
+    metadata.lineCount = lines.length;
+    
+    // Check if empty output is allowed
+    if (!output.trim() && !options.allowEmptyOutput) {
+      errors.push('Empty output not allowed');
+    }
+    
+    // Validate line count
+    if (options.minLines && lines.length < options.minLines) {
+      errors.push(`Expected at least ${options.minLines} lines, found ${lines.length}`);
+    }
+    if (options.maxLines && lines.length > options.maxLines) {
+      errors.push(`Expected at most ${options.maxLines} lines, found ${lines.length}`);
+    }
+    
+    // Pattern matching
+    if (options.patterns) {
+      const detectedPatterns: string[] = [];
+      for (const pattern of options.patterns) {
+        const matches = output.match(pattern.pattern);
+        if (matches) {
+          detectedPatterns.push(pattern.name);
+        } else if (pattern.required) {
+          errors.push(`Required pattern '${pattern.name}' not found: ${pattern.description || pattern.pattern.toString()}`);
+        }
+      }
+      metadata.detectedPatterns = detectedPatterns;
+    }
+    
+    return {
+      isValid: errors.length === 0,
+      errors,
+      warnings,
+      outputType: OutputType.PLAIN_TEXT,
+      metadata
+    };
+  }
+
+  /**
+   * Validate command execution result with comprehensive options
+   */
+  static validateResult(
+    result: CLIExecutionResult, 
+    expectedType?: OutputType, 
+    options?: JsonValidationOptions | YamlValidationOptions | TableValidationOptions | TextValidationOptions
+  ): ValidationResult {
+    // Check for command execution failure
+    if (result.exitCode !== 0) {
+      return {
+        isValid: false,
+        errors: [`Command failed with exit code ${result.exitCode}: ${result.stderr}`],
+        warnings: [],
+        outputType: OutputType.PLAIN_TEXT,
+        metadata: {
+          exitCode: result.exitCode,
+          executionTime: result.executionTime,
+          timedOut: result.timedOut
+        }
+      };
+    }
+
+    // Validate the output
+    return this.validateOutput(result.stdout, expectedType, options);
+  }
+
+  /**
+   * Create a detailed validation report
+   */
+  static createValidationReport(results: ValidationResult[]): string {
+    const report: string[] = [];
+    
+    report.push('=== Output Validation Report ===\n');
+    
+    let totalTests = results.length;
+    let passedTests = results.filter(r => r.isValid).length;
+    let failedTests = totalTests - passedTests;
+    
+    report.push(`Total Tests: ${totalTests}`);
+    report.push(`Passed: ${passedTests}`);
+    report.push(`Failed: ${failedTests}`);
+    report.push(`Success Rate: ${((passedTests / totalTests) * 100).toFixed(1)}%\n`);
+    
+    // Group by output type
+    const byType = results.reduce((acc, result, index) => {
+      if (!acc[result.outputType]) {
+        acc[result.outputType] = [];
+      }
+      acc[result.outputType].push({ result, index });
+      return acc;
+    }, {} as Record<OutputType, Array<{ result: ValidationResult; index: number }>>);
+    
+    for (const [type, typeResults] of Object.entries(byType)) {
+      report.push(`\n--- ${type.toUpperCase()} Results ---`);
+      
+      typeResults.forEach(({ result, index }) => {
+        const status = result.isValid ? '✓ PASS' : '✗ FAIL';
+        report.push(`Test ${index + 1}: ${status}`);
+        
+        if (result.errors.length > 0) {
+          report.push(`  Errors:`);
+          result.errors.forEach(error => report.push(`    - ${error}`));
+        }
+        
+        if (result.warnings.length > 0) {
+          report.push(`  Warnings:`);
+          result.warnings.forEach(warning => report.push(`    - ${warning}`));
+        }
+        
+        if (result.metadata) {
+          report.push(`  Metadata: ${JSON.stringify(result.metadata, null, 2)}`);
+        }
+      });
+    }
+    
+    return report.join('\n');
+  }
+
+  // Private helper methods
+  private static isValidJson(str: string): boolean {
+    try {
+      JSON.parse(str);
+      return true;
+    } catch {
+      return false;
+    }
+  }
+
+  private static looksLikeYaml(str: string): boolean {
+    const lines = str.split('\n');
+    
+    // YAML characteristics
+    const hasColons = str.includes(':');
+    const hasIndentation = lines.some(line => line.match(/^\s+/));
+    const hasListItems = lines.some(line => line.trim().startsWith('-'));
+    const hasDocumentSeparator = str.includes('---');
+    
+    // Must have colons and either indentation, list items, or document separators
+    return hasColons && (hasIndentation || hasListItems || hasDocumentSeparator);
+  }
+
+  private static looksLikeTable(str: string): boolean {
+    const lines = str.split('\n').filter(line => line.trim());
+    
+    if (lines.length < 2) return false;
+    
+    // Check for common table patterns
+    const hasPipes = lines.some(line => line.includes('|'));
+    const hasTabs = lines.some(line => line.includes('\t'));
+    const hasMultipleSpaces = lines.some(line => /\s{2,}/.test(line));
+    const hasConsistentColumns = this.hasConsistentColumnStructure(lines);
+    
+    return hasPipes || hasTabs || (hasMultipleSpaces && hasConsistentColumns);
+  }
+
+  private static hasConsistentColumnStructure(lines: string[]): boolean {
+    if (lines.length < 2) return false;
+    
+    // Check if lines have similar structure (similar number of words/columns)
+    const wordCounts = lines.map(line => line.trim().split(/\s+/).length);
+    const avgWordCount = wordCounts.reduce((a, b) => a + b, 0) / wordCounts.length;
+    
+    // Allow some variance but expect consistency
+    return wordCounts.every(count => Math.abs(count - avgWordCount) <= 2);
+  }
+
+  private static extractJsonKeys(obj: any, prefix = ''): string[] {
+    const keys: string[] = [];
+    
+    if (typeof obj === 'object' && obj !== null) {
+      if (Array.isArray(obj)) {
+        obj.forEach((item, index) => {
+          const itemKeys = this.extractJsonKeys(item, `${prefix}[${index}]`);
+          keys.push(...itemKeys);
+        });
+      } else {
+        Object.keys(obj).forEach(key => {
+          const fullKey = prefix ? `${prefix}.${key}` : key;
+          keys.push(fullKey);
+          
+          const nestedKeys = this.extractJsonKeys(obj[key], fullKey);
+          keys.push(...nestedKeys);
+        });
+      }
+    }
+    
+    return keys;
+  }
+
+  private static extractYamlKeys(yamlStr: string): string[] {
+    const keys: string[] = [];
+    const lines = yamlStr.split('\n');
+    
+    for (const line of lines) {
+      const trimmed = line.trim();
+      if (trimmed && trimmed.includes(':') && !trimmed.startsWith('#')) {
+        const colonIndex = trimmed.indexOf(':');
+        const key = trimmed.substring(0, colonIndex).trim();
+        if (key && !key.startsWith('-')) {
+          keys.push(key);
+        }
+      }
+    }
+    
+    return keys;
+  }
+
+  private static analyzeTableStructure(output: string, separator?: string | RegExp) {
+    const lines = output.split('\n');
+    const nonEmptyLines = lines.filter(line => line.trim());
+    
+    let headers: string[] | null = null;
+    let columnCount = 0;
+    let dataRows = 0;
+    let emptyRows = 0;
+    let hasHeader = false;
+    
+    if (nonEmptyLines.length > 0) {
+      // Assume first non-empty line is header
+      const firstLine = nonEmptyLines[0];
+      
+      if (separator) {
+        headers = firstLine.split(separator).map(h => h.trim());
+      } else {
+        // Auto-detect separator
+        if (firstLine.includes('|')) {
+          headers = firstLine.split('|').map(h => h.trim()).filter(h => h);
+        } else if (firstLine.includes('\t')) {
+          headers = firstLine.split('\t').map(h => h.trim());
+        } else {
+          headers = firstLine.split(/\s{2,}/).map(h => h.trim());
+        }
+      }
+      
+      columnCount = headers.length;
+      hasHeader = true;
+      
+      // Count actual data rows, excluding header and separator lines
+      dataRows = nonEmptyLines.slice(1).filter(line => {
+        // Skip markdown-style separator lines like |------|-----|------|
+        const trimmed = line.trim();
+        return !this.isTableSeparatorLine(trimmed);
+      }).length;
+    }
+    
+    emptyRows = lines.length - nonEmptyLines.length;
+    
+    return {
+      headers,
+      columnCount,
+      dataRows,
+      emptyRows,
+      hasHeader
+    };
+  }
+
+  private static isTableSeparatorLine(line: string): boolean {
+    // Check if line is a markdown-style table separator (e.g., |------|-----|------|)
+    const trimmed = line.trim();
+    if (trimmed.includes('|')) {
+      // Remove pipes and check if remaining content is mostly dashes and spaces
+      const withoutPipes = trimmed.replace(/\|/g, '');
+      return /^[\s\-:]*$/.test(withoutPipes);
+    }
+    return false;
+  }
+
+  private static validateJsonSchema(data: any, schema: SchemaValidationRule[]): string[] {
+    const errors: string[] = [];
+    
+    for (const rule of schema) {
+      const value = this.getValueByPath(data, rule.path);
+      
+      // Check if required field is missing
+      if (rule.required && (value === undefined || value === null)) {
+        errors.push(`Required field '${rule.path}' is missing`);
+        continue;
+      }
+      
+      // Skip validation if field is not required and missing
+      if (!rule.required && (value === undefined || value === null)) {
+        continue;
+      }
+      
+      // Type validation
+      if (!this.validateType(value, rule.type)) {
+        errors.push(`Field '${rule.path}' expected type ${rule.type}, got ${typeof value}`);
+        continue;
+      }
+      
+      // Additional validations based on type
+      if (rule.type === 'string' && typeof value === 'string') {
+        if (rule.pattern && !rule.pattern.test(value)) {
+          errors.push(`Field '${rule.path}' does not match pattern ${rule.pattern}`);
+        }
+        if (rule.minLength && value.length < rule.minLength) {
+          errors.push(`Field '${rule.path}' length ${value.length} is less than minimum ${rule.minLength}`);
+        }
+        if (rule.maxLength && value.length > rule.maxLength) {
+          errors.push(`Field '${rule.path}' length ${value.length} exceeds maximum ${rule.maxLength}`);
+        }
+      }
+      
+      if (rule.type === 'number' && typeof value === 'number') {
+        if (rule.min !== undefined && value < rule.min) {
+          errors.push(`Field '${rule.path}' value ${value} is less than minimum ${rule.min}`);
+        }
+        if (rule.max !== undefined && value > rule.max) {
+          errors.push(`Field '${rule.path}' value ${value} exceeds maximum ${rule.max}`);
+        }
+      }
+      
+      if (rule.enum && !rule.enum.includes(value)) {
+        errors.push(`Field '${rule.path}' value '${value}' is not in allowed values: ${rule.enum.join(', ')}`);
+      }
+    }
+    
+    return errors;
+  }
+
+  private static getValueByPath(obj: any, path: string): any {
+    return path.split('.').reduce((current, key) => {
+      return current && current[key] !== undefined ? current[key] : undefined;
+    }, obj);
+  }
+
+  private static validateType(value: any, expectedType: string): boolean {
+    switch (expectedType) {
+      case 'string':
+        return typeof value === 'string';
+      case 'number':
+        return typeof value === 'number' && !isNaN(value);
+      case 'boolean':
+        return typeof value === 'boolean';
+      case 'array':
+        return Array.isArray(value);
+      case 'object':
+        return typeof value === 'object' && value !== null && !Array.isArray(value);
+      default:
+        return false;
+    }
+  }
+
+  private static findAdditionalProperties(data: any, schema: SchemaValidationRule[]): string[] {
+    const definedPaths = new Set(schema.map(rule => rule.path));
+    const actualPaths = this.extractJsonKeys(data);
+    
+    return actualPaths.filter(path => !definedPaths.has(path));
+  }
+}

package/src/test/setup.ts

@@ -1,2 +1,4 @@
+import { vi } from 'vitest';
+
 // silence expected console.error()
-jest.spyOn(global.console, 'error').mockImplementation(() => jest.fn());
+vi.spyOn(global.console, 'error').mockImplementation(() => vi.fn());

package/src/test/tsconfig.json

@@ -0,0 +1,17 @@
+{
+  "extends": "../../tsconfig.json",
+  "include": [
+    "**/*.ts"
+  ],
+  "exclude": [
+    "node_modules"
+  ],
+  "compilerOptions": {
+    "types": ["vitest/globals", "node"],
+    "moduleResolution": "node",
+    "allowSyntheticDefaultImports": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "noEmit": true
+  }
+}

package/dist/test/setup.js

@@ -1,4 +0,0 @@
-"use strict";
-// silence expected console.error()
-jest.spyOn(global.console, 'error').mockImplementation(() => jest.fn());
-//# sourceMappingURL=setup.js.map

package/dist/test/setup.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"setup.js","sourceRoot":"","sources":["../../src/test/setup.ts"],"names":[],"mappings":";AAAA,mCAAmC;AACnC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC"}