@arclabs561/ai-visual-test 0.5.1

package/src/validators/hybrid-validator.mjs

@@ -0,0 +1,268 @@
+/**
+ * Hybrid Validator
+ * 
+ * Combines programmatic validation with VLLM evaluation.
+ * Programmatic data provides ground truth, VLLM provides semantic reasoning.
+ * 
+ * This follows the PROVE framework pattern: programmatic verification + LLM evaluation.
+ */
+
+import { validateScreenshot } from '../judge.mjs';
+import { ValidationError } from '../errors.mjs';
+import { assertString, assertObject } from '../type-guards.mjs';
+import {
+  checkAllTextContrast,
+  checkKeyboardNavigation
+} from './accessibility-programmatic.mjs';
+import {
+  validateStateProgrammatic
+} from './state-programmatic.mjs';
+
+// Allow dependency injection for testing
+let injectedValidateScreenshot = null;
+
+/**
+ * Inject validateScreenshot function for testing
+ * @internal
+ * @param {Function} fn - Mock validateScreenshot function
+ */
+export function _injectValidateScreenshot(fn) {
+  injectedValidateScreenshot = fn;
+}
+
+/**
+ * Reset injected function
+ * @internal
+ */
+export function _resetValidateScreenshot() {
+  injectedValidateScreenshot = null;
+}
+
+function getValidateScreenshot() {
+  return injectedValidateScreenshot || validateScreenshot;
+}
+
+/**
+ * Hybrid accessibility validation
+ * 
+ * Combines programmatic contrast/keyboard checks with VLLM semantic evaluation.
+ * Programmatic data provides ground truth, VLLM evaluates context and criticality.
+ * 
+ * @param {any} page - Playwright page object
+ * @param {string} screenshotPath - Path to screenshot
+ * @param {number} minContrast - Minimum contrast ratio (default: 4.5)
+ * @param {object} options - Validation options
+ * @returns {Promise<import('../index.mjs').ValidationResult & {programmaticData: object}>}
+ * @throws {ValidationError} If inputs are invalid
+ */
+export async function validateAccessibilityHybrid(
+  page,
+  screenshotPath,
+  minContrast = 4.5,
+  options = {}
+) {
+  // Validate inputs
+  if (!page || typeof page.evaluate !== 'function') {
+    throw new ValidationError('validateAccessibilityHybrid requires a Playwright Page object', {
+      received: typeof page,
+      hasEvaluate: typeof page?.evaluate === 'function'
+    });
+  }
+  
+  assertString(screenshotPath, 'screenshotPath');
+  
+  if (typeof minContrast !== 'number' || minContrast < 1 || minContrast > 21) {
+    throw new ValidationError('minContrast must be a number between 1 and 21', {
+      received: minContrast
+    });
+  }
+  
+  // Extract programmatic data
+  const programmaticData = {
+    contrast: await checkAllTextContrast(page, minContrast),
+    keyboard: await checkKeyboardNavigation(page)
+  };
+  
+  // Build prompt with programmatic context
+  const prompt = `
+ACCESSIBILITY EVALUATION
+
+PROGRAMMATIC DATA (GROUND TRUTH):
+- Contrast: ${programmaticData.contrast.passing}/${programmaticData.contrast.total} elements pass (required: ${minContrast}:1)
+- Violations: ${programmaticData.contrast.failing} elements fail
+${programmaticData.contrast.violations.length > 0 ? `
+  Top violations:
+${programmaticData.contrast.violations.slice(0, 5).map(v => `  - ${v.element}: ${v.ratio}:1 (required: ${v.required}:1)`).join('\n')}
+` : ''}
+- Keyboard: ${programmaticData.keyboard.focusableElements} focusable elements
+${programmaticData.keyboard.violations.length > 0 ? `
+  Violations:
+${programmaticData.keyboard.violations.map(v => `  - ${v.element}: ${v.issue}`).join('\n')}
+` : ''}
+
+EVALUATION TASK:
+Use this programmatic data as ground truth (no hallucinations about measurements).
+Evaluate semantic aspects:
+1. Is contrast adequate for readability in context? (ratio alone doesn't tell you if it's readable)
+2. Are contrast violations critical or minor? (some violations might be acceptable in context)
+3. Is keyboard navigation intuitive? (semantic evaluation beyond just focusable elements)
+4. Does overall accessibility support user goals? (holistic evaluation)
+5. Are there accessibility issues that programmatic checks don't capture? (visual, semantic, contextual)
+
+Provide actionable recommendations based on both programmatic and semantic analysis.
+`;
+  
+  // VLLM evaluation with programmatic grounding
+  const result = await getValidateScreenshot()(screenshotPath, prompt, {
+    testType: options.testType || 'accessibility-hybrid',
+    minContrast,
+    ...options,
+    programmaticData
+  });
+  
+  return {
+    ...result,
+    programmaticData
+  };
+}
+
+/**
+ * Hybrid state validation
+ * 
+ * Combines programmatic state extraction with VLLM semantic evaluation.
+ * Programmatic data provides ground truth, VLLM evaluates visual consistency and context.
+ * 
+ * @param {any} page - Playwright page object
+ * @param {string} screenshotPath - Path to screenshot
+ * @param {object} expectedState - Expected state object
+ * @param {object} options - Validation options
+ * @param {object} options.selectors - Map of state keys to CSS selectors
+ * @param {number} options.tolerance - Pixel tolerance (default: 5)
+ * @returns {Promise<import('../index.mjs').ValidationResult & {programmaticData: object}>}
+ * @throws {ValidationError} If inputs are invalid
+ */
+export async function validateStateHybrid(
+  page,
+  screenshotPath,
+  expectedState,
+  options = {}
+) {
+  // Validate inputs
+  if (!page || typeof page.evaluate !== 'function') {
+    throw new ValidationError('validateStateHybrid requires a Playwright Page object', {
+      received: typeof page,
+      hasEvaluate: typeof page?.evaluate === 'function'
+    });
+  }
+  
+  assertString(screenshotPath, 'screenshotPath');
+  assertObject(expectedState, 'expectedState');
+  
+  const selectors = options.selectors || {};
+  const tolerance = options.tolerance || 5;
+  
+  // Extract programmatic state
+  // Note: validateStateProgrammatic will extract gameState internally if available
+  // We also extract it separately for the prompt
+  const gameState = await page.evaluate(() => window.gameState || null);
+  
+  // validateStateProgrammatic doesn't throw by default, it returns matches: false
+  const visualState = await validateStateProgrammatic(
+    page,
+    expectedState,
+    { selectors, tolerance }
+  );
+  
+  // visualState already includes gameState if extracted, but we want it in programmaticData too
+  
+  const programmaticData = {
+    gameState,
+    visualState: visualState.visualState,
+    discrepancies: visualState.discrepancies,
+    matches: visualState.matches
+  };
+  
+  // Build prompt with programmatic context
+  const prompt = `
+STATE CONSISTENCY EVALUATION
+
+PROGRAMMATIC DATA (GROUND TRUTH):
+${gameState ? `Game State: ${JSON.stringify(gameState, null, 2)}` : 'Game State: Not available'}
+Visual State: ${JSON.stringify(visualState.visualState, null, 2)}
+Expected State: ${JSON.stringify(expectedState, null, 2)}
+${visualState.discrepancies.length > 0 ? `
+Discrepancies: ${visualState.discrepancies.join(', ')}
+` : 'No discrepancies found (programmatic check passed)'}
+
+EVALUATION TASK:
+Use this programmatic data as ground truth (no hallucinations about positions/state).
+Evaluate semantic aspects:
+1. Does visual representation match programmatic state? (semantic check beyond exact positions)
+2. Is game state consistent with gameplay? (context-aware evaluation)
+3. Are there visual bugs that state data doesn't capture? (holistic evaluation)
+4. Is the state transition smooth and coherent? (temporal/contextual evaluation)
+5. Are discrepancies critical or acceptable? (context-aware criticality assessment)
+
+Provide actionable recommendations based on both programmatic and semantic analysis.
+`;
+  
+  // VLLM evaluation with programmatic grounding
+  const result = await getValidateScreenshot()(screenshotPath, prompt, {
+    testType: options.testType || 'state-hybrid',
+    expectedState,
+    ...options,
+    programmaticData
+  });
+  
+  return {
+    ...result,
+    programmaticData
+  };
+}
+
+/**
+ * Generic hybrid validator helper
+ * 
+ * Combines any programmatic data with VLLM evaluation.
+ * 
+ * @param {string} screenshotPath - Path to screenshot
+ * @param {string} prompt - Base evaluation prompt
+ * @param {object} programmaticData - Programmatic validation data
+ * @param {object} options - Validation options
+ * @returns {Promise<import('../index.mjs').ValidationResult & {programmaticData: object}>}
+ */
+export async function validateWithProgrammaticContext(
+  screenshotPath,
+  prompt,
+  programmaticData,
+  options = {}
+) {
+  assertString(screenshotPath, 'screenshotPath');
+  assertString(prompt, 'prompt');
+  assertObject(programmaticData, 'programmaticData');
+  
+  // Build enhanced prompt with programmatic context
+  const enhancedPrompt = `
+${prompt}
+
+PROGRAMMATIC DATA (GROUND TRUTH):
+${JSON.stringify(programmaticData, null, 2)}
+
+EVALUATION INSTRUCTIONS:
+- Use programmatic data as ground truth (no hallucinations about measurements)
+- Evaluate semantic aspects: context, criticality, usability, consistency
+- Report any discrepancies between programmatic data and visual appearance
+- Provide actionable recommendations based on both programmatic and semantic analysis
+`;
+  
+  const result = await getValidateScreenshot()(screenshotPath, enhancedPrompt, {
+    ...options,
+    programmaticData
+  });
+  
+  return {
+    ...result,
+    programmaticData
+  };
+}
+

package/src/validators/index.mjs

@@ -0,0 +1,34 @@
+/**
+ * Validators Sub-Module
+ * 
+ * All validation-related functionality grouped together.
+ * 
+ * Import from 'ai-visual-test/validators'
+ */
+
+// Re-export everything from validators
+export { StateValidator } from './state-validator.mjs';
+export { AccessibilityValidator } from './accessibility-validator.mjs';
+export { PromptBuilder } from './prompt-builder.mjs';
+export { validateWithRubric } from './rubric.mjs';
+export { BatchValidator } from './batch-validator.mjs';
+
+// Programmatic validators (fast, deterministic)
+export {
+  getContrastRatio,
+  checkElementContrast,
+  checkAllTextContrast,
+  checkKeyboardNavigation
+} from './accessibility-programmatic.mjs';
+
+export {
+  validateStateProgrammatic,
+  validateElementPosition
+} from './state-programmatic.mjs';
+
+// Hybrid validators (programmatic + VLLM)
+export {
+  validateAccessibilityHybrid,
+  validateStateHybrid,
+  validateWithProgrammaticContext
+} from './hybrid-validator.mjs';

package/src/validators/prompt-builder.mjs

@@ -0,0 +1,218 @@
+/**
+ * Prompt Builder
+ * 
+ * Template-based prompt construction with rubric integration
+ * 
+ * Provides:
+ * - Reusable prompt templates
+ * - Rubric integration
+ * - Context injection
+ * - Variable substitution
+ */
+
+import { buildRubricPrompt } from '../rubrics.mjs';
+import { ValidationError } from '../errors.mjs';
+
+/**
+ * Generic prompt builder with template support
+ */
+export class PromptBuilder {
+  constructor(options = {}) {
+    // Validate templates
+    if (options.templates !== undefined) {
+      if (typeof options.templates !== 'object' || options.templates === null || Array.isArray(options.templates)) {
+        throw new ValidationError(
+          'templates must be a non-null object',
+          { received: typeof options.templates }
+        );
+      }
+      this.templates = options.templates;
+    } else {
+      this.templates = {};
+    }
+    
+    this.rubric = options.rubric || null;
+    
+    // Validate defaultContext
+    if (options.defaultContext !== undefined) {
+      if (typeof options.defaultContext !== 'object' || options.defaultContext === null || Array.isArray(options.defaultContext)) {
+        throw new ValidationError(
+          'defaultContext must be a non-null object',
+          { received: typeof options.defaultContext }
+        );
+      }
+      this.defaultContext = options.defaultContext;
+    } else {
+      this.defaultContext = {};
+    }
+  }
+
+  /**
+   * Build prompt with optional rubric
+   */
+  buildPrompt(basePrompt, options = {}) {
+    let prompt = basePrompt;
+    
+    // Add rubric if provided
+    if (options.rubric || this.rubric) {
+      const rubric = options.rubric || this.rubric;
+      const rubricPrompt = buildRubricPrompt(rubric, {
+        includeZeroTolerance: options.includeZeroTolerance !== false,
+        includeScoring: options.includeScoring !== false
+      });
+      prompt = `${prompt}\n\n${rubricPrompt}`;
+      
+      // Add zero tolerance enforcement if applicable
+      if (options.enforceZeroTolerance !== false && rubric.criteria?.some(c => c.zeroTolerance)) {
+        prompt = `${prompt}\n\nZERO TOLERANCE ENFORCEMENT:
+Any zero tolerance violation results in automatic failure.
+Score is automatically set to 0 if any zero tolerance violation is detected.`;
+      }
+    }
+    
+    // Add context if provided
+    if (options.context || this.defaultContext) {
+      const context = { ...this.defaultContext, ...options.context };
+      if (Object.keys(context).length > 0) {
+        prompt = `${prompt}\n\nCONTEXT:
+${JSON.stringify(context, null, 2)}`;
+      }
+    }
+    
+    return prompt;
+  }
+
+  /**
+   * Build prompt from template with support for conditionals and loops
+   * 
+   * Supports:
+   * - Variables: {{variable}}
+   * - Conditionals: {{#if condition}}...{{/if}}, {{#unless condition}}...{{/unless}}
+   * - Loops: {{#each items}}...{{/each}}
+   * - Nested templates: {{>templateName}}
+   */
+  buildFromTemplate(templateName, variables = {}, options = {}) {
+    const template = this.templates[templateName];
+    if (!template) {
+      throw new ValidationError(
+        `Template "${templateName}" not found. Available templates: ${Object.keys(this.templates).join(', ') || 'none'}`,
+        { templateName, availableTemplates: Object.keys(this.templates) }
+      );
+    }
+    
+    // Get template string
+    let templateStr = typeof template === 'function' 
+      ? template(variables)
+      : template;
+    
+    // Process nested templates (partials) first: {{>templateName}}
+    templateStr = templateStr.replace(/\{\{>([^}]+)\}\}/g, (match, partialName) => {
+      const trimmedName = partialName.trim();
+      if (this.templates[trimmedName]) {
+        return this.buildFromTemplate(trimmedName, variables, { ...options, skipRubric: true });
+      }
+      return match; // Return original if partial not found
+    });
+    
+    // Process loops: {{#each items}}...{{/each}}
+    // Need to process nested conditionals inside loops, so we need to handle this carefully
+    templateStr = templateStr.replace(/\{\{#each\s+([^}]+)\}\}([\s\S]*?)\{\{\/each\}\}/g, (match, arrayKey, loopBody) => {
+      const trimmedKey = arrayKey.trim();
+      const array = variables[trimmedKey];
+      if (Array.isArray(array)) {
+        return array.map((item, index) => {
+          const itemVars = { 
+            ...variables, 
+            '@index': index, 
+            '@first': index === 0, 
+            '@last': index === array.length - 1 
+          };
+          // If item is an object, merge its properties
+          if (typeof item === 'object' && item !== null) {
+            Object.assign(itemVars, item);
+          } else {
+            itemVars['@value'] = item;
+          }
+          // Process the loop body with item variables, including nested conditionals
+          let processedBody = loopBody;
+          // Process nested conditionals in loop body
+          processedBody = processedBody.replace(/\{\{#if\s+([^}]+)\}\}([\s\S]*?)\{\{\/if\}\}/g, (m, condKey, body) => {
+            const trimmedCond = condKey.trim();
+            const condValue = itemVars[trimmedCond];
+            const isTruthy = condValue !== undefined && condValue !== null && condValue !== false && condValue !== '';
+            return isTruthy ? this._processTemplate(body, itemVars) : '';
+          });
+          return this._processTemplate(processedBody, itemVars);
+        }).join('');
+      }
+      return ''; // Return empty if not an array
+    });
+    
+    // Process conditionals: {{#if condition}}...{{else}}...{{/if}} and {{#unless condition}}...{{/unless}}
+    templateStr = templateStr.replace(/\{\{#if\s+([^}]+)\}\}([\s\S]*?)\{\{\/if\}\}/g, (match, conditionKey, body) => {
+      const trimmedKey = conditionKey.trim();
+      const value = variables[trimmedKey];
+      const isTruthy = value !== undefined && value !== null && value !== false && value !== '';
+      
+      // Check for {{else}} block
+      const elseMatch = body.match(/^([\s\S]*?)\{\{else\}\}([\s\S]*)$/);
+      if (elseMatch) {
+        const trueBody = elseMatch[1];
+        const falseBody = elseMatch[2];
+        return isTruthy 
+          ? this._processTemplate(trueBody, variables)
+          : this._processTemplate(falseBody, variables);
+      }
+      
+      return isTruthy ? this._processTemplate(body, variables) : '';
+    });
+    
+    templateStr = templateStr.replace(/\{\{#unless\s+([^}]+)\}\}([\s\S]*?)\{\{\/unless\}\}/g, (match, conditionKey, body) => {
+      const trimmedKey = conditionKey.trim();
+      const value = variables[trimmedKey];
+      const isFalsy = value === undefined || value === null || value === false || value === '';
+      return isFalsy ? this._processTemplate(body, variables) : '';
+    });
+    
+    // Process variables: {{variable}}
+    let prompt = this._processTemplate(templateStr, variables);
+    
+    // Apply rubric and context (unless skipRubric is set for nested templates)
+    if (options.skipRubric) {
+      return prompt;
+    }
+    return this.buildPrompt(prompt, options);
+  }
+
+  /**
+   * Internal method to process template variables
+   * @private
+   */
+  _processTemplate(templateStr, variables) {
+    return templateStr.replace(/\{\{([^}]+)\}\}/g, (match, key) => {
+      const trimmedKey = key.trim();
+      // Support dot notation: {{object.property}}
+      if (trimmedKey.includes('.')) {
+        const parts = trimmedKey.split('.');
+        let value = variables;
+        for (const part of parts) {
+          if (value && typeof value === 'object' && part in value) {
+            value = value[part];
+          } else {
+            return match; // Return original if path not found
+          }
+        }
+        return value !== undefined ? String(value) : match;
+      }
+      return variables[trimmedKey] !== undefined ? String(variables[trimmedKey]) : match;
+    });
+  }
+
+  /**
+   * Register a template
+   */
+  registerTemplate(name, template) {
+    this.templates[name] = template;
+  }
+}
+

package/src/validators/rubric.mjs

@@ -0,0 +1,85 @@
+/**
+ * Rubric System
+ * 
+ * Generic rubric-based validation with zero tolerance support
+ * 
+ * Provides:
+ * - Rubric-based validation
+ * - Zero tolerance violation enforcement
+ * - Research-enhanced validation integration
+ */
+
+import { validateWithResearchEnhancements } from '../research-enhanced-validation.mjs';
+import { PromptBuilder } from './prompt-builder.mjs';
+import { ValidationError } from '../errors.mjs';
+import { assertString, assertObject } from '../type-guards.mjs';
+
+/**
+ * Validate with rubric (generic, not project-specific)
+ */
+export async function validateWithRubric(screenshotPath, prompt, rubric, context = {}, options = {}) {
+  // Input validation
+  assertString(screenshotPath, 'screenshotPath');
+  assertString(prompt, 'prompt');
+  assertObject(rubric, 'rubric');
+  
+  if (!rubric.score || !rubric.score.criteria) {
+    throw new ValidationError(
+      'Rubric must have score.criteria property',
+      { rubric: Object.keys(rubric) }
+    );
+  }
+  
+  const builder = new PromptBuilder({ rubric });
+  const enhancedPrompt = builder.buildPrompt(prompt, {
+    enforceZeroTolerance: options.enforceZeroTolerance !== false,
+    ...options
+  });
+  
+  try {
+    const result = await validateWithResearchEnhancements(
+      screenshotPath,
+      enhancedPrompt,
+      {
+        ...context,
+        rubric,
+        testType: context.testType || 'rubric-validation'
+      }
+    );
+  
+    // Check for zero tolerance violations if rubric has them
+    const hasZeroTolerance = rubric?.criteria?.some(c => c.zeroTolerance) || false;
+    if (hasZeroTolerance && options.enforceZeroTolerance !== false) {
+      // Ensure issues is an array before calling .some()
+      const issues = Array.isArray(result.issues) ? result.issues : [];
+      const hasZeroToleranceViolation = issues.some(issue => 
+        typeof issue === 'string' && (
+          issue.toLowerCase().includes('zero tolerance') ||
+          issue.toLowerCase().includes('instant fail') ||
+          rubric.criteria.some(c => c.zeroTolerance && issue.includes(c.id))
+        )
+      );
+      
+      if (hasZeroToleranceViolation) {
+        return {
+          ...result,
+          score: 0,
+          assessment: 'fail',
+          zeroToleranceViolation: true
+        };
+      }
+    }
+    
+    return result;
+  } catch (error) {
+    // Re-throw ValidationError as-is, wrap others
+    if (error instanceof ValidationError) {
+      throw error;
+    }
+    throw new ValidationError(
+      `Rubric validation failed: ${error.message}`,
+      { screenshotPath, rubricName: rubric.name, originalError: error.message }
+    );
+  }
+}
+