@arclabs561/ai-visual-test 0.5.1

package/src/validation-framework.mjs

@@ -0,0 +1,321 @@
+/**
+ * Validation Framework
+ * 
+ * Provides comprehensive validation of:
+ * 1. Temporal perception accuracy (human time scales)
+ * 2. VLLM judgment accuracy (against human ground truth)
+ * 3. Gameplay temporal experience correctness
+ * 4. Webpage evaluation correctness
+ * 
+ * This framework validates that our systems produce correct results,
+ * not just that they work.
+ */
+
+import { humanPerceptionTime } from './temporal-decision.mjs';
+import { TIME_SCALES } from './temporal-constants.mjs';
+import { aggregateTemporalNotes, calculateCoherenceExported as calculateCoherence } from './temporal.mjs';
+import {
+  compareJudgments,
+  collectHumanJudgment,
+  loadHumanJudgment
+} from '../evaluation/human-validation/human-validation.mjs';
+import { getHumanValidationManager } from './human-validation-manager.mjs';
+import { log, warn } from './logger.mjs';
+
+/**
+ * Validate temporal perception against research values
+ * 
+ * @param {Object} options - Validation options
+ * @returns {Object} Validation results
+ */
+export function validateTemporalPerception(options = {}) {
+  const results = {
+    researchAlignment: {},
+    consistency: {},
+    recommendations: []
+  };
+  
+  // Validate visual appeal time (50ms research base)
+  const visualAppealTime = humanPerceptionTime('visual-appeal', {
+    attentionLevel: 'focused',
+    actionComplexity: 'simple'
+  });
+  
+  results.researchAlignment.visualAppeal = {
+    researchValue: 50, // Lindgaard research
+    actualValue: visualAppealTime,
+    aligned: visualAppealTime >= 50 && visualAppealTime <= 200,
+    note: visualAppealTime >= 100 ? 'Enforces 100ms minimum (implementation constraint)' : 'Matches research (50ms)'
+  };
+  
+  // Validate instant threshold (100ms research)
+  results.researchAlignment.instantThreshold = {
+    researchValue: 100, // NN/g research
+    actualValue: TIME_SCALES.INSTANT,
+    aligned: TIME_SCALES.INSTANT === 100,
+    note: TIME_SCALES.INSTANT === 100 ? 'Matches research' : 'Does not match research'
+  };
+  
+  // Validate reading time scales with content
+  const shortReading = humanPerceptionTime('reading', { contentLength: 100 });
+  const longReading = humanPerceptionTime('reading', { contentLength: 1000 });
+  
+  results.researchAlignment.readingTime = {
+    scalesWithContent: longReading > shortReading,
+    shortContent: shortReading,
+    longContent: longReading,
+    note: longReading > shortReading ? 'Reading time scales with content' : 'Reading time does not scale correctly'
+  };
+  
+  // Generate recommendations
+  if (!results.researchAlignment.visualAppeal.aligned) {
+    results.recommendations.push('Visual appeal time does not align with research (50ms)');
+  }
+  if (!results.researchAlignment.instantThreshold.aligned) {
+    results.recommendations.push('Instant threshold does not match research (100ms)');
+  }
+  if (!results.researchAlignment.readingTime.scalesWithContent) {
+    results.recommendations.push('Reading time does not scale correctly with content length');
+  }
+  
+  return results;
+}
+
+/**
+ * Validate VLLM judgment accuracy against human ground truth
+ * 
+ * @param {Array} humanJudgments - Human judgment ground truth
+ * @param {Array} vllmJudgments - VLLM judgments to validate
+ * @param {Object} options - Validation options
+ * @returns {Object} Validation results
+ */
+export function validateVLLMAccuracy(humanJudgments, vllmJudgments, options = {}) {
+  const {
+    minCorrelation = 0.7,
+    maxMAE = 1.0,
+    minKappa = 0.6
+  } = options;
+  
+  try {
+    const calibration = compareJudgments(humanJudgments, vllmJudgments);
+    
+    const results = {
+      calibration,
+      isValid: false,
+      issues: [],
+      recommendations: []
+    };
+    
+    // Check correlation
+    if (calibration.agreement.pearson < minCorrelation) {
+      results.issues.push(`Low correlation (${calibration.agreement.pearson.toFixed(3)} < ${minCorrelation})`);
+      results.isValid = false;
+    }
+    
+    // Check MAE
+    if (calibration.agreement.mae > maxMAE) {
+      results.issues.push(`High MAE (${calibration.agreement.mae.toFixed(2)} > ${maxMAE})`);
+      results.isValid = false;
+    }
+    
+    // Check Kappa
+    if (calibration.agreement.kappa < minKappa) {
+      results.issues.push(`Low Kappa (${calibration.agreement.kappa.toFixed(3)} < ${minKappa})`);
+      results.isValid = false;
+    }
+    
+    // If all checks pass
+    if (results.issues.length === 0) {
+      results.isValid = true;
+      results.recommendations.push('VLLM judgments align well with human ground truth');
+    } else {
+      results.recommendations.push(...calibration.recommendations);
+    }
+    
+    return results;
+  } catch (error) {
+    return {
+      error: error.message,
+      isValid: false,
+      issues: ['Failed to compare judgments'],
+      recommendations: ['Ensure human and VLLM judgments are properly matched']
+    };
+  }
+}
+
+/**
+ * Validate gameplay temporal experience
+ * 
+ * @param {Array} gameplayNotes - Temporal notes from gameplay
+ * @param {Object} options - Validation options
+ * @returns {Object} Validation results
+ */
+export function validateGameplayTemporal(gameplayNotes, options = {}) {
+  const {
+    minCoherenceForSmooth = 0.7,
+    maxCoherenceForErratic = 0.5
+  } = options;
+  
+  if (!gameplayNotes || gameplayNotes.length === 0) {
+    return {
+      isValid: false,
+      issues: ['No gameplay notes provided'],
+      recommendations: ['Provide gameplay notes for validation']
+    };
+  }
+  
+  const aggregated = aggregateTemporalNotes(gameplayNotes, options);
+  
+  const results = {
+    aggregated,
+    isValid: true,
+    issues: [],
+    recommendations: []
+  };
+  
+  // Check coherence
+  if (aggregated.coherence < minCoherenceForSmooth && aggregated.coherence > maxCoherenceForErratic) {
+    results.issues.push(`Moderate coherence (${aggregated.coherence.toFixed(3)}) - neither smooth nor clearly erratic`);
+    results.recommendations.push('Review gameplay notes for consistency issues');
+  }
+  
+  // Check for conflicts
+  if (aggregated.conflicts && aggregated.conflicts.length > 0) {
+    results.issues.push(`Detected ${aggregated.conflicts.length} conflicts in gameplay notes`);
+    results.recommendations.push('Review conflicting observations in gameplay notes');
+  }
+  
+  // Check window count
+  if (aggregated.windows.length < 2) {
+    results.issues.push('Insufficient windows for temporal analysis (need at least 2)');
+    results.recommendations.push('Collect more gameplay notes or use smaller window size');
+  }
+  
+  return results;
+}
+
+/**
+ * Validate webpage evaluation correctness
+ * 
+ * Validates that VLLM judgments about webpages align with human expectations.
+ * 
+ * @param {Array} evaluations - Array of evaluation results
+ * @param {Object} groundTruth - Ground truth data (if available)
+ * @param {Object} options - Validation options
+ * @returns {Object} Validation results
+ */
+export function validateWebpageEvaluation(evaluations, groundTruth = null, options = {}) {
+  const results = {
+    evaluations,
+    isValid: true,
+    issues: [],
+    recommendations: []
+  };
+  
+  // If ground truth available, compare
+  if (groundTruth) {
+    const humanJudgments = groundTruth.humanJudgments || [];
+    const vllmJudgments = evaluations.map(eval => ({
+      id: eval.id || `eval-${Date.now()}`,
+      vllmScore: eval.score,
+      vllmIssues: eval.issues || [],
+      vllmReasoning: eval.reasoning || '',
+      provider: eval.provider || 'unknown',
+      timestamp: eval.timestamp || new Date().toISOString()
+    }));
+    
+    if (humanJudgments.length > 0 && vllmJudgments.length > 0) {
+      const accuracy = validateVLLMAccuracy(humanJudgments, vllmJudgments, options);
+      results.accuracy = accuracy;
+      results.isValid = accuracy.isValid;
+      results.issues.push(...accuracy.issues);
+      results.recommendations.push(...accuracy.recommendations);
+    }
+  }
+  
+  // Validate evaluation structure
+  for (const eval of evaluations) {
+    if (eval.score === null || eval.score === undefined) {
+      results.issues.push(`Evaluation ${eval.id || 'unknown'} has null/undefined score`);
+      results.isValid = false;
+    }
+    if (eval.score !== null && (eval.score < 0 || eval.score > 10)) {
+      results.issues.push(`Evaluation ${eval.id || 'unknown'} has invalid score: ${eval.score}`);
+      results.isValid = false;
+    }
+    if (!Array.isArray(eval.issues)) {
+      results.issues.push(`Evaluation ${eval.id || 'unknown'} has non-array issues`);
+      results.isValid = false;
+    }
+  }
+  
+  return results;
+}
+
+/**
+ * Comprehensive validation report
+ * 
+ * Validates all aspects: temporal perception, VLLM accuracy, gameplay, webpage evaluation
+ * 
+ * @param {Object} data - Validation data
+ * @param {Object} options - Validation options
+ * @returns {Object} Comprehensive validation report
+ */
+export function validateComprehensive(data, options = {}) {
+  const report = {
+    temporalPerception: null,
+    vllmAccuracy: null,
+    gameplayTemporal: null,
+    webpageEvaluation: null,
+    overall: {
+      isValid: true,
+      issues: [],
+      recommendations: []
+    }
+  };
+  
+  // Validate temporal perception
+  if (data.temporalPerception !== false) {
+    report.temporalPerception = validateTemporalPerception(options);
+    if (report.temporalPerception.recommendations.length > 0) {
+      report.overall.issues.push('Temporal perception validation issues');
+      report.overall.recommendations.push(...report.temporalPerception.recommendations);
+    }
+  }
+  
+  // Validate VLLM accuracy
+  if (data.humanJudgments && data.vllmJudgments) {
+    report.vllmAccuracy = validateVLLMAccuracy(data.humanJudgments, data.vllmJudgments, options);
+    if (!report.vllmAccuracy.isValid) {
+      report.overall.isValid = false;
+      report.overall.issues.push('VLLM accuracy validation failed');
+      report.overall.recommendations.push(...report.vllmAccuracy.recommendations);
+    }
+  }
+  
+  // Validate gameplay temporal
+  if (data.gameplayNotes) {
+    report.gameplayTemporal = validateGameplayTemporal(data.gameplayNotes, options);
+    if (report.gameplayTemporal.issues.length > 0) {
+      report.overall.issues.push('Gameplay temporal validation issues');
+      report.overall.recommendations.push(...report.gameplayTemporal.recommendations);
+    }
+  }
+  
+  // Validate webpage evaluation
+  if (data.evaluations) {
+    report.webpageEvaluation = validateWebpageEvaluation(
+      data.evaluations,
+      data.groundTruth,
+      options
+    );
+    if (!report.webpageEvaluation.isValid) {
+      report.overall.isValid = false;
+      report.overall.issues.push('Webpage evaluation validation failed');
+      report.overall.recommendations.push(...report.webpageEvaluation.recommendations);
+    }
+  }
+  
+  return report;
+}
+

package/src/validation-result-normalizer.mjs

@@ -0,0 +1,64 @@
+/**
+ * Validation Result Normalizer
+ * 
+ * Ensures validation results have consistent structure.
+ * Centralizes normalization logic to avoid duplication.
+ */
+
+import { warn } from './logger.mjs';
+
+/**
+ * Normalize validation result to ensure consistent structure
+ * 
+ * @param {Object} result - Validation result from validateScreenshot
+ * @param {string} [source] - Source function name for logging
+ * @returns {Object} Normalized validation result
+ */
+export function normalizeValidationResult(result, source = 'unknown') {
+  if (!result) {
+    warn(`[Normalizer] ${source}: result is null/undefined`);
+    return {
+      enabled: false,
+      score: null,
+      issues: [],
+      reasoning: 'Result was null or undefined',
+      assessment: null
+    };
+  }
+
+  // Create a copy to avoid mutating the original
+  const normalized = { ...result };
+
+  // Ensure enabled field is always present (default to true if not specified)
+  if (normalized.enabled === undefined) {
+    // If enabled is not specified, infer from presence of other fields
+    normalized.enabled = normalized.score !== null || normalized.judgment || normalized.provider !== undefined;
+  }
+
+  // Ensure score is always present (may be null)
+  if (normalized.score === null || normalized.score === undefined) {
+    if (normalized.score === undefined) {
+      warn(`[Normalizer] ${source}: score is undefined, defaulting to null`);
+    }
+    normalized.score = null;
+  }
+
+  // Ensure issues is always an array
+  if (!Array.isArray(normalized.issues)) {
+    warn(`[Normalizer] ${source}: issues is not an array, defaulting to empty array`);
+    normalized.issues = [];
+  }
+
+  // Ensure reasoning is always present
+  if (!normalized.reasoning) {
+    normalized.reasoning = normalized.judgment || normalized.message || 'No reasoning provided';
+  }
+
+  // Ensure assessment is present (may be null)
+  if (normalized.assessment === undefined) {
+    normalized.assessment = null;
+  }
+
+  return normalized;
+}
+

package/src/validation.mjs

@@ -0,0 +1,243 @@
+/**
+ * Input Validation Utilities
+ * 
+ * Provides validation functions for common input types to prevent
+ * security vulnerabilities and improve error messages.
+ */
+
+import { ValidationError } from './errors.mjs';
+import { existsSync } from 'fs';
+import { normalize, resolve } from 'path';
+
+/**
+ * Validate image file path
+ * 
+ * @param {string} imagePath - Path to image file
+ * @returns {true} Always returns true if valid
+ * @throws {ValidationError} If path is invalid, empty, or contains path traversal
+ */
+export function validateImagePath(imagePath) {
+  if (typeof imagePath !== 'string') {
+    throw new ValidationError('Image path must be a string', null, {
+      received: typeof imagePath
+    });
+  }
+  
+  if (imagePath.length === 0) {
+    throw new ValidationError('Image path cannot be empty');
+  }
+  
+  // Check for path traversal attempts
+  const normalized = normalize(imagePath);
+  if (normalized.includes('..')) {
+    throw new ValidationError('Invalid image path: path traversal detected', imagePath);
+  }
+  
+  // Validate file extension
+  const validExtensions = ['.png', '.jpg', '.jpeg', '.gif', '.webp'];
+  const hasValidExtension = validExtensions.some(ext => 
+    imagePath.toLowerCase().endsWith(ext)
+  );
+  
+  if (!hasValidExtension) {
+    throw new ValidationError('Invalid image format. Supported: png, jpg, jpeg, gif, webp', imagePath);
+  }
+  
+  // Check if file exists (optional - may not exist in all contexts)
+  // This is a soft check - don't throw if file doesn't exist yet
+  // The actual file operations will handle missing files
+  
+  return true;
+}
+
+/**
+ * Validate prompt string
+ * 
+ * @param {string} prompt - Prompt text to validate
+ * @param {number} [maxLength=10000] - Maximum allowed length
+ * @returns {true} Always returns true if valid
+ * @throws {ValidationError} If prompt is invalid, empty, or too long
+ */
+export function validatePrompt(prompt, maxLength = 10000) {
+  if (typeof prompt !== 'string') {
+    throw new ValidationError('Prompt must be a string', null, {
+      received: typeof prompt
+    });
+  }
+  
+  if (prompt.length === 0) {
+    throw new ValidationError('Prompt cannot be empty');
+  }
+  
+  if (prompt.length > maxLength) {
+    throw new ValidationError(`Prompt too long (max ${maxLength} characters)`, null, {
+      length: prompt.length,
+      maxLength
+    });
+  }
+  
+  return true;
+}
+
+/**
+ * Validate context object
+ * 
+ * @param {unknown} context - Context object to validate (can be null/undefined)
+ * @param {number} [maxSize=50000] - Maximum serialized size in bytes
+ * @returns {true} Always returns true if valid
+ * @throws {ValidationError} If context is invalid type, too large, or non-serializable
+ */
+export function validateContext(context, maxSize = 50000) {
+  if (context === null || context === undefined) {
+    return true; // Context is optional
+  }
+  
+  if (typeof context !== 'object' || Array.isArray(context)) {
+    throw new ValidationError('Context must be an object', null, {
+      received: Array.isArray(context) ? 'array' : typeof context
+    });
+  }
+  
+  // Check size by stringifying
+  try {
+    const contextString = JSON.stringify(context);
+    if (contextString.length > maxSize) {
+      throw new ValidationError(`Context too large (max ${maxSize} bytes)`, null, {
+        size: contextString.length,
+        maxSize
+      });
+    }
+  } catch (error) {
+    if (error instanceof ValidationError) {
+      throw error;
+    }
+    throw new ValidationError('Context contains non-serializable data', null, {
+      originalError: error.message
+    });
+  }
+  
+  return true;
+}
+
+/**
+ * Validate timeout value
+ * 
+ * @param {number} timeout - Timeout value in milliseconds
+ * @param {number} [min=1000] - Minimum allowed timeout
+ * @param {number} [max=300000] - Maximum allowed timeout
+ * @returns {true} Always returns true if valid
+ * @throws {ValidationError} If timeout is invalid, too short, or too long
+ */
+export function validateTimeout(timeout, min = 1000, max = 300000) {
+  if (typeof timeout !== 'number') {
+    throw new ValidationError('Timeout must be a number', null, {
+      received: typeof timeout
+    });
+  }
+  
+  if (timeout < min) {
+    throw new ValidationError(`Timeout too short (min ${min}ms)`, null, {
+      timeout,
+      min
+    });
+  }
+  
+  if (timeout > max) {
+    throw new ValidationError(`Timeout too long (max ${max}ms)`, null, {
+      timeout,
+      max
+    });
+  }
+  
+  return true;
+}
+
+/**
+ * Validate schema object for data extraction
+ * 
+ * @param {unknown} schema - Schema object to validate
+ * @returns {true} Always returns true if valid
+ * @throws {ValidationError} If schema is invalid or has invalid field types
+ */
+export function validateSchema(schema) {
+  if (!schema || typeof schema !== 'object' || Array.isArray(schema)) {
+    throw new ValidationError('Schema must be a non-empty object', null, {
+      received: Array.isArray(schema) ? 'array' : typeof schema
+    });
+  }
+  
+  const validTypes = ['string', 'number', 'boolean', 'object', 'array'];
+  
+  for (const [key, field] of Object.entries(schema)) {
+    if (typeof field !== 'object' || Array.isArray(field)) {
+      throw new ValidationError(`Schema field "${key}" must be an object`, null, {
+        field
+      });
+    }
+    
+    if (!field.type || !validTypes.includes(field.type)) {
+      throw new ValidationError(`Schema field "${key}" has invalid type`, null, {
+        type: field.type,
+        validTypes
+      });
+    }
+  }
+  
+  return true;
+}
+
+/**
+ * Validate file path (general purpose)
+ * 
+ * @param {string} filePath - File path to validate
+ * @param {{
+   *   mustExist?: boolean;
+   *   allowedExtensions?: string[] | null;
+   *   maxLength?: number;
+   * }} [options={}] - Validation options
+ * @returns {true} Always returns true if valid
+ * @throws {ValidationError} If path is invalid, empty, too long, has path traversal, wrong extension, or doesn't exist (if required)
+ */
+export function validateFilePath(filePath, options = {}) {
+  const {
+    mustExist = false,
+    allowedExtensions = null,
+    maxLength = 4096
+  } = options;
+  
+  if (typeof filePath !== 'string') {
+    throw new ValidationError('File path must be a string', null, {
+      received: typeof filePath
+    });
+  }
+  
+  if (filePath.length === 0) {
+    throw new ValidationError('File path cannot be empty');
+  }
+  
+  if (filePath.length > maxLength) {
+    throw new ValidationError(`File path too long (max ${maxLength} characters)`, filePath);
+  }
+  
+  // Check for path traversal
+  const normalized = normalize(filePath);
+  if (normalized.includes('..')) {
+    throw new ValidationError('Invalid file path: path traversal detected', filePath);
+  }
+  
+  // Check extension if specified
+  if (allowedExtensions) {
+    const ext = normalized.substring(normalized.lastIndexOf('.'));
+    if (!allowedExtensions.includes(ext.toLowerCase())) {
+      throw new ValidationError(`Invalid file extension. Allowed: ${allowedExtensions.join(', ')}`, filePath);
+    }
+  }
+  
+  // Check existence if required
+  if (mustExist && !existsSync(filePath)) {
+    throw new ValidationError('File does not exist', filePath);
+  }
+  
+  return true;
+}
+