@arclabs561/ai-visual-test 0.5.1

package/src/latency-aware-batch-optimizer.mjs

@@ -0,0 +1,192 @@
+/**
+ * Latency-Aware Batch Optimizer
+ *
+ * Adaptive batching that considers latency requirements for fast reactive games.
+ *
+ * Key features:
+ * - Bypasses batching for critical requests (<100ms requirement)
+ * - Adaptive batch sizing based on latency requirements
+ * - Deadline-based scheduling
+ * - Priority queue for fast games
+ */
+
+import { BatchOptimizer } from './batch-optimizer.mjs';
+
+/**
+ * Latency-Aware Batch Optimizer
+ *
+ * Extends BatchOptimizer with latency awareness for fast reactive games.
+ */
+export class LatencyAwareBatchOptimizer extends BatchOptimizer {
+  /**
+   * @param {{
+   *   maxConcurrency?: number;
+   *   batchSize?: number;
+   *   cacheEnabled?: boolean;
+   *   defaultMaxLatency?: number;
+   *   adaptiveBatchSize?: boolean;
+   * }} [options={}] - Optimizer options
+   */
+  constructor(options = {}) {
+    super(options);
+    this.defaultMaxLatency = options.defaultMaxLatency || 1000; // Default 1 second
+    this.adaptiveBatchSize = options.adaptiveBatchSize !== false;
+    this.criticalRequests = new Set(); // Track critical requests
+  }
+
+  /**
+   * Add request with latency requirement
+   *
+   * @param {string} imagePath - Screenshot path
+   * @param {string} prompt - Validation prompt
+   * @param {import('./index.mjs').ValidationContext} [context={}] - Validation context
+   * @param {number} [maxLatency=null] - Maximum acceptable latency in ms (null = use default)
+   * @returns {Promise<import('./index.mjs').ValidationResult>} Validation result
+   */
+  async addRequest(imagePath, prompt, context = {}, maxLatency = null) {
+    const latencyRequirement = maxLatency || context.maxLatency || this.defaultMaxLatency;
+    const isCritical = latencyRequirement < 200; // Critical if <200ms
+
+    if (isCritical) {
+      this.criticalRequests.add(imagePath);
+    }
+
+    // If latency requirement is very tight, bypass batching
+    if (latencyRequirement < 100) {
+      // Process immediately, no batching
+      return this._processRequest(imagePath, prompt, {
+        ...context,
+        maxLatency: latencyRequirement,
+        critical: true
+      });
+    }
+
+    // For slightly less critical requests, use adaptive batch size
+    if (this.adaptiveBatchSize && latencyRequirement < 200) {
+      // Use smaller batch size for fast games
+      const originalBatchSize = this.batchSize;
+      this.batchSize = 1; // Process one at a time for very fast games
+
+      try {
+        return await this._queueRequest(imagePath, prompt, {
+          ...context,
+          maxLatency: latencyRequirement,
+          critical: isCritical
+        });
+      } finally {
+        this.batchSize = originalBatchSize;
+      }
+    }
+
+    // For normal requests, use standard batching
+    return this._queueRequest(imagePath, prompt, {
+      ...context,
+      maxLatency: latencyRequirement
+    });
+  }
+
+  /**
+   * Process queue with latency awareness
+   */
+  async _processQueue() {
+    if (this.processing || this.queue.length === 0 || this.activeRequests >= this.maxConcurrency) {
+      return;
+    }
+
+    this.processing = true;
+
+    try {
+      // Sort queue by latency requirement (critical first)
+      const sortedQueue = [...this.queue].sort((a, b) => {
+        const latencyA = a.context?.maxLatency || this.defaultMaxLatency;
+        const latencyB = b.context?.maxLatency || this.defaultMaxLatency;
+
+        // Critical requests (low latency) come first
+        if (latencyA < latencyB) return -1;
+        if (latencyA > latencyB) return 1;
+
+        // If same latency, process in order
+        return 0;
+      });
+
+      while (sortedQueue.length > 0 && this.activeRequests < this.maxConcurrency) {
+        // Calculate adaptive batch size based on latency requirements
+        const batchSize = this.adaptiveBatchSize
+          ? this._calculateAdaptiveBatchSize(sortedQueue)
+          : this.batchSize;
+
+        const batch = sortedQueue.splice(0, batchSize);
+
+        // Remove from original queue
+        batch.forEach(item => {
+          const index = this.queue.findIndex(q => q.imagePath === item.imagePath);
+          if (index >= 0) this.queue.splice(index, 1);
+        });
+
+        // Process batch
+        const promises = batch.map(async ({ imagePath, prompt, context, validateFn, resolve, reject }) => {
+          try {
+            // Check cache
+            if (this.cache) {
+              const cacheKey = this._getCacheKey(imagePath, prompt, context);
+              if (this.cache.has(cacheKey)) {
+                resolve(this.cache.get(cacheKey));
+                return;
+              }
+            }
+
+            const result = await this._processRequest(imagePath, prompt, context, validateFn);
+            resolve(result);
+          } catch (error) {
+            reject(error);
+          } finally {
+            // Remove from critical requests if processed
+            this.criticalRequests.delete(imagePath);
+          }
+        });
+
+        await Promise.allSettled(promises);
+      }
+    } finally {
+      this.processing = false;
+    }
+  }
+
+  /**
+   * Calculate adaptive batch size based on latency requirements
+   */
+  _calculateAdaptiveBatchSize(queue) {
+    if (queue.length === 0) return this.batchSize;
+
+    // Get latency requirement of first request
+    const firstLatency = queue[0].context?.maxLatency || this.defaultMaxLatency;
+
+    // Very fast games (<100ms) - no batching
+    if (firstLatency < 100) return 1;
+
+    // Fast games (<200ms) - small batches
+    if (firstLatency < 200) return 2;
+
+    // Normal games - standard batch size
+    return this.batchSize;
+  }
+
+  /**
+   * Get latency-aware statistics
+   */
+  getLatencyStats() {
+    return {
+      ...this.getCacheStats(),
+      criticalRequests: this.criticalRequests.size,
+      queueLatencyRequirements: this.queue.map(q => ({
+        imagePath: q.imagePath,
+        maxLatency: q.context?.maxLatency || this.defaultMaxLatency,
+        critical: (q.context?.maxLatency || this.defaultMaxLatency) < 200
+      }))
+    };
+  }
+}
+
+
+
+

package/src/load-env.mjs

@@ -0,0 +1,159 @@
+/**
+ * Environment Variable Loader
+ * 
+ * Loads environment variables from .env file if it exists.
+ * Works in both local development and deployed environments.
+ */
+
+import { readFileSync, existsSync } from 'fs';
+import { join, dirname } from 'path';
+import { fileURLToPath } from 'url';
+import { warn } from './logger.mjs';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+
+// SECURITY: Whitelist allowed environment variable keys to prevent injection
+// Only allow keys that are actually used by this application
+const ALLOWED_ENV_KEYS = [
+  'GEMINI_API_KEY',
+  'OPENAI_API_KEY',
+  'ANTHROPIC_API_KEY',
+  'GROQ_API_KEY',  // Added for Groq integration (high-frequency decisions)
+  'API_KEY',
+  'VLLM_API_KEY',
+  'VLM_PROVIDER',
+  'VLM_MODEL',
+  'VLM_MODEL_TIER',
+  'RATE_LIMIT_MAX_REQUESTS',
+  'REQUIRE_AUTH'
+];
+
+// Valid values for VLM_PROVIDER
+// Groq added for high-frequency decisions (10-60Hz temporal decisions)
+const VALID_PROVIDERS = ['gemini', 'openai', 'claude', 'groq'];
+
+// Validation functions for environment variables
+function validateRateLimitMaxRequests(value) {
+  const num = parseInt(value, 10);
+  if (isNaN(num) || num < 1 || num > 1000) {
+    warn(`[LoadEnv] Invalid RATE_LIMIT_MAX_REQUESTS: ${value}. Must be between 1 and 1000. Using default.`);
+    return null; // Will use default
+  }
+  return num;
+}
+
+function validateVLMProvider(value) {
+  const normalized = value?.toLowerCase().trim();
+  if (normalized && !VALID_PROVIDERS.includes(normalized)) {
+    warn(`[LoadEnv] Invalid VLM_PROVIDER: ${value}. Must be one of: ${VALID_PROVIDERS.join(', ')}. Ignoring.`);
+    return null; // Ignore invalid provider
+  }
+  return normalized;
+}
+
+function validateRequireAuth(value) {
+  if (value === 'true' || value === '1' || value === 'yes') {
+    return true;
+  }
+  if (value === 'false' || value === '0' || value === 'no' || value === '') {
+    return false;
+  }
+  warn(`[LoadEnv] Invalid REQUIRE_AUTH: ${value}. Must be 'true' or 'false'. Using default.`);
+  return null; // Will use default
+}
+
+/**
+ * Load environment variables from .env file
+ * 
+ * @param {string | null} [basePath=null] - Base path to search for .env file (optional)
+ * @returns {boolean} True if .env file was found and loaded
+ */
+export function loadEnv(basePath = null) {
+  // Try multiple locations for .env file
+  const possiblePaths = basePath 
+    ? [
+        join(basePath, '.env'),
+        join(basePath, '..', '.env'),
+        join(basePath, '../..', '.env')
+      ]
+    : [
+        join(process.cwd(), '.env'),
+        join(__dirname, '..', '.env'),
+        join(__dirname, '../../..', '.env'),
+        join(__dirname, '../../../..', '.env')
+      ];
+
+  for (const envPath of possiblePaths) {
+    if (existsSync(envPath)) {
+      try {
+        const envContent = readFileSync(envPath, 'utf8');
+        const lines = envContent.split('\n');
+        
+        for (const line of lines) {
+          const trimmed = line.trim();
+          
+          // Skip comments and empty lines
+          if (!trimmed || trimmed.startsWith('#')) {
+            continue;
+          }
+          
+          // Parse KEY=VALUE format
+          const match = trimmed.match(/^([^=]+)=(.*)$/);
+          if (match) {
+            const key = match[1].trim();
+            let value = match[2].trim();
+            
+            // SECURITY: Only allow whitelisted environment variable keys
+            // Prevents malicious .env files from setting arbitrary variables
+            if (!ALLOWED_ENV_KEYS.includes(key)) {
+              warn(`[LoadEnv] Ignoring unknown environment variable key: ${key}`);
+              continue;
+            }
+            
+            // Remove quotes if present
+            if ((value.startsWith('"') && value.endsWith('"')) ||
+                (value.startsWith("'") && value.endsWith("'"))) {
+              value = value.slice(1, -1);
+            }
+            
+            // Validate and transform values based on key
+            let validatedValue = value;
+            if (key === 'RATE_LIMIT_MAX_REQUESTS') {
+              const validated = validateRateLimitMaxRequests(value);
+              if (validated === null) {
+                continue; // Skip invalid value
+              }
+              validatedValue = String(validated);
+            } else if (key === 'VLM_PROVIDER') {
+              const validated = validateVLMProvider(value);
+              if (validated === null) {
+                continue; // Skip invalid value
+              }
+              validatedValue = validated;
+            } else if (key === 'REQUIRE_AUTH') {
+              const validated = validateRequireAuth(value);
+              if (validated === null) {
+                continue; // Skip invalid value
+              }
+              validatedValue = String(validated);
+            }
+            
+            // Only set if not already set (env vars take precedence)
+            if (!process.env[key]) {
+              process.env[key] = validatedValue;
+            }
+          }
+        }
+        
+        return true;
+      } catch (err) {
+        // Silently fail - .env is optional
+        return false;
+      }
+    }
+  }
+  
+  return false;
+}
+

package/src/logger.mjs

@@ -0,0 +1,55 @@
+/**
+ * Simple logger utility
+ * 
+ * Provides conditional logging that respects debug mode.
+ * In production, warnings are silent unless explicitly enabled.
+ */
+
+let DEBUG_ENABLED = false;
+
+/**
+ * Enable debug logging
+ */
+export function enableDebug() {
+  DEBUG_ENABLED = true;
+}
+
+/**
+ * Disable debug logging
+ */
+export function disableDebug() {
+  DEBUG_ENABLED = false;
+}
+
+/**
+ * Check if debug is enabled
+ */
+export function isDebugEnabled() {
+  return DEBUG_ENABLED;
+}
+
+/**
+ * Log a warning (only if debug enabled)
+ */
+export function warn(...args) {
+  if (DEBUG_ENABLED) {
+    console.warn(...args);
+  }
+}
+
+/**
+ * Log info (only if debug enabled)
+ */
+export function log(...args) {
+  if (DEBUG_ENABLED) {
+    console.log(...args);
+  }
+}
+
+/**
+ * Log error (always logged)
+ */
+export function error(...args) {
+  console.error(...args);
+}
+

package/src/metrics.mjs

@@ -0,0 +1,187 @@
+/**
+ * Evaluation Metrics
+ * 
+ * Provides comprehensive metrics for evaluation results, including:
+ * - Spearman's rank correlation (for ordinal ratings)
+ * - Pearson's correlation
+ * - Agreement metrics
+ * - Rank-based metrics
+ * 
+ * Research: Spearman's ρ is more appropriate than Pearson's r for LLM evaluation
+ * (arXiv:2506.02945).
+ */
+
+/**
+ * Calculate Spearman's rank correlation coefficient
+ * 
+ * @param {Array<number>} x - First set of values
+ * @param {Array<number>} y - Second set of values
+ * @returns {number | null} Spearman's ρ (rho), or null if insufficient data
+ */
+export function spearmanCorrelation(x, y) {
+  if (x.length !== y.length || x.length < 2) {
+    return null;
+  }
+  
+  // Remove pairs with null/undefined values
+  const pairs = x.map((xi, i) => [xi, y[i]])
+    .filter(([xi, yi]) => xi != null && yi != null);
+  
+  if (pairs.length < 2) {
+    return null;
+  }
+  
+  const xValues = pairs.map(p => p[0]);
+  const yValues = pairs.map(p => p[1]);
+  
+  // Rank the values
+  const xRanks = rank(xValues);
+  const yRanks = rank(yValues);
+  
+  // Calculate Pearson correlation on ranks
+  return pearsonCorrelation(xRanks, yRanks);
+}
+
+/**
+ * Calculate Pearson's correlation coefficient
+ * 
+ * @param {Array<number>} x - First set of values
+ * @param {Array<number>} y - Second set of values
+ * @returns {number | null} Pearson's r, or null if insufficient data
+ */
+export function pearsonCorrelation(x, y) {
+  if (x.length !== y.length || x.length < 2) {
+    return null;
+  }
+  
+  const n = x.length;
+  const xMean = x.reduce((a, b) => a + b, 0) / n;
+  const yMean = y.reduce((a, b) => a + b, 0) / n;
+  
+  let numerator = 0;
+  let xVariance = 0;
+  let yVariance = 0;
+  
+  for (let i = 0; i < n; i++) {
+    const xDiff = x[i] - xMean;
+    const yDiff = y[i] - yMean;
+    numerator += xDiff * yDiff;
+    xVariance += xDiff * xDiff;
+    yVariance += yDiff * yDiff;
+  }
+  
+  const denominator = Math.sqrt(xVariance * yVariance);
+  
+  if (denominator === 0) {
+    return null; // No variance
+  }
+  
+  return numerator / denominator;
+}
+
+/**
+ * Rank values (handle ties by averaging)
+ * 
+ * @param {Array<number>} values - Values to rank
+ * @returns {Array<number>} Ranks (1-indexed)
+ */
+function rank(values) {
+  const indexed = values.map((v, i) => ({ value: v, index: i }));
+  indexed.sort((a, b) => a.value - b.value);
+  
+  const ranks = new Array(values.length);
+  let currentRank = 1;
+  
+  for (let i = 0; i < indexed.length; i++) {
+    // Check for ties
+    let tieCount = 1;
+    let tieSum = currentRank;
+    
+    while (i + tieCount < indexed.length && 
+           indexed[i].value === indexed[i + tieCount].value) {
+      tieSum += currentRank + tieCount;
+      tieCount++;
+    }
+    
+    // Average rank for ties
+    const avgRank = tieSum / tieCount;
+    
+    for (let j = 0; j < tieCount; j++) {
+      ranks[indexed[i + j].index] = avgRank;
+    }
+    
+    i += tieCount - 1;
+    currentRank += tieCount;
+  }
+  
+  return ranks;
+}
+
+/**
+ * Calculate agreement between two rankings
+ * 
+ * @param {Array<number>} ranking1 - First ranking (indices or scores)
+ * @param {Array<number>} ranking2 - Second ranking (indices or scores)
+ * @returns {{
+ *   spearman: number | null;
+ *   pearson: number | null;
+ *   kendall: number | null;
+ *   exactMatches: number;
+ *   totalItems: number;
+ * }} Agreement metrics
+ */
+export function calculateRankAgreement(ranking1, ranking2) {
+  const spearman = spearmanCorrelation(ranking1, ranking2);
+  const pearson = pearsonCorrelation(ranking1, ranking2);
+  const kendall = kendallTau(ranking1, ranking2);
+  
+  // Count exact matches
+  const exactMatches = ranking1.filter((r1, i) => r1 === ranking2[i]).length;
+  
+  return {
+    spearman,
+    pearson,
+    kendall,
+    exactMatches,
+    totalItems: ranking1.length,
+    agreementRate: exactMatches / ranking1.length
+  };
+}
+
+/**
+ * Calculate Kendall's tau (rank correlation)
+ * 
+ * @param {Array<number>} x - First ranking
+ * @param {Array<number>} y - Second ranking
+ * @returns {number | null} Kendall's τ, or null if insufficient data
+ */
+function kendallTau(x, y) {
+  if (x.length !== y.length || x.length < 2) {
+    return null;
+  }
+  
+  let concordant = 0;
+  let discordant = 0;
+  
+  for (let i = 0; i < x.length; i++) {
+    for (let j = i + 1; j < x.length; j++) {
+      const xOrder = x[i] - x[j];
+      const yOrder = y[i] - y[j];
+      
+      if (xOrder * yOrder > 0) {
+        concordant++;
+      } else if (xOrder * yOrder < 0) {
+        discordant++;
+      }
+      // Ties (xOrder === 0 or yOrder === 0) are ignored
+    }
+  }
+  
+  const total = concordant + discordant;
+  if (total === 0) {
+    return null;
+  }
+  
+  return (concordant - discordant) / total;
+}
+