@arclabs561/ai-visual-test 0.5.1

package/src/cost-tracker.mjs

@@ -0,0 +1,257 @@
+/**
+ * Cost Tracking Utilities
+ * 
+ * Tracks API costs over time, provides cost estimates, and helps optimize spending.
+ */
+
+import { getCached, setCached } from './cache.mjs';
+
+/**
+ * Cost Tracker Class
+ * 
+ * Tracks costs across multiple validations and provides cost analysis.
+ */
+export class CostTracker {
+  constructor(options = {}) {
+    this.storageKey = options.storageKey || 'ai-visual-test-costs';
+    this.maxHistory = options.maxHistory || 1000;
+    this.costs = this.loadCosts();
+  }
+
+  /**
+   * Load costs from cache/storage
+   */
+  loadCosts() {
+    try {
+      const cached = getCached(this.storageKey, 'cost-tracker', {});
+      return cached || { history: [], totals: {}, byProvider: {} };
+    } catch {
+      return { history: [], totals: {}, byProvider: {} };
+    }
+  }
+
+  /**
+   * Save costs to cache/storage
+   */
+  saveCosts() {
+    try {
+      setCached(this.storageKey, 'cost-tracker', this.costs, {});
+    } catch {
+      // Silently fail if cache unavailable
+    }
+  }
+
+  /**
+   * Record a cost
+   * 
+   * @param {{
+   *   provider: string;
+   *   cost: number;
+   *   inputTokens?: number;
+   *   outputTokens?: number;
+   *   timestamp?: number;
+   *   testName?: string;
+   * }} costData - Cost data to record
+   */
+  recordCost(costData) {
+    const {
+      provider,
+      cost,
+      inputTokens = 0,
+      outputTokens = 0,
+      timestamp = Date.now(),
+      testName = 'unknown'
+    } = costData;
+
+    if (cost === null || cost === undefined) return; // Skip null costs
+
+    const entry = {
+      provider,
+      cost,
+      inputTokens,
+      outputTokens,
+      timestamp,
+      testName,
+      date: new Date(timestamp).toISOString().split('T')[0] // YYYY-MM-DD
+    };
+
+    // Add to history
+    this.costs.history.push(entry);
+
+    // Trim history if too long
+    if (this.costs.history.length > this.maxHistory) {
+      this.costs.history = this.costs.history.slice(-this.maxHistory);
+    }
+
+    // Update totals
+    this.costs.totals.total = (this.costs.totals.total || 0) + cost;
+    this.costs.totals.count = (this.costs.totals.count || 0) + 1;
+
+    // Update by provider
+    if (!this.costs.byProvider[provider]) {
+      this.costs.byProvider[provider] = { total: 0, count: 0, inputTokens: 0, outputTokens: 0 };
+    }
+    this.costs.byProvider[provider].total += cost;
+    this.costs.byProvider[provider].count += 1;
+    this.costs.byProvider[provider].inputTokens += inputTokens;
+    this.costs.byProvider[provider].outputTokens += outputTokens;
+
+    // Update by date
+    if (!this.costs.byDate) {
+      this.costs.byDate = {};
+    }
+    if (!this.costs.byDate[entry.date]) {
+      this.costs.byDate[entry.date] = { total: 0, count: 0 };
+    }
+    this.costs.byDate[entry.date].total += cost;
+    this.costs.byDate[entry.date].count += 1;
+
+    this.saveCosts();
+  }
+
+  /**
+   * Get cost statistics
+   * 
+   * @returns {{
+   *   total: number;
+   *   count: number;
+   *   average: number;
+   *   byProvider: Record<string, { total: number; count: number; average: number }>;
+   *   byDate: Record<string, { total: number; count: number }>;
+   *   recent: Array<{ provider: string; cost: number; timestamp: number }>;
+   * }} Cost statistics
+   */
+  getStats() {
+    const stats = {
+      total: this.costs.totals.total || 0,
+      count: this.costs.totals.count || 0,
+      average: 0,
+      byProvider: {},
+      byDate: this.costs.byDate || {},
+      recent: this.costs.history.slice(-10).map(e => ({
+        provider: e.provider,
+        cost: e.cost,
+        timestamp: e.timestamp,
+        testName: e.testName
+      }))
+    };
+
+    if (stats.count > 0) {
+      stats.average = stats.total / stats.count;
+    }
+
+    // Calculate averages by provider
+    for (const [provider, data] of Object.entries(this.costs.byProvider)) {
+      stats.byProvider[provider] = {
+        total: data.total,
+        count: data.count,
+        average: data.count > 0 ? data.total / data.count : 0,
+        inputTokens: data.inputTokens,
+        outputTokens: data.outputTokens
+      };
+    }
+
+    return stats;
+  }
+
+  /**
+   * Get cost projection
+   * 
+   * @param {number} [days=30] - Number of days to project
+   * @returns {{ projected: number; dailyAverage: number; trend: 'increasing' | 'decreasing' | 'stable' }} Projection
+   */
+  getProjection(days = 30) {
+    const stats = this.getStats();
+    const dailyAverage = stats.byDate ? 
+      Object.values(stats.byDate).reduce((sum, day) => sum + day.total, 0) / 
+      Math.max(Object.keys(stats.byDate).length, 1) : 0;
+    
+    const projected = dailyAverage * days;
+
+    // Simple trend detection (last 7 days vs previous 7 days)
+    const dates = Object.keys(stats.byDate || {}).sort().slice(-14);
+    let trend = 'stable';
+    if (dates.length >= 14) {
+      const recent = dates.slice(-7).reduce((sum, d) => sum + (stats.byDate[d]?.total || 0), 0);
+      const previous = dates.slice(0, 7).reduce((sum, d) => sum + (stats.byDate[d]?.total || 0), 0);
+      if (recent > previous * 1.1) trend = 'increasing';
+      else if (recent < previous * 0.9) trend = 'decreasing';
+    }
+
+    return { projected, dailyAverage, trend };
+  }
+
+  /**
+   * Check if cost exceeds threshold
+   * 
+   * @param {number} threshold - Cost threshold
+   * @returns {{ exceeded: boolean; current: number; remaining: number }} Threshold check
+   */
+  checkThreshold(threshold) {
+    const current = this.getStats().total;
+    return {
+      exceeded: current >= threshold,
+      current,
+      remaining: Math.max(0, threshold - current)
+    };
+  }
+
+  /**
+   * Reset cost tracking
+   */
+  reset() {
+    this.costs = { history: [], totals: {}, byProvider: {}, byDate: {} };
+    this.saveCosts();
+  }
+
+  /**
+   * Export cost data
+   * 
+   * @returns {object} Cost data for export
+   */
+  export() {
+    return {
+      ...this.costs,
+      stats: this.getStats(),
+      projection: this.getProjection(30)
+    };
+  }
+}
+
+/**
+ * Global cost tracker instance
+ */
+let globalCostTracker = null;
+
+/**
+ * Get or create global cost tracker
+ * 
+ * @param {object} [options] - Cost tracker options
+ * @returns {CostTracker} Cost tracker instance
+ */
+export function getCostTracker(options = {}) {
+  if (!globalCostTracker) {
+    globalCostTracker = new CostTracker(options);
+  }
+  return globalCostTracker;
+}
+
+/**
+ * Record cost (convenience function)
+ * 
+ * @param {object} costData - Cost data
+ */
+export function recordCost(costData) {
+  const tracker = getCostTracker();
+  tracker.recordCost(costData);
+}
+
+/**
+ * Get cost statistics (convenience function)
+ * 
+ * @returns {object} Cost statistics
+ */
+export function getCostStats() {
+  return getCostTracker().getStats();
+}
+

package/src/cross-modal-consistency.mjs

@@ -0,0 +1,170 @@
+/**
+ * Cross-Modal Consistency Checker
+ * 
+ * Validates consistency between screenshot (visual) and HTML/CSS (structural).
+ * Detects mismatches between what's shown visually and what the code structure indicates.
+ * 
+ * Research:
+ * - "Cross-Modal Consistency in Multimodal Large Language Models" - Consistency issues in GPT-4V
+ * - "Verifying Cross-modal Entity Consistency in News using Vision-language Models" (LVLM4CEC)
+ * - "Hallucination Detection in Vision-Language Models" - Multiple papers on detecting unfaithful outputs
+ * 
+ * Key findings: Consistency is a major challenge. Entity verification is critical.
+ * Hallucination detection is needed. Modality gap contributes to issues.
+ * 
+ * Note: This implementation uses heuristic-based checking. Full research implementation
+ * would use VLLM-based verification and entity-level consistency checking as in LVLM4CEC.
+ */
+
+import { warn, log } from './logger.mjs';
+
+/**
+ * Check consistency between screenshot and HTML/CSS
+ * 
+ * @param {Object} options - Consistency check options
+ * @param {string} [options.screenshot] - Screenshot path
+ * @param {Object} [options.renderedCode] - Rendered code (HTML/CSS/DOM)
+ * @param {Object} [options.gameState] - Game state
+ * @param {Object} [options.pageState] - Page state
+ * @param {boolean} [options.strict=false] - Strict mode (warn on all inconsistencies)
+ * @returns {Object} Consistency check result
+ */
+export function checkCrossModalConsistency(options = {}) {
+  const {
+    screenshot,
+    renderedCode,
+    gameState,
+    pageState,
+    strict = false
+  } = options;
+
+  const issues = [];
+  const warnings = [];
+  const checks = {
+    hasScreenshot: !!screenshot,
+    hasRenderedCode: !!renderedCode,
+    hasHTML: !!renderedCode?.html,
+    hasCSS: !!renderedCode?.criticalCSS,
+    hasDOM: !!renderedCode?.domStructure,
+    hasGameState: !!gameState,
+    hasPageState: !!pageState
+  };
+
+  // Check 1: Basic presence
+  if (!screenshot && !renderedCode) {
+    issues.push('Missing both screenshot and rendered code - cannot check consistency');
+    return {
+      isConsistent: false,
+      issues,
+      warnings,
+      checks,
+      score: 0
+    };
+  }
+
+  // Check 2: HTML structure vs visual expectations
+  if (renderedCode?.html && renderedCode?.domStructure) {
+    // Check if key elements exist in DOM but might not be visible
+    const domElements = Object.keys(renderedCode.domStructure);
+    if (domElements.length === 0) {
+      warnings.push('DOM structure is empty - may indicate missing elements');
+    }
+
+    // Check for hidden elements that should be visible
+    if (renderedCode.domStructure.prideParade && !renderedCode.domStructure.prideParade.exists) {
+      warnings.push('Pride parade element missing from DOM structure');
+    }
+  }
+
+  // Check 3: CSS positioning vs visual layout
+  if (renderedCode?.criticalCSS) {
+    const cssElements = Object.keys(renderedCode.criticalCSS);
+    
+    // Check for positioning issues
+    for (const [selector, styles] of Object.entries(renderedCode.criticalCSS)) {
+      if (styles.position === 'absolute' && (styles.top === 'auto' || styles.left === 'auto')) {
+        warnings.push(`Element '${selector}' has absolute positioning but auto top/left - may cause layout issues`);
+      }
+      
+      if (styles.display === 'none' && selector.includes('game')) {
+        warnings.push(`Game element '${selector}' has display:none - may not be visible`);
+      }
+      
+      if (styles.visibility === 'hidden' && selector.includes('game')) {
+        warnings.push(`Game element '${selector}' has visibility:hidden - may not be visible`);
+      }
+    }
+  }
+
+  // Check 4: Game state vs visual display
+  if (gameState && renderedCode?.domStructure) {
+    // Check if game state indicates active game but DOM doesn't show game elements
+    if (gameState.gameActive && !renderedCode.domStructure.game) {
+      warnings.push('Game state indicates active game but game elements not found in DOM');
+    }
+
+    // Check score consistency (if score is in game state and visible in DOM)
+    if (gameState.score !== undefined) {
+      // This would require VLLM to extract score from screenshot
+      // For now, just note that we should check this
+      if (strict) {
+        warnings.push('Game state has score - should verify it matches visual display');
+      }
+    }
+  }
+
+  // Check 5: Page state vs rendered code
+  if (pageState && renderedCode?.html) {
+    // Check if page title matches
+    if (pageState.title && !renderedCode.html.includes(pageState.title)) {
+      warnings.push(`Page title '${pageState.title}' not found in HTML`);
+    }
+  }
+
+  // Calculate consistency score
+  const totalChecks = Object.values(checks).filter(Boolean).length;
+  const issueCount = issues.length;
+  const warningCount = warnings.length;
+  const maxIssues = 5; // Normalize to 0-1 scale
+  const consistencyScore = Math.max(0, 1 - (issueCount / maxIssues) - (warningCount / (maxIssues * 2)));
+
+  const isConsistent = issueCount === 0 && (strict ? warningCount === 0 : true);
+
+  // Log warnings if any
+  if (warnings.length > 0) {
+    log(`[Cross-Modal Consistency] ${warnings.length} warning(s):`, warnings);
+  }
+
+  if (issues.length > 0) {
+    warn(`[Cross-Modal Consistency] ${issues.length} issue(s):`, issues);
+  }
+
+  return {
+    isConsistent,
+    issues,
+    warnings,
+    checks,
+    score: consistencyScore,
+    summary: issueCount === 0 && warningCount === 0
+      ? 'All consistency checks passed'
+      : `${issueCount} issue(s), ${warningCount} warning(s)`
+  };
+}
+
+/**
+ * Validate experience consistency (convenience function)
+ * 
+ * @param {Object} experience - Experience object from experiencePageAsPersona
+ * @param {Object} [options={}] - Validation options
+ * @returns {Object} Consistency check result
+ */
+export function validateExperienceConsistency(experience, options = {}) {
+  return checkCrossModalConsistency({
+    screenshot: experience.screenshots?.[0]?.path,
+    renderedCode: experience.renderedCode,
+    gameState: experience.pageState?.gameState || experience.gameState,
+    pageState: experience.pageState,
+    ...options
+  });
+}
+

package/src/data-extractor.mjs

@@ -0,0 +1,232 @@
+/**
+ * Structured Data Extractor
+ * 
+ * Extracts structured data from VLLM responses using multiple strategies:
+ * - JSON parsing (if response contains JSON)
+ * - LLM extraction (if LLM is available)
+ * - Regex fallback (simple patterns)
+ * 
+ * General-purpose utility - no domain-specific logic.
+ */
+
+import { createConfig } from './config.mjs';
+import { loadEnv } from './load-env.mjs';
+import { warn } from './logger.mjs';
+// Load env before LLM utils
+loadEnv();
+// Use shared LLM utility library for text-only calls (optional dependency)
+// Note: This module uses Claude Sonnet (advanced tier) for data extraction
+// which requires higher quality than simple validation tasks
+// Import is handled dynamically to make it optional
+
+/**
+ * Extract structured data from text using multiple strategies
+ * 
+ * @param {string} text - Text to extract data from
+ * @param {Record<string, { type: string; [key: string]: unknown }>} schema - Schema definition for extraction
+ * @param {{
+ *   method?: 'json' | 'llm' | 'regex';
+ *   provider?: string;
+ *   apiKey?: string;
+ *   fallback?: 'llm' | 'regex' | 'json' | 'auto';
+ * }} [options={}] - Extraction options
+ * @returns {Promise<unknown>} Extracted structured data matching schema, or null if extraction fails
+ */
+export async function extractStructuredData(text, schema, options = {}) {
+  if (!text) return null;
+  
+  const {
+    fallback = 'llm', // 'llm' | 'regex' | 'json'
+    provider = null
+  } = options;
+  
+  // Strategy 1: Try JSON parsing first (fastest)
+  try {
+    const jsonMatch = text.match(/\{[\s\S]*\}/);
+    if (jsonMatch) {
+      const parsed = JSON.parse(jsonMatch[0]);
+      // Validate against schema
+      if (validateSchema(parsed, schema)) {
+        return parsed;
+      }
+    }
+  } catch (error) {
+    // JSON parsing failed, try next strategy
+  }
+  
+  // Strategy 2: Try LLM extraction (if available and requested)
+  if (fallback === 'llm' || fallback === 'auto') {
+    try {
+      const config = createConfig({ provider });
+      if (config.enabled) {
+        const extracted = await extractWithLLM(text, schema, config);
+        if (extracted) {
+          return extracted;
+        }
+      }
+    } catch (error) {
+      warn(`[DataExtractor] LLM extraction failed: ${error.message}`);
+    }
+  }
+  
+  // Strategy 3: Try regex fallback
+  if (fallback === 'regex' || fallback === 'auto') {
+    try {
+      const extracted = extractWithRegex(text, schema);
+      if (extracted) {
+        return extracted;
+      }
+    } catch (error) {
+      warn(`[DataExtractor] Regex extraction failed: ${error.message}`);
+    }
+  }
+  
+  return null;
+}
+
+/**
+ * Extract structured data using LLM
+ */
+async function extractWithLLM(text, schema, config) {
+  const prompt = `Extract structured data from the following text. Return ONLY valid JSON matching this schema:
+
+Schema:
+${JSON.stringify(schema, null, 2)}
+
+Text to extract from:
+${text}
+
+Return ONLY the JSON object, no other text.`;
+
+  try {
+    const response = await callLLMForExtraction(prompt, config);
+    // Try to extract JSON from response
+    let parsed;
+    try {
+      const llmUtils = await import('@arclabs561/llm-utils');
+      parsed = llmUtils.extractJSON(response);
+    } catch (error) {
+      // Fallback: try to parse as JSON directly
+      const jsonMatch = response.match(/\{[\s\S]*\}/);
+      if (jsonMatch) {
+        parsed = JSON.parse(jsonMatch[0]);
+      } else {
+        throw new Error('Could not extract JSON from response');
+      }
+    }
+    if (parsed && validateSchema(parsed, schema)) {
+      return parsed;
+    }
+  } catch (error) {
+    warn(`[DataExtractor] LLM extraction error: ${error.message}`);
+  }
+  
+  return null;
+}
+
+/**
+ * Call LLM API (text-only, no vision)
+ * Uses shared utility with advanced tier for better extraction quality
+ */
+async function callLLMForExtraction(prompt, config) {
+  const apiKey = config.apiKey;
+  const provider = config.provider || 'gemini';
+  
+  // Try to use optional llm-utils library if available
+  try {
+    const llmUtils = await import('@arclabs561/llm-utils');
+    const callLLMUtil = llmUtils.callLLM;
+    // Use advanced tier for data extraction (needs higher quality)
+    return await callLLMUtil(prompt, provider, apiKey, {
+      tier: 'advanced', // Data extraction benefits from better models
+      temperature: 0.1,
+      maxTokens: 1000,
+    });
+  } catch (error) {
+    // Fallback: use local implementation or throw
+    throw new Error(`LLM extraction requires @arclabs561/llm-utils package: ${error.message}`);
+  }
+}
+
+/**
+ * Escape special regex characters to prevent ReDoS
+ */
+function escapeRegex(str) {
+  return str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+
+/**
+ * Extract structured data using regex patterns
+ */
+function extractWithRegex(text, schema) {
+  const result = {};
+  
+  for (const [key, field] of Object.entries(schema)) {
+    const { type, required = false } = field;
+    
+    // Escape key to prevent ReDoS attacks
+    const escapedKey = escapeRegex(key);
+    
+    // Try to find value in text
+    let value = null;
+    
+    if (type === 'number') {
+      // Look for number patterns
+      const match = text.match(new RegExp(`${escapedKey}[\\s:=]+([0-9.]+)`, 'i'));
+      if (match) {
+        value = parseFloat(match[1]);
+      }
+    } else if (type === 'string') {
+      // Look for string patterns
+      const match = text.match(new RegExp(`${escapedKey}[\\s:=]+([^\\n,]+)`, 'i'));
+      if (match) {
+        value = match[1].trim();
+      }
+    } else if (type === 'boolean') {
+      // Look for boolean patterns
+      const match = text.match(new RegExp(`${escapedKey}[\\s:=]+(true|false|yes|no)`, 'i'));
+      if (match) {
+        value = match[1].toLowerCase() === 'true' || match[1].toLowerCase() === 'yes';
+      }
+    }
+    
+    if (value !== null) {
+      result[key] = value;
+    } else if (required) {
+      // Required field not found
+      return null;
+    }
+  }
+  
+  return Object.keys(result).length > 0 ? result : null;
+}
+
+/**
+ * Validate extracted data against schema
+ */
+function validateSchema(data, schema) {
+  if (!data || typeof data !== 'object') return false;
+  
+  for (const [key, field] of Object.entries(schema)) {
+    const { type, required = false } = field;
+    
+    if (required && !(key in data)) {
+      return false;
+    }
+    
+    if (key in data) {
+      const value = data[key];
+      
+      if (type === 'number' && typeof value !== 'number') {
+        return false;
+      } else if (type === 'string' && typeof value !== 'string') {
+        return false;
+      } else if (type === 'boolean' && typeof value !== 'boolean') {
+        return false;
+      }
+    }
+  }
+  
+  return true;
+}
+