@arclabs561/ai-visual-test 0.5.1 → 0.7.3

package/src/utils/capability-stratifier.mjs

@@ -0,0 +1,108 @@
+/**
+ * Capability Stratifier
+ * 
+ * Tests VLLM capabilities at different levels (low/mid/high)
+ * 
+ * Research context:
+ * - VLMs exhibit widespread deficits in low- and mid-level visual abilities
+ * - High-level object recognition performance cannot predict low-level capabilities
+ * - Need stratified testing to identify capability gaps
+ */
+
+import { validateScreenshot } from '../judge.mjs';
+
+/**
+ * Test capability at specific level
+ * 
+ * @param {string} level - 'low', 'mid', or 'high'
+ * @param {Array<{imagePath: string, prompt: string, expected: any}>} testCases
+ * @param {Object} options - Test options
+ * @returns {Promise<Object>} Capability test result
+ */
+export async function testCapabilityLevel(level, testCases, options = {}) {
+  const results = await Promise.all(
+    testCases.map(async (tc) => {
+      const result = await validateScreenshot(tc.imagePath, tc.prompt, {
+        testType: `capability-${level}`,
+        ...options
+      });
+
+      const extractedValue = result.extractedValue || result.score;
+      const correct = extractedValue === tc.expected ||
+                     (typeof extractedValue === 'number' && typeof tc.expected === 'number' &&
+                      Math.abs(extractedValue - tc.expected) < 0.1);
+
+      return {
+        testCase: tc,
+        result,
+        correct,
+        extractedValue,
+        expected: tc.expected
+      };
+    })
+  );
+
+  const accuracy = results.filter(r => r.correct).length / results.length;
+
+  return {
+    level,
+    accuracy,
+    total: results.length,
+    correct: results.filter(r => r.correct).length,
+    results,
+    recommendation: accuracy < 0.7
+      ? `Low ${level}-level capability accuracy. VLLM may struggle with ${level}-level visual tasks.`
+      : `${level}-level capability appears adequate.`
+  };
+}
+
+/**
+ * Stratified capability testing (all levels)
+ * 
+ * @param {Object} testSuites - {low: [...], mid: [...], high: [...]}
+ * @param {Object} options - Test options
+ * @returns {Promise<Object>} Stratified test results
+ */
+export async function testStratifiedCapabilities(testSuites, options = {}) {
+  const levels = ['low', 'mid', 'high'];
+  const results = {};
+
+  for (const level of levels) {
+    if (testSuites[level] && testSuites[level].length > 0) {
+      results[level] = await testCapabilityLevel(level, testSuites[level], options);
+    }
+  }
+
+  // Detect gaps (high-level >0.9 but low-level <0.7)
+  const gaps = [];
+  if (results.high && results.low) {
+    if (results.high.accuracy > 0.9 && results.low.accuracy < 0.7) {
+      gaps.push({
+        type: 'high-low-gap',
+        highAccuracy: results.high.accuracy,
+        lowAccuracy: results.low.accuracy,
+        recommendation: 'High-level performance does not predict low-level capabilities. Validate low-level tasks separately.'
+      });
+    }
+  }
+
+  if (results.high && results.mid) {
+    if (results.high.accuracy > 0.9 && results.mid.accuracy < 0.7) {
+      gaps.push({
+        type: 'high-mid-gap',
+        highAccuracy: results.high.accuracy,
+        midAccuracy: results.mid.accuracy,
+        recommendation: 'High-level performance does not predict mid-level capabilities. Validate mid-level tasks separately.'
+      });
+    }
+  }
+
+  return {
+    results,
+    gaps,
+    overallRecommendation: gaps.length > 0
+      ? 'Capability gaps detected. High-level performance cannot predict low/mid-level capabilities.'
+      : 'Capability levels appear consistent.'
+  };
+}
+

package/src/utils/counterfactual-tester.mjs

@@ -0,0 +1,83 @@
+/**
+ * Counterfactual Tester
+ * 
+ * Tests whether VLLM uses visual analysis vs. memorized knowledge
+ * 
+ * Research context:
+ * - VLMs achieve only 58.57% accuracy on basic visual tasks
+ * - When counterfactual images contradict training data, accuracy drops to 17.05%
+ * - 75.70% of errors are bias-aligned rather than random
+ * 
+ * This utility helps detect when VLLM is using memorization vs. visual analysis
+ */
+
+import { validateScreenshot } from '../judge.mjs';
+
+/**
+ * Test counterfactual scenario
+ * 
+ * @param {string} imagePath - Path to counterfactual image
+ * @param {string} prompt - Question about the image
+ * @param {any} expectedMemorized - What memorized knowledge would predict
+ * @param {any} expectedVisual - What visual analysis should find
+ * @param {Object} options - Test options
+ * @returns {Promise<Object>} Test result
+ */
+export async function testCounterfactual(imagePath, prompt, expectedMemorized, expectedVisual, options = {}) {
+  const result = await validateScreenshot(imagePath, prompt, {
+    testType: 'counterfactual',
+    ...options
+  });
+
+  const extractedValue = result.extractedValue || result.score;
+  const usesVisual = extractedValue === expectedVisual || 
+                     (typeof extractedValue === 'number' && typeof expectedVisual === 'number' && 
+                      Math.abs(extractedValue - expectedVisual) < 0.1);
+  const usesMemorization = extractedValue === expectedMemorized ||
+                          (typeof extractedValue === 'number' && typeof expectedMemorized === 'number' &&
+                           Math.abs(extractedValue - expectedMemorized) < 0.1);
+
+  return {
+    extractedValue,
+    expectedMemorized,
+    expectedVisual,
+    usesVisual,
+    usesMemorization,
+    biasAligned: usesMemorization && !usesVisual,
+    result,
+    recommendation: usesMemorization 
+      ? 'VLLM appears to use memorized knowledge. Consider visual analysis validation.'
+      : 'VLLM appears to use visual analysis.'
+  };
+}
+
+/**
+ * Batch test counterfactual scenarios
+ * 
+ * @param {Array<{imagePath: string, prompt: string, expectedMemorized: any, expectedVisual: any}>} testCases
+ * @param {Object} options - Test options
+ * @returns {Promise<Object>} Batch test results
+ */
+export async function batchTestCounterfactual(testCases, options = {}) {
+  const results = await Promise.all(
+    testCases.map(tc => 
+      testCounterfactual(tc.imagePath, tc.prompt, tc.expectedMemorized, tc.expectedVisual, options)
+    )
+  );
+
+  const visualCount = results.filter(r => r.usesVisual).length;
+  const memorizationCount = results.filter(r => r.usesMemorization).length;
+  const biasAlignedCount = results.filter(r => r.biasAligned).length;
+
+  return {
+    total: results.length,
+    visualAccuracy: visualCount / results.length,
+    memorizationRate: memorizationCount / results.length,
+    biasAlignedRate: biasAlignedCount / results.length,
+    results,
+    recommendation: biasAlignedCount > results.length * 0.5
+      ? 'High bias-aligned error rate. VLLM may be relying on memorized knowledge.'
+      : 'VLLM appears to use visual analysis appropriately.'
+  };
+}
+

package/src/utils/error-recovery.mjs

@@ -0,0 +1,117 @@
+/**
+ * Error Recovery for Browser Automation
+ * 
+ * Simple retry logic: wait and retry, or try alternative action.
+ * 
+ * Research Context:
+ * - Error recovery success rate >70% is often cited as critical for browser automation agents
+ * - Agents should gracefully handle failures and try alternatives
+ * - Need to avoid infinite retry loops
+ * 
+ * Implementation:
+ * - Most errors are timeouts or element not found - simple wait + retry handles these
+ * - Complex error classification adds complexity without clear benefit
+ * - The VLLM can handle complex error recovery during action execution
+ * 
+ * See docs/research/IMPLEMENTATION_VS_RESEARCH.md for detailed research context.
+ * 
+ * @module error-recovery
+ */
+
+/**
+ * Error recovery strategy
+ */
+export class ErrorRecoveryStrategy {
+  constructor(options = {}) {
+    this.maxRetries = options.maxRetries || 3;
+    this.retryDelay = options.retryDelay || 1000;
+    this.recoveryHistory = [];
+  }
+  
+  /**
+   * Attempt to recover from error
+   * 
+   * @param {Error} error - The error that occurred
+   * @param {Object} action - The action that failed
+   * @param {Object} context - Current context (page, state, etc.)
+   * @returns {Promise<Object|null>} Recovery action or null if no recovery possible
+   */
+  async attemptRecovery(error, action, context = {}) {
+    if (this.recoveryHistory.length >= this.maxRetries) {
+      return null; // Max retries reached
+    }
+    
+    const recovery = this.generateRecoveryAction(error, action, context);
+    
+    if (!recovery) {
+      return null; // No recovery strategy available
+    }
+    
+    this.recoveryHistory.push({
+      error: error.message,
+      action,
+      recovery,
+      timestamp: Date.now()
+    });
+    
+    return recovery;
+  }
+  
+  /**
+   * Generate recovery action based on error type
+   * 
+   * Simple strategy: wait longer for timeouts/network, wait and retry for others.
+   */
+  generateRecoveryAction(error, action, context) {
+    const errorMessage = error.message.toLowerCase();
+    
+    // Timeout or network errors: wait longer
+    if (errorMessage.includes('timeout') || errorMessage.includes('network')) {
+      return {
+        type: 'wait',
+        duration: this.retryDelay * 2,
+        reason: 'Timeout/network error, waiting longer',
+        originalAction: action
+      };
+    }
+    
+    // Everything else: wait and retry
+    return {
+      type: 'wait',
+      duration: this.retryDelay,
+      reason: 'Error occurred, waiting and retrying',
+      originalAction: action
+    };
+  }
+  
+  /**
+   * Reset recovery state
+   */
+  reset() {
+    this.recoveryHistory = [];
+  }
+  
+  /**
+   * Get recovery statistics
+   */
+  getStats() {
+    const successful = this.recoveryHistory.filter(r => r.success).length;
+    const total = this.recoveryHistory.length;
+    const successRate = total > 0 ? successful / total : 0;
+    
+    return {
+      totalRecoveries: total,
+      successfulRecoveries: successful,
+      successRate,
+      recoveries: this.recoveryHistory
+    };
+  }
+}
+
+/**
+ * Create error recovery strategy
+ */
+export function createErrorRecoveryStrategy(options = {}) {
+  return new ErrorRecoveryStrategy(options);
+}
+

package/src/utils/explainability-scorer.mjs

@@ -0,0 +1,119 @@
+/**
+ * Explainability Scoring
+ * 
+ * Simple heuristic: checks if reasoning exists, mentions the action, and isn't too technical.
+ * 
+ * Research Context:
+ * - Explainability score >80% is often cited as critical for browser automation agents
+ * - Users need to understand agent reasoning for trust and debugging
+ * - Transparency scores measure communication quality
+ * 
+ * Implementation:
+ * - Simple checks (has action, has target, not too technical, reasonable length) are sufficient
+ * - Complex scoring adds computation without clear benefit
+ * - The VLLM's reasoning is already human-readable
+ * 
+ * See docs/research/IMPLEMENTATION_VS_RESEARCH.md for detailed research context.
+ * 
+ * @module explainability-scorer
+ */
+
+/**
+ * Score explainability of action reasoning
+ * 
+ * @param {string} reasoning - Agent's reasoning for an action
+ * @param {Object} action - The action taken
+ * @param {Object} [options] - Scoring options
+ * @returns {Object} Explainability score and analysis
+ */
+export function scoreExplainability(reasoning, action, options = {}) {
+  if (!reasoning || reasoning.trim().length === 0) {
+    return {
+      score: 0,
+      clarity: 0,
+      completeness: 0,
+      relevance: 0,
+      issues: ['No reasoning provided'],
+      recommendation: 'Add reasoning to explain why this action was taken'
+    };
+  }
+  
+  // Simple scoring: has reasoning, mentions action, not too technical
+  const hasAction = action.type && reasoning.toLowerCase().includes(action.type.toLowerCase());
+  
+  // Check for target: selector, key, or URL - also check for semantic mentions (e.g., "submit button" for selector "#submit")
+  let hasTarget = false;
+  if (action.selector) {
+    // Check for exact selector match or semantic match (e.g., "submit button" for "#submit")
+    const selectorLower = action.selector.toLowerCase().replace(/[#.]/g, '');
+    const reasoningLower = reasoning.toLowerCase();
+    hasTarget = reasoning.includes(action.selector) || 
+                (selectorLower && reasoningLower.includes(selectorLower));
+  } else if (action.key) {
+    hasTarget = reasoning.includes(action.key);
+  } else if (action.url) {
+    hasTarget = reasoning.includes(action.url);
+  }
+  
+  const notTooTechnical = !reasoning.match(/\b(algorithm|implementation|optimization|paradigm)\b/gi);
+  const reasonableLength = reasoning.length > 20 && reasoning.length < 500;
+  
+  // Completeness: considers both action and target, plus reasoning depth
+  const hasDepth = reasoning.split(/[.!?]/).length > 2; // Multiple sentences indicate depth
+  const completeness = (hasAction && hasTarget && hasDepth) ? 0.9 :
+                       (hasAction && hasTarget) ? 0.8 :
+                       (hasAction || hasTarget) ? 0.6 : 0.4;
+  
+  const score = (hasAction ? 0.4 : 0) + 
+                (hasTarget ? 0.3 : 0) + 
+                (notTooTechnical ? 0.2 : 0) + 
+                (reasonableLength ? 0.1 : 0);
+  
+  const issues = [];
+  if (!hasAction) issues.push('Reasoning does not mention action type');
+  if (!hasTarget) issues.push('Reasoning does not mention action target');
+  if (!notTooTechnical) issues.push('Reasoning uses technical jargon');
+  if (!reasonableLength) issues.push('Reasoning is too short or too long');
+  
+  return {
+    score,
+    clarity: notTooTechnical && reasonableLength ? 0.8 : 0.5,
+    completeness,
+    relevance: hasAction ? 0.8 : 0.5,
+    issues,
+    recommendation: score >= 0.7
+      ? 'Reasoning is clear and relevant'
+      : 'Add more context about the action and its target'
+  };
+}
+
+
+/**
+ * Batch score explainability
+ */
+export function batchScoreExplainability(reasonings, actions, options = {}) {
+  const scores = reasonings.map((reasoning, i) => 
+    scoreExplainability(reasoning, actions[i] || {}, options)
+  );
+  
+  const avgScore = scores.reduce((sum, s) => sum + s.score, 0) / scores.length;
+  const avgClarity = scores.reduce((sum, s) => sum + s.clarity, 0) / scores.length;
+  const avgCompleteness = scores.reduce((sum, s) => sum + s.completeness, 0) / scores.length;
+  const avgRelevance = scores.reduce((sum, s) => sum + s.relevance, 0) / scores.length;
+  
+  const meetsTarget = avgScore >= 0.8;
+  
+  return {
+    total: scores.length,
+    averageScore: avgScore,
+    averageClarity: avgClarity,
+    averageCompleteness: avgCompleteness,
+    averageRelevance: avgRelevance,
+    meetsTarget,
+    scores,
+    recommendation: meetsTarget
+      ? 'Explainability meets target (>80%)'
+      : `Explainability ${(avgScore * 100).toFixed(1)}% below target. Improve reasoning clarity, completeness, or relevance.`
+  };
+}
+

package/src/utils/exploratory-automation.mjs

@@ -0,0 +1,131 @@
+/**
+ * Exploratory Automation
+ * 
+ * Tries alternative approaches when actions fail.
+ * Simple strategy: wait, try different action type, or give up after max attempts.
+ * 
+ * Research Context:
+ * - Exploratory success rate >60% is often cited as critical for browser automation agents
+ * - Agents should try alternative approaches when initial attempts fail
+ * - Need to track exploration attempts and avoid infinite loops
+ * 
+ * Implementation:
+ * - Simple wait + alternative action type is sufficient for most failures
+ * - Complex exploration strategies add complexity without clear benefit
+ * - The VLLM can handle complex decision-making during action execution
+ * 
+ * See docs/research/IMPLEMENTATION_VS_RESEARCH.md for detailed research context.
+ * 
+ * @module exploratory-automation
+ */
+
+/**
+ * Exploration strategy
+ */
+export class ExploratoryStrategy {
+  constructor(options = {}) {
+    this.maxAttempts = options.maxAttempts || 5;
+    this.attemptHistory = [];
+    this.alternativeActions = [];
+  }
+  
+  /**
+   * Get next exploration action
+   * 
+   * @param {Object} currentState - Current browser state
+   * @param {Array} failedActions - Actions that have failed
+   * @param {string} goal - Current goal
+   * @returns {Object|null} Next action to try, or null if no more alternatives
+   */
+  getNextAction(currentState, failedActions = [], goal = '') {
+    if (this.attemptHistory.length >= this.maxAttempts) {
+      return null; // Max attempts reached
+    }
+    
+    // Generate alternative actions based on goal and failed actions
+    const alternatives = this.generateAlternatives(currentState, failedActions, goal);
+    
+    // Filter out already attempted actions
+    const untried = alternatives.filter(alt => 
+      !this.attemptHistory.some(attempt => 
+        JSON.stringify(attempt.action) === JSON.stringify(alt)
+      )
+    );
+    
+    if (untried.length === 0) {
+      return null; // No more alternatives
+    }
+    
+    // Select next action (prefer actions that haven't been tried)
+    const nextAction = untried[0];
+    this.attemptHistory.push({
+      action: nextAction,
+      timestamp: Date.now(),
+      state: currentState
+    });
+    
+    return nextAction;
+  }
+  
+  /**
+   * Generate alternative actions
+   * 
+   * Simple strategy: wait, then try a different action type if available.
+   */
+  generateAlternatives(currentState, failedActions, goal) {
+    const alternatives = [];
+    const lastFailed = failedActions[failedActions.length - 1];
+    
+    if (!lastFailed) {
+      return alternatives;
+    }
+    
+    // If click failed, try wait then retry
+    if (lastFailed.type === 'click') {
+      alternatives.push(
+        { type: 'wait', duration: 1000 },
+        { type: 'keyboard', key: 'Tab' } // Try keyboard navigation
+      );
+    }
+    
+    // If keyboard failed, try wait
+    if (lastFailed.type === 'keyboard') {
+      alternatives.push({ type: 'wait', duration: 1000 });
+    }
+    
+    // Always have wait as fallback
+    if (alternatives.length === 0) {
+      alternatives.push({ type: 'wait', duration: 1000 });
+    }
+    
+    return alternatives;
+  }
+  
+  /**
+   * Reset exploration state
+   */
+  reset() {
+    this.attemptHistory = [];
+    this.alternativeActions = [];
+  }
+  
+  /**
+   * Get exploration statistics
+   */
+  getStats() {
+    return {
+      totalAttempts: this.attemptHistory.length,
+      maxAttempts: this.maxAttempts,
+      remainingAttempts: this.maxAttempts - this.attemptHistory.length,
+      attempts: this.attemptHistory
+    };
+  }
+}
+
+/**
+ * Create exploratory strategy
+ */
+export function createExploratoryStrategy(options = {}) {
+  return new ExploratoryStrategy(options);
+}
+

package/src/utils/index.mjs

@@ -173,3 +173,13 @@ export {
   initHumanValidation
 } from '../human-validation-manager.mjs';
 
+// Browser automation utilities
+export * from './counterfactual-tester.mjs';
+export * from './capability-stratifier.mjs';
+export * from './baseline-validator.mjs';
+export * from './intent-recognizer.mjs';
+export * from './action-hallucination-detector.mjs';
+export * from './exploratory-automation.mjs';
+export * from './error-recovery.mjs';
+export * from './explainability-scorer.mjs';
+