@mseep/ai-tech-app-agent 1.0.0

package/src/models/testSpec.ts

@@ -0,0 +1,85 @@
+export interface TestStep {
+  step: string;
+  description?: string;
+  expectedOutcome?: string;
+  timeout?: number;
+}
+
+export interface TestSpec {
+  name: string;
+  description?: string;
+  platform: 'android' | 'ios' | 'both';
+  appPackage?: string;
+  appActivity?: string;
+  bundleId?: string;
+  device?: {
+    deviceName?: string;
+    platformVersion?: string;
+  };
+  setup?: TestStep[];
+  steps: TestStep[];
+  teardown?: TestStep[];
+}
+
+export interface ActionPlan {
+  actions: Action[];
+  reasoning?: string;
+}
+
+export interface Action {
+  type: 'tap' | 'type' | 'scroll' | 'swipe' | 'wait' | 'assert' | 'screenshot' | 'launch';
+  selector?: Selector;
+  value?: string;
+  direction?: 'up' | 'down' | 'left' | 'right';
+  duration?: number;
+  assertionType?: 'exists' | 'visible' | 'text' | 'enabled';
+  expectedValue?: string;
+}
+
+export interface Selector {
+  strategy: 'text' | 'id' | 'accessibility-id' | 'xpath' | 'class' | 'coordinates';
+  value: string;
+  index?: number;
+}
+
+export interface UIContext {
+  pageSource: string;
+  currentActivity?: string;
+  currentPackage?: string;
+  visibleElements?: UIElement[];
+  platform?: 'android' | 'ios';
+  screenSize?: {
+    width: number;
+    height: number;
+  };
+}
+
+export interface UIElement {
+  type: string;
+  text?: string;
+  resourceId?: string;
+  contentDesc?: string;
+  className?: string;
+  bounds?: string;
+  clickable?: boolean;
+  enabled?: boolean;
+  index?: number;
+}
+
+export interface TestResult {
+  specName: string;
+  status: 'passed' | 'failed' | 'skipped';
+  duration: number;
+  steps: StepResult[];
+  screenshots?: string[];
+  error?: string;
+}
+
+export interface StepResult {
+  step: string;
+  status: 'passed' | 'failed' | 'skipped';
+  duration: number;
+  actions: Action[];
+  error?: string;
+  screenshot?: string;
+}

package/src/orchestrator/aiTestRunner.ts

@@ -0,0 +1,286 @@
+import fs from 'fs';
+import yaml from 'js-yaml';
+import path from 'path';
+import logger from '../utils/logger';
+import LlamaClient from '../llm/llamaClient';
+import MCPClient from '../mcp/mcpClient';
+import {
+  TestSpec,
+  TestResult,
+  StepResult,
+  TestStep,
+  Action,
+} from '../models/testSpec';
+
+export class AITestRunner {
+  private llamaClient: LlamaClient;
+  private mcpClient: MCPClient;
+  private appKnowledge: string = '';
+  private maxRetries: number;
+  private screenshotOnFailure: boolean;
+
+  constructor() {
+    this.llamaClient = new LlamaClient();
+    this.mcpClient = new MCPClient();
+    this.maxRetries = parseInt(process.env.MAX_RETRY_ATTEMPTS || '2');
+    this.screenshotOnFailure = process.env.SCREENSHOT_ON_FAILURE === 'true';
+  }
+
+  async loadAppKnowledge(knowledgePath?: string): Promise<void> {
+    if (!knowledgePath) {
+      logger.info('No app knowledge file provided, using defaults');
+      return;
+    }
+
+    try {
+      this.appKnowledge = fs.readFileSync(knowledgePath, 'utf-8');
+      logger.info(`Loaded app knowledge from: ${knowledgePath}`);
+    } catch (error: any) {
+      logger.warn(`Failed to load app knowledge: ${error.message}`);
+    }
+  }
+
+  async runTestSpec(specPath: string, sessionId?: string): Promise<TestResult> {
+    const startTime = Date.now();
+    
+    try {
+      // Load test spec
+      const testSpec = this.loadTestSpec(specPath);
+      logger.info(`Running test spec: ${testSpec.name}`);
+
+      // Initialize MCP client
+      await this.mcpClient.initialize();
+      if (sessionId) {
+        this.mcpClient.setSessionId(sessionId);
+      }
+
+      // Health check LLM
+      const llmHealthy = await this.llamaClient.healthCheck();
+      if (!llmHealthy) {
+        throw new Error('LLM is not available');
+      }
+
+      // Execute setup steps
+      const setupResults: StepResult[] = [];
+      if (testSpec.setup) {
+        logger.info('Executing setup steps...');
+        for (const step of testSpec.setup) {
+          const result = await this.executeStep(step);
+          setupResults.push(result);
+          if (result.status === 'failed') {
+            throw new Error(`Setup failed at step: ${step.step}`);
+          }
+        }
+      }
+
+      // Execute main test steps
+      const stepResults: StepResult[] = [];
+      for (const step of testSpec.steps) {
+        const result = await this.executeStep(step);
+        stepResults.push(result);
+
+        if (result.status === 'failed' && !process.env.CONTINUE_ON_FAILURE) {
+          logger.error(`Test failed at step: ${step.step}`);
+          break;
+        }
+      }
+
+      // Execute teardown steps
+      if (testSpec.teardown) {
+        logger.info('Executing teardown steps...');
+        for (const step of testSpec.teardown) {
+          await this.executeStep(step);
+        }
+      }
+
+      // Determine overall test status
+      const allSteps = [...setupResults, ...stepResults];
+      const testStatus = allSteps.every(s => s.status === 'passed')
+        ? 'passed'
+        : allSteps.some(s => s.status === 'failed')
+        ? 'failed'
+        : 'skipped';
+
+      const duration = Date.now() - startTime;
+
+      const result: TestResult = {
+        specName: testSpec.name,
+        status: testStatus,
+        duration,
+        steps: allSteps,
+      };
+
+      logger.info(`Test ${testSpec.name} completed with status: ${testStatus}`);
+      return result;
+
+    } catch (error: any) {
+      logger.error(`Test execution failed: ${error.message}`);
+      
+      return {
+        specName: path.basename(specPath),
+        status: 'failed',
+        duration: Date.now() - startTime,
+        steps: [],
+        error: error.message,
+      };
+    } finally {
+      await this.mcpClient.cleanup();
+    }
+  }
+
+  private async executeStep(testStep: TestStep): Promise<StepResult> {
+    const startTime = Date.now();
+    let attempts = 0;
+    let lastError: string | undefined;
+
+    logger.info(`Executing step: "${testStep.step}"`);
+
+    while (attempts <= this.maxRetries) {
+      attempts++;
+
+      try {
+        // Get current UI context
+        const uiContext = await this.mcpClient.getUIContext();
+        const uiContextStr = this.formatUIContext(uiContext);
+
+        // Generate action plan from LLM
+        const actionPlan = await this.llamaClient.generateActionPlan(
+          testStep.step,
+          uiContextStr,
+          this.appKnowledge
+        );
+
+        if (actionPlan.actions.length === 0) {
+          throw new Error('LLM generated no actions for this step');
+        }
+
+        logger.info(`Generated ${actionPlan.actions.length} actions`);
+        if (actionPlan.reasoning) {
+          logger.debug(`LLM reasoning: ${actionPlan.reasoning}`);
+        }
+
+        // Execute each action
+        let allActionsSucceeded = true;
+        for (const action of actionPlan.actions) {
+          const success = await this.mcpClient.executeAction(action);
+          if (!success) {
+            allActionsSucceeded = false;
+            throw new Error(`Action failed: ${action.type}`);
+          }
+
+          // Small delay between actions for stability
+          await this.delay(500);
+        }
+
+        // Step succeeded
+        const duration = Date.now() - startTime;
+        return {
+          step: testStep.step,
+          status: 'passed',
+          duration,
+          actions: actionPlan.actions,
+        };
+
+      } catch (error: any) {
+        lastError = error.message;
+        logger.warn(`Step attempt ${attempts} failed: ${lastError}`);
+
+        if (attempts > this.maxRetries) {
+          // Final failure
+          if (this.screenshotOnFailure) {
+            await this.mcpClient.executeAction({ type: 'screenshot' });
+          }
+
+          const duration = Date.now() - startTime;
+          return {
+            step: testStep.step,
+            status: 'failed',
+            duration,
+            actions: [],
+            error: lastError,
+          };
+        }
+
+        // Wait before retry
+        await this.delay(2000);
+      }
+    }
+
+    // Should not reach here, but handling it
+    return {
+      step: testStep.step,
+      status: 'failed',
+      duration: Date.now() - startTime,
+      actions: [],
+      error: lastError || 'Unknown error',
+    };
+  }
+
+  private loadTestSpec(specPath: string): TestSpec {
+    try {
+      const fileContent = fs.readFileSync(specPath, 'utf-8');
+      const ext = path.extname(specPath).toLowerCase();
+
+      if (ext === '.yaml' || ext === '.yml') {
+        return yaml.load(fileContent) as TestSpec;
+      } else if (ext === '.json') {
+        return JSON.parse(fileContent) as TestSpec;
+      } else {
+        throw new Error(`Unsupported spec file format: ${ext}`);
+      }
+    } catch (error: any) {
+      logger.error(`Failed to load test spec: ${error.message}`);
+      throw error;
+    }
+  }
+
+  private formatUIContext(uiContext: any): string {
+    // Format UI context for LLM consumption
+    // Keep it concise to save tokens
+    const elements = uiContext.visibleElements || [];
+    
+    let formatted = `Current Activity: ${uiContext.currentActivity || 'Unknown'}\n\n`;
+    formatted += `Visible UI Elements (${elements.length}):\n`;
+
+    elements.slice(0, 30).forEach((el: any, idx: number) => {
+      formatted += `${idx + 1}. ${el.type}`;
+      if (el.text) formatted += ` text="${el.text}"`;
+      if (el.resourceId) formatted += ` id="${el.resourceId}"`;
+      if (el.contentDesc) formatted += ` desc="${el.contentDesc}"`;
+      if (el.clickable) formatted += ` [clickable]`;
+      formatted += '\n';
+    });
+
+    if (elements.length > 30) {
+      formatted += `... and ${elements.length - 30} more elements\n`;
+    }
+
+    return formatted;
+  }
+
+  private delay(ms: number): Promise<void> {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+
+  async generateReport(results: TestResult[], outputPath: string): Promise<void> {
+    try {
+      const report = {
+        timestamp: new Date().toISOString(),
+        summary: {
+          total: results.length,
+          passed: results.filter(r => r.status === 'passed').length,
+          failed: results.filter(r => r.status === 'failed').length,
+          skipped: results.filter(r => r.status === 'skipped').length,
+        },
+        results,
+      };
+
+      fs.writeFileSync(outputPath, JSON.stringify(report, null, 2));
+      logger.info(`Test report generated: ${outputPath}`);
+    } catch (error: any) {
+      logger.error(`Failed to generate report: ${error.message}`);
+    }
+  }
+}
+
+export default AITestRunner;

package/src/utils/logger.ts

@@ -0,0 +1,59 @@
+import winston from 'winston';
+import path from 'path';
+import dotenv from 'dotenv';
+import fs from 'fs';
+
+dotenv.config();
+
+const logDir = process.env.LOG_DIR || './logs';
+
+// Ensure log directory exists
+if (!fs.existsSync(logDir)) {
+  fs.mkdirSync(logDir, { recursive: true });
+}
+
+const logger = winston.createLogger({
+  level: process.env.LOG_LEVEL || 'info',
+  format: winston.format.combine(
+    winston.format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }),
+    winston.format.errors({ stack: true }),
+    winston.format.splat(),
+    winston.format.json()
+  ),
+  defaultMeta: { service: 'ai-test-runner' },
+  transports: [
+    // Error log file
+    new winston.transports.File({
+      filename: path.join(logDir, 'error.log'),
+      level: 'error',
+      maxsize: 5242880, // 5MB
+      maxFiles: 5,
+    }),
+    // Combined log file
+    new winston.transports.File({
+      filename: path.join(logDir, 'combined.log'),
+      maxsize: 5242880,
+      maxFiles: 5,
+    }),
+  ],
+});
+
+// Also log to console in non-production environments
+if (process.env.NODE_ENV !== 'production') {
+  logger.add(
+    new winston.transports.Console({
+      format: winston.format.combine(
+        winston.format.colorize(),
+        winston.format.printf(({ level, message, timestamp, ...meta }) => {
+          let metaStr = '';
+          if (Object.keys(meta).length > 0 && meta.service !== 'ai-test-runner') {
+            metaStr = ' ' + JSON.stringify(meta);
+          }
+          return `${timestamp} [${level}]: ${message}${metaStr}`;
+        })
+      ),
+    })
+  );
+}
+
+export default logger;