@mseep/ai-tech-app-agent 1.0.0

package/src/utils/promptBuilder.ts

@@ -0,0 +1,384 @@
+import llmConfig, { PromptTemplates } from '../config/llm.config';
+import { UIContext, UIElement } from '../models/testSpec';
+
+export class PromptBuilder {
+  private maxUIElements: number;
+  private maxContextTokens: number;
+  private includeAppContext: boolean;
+  private includeUIContext: boolean;
+
+  constructor() {
+    this.maxUIElements = llmConfig.prompt.maxUIElements;
+    this.maxContextTokens = llmConfig.prompt.maxContextTokens;
+    this.includeAppContext = llmConfig.prompt.includeAppContext;
+    this.includeUIContext = llmConfig.prompt.includeUIContext;
+  }
+
+  /**
+   * Build the main action planning prompt
+   */
+  buildActionPlanningPrompt(
+    stepText: string,
+    uiContext: UIContext,
+    appKnowledge?: string,
+    platform?: 'android' | 'ios'
+  ): string {
+    const sections: string[] = [];
+
+    // System context
+    sections.push(PromptTemplates.actionPlanning);
+    sections.push('');
+
+    // App-specific knowledge
+    if (this.includeAppContext && appKnowledge) {
+      sections.push('**App-Specific Knowledge:**');
+      sections.push(this.truncateText(appKnowledge, 1000));
+      sections.push('');
+    }
+
+    // Platform information
+    if (platform) {
+      sections.push(`**Platform:** ${platform.toUpperCase()}`);
+      sections.push('');
+    }
+
+    // Current UI state
+    if (this.includeUIContext) {
+      sections.push('**Current UI State:**');
+      sections.push(this.formatUIContext(uiContext));
+      sections.push('');
+    }
+
+    // Test step to execute
+    sections.push('**Test Step to Execute:**');
+    sections.push(`"${stepText}"`);
+    sections.push('');
+
+    // Output format instructions
+    sections.push(this.getOutputFormatInstructions());
+
+    return sections.join('\n');
+  }
+
+  /**
+   * Build prompt for context analysis
+   */
+  buildContextAnalysisPrompt(
+    uiContext: UIContext,
+    focusArea?: string
+  ): string {
+    const sections: string[] = [];
+
+    sections.push(PromptTemplates.contextAnalysis);
+    sections.push('');
+
+    if (focusArea) {
+      sections.push(`**Focus Area:** ${focusArea}`);
+      sections.push('');
+    }
+
+    sections.push('**Current UI State:**');
+    sections.push(this.formatUIContext(uiContext));
+    sections.push('');
+
+    sections.push('Identify and list the most relevant interactive elements with their best selectors.');
+
+    return sections.join('\n');
+  }
+
+  /**
+   * Build prompt for error recovery
+   */
+  buildErrorRecoveryPrompt(
+    stepText: string,
+    previousError: string,
+    uiContext: UIContext,
+    attemptNumber: number
+  ): string {
+    const sections: string[] = [];
+
+    sections.push(PromptTemplates.errorRecovery);
+    sections.push('');
+
+    sections.push(`**Original Test Step:** "${stepText}"`);
+    sections.push(`**Attempt Number:** ${attemptNumber}`);
+    sections.push(`**Previous Error:** ${previousError}`);
+    sections.push('');
+
+    sections.push('**Current UI State:**');
+    sections.push(this.formatUIContext(uiContext));
+    sections.push('');
+
+    sections.push('**Instructions:**');
+    sections.push('1. Analyze why the previous attempt failed');
+    sections.push('2. Check if the UI state has changed');
+    sections.push('3. Suggest alternative actions or selectors');
+    sections.push('4. If the step is impossible, return an empty actions array with reasoning');
+    sections.push('');
+
+    sections.push(this.getOutputFormatInstructions());
+
+    return sections.join('\n');
+  }
+
+  /**
+   * Build prompt for assertion generation
+   */
+  buildAssertionPrompt(
+    expectedOutcome: string,
+    uiContext: UIContext
+  ): string {
+    const sections: string[] = [];
+
+    sections.push(PromptTemplates.assertionGeneration);
+    sections.push('');
+
+    sections.push(`**Expected Outcome:** "${expectedOutcome}"`);
+    sections.push('');
+
+    sections.push('**Current UI State:**');
+    sections.push(this.formatUIContext(uiContext));
+    sections.push('');
+
+    sections.push('Generate assertion actions to verify the expected outcome.');
+    sections.push('');
+
+    sections.push(this.getOutputFormatInstructions());
+
+    return sections.join('\n');
+  }
+
+  /**
+   * Format UI context for inclusion in prompts
+   */
+  private formatUIContext(uiContext: UIContext): string {
+    const lines: string[] = [];
+
+    // Current screen/activity
+    if (uiContext.currentActivity) {
+      lines.push(`Current Screen: ${uiContext.currentActivity}`);
+      lines.push('');
+    }
+
+    // Visible elements
+    const elements = uiContext.visibleElements || [];
+    
+    if (elements.length === 0) {
+      lines.push('No visible elements detected.');
+      return lines.join('\n');
+    }
+
+    lines.push(`Visible Elements (showing ${Math.min(elements.length, this.maxUIElements)} of ${elements.length}):`);
+    lines.push('');
+
+    // Filter and sort elements by relevance
+    const relevantElements = this.filterRelevantElements(elements);
+    const topElements = relevantElements.slice(0, this.maxUIElements);
+
+    topElements.forEach((element, index) => {
+      lines.push(this.formatElement(element, index + 1));
+    });
+
+    return lines.join('\n');
+  }
+
+  /**
+   * Format a single UI element
+   */
+  private formatElement(element: UIElement, index: number): string {
+    const parts: string[] = [`${index}.`];
+
+    // Element type
+    parts.push(`[${element.type}]`);
+
+    // Text content
+    if (element.text && element.text.trim().length > 0) {
+      parts.push(`text="${this.truncateText(element.text, 50)}"`);
+    }
+
+    // Resource ID (Android) or Name (iOS)
+    if (element.resourceId) {
+      const shortId = element.resourceId.split('/').pop() || element.resourceId;
+      parts.push(`id="${shortId}"`);
+    }
+
+    // Content description / accessibility label
+    if (element.contentDesc) {
+      parts.push(`desc="${this.truncateText(element.contentDesc, 30)}"`);
+    }
+
+    // Attributes
+    const attrs: string[] = [];
+    if (element.clickable) attrs.push('clickable');
+    if (element.enabled) attrs.push('enabled');
+    if (!element.enabled) attrs.push('disabled');
+
+    if (attrs.length > 0) {
+      parts.push(`[${attrs.join(', ')}]`);
+    }
+
+    // Bounds (if useful)
+    if (element.bounds) {
+      parts.push(`bounds=${element.bounds}`);
+    }
+
+    return parts.join(' ');
+  }
+
+  /**
+   * Filter elements to show most relevant ones
+   */
+  private filterRelevantElements(elements: UIElement[]): UIElement[] {
+    // Prioritize elements that are:
+    // 1. Interactive (clickable, editable)
+    // 2. Have meaningful text or labels
+    // 3. Are common UI patterns (buttons, inputs, etc.)
+
+    const scored = elements.map(element => ({
+      element,
+      score: this.calculateElementRelevance(element),
+    }));
+
+    // Sort by score (descending)
+    scored.sort((a, b) => b.score - a.score);
+
+    return scored.map(s => s.element);
+  }
+
+  /**
+   * Calculate relevance score for an element
+   */
+  private calculateElementRelevance(element: UIElement): number {
+    let score = 0;
+
+    // Interactive elements get high priority
+    if (element.clickable) score += 10;
+
+    // Elements with text are usually important
+    if (element.text && element.text.trim().length > 0) {
+      score += 8;
+    }
+
+    // Elements with IDs are identifiable
+    if (element.resourceId) score += 5;
+
+    // Elements with accessibility labels
+    if (element.contentDesc) score += 5;
+
+    // Common interactive types
+    const interactiveTypes = [
+      'Button', 'EditText', 'TextView', 'ImageButton',
+      'CheckBox', 'RadioButton', 'Switch', 'Spinner',
+      'UIButton', 'UITextField', 'UITextView', 'UILabel',
+    ];
+
+    if (interactiveTypes.some(type => element.type.includes(type))) {
+      score += 7;
+    }
+
+    // Enabled elements
+    if (element.enabled) score += 3;
+
+    // Penalize common container types (unless they have text)
+    const containerTypes = ['ViewGroup', 'LinearLayout', 'FrameLayout', 'View', 'UIView'];
+    if (containerTypes.some(type => element.type.includes(type)) && !element.text) {
+      score -= 5;
+    }
+
+    return Math.max(0, score);
+  }
+
+  /**
+   * Get output format instructions
+   */
+  private getOutputFormatInstructions(): string {
+    return `**Output Format (JSON only, no markdown):**
+{
+  "actions": [
+    {
+      "type": "tap|type|scroll|swipe|wait|assert|screenshot",
+      "selector": {
+        "strategy": "text|id|accessibility-id|xpath|class",
+        "value": "selector_value",
+        "index": 0
+      },
+      "value": "text_to_type (only for type action)",
+      "direction": "up|down|left|right (only for scroll/swipe)",
+      "duration": 1000,
+      "assertionType": "exists|visible|text|enabled (only for assert)",
+      "expectedValue": "expected_value (only for assert)"
+    }
+  ],
+  "reasoning": "Brief explanation of the action sequence and why these selectors were chosen"
+}
+
+**Critical Rules:**
+1. Return ONLY valid JSON, no markdown code blocks
+2. Always prefer text-based selectors over XPath
+3. Use accessibility-id or resource-id when available
+4. Include reasoning for your selector choices
+5. If no action is possible, return empty actions array with explanation
+6. Keep action sequences minimal (1-3 actions preferred)
+7. Add wait actions if UI transitions are expected
+
+Generate the JSON action plan now:`;
+  }
+
+  /**
+   * Truncate text to max length
+   */
+  private truncateText(text: string, maxLength: number): string {
+    if (text.length <= maxLength) {
+      return text;
+    }
+    return text.substring(0, maxLength - 3) + '...';
+  }
+
+  /**
+   * Estimate token count (rough approximation)
+   */
+  private estimateTokens(text: string): number {
+    // Rough estimate: 1 token ≈ 4 characters
+    return Math.ceil(text.length / 4);
+  }
+
+  /**
+   * Build a few-shot learning prompt with examples
+   */
+  buildFewShotPrompt(
+    stepText: string,
+    uiContext: UIContext,
+    examples: Array<{ step: string; actions: any; context?: string }>
+  ): string {
+    const sections: string[] = [];
+
+    sections.push(PromptTemplates.actionPlanning);
+    sections.push('');
+
+    sections.push('**Example Scenarios:**');
+    sections.push('');
+
+    examples.forEach((example, index) => {
+      sections.push(`Example ${index + 1}:`);
+      sections.push(`Step: "${example.step}"`);
+      if (example.context) {
+        sections.push(`Context: ${example.context}`);
+      }
+      sections.push(`Actions: ${JSON.stringify(example.actions, null, 2)}`);
+      sections.push('');
+    });
+
+    sections.push('**Now, solve this:**');
+    sections.push('');
+    sections.push('**Current UI State:**');
+    sections.push(this.formatUIContext(uiContext));
+    sections.push('');
+    sections.push(`**Test Step:** "${stepText}"`);
+    sections.push('');
+    sections.push(this.getOutputFormatInstructions());
+
+    return sections.join('\n');
+  }
+}
+
+export default PromptBuilder;

package/tests/nlp-specs/login-flow.yaml

@@ -0,0 +1,31 @@
+name: Technician App Login Flow
+description: Test the complete login flow with OTP verification
+platform: Android
+
+setup:
+  - step: Launch the DmgPro app
+    timeout: 10000
+
+steps:
+  - step: Click on 'Allow' if notifications dialog displayed
+    expectedOutcome: Get Started Screen should appear
+    timeout: 5000
+  
+  - step: Click on 'Get Started' button
+    expectedOutcome: Login screen should appear
+    timeout: 5000
+
+  - step: Input the mobile number "112233445" into the Phone Number input field
+    expectedOutcome: OTP screen should appear
+    timeout: 5000
+
+  - step: Input OTP "1111"
+    expectedOutcome: Home screen should load
+    timeout: 5000
+
+  - step: Enable location access
+    expectedOutcome: Location permission granted
+    timeout: 5000
+
+teardown:
+  - step: Close the app

package/tsconfig.json

@@ -0,0 +1,31 @@
+{
+    "compilerOptions": {
+        "target": "ES2020",
+        "module": "commonjs",
+        "lib": [
+            "ES2020"
+        ],
+        "outDir": "./dist",
+        "rootDir": "./src",
+        "strict": true,
+        "esModuleInterop": true,
+        "skipLibCheck": true,
+        "forceConsistentCasingInFileNames": true,
+        "resolveJsonModule": true,
+        "moduleResolution": "node",
+        "declaration": true,
+        "declarationMap": true,
+        "sourceMap": true,
+        "types": [
+            "node"
+        ]
+    },
+    "include": [
+        "src/**/*"
+    ],
+    "exclude": [
+        "node_modules",
+        "dist",
+        "tests"
+    ]
+}