@diyor28/qa-core 0.1.0

package/src/agent/loop.ts

@@ -0,0 +1,147 @@
+import { createOpenAI } from '@ai-sdk/openai';
+import { generateText } from 'ai';
+import type { BrowserController } from '../browser/controller.js';
+import type { VisionAnalyzer } from '../vision/analyzer.js';
+import type { QaFinding, QaProgressEvent } from '../types.js';
+import { createAgentTools } from './tools/index.js';
+import { buildSystemPrompt } from './system-prompt.js';
+import { QaContextService } from './context/index.js';
+
+export interface AgentLoopConfig {
+  cerebrasApiKey: string;
+  browserController: BrowserController;
+  visionAnalyzer: VisionAnalyzer;
+  scenarioName: string;
+  scenarioContent: string;
+  baseUrl: string;
+  defectsToCheck: string[];
+  maxIterations?: number;
+  onProgress?: (event: QaProgressEvent) => void;
+}
+
+export interface AgentLoopResult {
+  findings: QaFinding[];
+  summary: string;
+  status: 'passed' | 'failed' | 'issues_found';
+}
+
+export class AgentLoop {
+  private config: AgentLoopConfig;
+  private contextService: QaContextService;
+
+  constructor(config: AgentLoopConfig) {
+    this.config = config;
+    this.contextService = new QaContextService();
+  }
+
+  async run(): Promise<AgentLoopResult> {
+    const cerebras = createOpenAI({
+      baseURL: 'https://inference.cerebras.ai/v1',
+      apiKey: this.config.cerebrasApiKey,
+    });
+
+    const model = cerebras('glm-4.7');
+
+    const tools = createAgentTools(this.config.browserController, this.config.visionAnalyzer);
+    const systemPrompt = buildSystemPrompt(this.config.defectsToCheck, this.config.scenarioContent);
+
+    try {
+      this.emitProgress('step_started', { step: 'agent_execution', message: 'Starting QA agent loop' });
+
+      const result = await generateText({
+        model,
+        system: systemPrompt,
+        prompt: `Execute the test scenario. Start by navigating to the base URL: ${this.config.baseUrl}`,
+        tools,
+        maxSteps: this.config.maxIterations || 15,
+        temperature: 0.15,
+        maxTokens: 1000,
+      });
+
+      this.emitProgress('step_completed', {
+        step: 'agent_execution',
+        message: 'QA agent loop completed',
+        usage: result.usage,
+      });
+
+      // Parse findings from agent's final response
+      const findings = this.parseFindingsFromResponse(result.text);
+
+      // Determine status
+      let status: 'passed' | 'failed' | 'issues_found' = 'passed';
+      if (findings.some((f) => f.severity === 'critical' || f.severity === 'high')) {
+        status = 'failed';
+      } else if (findings.length > 0) {
+        status = 'issues_found';
+      }
+
+      return {
+        findings,
+        summary: result.text,
+        status,
+      };
+    } catch (error) {
+      console.error('[AgentLoop] Error during execution:', error);
+
+      return {
+        findings: [
+          {
+            severity: 'critical',
+            title: 'Agent execution failed',
+            description: `Error during test execution: ${error instanceof Error ? error.message : String(error)}`,
+            reproSteps: ['Agent loop encountered an error'],
+            category: 'functional',
+          },
+        ],
+        summary: `Test execution failed: ${error instanceof Error ? error.message : String(error)}`,
+        status: 'failed',
+      };
+    }
+  }
+
+  private parseFindingsFromResponse(responseText: string): QaFinding[] {
+    const findings: QaFinding[] = [];
+
+    // Look for structured findings in the response
+    // Pattern 1: JSON findings array
+    const jsonMatch = responseText.match(/\{[\s\S]*"findings"[\s\S]*\}/);
+    if (jsonMatch) {
+      try {
+        const parsed = JSON.parse(jsonMatch[0]);
+        if (Array.isArray(parsed.findings)) {
+          findings.push(...parsed.findings);
+        }
+      } catch {
+        // Not valid JSON, continue
+      }
+    }
+
+    // Pattern 2: Look for severity keywords in text
+    const severityPattern = /(critical|high|medium|low)\s*(issue|bug|defect|problem):\s*(.+)/gi;
+    let match;
+    while ((match = severityPattern.exec(responseText)) !== null) {
+      const severity = match[1].toLowerCase() as 'low' | 'medium' | 'high' | 'critical';
+      const description = match[3];
+
+      findings.push({
+        severity,
+        title: description.substring(0, 100),
+        description,
+        reproSteps: ['Extracted from agent response'],
+        category: 'functional',
+      });
+    }
+
+    return findings;
+  }
+
+  private emitProgress(type: QaProgressEvent['type'], data: Record<string, unknown>) {
+    if (this.config.onProgress) {
+      this.config.onProgress({
+        type,
+        data,
+        timestamp: Date.now(),
+      });
+    }
+  }
+}

package/src/agent/system-prompt.ts

@@ -0,0 +1,39 @@
+export function buildSystemPrompt(defectsToCheck: string[], scenarioContent: string): string {
+  return `You are a QA testing agent executing browser-based test scenarios.
+
+## Your Mission
+Execute the test scenario step-by-step, validate behavior, and report any issues discovered.
+
+## Execution Guidelines
+1. Start by navigating to the base URL
+2. Follow the scenario instructions precisely
+3. After each significant action (navigation, form submission), take a screenshot
+4. Use the analyze_ui_ux tool after taking screenshots to check for visual/UX issues
+5. Verify expected outcomes with assertions
+6. When you find an issue:
+   - Take a screenshot
+   - Document expected vs actual behavior with severity (low, medium, high, critical)
+   - Provide clear reproduction steps
+7. Use parallel tool calls when actions are independent (e.g., multiple assertions)
+
+## Defects to Check
+${defectsToCheck.map((d) => `- ${d}`).join('\n')}
+
+## Important Rules
+- Use getByRole/getByText locators when possible (more reliable than CSS selectors)
+- Wait for elements before interacting
+- Take screenshots at key checkpoints for visual verification
+- Request UX analysis on key screens using analyze_ui_ux tool
+- If a tool fails, analyze the error and try an alternative approach (different selector, wait longer, etc.)
+
+## On Completion
+When you've completed all scenario steps, respond with a summary of:
+- Test result (passed/failed)
+- Issues discovered with severity and details
+- Overall assessment
+
+Do not make additional tool calls after providing your final summary.
+
+## Scenario to Execute
+${scenarioContent}`;
+}

package/src/agent/tools/index.ts

@@ -0,0 +1,162 @@
+import { z } from 'zod';
+import type { CoreTool } from 'ai';
+import type { BrowserController } from '../../browser/controller.js';
+import type { VisionAnalyzer } from '../../vision/analyzer.js';
+
+export function createAgentTools(
+  browserController: BrowserController,
+  visionAnalyzer: VisionAnalyzer
+): Record<string, CoreTool> {
+  return {
+    navigate: {
+      description: 'Navigate to a URL',
+      parameters: z.object({
+        url: z.string().describe('The URL to navigate to'),
+      }),
+      execute: async ({ url }: { url: string }) => {
+        return await browserController.navigate(url);
+      },
+    },
+    click: {
+      description: 'Click an element by selector or text. Use role:button, text:Login, or CSS selectors.',
+      parameters: z.object({
+        selector: z.string().describe('CSS selector, role:type, or text:content to find the element'),
+      }),
+      execute: async ({ selector }: { selector: string }) => {
+        return await browserController.click(selector);
+      },
+    },
+    fill: {
+      description: 'Fill an input field with a value',
+      parameters: z.object({
+        selector: z.string().describe('Selector for the input field'),
+        value: z.string().describe('Value to fill'),
+      }),
+      execute: async ({ selector, value }: { selector: string; value: string }) => {
+        return await browserController.fill(selector, value);
+      },
+    },
+    type: {
+      description: 'Type text into an element with key events (simulates typing)',
+      parameters: z.object({
+        selector: z.string().describe('Selector for the element'),
+        text: z.string().describe('Text to type'),
+      }),
+      execute: async ({ selector, text }: { selector: string; text: string }) => {
+        return await browserController.type(selector, text);
+      },
+    },
+    screenshot: {
+      description: 'Take a screenshot with a descriptive name',
+      parameters: z.object({
+        name: z.string().describe('Descriptive name for the screenshot (e.g., "login-page", "after-submit")'),
+        context: z.string().optional().describe('What is happening in this screenshot'),
+      }),
+      execute: async ({ name, context }: { name: string; context?: string }) => {
+        return await browserController.screenshot(name, context);
+      },
+    },
+    analyze_ui_ux: {
+      description: 'Analyze the current or a specific screenshot for UX/UI issues using AI vision. Returns findings about visual overflow, poor contrast, bad UX, and accessibility.',
+      parameters: z.object({
+        screenshotName: z
+          .string()
+          .optional()
+          .describe('Name of specific screenshot to analyze. If omitted, analyzes the most recent screenshot.'),
+        focus: z
+          .enum(['visual', 'accessibility', 'ux', 'all'])
+          .optional()
+          .default('all')
+          .describe('What aspect to focus the analysis on'),
+      }),
+      execute: async ({ screenshotName, focus }: { screenshotName?: string; focus?: string }) => {
+        // Get screenshot from history
+        let screenshot = screenshotName
+          ? browserController.getScreenshotByName(screenshotName)
+          : browserController.getLatestScreenshot();
+
+        if (!screenshot) {
+          // No screenshot available, take one automatically
+          await browserController.screenshot('auto-capture', 'Auto-captured for UX analysis');
+          screenshot = browserController.getLatestScreenshot();
+          if (!screenshot) {
+            return { error: 'Failed to capture screenshot' };
+          }
+        }
+
+        // Analyze with Gemini
+        const findings = await visionAnalyzer.analyzeScreenshot(screenshot.buffer, screenshot.context, screenshot.url);
+
+        return {
+          findings,
+          analyzed_screenshot: screenshot.name,
+          analyzed_url: screenshot.url,
+        };
+      },
+    },
+    assert_visible: {
+      description: 'Assert that an element is visible on the page',
+      parameters: z.object({
+        selector: z.string().describe('Selector for the element'),
+      }),
+      execute: async ({ selector }: { selector: string }) => {
+        return await browserController.assertVisible(selector);
+      },
+    },
+    assert_text: {
+      description: 'Assert that an element contains specific text',
+      parameters: z.object({
+        selector: z.string().describe('Selector for the element'),
+        expected: z.string().describe('Expected text content'),
+      }),
+      execute: async ({ selector, expected }: { selector: string; expected: string }) => {
+        return await browserController.assertText(selector, expected);
+      },
+    },
+    get_text: {
+      description: 'Get text content from an element',
+      parameters: z.object({
+        selector: z.string().describe('Selector for the element'),
+      }),
+      execute: async ({ selector }: { selector: string }) => {
+        return await browserController.getText(selector);
+      },
+    },
+    wait_for: {
+      description: 'Wait for an element to appear on the page',
+      parameters: z.object({
+        selector: z.string().describe('Selector for the element'),
+        timeout: z.number().optional().default(5000).describe('Timeout in milliseconds'),
+      }),
+      execute: async ({ selector, timeout }: { selector: string; timeout?: number }) => {
+        return await browserController.waitFor(selector, timeout);
+      },
+    },
+    scroll: {
+      description: 'Scroll the page up or down',
+      parameters: z.object({
+        direction: z.enum(['up', 'down']).describe('Scroll direction'),
+        amount: z.number().optional().describe('Scroll amount in pixels (default: 500)'),
+      }),
+      execute: async ({ direction, amount }: { direction: 'up' | 'down'; amount?: number }) => {
+        return await browserController.scroll(direction, amount);
+      },
+    },
+    evaluate_js: {
+      description: 'Execute JavaScript code in the browser context',
+      parameters: z.object({
+        code: z.string().describe('JavaScript code to execute'),
+      }),
+      execute: async ({ code }: { code: string }) => {
+        return await browserController.evaluateJs(code);
+      },
+    },
+    accessibility_snapshot: {
+      description: 'Capture an accessibility tree snapshot for analysis',
+      parameters: z.object({}),
+      execute: async () => {
+        return await browserController.accessibilitySnapshot();
+      },
+    },
+  };
+}

package/src/browser/controller.ts

@@ -0,0 +1,321 @@
+import type { Browser, BrowserContext, Page } from 'playwright';
+import { chromium } from 'playwright';
+import type { ScreenshotEntry } from '../types.js';
+
+export interface BrowserControllerConfig {
+  headless?: boolean;
+  viewport: { width: number; height: number };
+  enableTrace?: boolean;
+  traceDir?: string;
+}
+
+export class BrowserController {
+  private browser: Browser | null = null;
+  private context: BrowserContext | null = null;
+  private page: Page | null = null;
+  private screenshotHistory: ScreenshotEntry[] = [];
+  private config: BrowserControllerConfig;
+
+  constructor(config: BrowserControllerConfig) {
+    this.config = config;
+  }
+
+  async launch(): Promise<void> {
+    this.browser = await chromium.launch({
+      headless: this.config.headless ?? true,
+    });
+
+    this.context = await this.browser.newContext({
+      viewport: this.config.viewport,
+    });
+
+    if (this.config.enableTrace) {
+      await this.context.tracing.start({
+        screenshots: true,
+        snapshots: true,
+        sources: true,
+      });
+    }
+
+    this.page = await this.context.newPage();
+  }
+
+  async close(): Promise<void> {
+    if (this.config.enableTrace && this.context) {
+      const tracePath = `${this.config.traceDir}/trace-${Date.now()}.zip`;
+      await this.context.tracing.stop({ path: tracePath });
+    }
+
+    await this.context?.close();
+    await this.browser?.close();
+  }
+
+  async navigate(url: string): Promise<{ success: boolean; message: string }> {
+    if (!this.page) throw new Error('Browser not launched');
+
+    try {
+      await this.page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30000 });
+      return { success: true, message: `Navigated to ${url}` };
+    } catch (error) {
+      return {
+        success: false,
+        message: `Failed to navigate to ${url}: ${error instanceof Error ? error.message : String(error)}`,
+      };
+    }
+  }
+
+  async click(selector: string): Promise<{ success: boolean; message: string }> {
+    if (!this.page) throw new Error('Browser not launched');
+
+    try {
+      // Try intelligent locator strategies
+      let element = null;
+
+      // Strategy 1: getByRole
+      if (selector.includes('role:')) {
+        const role = selector.replace('role:', '');
+        element = this.page.getByRole(role as any);
+      }
+      // Strategy 2: getByText
+      else if (selector.includes('text:')) {
+        const text = selector.replace('text:', '');
+        element = this.page.getByText(text);
+      }
+      // Strategy 3: CSS/XPath
+      else {
+        element = this.page.locator(selector);
+      }
+
+      await element.click({ timeout: 10000 });
+      return { success: true, message: `Clicked ${selector}` };
+    } catch (error) {
+      // Retry with alternative strategies
+      try {
+        await this.page.locator(selector).first().click({ timeout: 5000 });
+        return { success: true, message: `Clicked ${selector} (fallback)` };
+      } catch {
+        return {
+          success: false,
+          message: `Failed to click ${selector}: ${error instanceof Error ? error.message : String(error)}`,
+        };
+      }
+    }
+  }
+
+  async fill(selector: string, value: string): Promise<{ success: boolean; message: string }> {
+    if (!this.page) throw new Error('Browser not launched');
+
+    try {
+      await this.page.locator(selector).fill(value, { timeout: 10000 });
+      return { success: true, message: `Filled ${selector} with "${value}"` };
+    } catch (error) {
+      return {
+        success: false,
+        message: `Failed to fill ${selector}: ${error instanceof Error ? error.message : String(error)}`,
+      };
+    }
+  }
+
+  async type(selector: string, text: string): Promise<{ success: boolean; message: string }> {
+    if (!this.page) throw new Error('Browser not launched');
+
+    try {
+      await this.page.locator(selector).pressSequentially(text, { timeout: 10000 });
+      return { success: true, message: `Typed "${text}" into ${selector}` };
+    } catch (error) {
+      return {
+        success: false,
+        message: `Failed to type into ${selector}: ${error instanceof Error ? error.message : String(error)}`,
+      };
+    }
+  }
+
+  async screenshot(name: string, context?: string): Promise<{ success: boolean; message: string; screenshotName: string }> {
+    if (!this.page) throw new Error('Browser not launched');
+
+    try {
+      const buffer = await this.page.screenshot({ fullPage: false });
+      const url = this.page.url();
+
+      this.screenshotHistory.push({
+        timestamp: Date.now(),
+        name,
+        buffer,
+        context: context || `Screenshot: ${name}`,
+        url,
+      });
+
+      return { success: true, message: `Captured screenshot: ${name}`, screenshotName: name };
+    } catch (error) {
+      return {
+        success: false,
+        message: `Failed to capture screenshot: ${error instanceof Error ? error.message : String(error)}`,
+        screenshotName: name,
+      };
+    }
+  }
+
+  async assertVisible(selector: string): Promise<{ success: boolean; message: string }> {
+    if (!this.page) throw new Error('Browser not launched');
+
+    try {
+      const element = this.page.locator(selector);
+      await element.waitFor({ state: 'visible', timeout: 10000 });
+      const isVisible = await element.isVisible();
+
+      if (isVisible) {
+        return { success: true, message: `Element ${selector} is visible` };
+      } else {
+        return { success: false, message: `Element ${selector} is not visible` };
+      }
+    } catch (error) {
+      return {
+        success: false,
+        message: `Element ${selector} not found or not visible: ${error instanceof Error ? error.message : String(error)}`,
+      };
+    }
+  }
+
+  async assertText(selector: string, expected: string): Promise<{ success: boolean; message: string }> {
+    if (!this.page) throw new Error('Browser not launched');
+
+    try {
+      const element = this.page.locator(selector);
+      const actual = await element.textContent();
+
+      if (actual?.includes(expected)) {
+        return { success: true, message: `Text "${expected}" found in ${selector}` };
+      } else {
+        return {
+          success: false,
+          message: `Expected "${expected}" but got "${actual}" in ${selector}`,
+        };
+      }
+    } catch (error) {
+      return {
+        success: false,
+        message: `Failed to assert text in ${selector}: ${error instanceof Error ? error.message : String(error)}`,
+      };
+    }
+  }
+
+  async waitFor(selector: string, timeout: number = 5000): Promise<{ success: boolean; message: string }> {
+    if (!this.page) throw new Error('Browser not launched');
+
+    try {
+      await this.page.locator(selector).waitFor({ state: 'visible', timeout });
+      return { success: true, message: `Element ${selector} appeared` };
+    } catch (error) {
+      return {
+        success: false,
+        message: `Element ${selector} did not appear within ${timeout}ms`,
+      };
+    }
+  }
+
+  async getText(selector: string): Promise<{ success: boolean; message: string; text?: string }> {
+    if (!this.page) throw new Error('Browser not launched');
+
+    try {
+      const text = await this.page.locator(selector).textContent();
+      return { success: true, message: `Got text from ${selector}`, text: text || '' };
+    } catch (error) {
+      return {
+        success: false,
+        message: `Failed to get text from ${selector}: ${error instanceof Error ? error.message : String(error)}`,
+      };
+    }
+  }
+
+  async scroll(direction: 'up' | 'down', amount?: number): Promise<{ success: boolean; message: string }> {
+    if (!this.page) throw new Error('Browser not launched');
+
+    try {
+      const delta = amount || 500;
+      const scrollAmount = direction === 'down' ? delta : -delta;
+
+      await this.page.evaluate((scroll) => {
+        window.scrollBy(0, scroll);
+      }, scrollAmount);
+
+      return { success: true, message: `Scrolled ${direction} by ${Math.abs(scrollAmount)}px` };
+    } catch (error) {
+      return {
+        success: false,
+        message: `Failed to scroll: ${error instanceof Error ? error.message : String(error)}`,
+      };
+    }
+  }
+
+  async evaluateJs(code: string): Promise<{ success: boolean; message: string; result?: any }> {
+    if (!this.page) throw new Error('Browser not launched');
+
+    try {
+      const result = await this.page.evaluate((jsCode) => {
+        return eval(jsCode);
+      }, code);
+
+      return { success: true, message: 'JavaScript executed', result };
+    } catch (error) {
+      return {
+        success: false,
+        message: `Failed to execute JavaScript: ${error instanceof Error ? error.message : String(error)}`,
+      };
+    }
+  }
+
+  async accessibilitySnapshot(): Promise<{ success: boolean; message: string; snapshot?: any }> {
+    if (!this.page) throw new Error('Browser not launched');
+
+    try {
+      // Get ARIA tree via evaluate
+      const snapshot = await this.page.evaluate(() => {
+        const getAriaTree = (element: Element): any => {
+          const role = element.getAttribute('role');
+          const ariaLabel = element.getAttribute('aria-label');
+          const ariaLabelledby = element.getAttribute('aria-labelledby');
+
+          return {
+            role: role || element.tagName.toLowerCase(),
+            name: ariaLabel || ariaLabelledby || element.textContent?.substring(0, 50),
+            children: Array.from(element.children).map((child) => getAriaTree(child)),
+          };
+        };
+
+        return getAriaTree(document.body);
+      });
+
+      return { success: true, message: 'Accessibility snapshot captured', snapshot };
+    } catch (error) {
+      return {
+        success: false,
+        message: `Failed to capture accessibility snapshot: ${error instanceof Error ? error.message : String(error)}`,
+      };
+    }
+  }
+
+  // Screenshot history access methods
+  getLatestScreenshot(): ScreenshotEntry | null {
+    return this.screenshotHistory[this.screenshotHistory.length - 1] || null;
+  }
+
+  getScreenshotByName(name: string): ScreenshotEntry | null {
+    return this.screenshotHistory.find((s) => s.name === name) || null;
+  }
+
+  getAllScreenshots(): ScreenshotEntry[] {
+    return [...this.screenshotHistory];
+  }
+
+  getCurrentUrl(): string {
+    return this.page?.url() || '';
+  }
+
+  getViewport(): { width: number; height: number } {
+    return this.config.viewport;
+  }
+
+  getScreenshotCount(): number {
+    return this.screenshotHistory.length;
+  }
+}

package/src/index.ts

@@ -0,0 +1,19 @@
+// Main exports
+export { QaRunner } from './runner.js';
+
+// Type exports
+export type {
+  QaScenario,
+  QaFinding,
+  QaArtifact,
+  QaReport,
+  QaProgressEvent,
+  QaRunnerConfig,
+  ScreenshotEntry,
+} from './types.js';
+
+// Component exports (for advanced usage)
+export { BrowserController } from './browser/controller.js';
+export { VisionAnalyzer } from './vision/analyzer.js';
+export { AgentLoop } from './agent/loop.js';
+export { ReportBuilder } from './report/builder.js';