@cotestdev/ai-runner 0.0.5 → 0.0.6

package/src/agents/tools/playwright-mcp-types.d.ts

@@ -0,0 +1,71 @@
+/**
+ * Playwright MCP 模块类型定义
+ *
+ * 为动态加载的 Playwright 内部模块提供 TypeScript 类型支持
+ */
+
+import type { BrowserContext } from 'playwright';
+
+export interface BrowserContextFactory {
+  createContext: (
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    clientInfo: any,
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    abortSignal: any,
+    toolName: string
+  ) => Promise<{
+    browserContext: BrowserContext;
+    close: () => Promise<void>;
+  }>;
+}
+
+export interface BackendConfig {
+  capabilities?: string[];
+  saveSession?: boolean;
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  [key: string]: any;
+}
+
+export interface BrowserServerBackendInstance {
+  initialize(clientInfo: { name: string; version: string }): Promise<void>;
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  listTools(): Promise<Array<{ name: string; description: string; inputSchema: any }>>;
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  callTool(name: string, params: any): Promise<any>;
+  serverClosed(): void;
+}
+
+export interface BrowserServerBackend {
+  new (config: BackendConfig, factory: BrowserContextFactory): BrowserServerBackendInstance;
+}
+
+export interface DefaultConfig {
+  capabilities?: string[];
+  saveSession?: boolean;
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  [key: string]: any;
+}
+
+/**
+ * 官方的 identityFactory 函数类型
+ * 参考：playwright/lib/mcp/test/browserBackend.js 第 94-104 行
+ */
+export type IdentityFactory = (browserContext: BrowserContext) => BrowserContextFactory;
+
+export interface MCPTool {
+  schema: {
+    name: string;
+    description: string;
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    inputSchema: any;
+  };
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  handle: (context: any, params: any, response: any) => Promise<void>;
+}
+
+// 动态加载的模块类型
+export interface PlaywrightMCPModules {
+  BrowserServerBackend: BrowserServerBackend;
+  defaultConfig: DefaultConfig;
+  identityFactory: IdentityFactory;
+}

package/src/agents/types.ts

@@ -0,0 +1,80 @@
+/**
+ * Playwright Agent Type Definitions
+ */
+
+import type { ModelConfig } from '../types/external';
+import type { ReActStep } from '../types';
+import { BrowserContext, Page } from '@playwright/test';
+import { Logger } from './logger';
+import { PlaywrightGroup } from '../tools/playwright-groups';
+import { PlaywrightBackendAdapter } from './tools/playwright-backend-adapter';
+import { StorageService } from '@cotestdev/core-infra';
+import { BaseLanguageModel } from '@langchain/core/language_models/base';
+
+// ============================================================================
+// Agent Execution Mode
+// ============================================================================
+
+/**
+ * Agent execution mode
+ */
+export type AgentMode = 'heal' | 'execute';
+
+// ============================================================================
+// Agent State
+// ============================================================================
+
+/**
+ * Playwright Agent State
+ */
+export interface PlaywrightAgentState {
+  logger: Logger;
+  /** Playwright Page object */
+  page: Page;
+  /** Playwright BrowserContext object */
+  context: BrowserContext;
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  variables: Map<string, any>;
+  model?: BaseLanguageModel;
+  originalScript: string;
+  playwrightGroup: PlaywrightGroup;
+  adapter: PlaywrightBackendAdapter;
+  storageService: StorageService;
+  testCaseId: string;
+}
+
+// ============================================================================
+// Agent Configuration
+// ============================================================================
+
+/**
+ * Playwright Agent Configuration
+ */
+export interface PlaywrightAgentConfig {
+  /** Model configuration (for LLM Agent) */
+  model: ModelConfig;
+}
+
+// ============================================================================
+// Agent Result
+// ============================================================================
+
+/**
+ * Playwright Agent Execution Result
+ */
+export interface PlaywrightAgentResult {
+  /** Whether the goal was successfully completed */
+  success: boolean;
+
+  /** Agent's reasoning process */
+  reasoning?: string;
+
+  /** If failed, the error reason */
+  errorMessage?: string;
+
+  /** How many steps were executed */
+  steps?: number;
+
+  /** Detailed step records of ReAct loop */
+  reactSteps?: ReActStep[];
+}

package/src/index.ts

@@ -0,0 +1,27 @@
+/**
+ * ai-runner SDK Main Entry Point
+ * Provides AI-driven test execution capabilities
+ */
+
+// Types
+export type {
+  RunnerConfig,
+  ExecutionResult,
+  HealContext,
+  HealSummary,
+  HealingDetail,
+  RunStepOptions,
+} from './types';
+
+
+// Core Classes
+export { Runner } from './runner';
+
+
+// Agent
+export type {
+  PlaywrightAgentState,
+  PlaywrightAgentResult,
+  PlaywrightAgentConfig,
+} from './agents/types';
+

package/src/runner.ts

@@ -0,0 +1,224 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+import type {
+  RunnerConfig,
+  HealContext,
+} from './types';
+import type { Page, BrowserContext } from 'playwright';
+import { createModel, StorageService } from '@cotestdev/core-infra';
+import { expect } from '@playwright/test';
+import { PlaywrightAgentState } from './agents/types';
+import { Logger } from './agents/logger';
+import { PlaywrightExecutor } from './agents/playwright-executor';
+import { PlaywrightGroup } from './tools/playwright-groups';
+import { PlaywrightBackendAdapter } from './agents/tools/playwright-backend-adapter';
+import { BaseLanguageModel } from '@langchain/core/language_models/base';
+
+export class Runner {
+  private config: RunnerConfig;
+  private model?: BaseLanguageModel;
+  private storageService: StorageService;
+  private logger: Logger;
+  private canHeal: boolean = false;
+  private executor: PlaywrightExecutor | undefined;
+  private state: PlaywrightAgentState | undefined;
+  private script?: string;
+  private variables: Map<string, any> = new Map<string, any>();
+
+  private constructor(config: RunnerConfig) {
+    this.config = config;
+    this.storageService = StorageService.getInstance(config.projectId);
+    this.logger = new Logger();
+  }
+
+  static NewInstance(projectId: string, testCaseId: string): Runner {
+    const runner = new Runner({ projectId, testCaseId });
+
+    try {
+      const script = runner.loadOriginalScript();
+      if (!script) {
+        runner.logger.error('No script found for the specified test case.');
+        return runner;
+      }
+      runner.script = script;
+
+      // 同步加载模型配置
+      const allConfigs = runner.storageService.listModelConfigsSync();
+      const config = allConfigs.find(c => c.isDefault === true);
+      if (config) {
+        runner.model = createModel(config);
+        runner.canHeal = true;
+      } else {
+        runner.logger.error('No default model configuration available in the system.');
+      }
+    } catch (error) {
+      runner.logger.error(`Failed to initialize runner: ${error}`);
+    }
+
+    return runner;
+  }
+
+  async init(page: Page, context: BrowserContext, data?: Record<string, any>): Promise<void> {
+    const adapter = await PlaywrightBackendAdapter.create(context);
+    this.state = {
+      page,
+      context,
+      variables: this.variables,
+      logger: this.logger,
+      model: this.model,
+      originalScript: this.state?.originalScript || this.script || '',
+      adapter: adapter,
+      playwrightGroup: new PlaywrightGroup(adapter),
+      testCaseId: this.config.testCaseId,
+      storageService: this.storageService,
+    };
+    if (data) {
+      for (const [key, value] of Object.entries(data)) {
+        this.set(key, value);
+      }
+    }
+    this.executor = new PlaywrightExecutor(this.state);
+  }
+
+  private loadOriginalScript(): string {
+    if (!this.config.testCaseId) {
+      throw new Error('Cannot load original script: testCaseId is not provided');
+    }
+
+    this.logger.log(`Attempting to load script from database: ${this.config.projectId}/${this.config.testCaseId}`);
+    const script = this.storageService.loadTestScript(this.config.testCaseId);
+    if (!script) {
+      throw new Error(`Cannot load original script: Test case not found in database (ID: ${this.config.testCaseId})`);
+    }
+
+    return script;
+  }
+
+  forEach(data: Record<string, any>[], callback: (index: number) => void) {
+    for (const [index, item] of data.entries()) {
+      for (const [key, value] of Object.entries(item)) {
+        this.set(key, value);
+      }
+
+      callback(index);
+    }
+  }
+
+  async runStep(description: string, fn: () => Promise<void>) {
+    const startTime = Date.now();
+    const context: HealContext = {
+      stepDescription: description,
+      originalCode: fn.toString(),
+      success: false
+    };
+
+    try {
+      await fn();
+      context.success = true;
+      return;
+    } catch (error) {
+      if (!this.canHeal || !(error instanceof Error)) {
+        throw error;
+      }
+
+      const healable = this.isHealableError(error);
+      if (!healable) {
+        throw error;
+      }
+
+      context.error = (error as Error).message;
+      context.stackTrace = (error as Error).stack;
+
+      this.logger.error(`Step failed: ${description}, Error: ${context.error}`);
+      this.logger.log(`Agent will attempt to self-heal...`);
+
+      await this.executor!.execute(context);
+
+      const duration = (Date.now() - startTime) / 1000;
+      if (context.success) {
+        this.logger.log(`Test step healed, duration: ${duration}s`);
+        return;
+      }
+
+      this.logger.error(`Failed to fix step: ${context.healError}, duration: ${duration}s`);
+      throw new Error(`original error: ${context.error}; heal error: ${context.healError}`);
+    }
+  }
+
+  private isHealableError(err: Error): boolean {
+    // 方式1：枚举所有 Symbol 属性（最常见尝试）
+    const symbols = Object.getOwnPropertySymbols(err);
+
+    for (const sym of symbols) {
+      try {
+        const value = (err as any)[sym];
+
+        // 如果有 category 字段，大概率就是我们要找的 step
+        if (value && typeof value === 'object' && 'category' in value) {
+          if (value.category === 'expect' || value.category === 'pw:api') {
+            return true;
+          }
+        }
+      } catch (e) {
+        // 访问可能会抛错，忽略即可
+      }
+    }
+
+    return false;
+  }
+
+  set(key: string, value: any): void {
+    this.variables.set(key, value);
+  }
+
+  get(key: string): any {
+    return this.variables.get(key);
+  }
+
+  has(key: string): boolean {
+    return this.variables.has(key);
+  }
+
+  delete(key: string): boolean {
+    return this.variables.delete(key);
+  }
+
+  clear(): void {
+    this.variables.clear();
+  }
+
+  keys(): string[] {
+    return Array.from(this.variables.keys());
+  }
+
+  getAll(): Record<string, unknown> {
+    return Object.fromEntries(this.variables);
+  }
+
+  async reuseTest(useTestCaseId: string, description: string): Promise<Map<string, unknown>> {
+    const scriptContent = this.storageService.loadTestScript(useTestCaseId);
+    if (!scriptContent) {
+      throw new Error(`Reusable test script not found: ${useTestCaseId}`);
+    }
+
+    const reuseRunner = Runner.NewInstance(this.config.projectId, useTestCaseId);
+    await reuseRunner.init(this.state!.page, this.state!.context);
+
+    this.logger.log(`Starting to execute reusable test case: ${description}`);
+
+    try {
+      const executeScript = new Function('runner', 'page', 'context', 'expect', `
+        return (async () => {
+          ${scriptContent}
+        })();
+      `);
+
+      await executeScript(reuseRunner, this.state!.page, this.state!.context, expect);
+
+      this.logger.log(`Reusable test case executed successfully`);
+
+      return reuseRunner.state?.variables || new Map<string, unknown>();
+    } catch (error) {
+      throw new Error(`reuseTest execution failed, error: ${error instanceof Error ? error.message : String(error)}`);
+    }
+  }
+}

package/src/tools/index.ts

@@ -0,0 +1,48 @@
+import { DynamicStructuredTool } from "langchain";
+
+export interface ToolGroup {
+    name: string;
+    description: string;
+    tools?: string[];
+    loadByDefault: boolean;
+}
+
+export abstract class AbstractGroup {
+    groups: ToolGroup[];
+
+    constructor(groups: ToolGroup[]) {
+        this.groups = groups;
+    }
+
+    abstract getAllTools(): DynamicStructuredTool[];
+
+    getDefaultTools(): DynamicStructuredTool[] {
+        return this.getAllTools().filter(tool => {
+            return this.groups.some(group => group.loadByDefault && (!group.tools || group.tools.includes(tool.name)));
+        });
+    }
+
+    getAvailableTools(groupName: string): DynamicStructuredTool[] {
+        const group = this.groups.find(g => g.name === groupName);
+        if (group) {
+            const toolNames = group.tools;
+            const all = this.getAllTools();
+            if (!toolNames) {
+                return all;
+            }
+
+            return toolNames.filter(name => all.some(tool => tool.name === name)).map(name => all.find(tool => tool.name === name)!);
+        }
+
+        throw new Error(`Tool group ${groupName} not found`);
+    }
+
+    getNonDefaultGroupInfo(): Record<string, string> {
+        return this.groups
+            .filter(group => !group.loadByDefault)
+            .reduce((acc, group) => {
+                acc[group.name] = group.description;
+                return acc;
+            }, {} as Record<string, string>);
+    }
+}

package/src/tools/playwright-groups.ts

@@ -0,0 +1,54 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+/**
+ * MCP (Model Context Protocol) Client for Playwright integration
+ * Integrates with https://github.com/microsoft/playwright-mcp
+ */
+import { DynamicStructuredTool } from 'langchain';
+import { AbstractGroup } from '.';
+import { PlaywrightBackendAdapter } from 'src/agents/tools/playwright-backend-adapter';
+export interface PlaywrightMCPOpts {
+  language: string,
+  headless?: boolean,
+}
+
+export class PlaywrightGroup extends AbstractGroup {
+  private allTools: DynamicStructuredTool[];
+  private adapter: PlaywrightBackendAdapter;
+
+  constructor(adapter: PlaywrightBackendAdapter) {
+    super([{
+      name: 'playwright_basic',
+      description: 'Basic Playwright tools for browser interactions',
+      tools: ['browser_navigate', 'browser_click', 'browser_fill_form', 'browser_press_key', 'browser_run_code', 'browser_select_option', 'browser_snapshot', 'browser_type', 'browser_wait_for'],
+      loadByDefault: true,
+    },
+    {
+      name: 'playwright_browser_manage',
+      description: `Playwright tools tab management, including: list, create, close, select a tab`,
+      tools: ['browser_tabs'],
+      loadByDefault: false,
+    },
+    {
+      name: 'playwright_other',
+      description: `Other Playwright tools for browser interactions, including: mouse hover on element, drag and drop an element, upload file, handle dialog, evaluate JavaScript on page or element
+      1. mouse hover on element
+      2. drag and drop an element
+      3. upload file
+      4. handle dialog
+      5. evaluate JavaScript on page or element`,
+      tools: ['browser_hover', 'browser_drag', 'browser_file_upload', 'browser_handle_dialog', 'browser_evaluate'],
+      loadByDefault: false,
+    }
+    ]);
+    this.adapter = adapter;
+    this.allTools = this.adapter.getTools();
+  }
+
+  async callTool(toolName: string, args: any): Promise<any> {
+    return this.adapter.callTool(toolName, args);
+  }
+
+  getAllTools(): DynamicStructuredTool[] {
+    return this.allTools
+  }
+}

package/src/types/external.ts

@@ -0,0 +1,7 @@
+/**
+ * External Type Definitions
+ * Import directly from core-infra
+ */
+
+// Re-export ModelConfig related types from core-infra
+export type { ModelConfig, ModelProvider } from '@cotestdev/core-infra';

package/src/types/index.ts

@@ -0,0 +1,118 @@
+/**
+ * ai-heal Core Type Definitions (MVP)
+ * Maintain simplicity, include only necessary types
+ */
+
+// Playwright types are used in type definitions below
+
+export interface RunnerConfig {
+  /** Test Case ID */
+  testCaseId: string;
+  /** Project ID */
+  projectId: string;
+}
+
+// ============================================================================
+// Execution Context
+// ============================================================================
+
+export interface RunStepOptions {
+  /** Step description for AI to understand */
+  description: string;
+}
+
+export interface ReActStep {
+  /** Step number */
+  stepNumber: number;
+  /** Step type */
+  stepType: 'thought' | 'action' | 'observation' | 'reflection';
+  /** Timestamp */
+  timestamp: number;
+  /** LLM thought process (thought step) */
+  thought?: string;
+  /** Tool call information (action step) */
+  toolCall?: {
+    name: string;
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    input: any;
+  };
+  /** Tool execution result (observation step) */
+  toolResult?: {
+    output: string;
+    success: boolean;
+    duration?: number;
+  };
+  /** Reflection content (reflection step) */
+  reflection?: string;
+}
+
+/**
+ * Context information for a single self-healing process
+ */
+export interface HealContext {
+  /** Step description */
+  stepDescription: string;
+  /** original error */
+  error?: string;
+  /** Stack trace */
+  stackTrace?: string;
+  /** Original code */
+  originalCode: string;
+  /** Whether successful */
+  success: boolean;
+  /** Total steps executed by Agent */
+  agentSteps?: number;
+  healError?: string;
+}
+
+// ============================================================================
+// Execution Result
+// ============================================================================
+
+/**
+ * Code execution result
+ */
+export interface ExecutionResult {
+  /** Whether successful */
+  success: boolean;
+  /** Return value */
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  returnValue: any;
+  /** Local variables */
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  locals: Record<string, any>;
+  /** Error message */
+  error?: Error;
+}
+
+// ============================================================================
+// Self-Healing Summary
+// ============================================================================
+
+/**
+ * Self-healing summary information
+ */
+export interface HealSummary {
+  /** Total number of steps */
+  totalSteps: number;
+  /** Number of successfully healed steps */
+  healedSteps: number;
+  /** Fixed script */
+  modifiedScript?: string;
+  /** List of self-healing details */
+  healingDetails: HealingDetail[];
+}
+
+/**
+ * Single self-healing detail
+ */
+export interface HealingDetail {
+  /** Step description */
+  step: string;
+  /** Error message */
+  error: string;
+  /** Recovery time */
+  recoveryTime: number;
+  /** Number of LLM interactions */
+  llmInteractions: number;
+}

package/tests/agent/test-heal-agent.spec.ts

@@ -0,0 +1,54 @@
+
+import { test, expect } from '@playwright/test';
+import { Runner } from '@cotestdev/ai-runner';
+
+test.describe(`Register user`, () => {
+  const runner = Runner.NewInstance('43af4db4-3249-4135-b279-97b24f27106b', '2e189c6f-6c2b-4086-8fe7-837bf1a6b1df');
+
+  test.afterEach(async () => {
+    runner.clear();
+  });
+
+  [
+    {
+      "password": "abcd1234"
+    }
+  ].forEach((data, index) => {
+    test(`#${index + 1}`, async ({ page, context }) => {
+      await runner.init(page, context, data);
+
+      await runner.runStep('进入注册页面', async () => {
+        await page.goto('https://practice.expandtesting.com/register');
+      });
+
+      await runner.runStep('填写并提交注册信息，用户名使用唯一格式（如未提供则自动生成）', async () => {
+        // 获取测试参数中的用户名和密码，如未提供用户名则自动生成
+        let username: string;
+        if (runner.has('username')) {
+          username = runner.get('username');
+        } else {
+          const timestamp = Date.now();
+          username = `user${timestamp}`;
+        }
+        const password = runner.get('password') || 'abcd1234';
+
+        // 填写注册表单
+        await page.locator('input[type="text"]').first().fill(username);
+        await page.locator('input[type="password"]').first().fill(password);
+        await page.locator('input[type="password"]').nth(1).fill(password);
+
+        // 提交注册表单
+        await page.getByRole('button', { name: 'Register' }).click();
+
+        // 设置输出参数
+        runner.set('username', username);
+      });
+
+      await runner.runStep('确认注册成功', async () => {
+        // 验证注册成功后跳转到登录页面或显示成功提示
+        await expect(page).toHaveURL(/login/);
+        await expect(page.locator('.alert-success')).toContainText('Registration successful');
+      });
+    });
+  });
+});