@cotestdev/ai-runner 0.0.5 → 0.0.7

package/src/agents/playwright-executor.ts

@@ -0,0 +1,284 @@
+/**
+ * Playwright Executor - Uses LangChain's createAgent API to implement ReAct loop
+ */
+/* eslint-disable @typescript-eslint/no-explicit-any */
+
+import { AIMessage, BaseMessage, createAgent, ToolMessage } from 'langchain';
+import { DynamicStructuredTool } from '@langchain/core/tools';
+import type { HealContext } from '../types';
+import z, { uuid } from 'zod';
+import { PlaywrightAgentState } from './types';
+import { uuid4 } from 'zod/v4/core/regexes.cjs';
+
+export class PlaywrightExecutor {
+  private state: PlaywrightAgentState;
+  private signal: AbortSignal;
+
+  constructor(state: PlaywrightAgentState, signal: AbortSignal) {
+    this.state = state;
+    this.signal = signal;
+  }
+
+  private async createPlaywrightTools(healContext: HealContext): Promise<DynamicStructuredTool[]> {
+    const standardTools = this.state.playwrightGroup.getDefaultTools();
+    const playwrightToolGroups = this.state.playwrightGroup.getNonDefaultGroupInfo();
+
+    const getToolGroupsTool = {
+      name: "get_tools",
+      description: "Get available tools in a group.",
+      schema: z.object({
+        group_name: z.string().describe("The name of the tool group."),
+      }),
+      func: async (param: any) => {
+        if (!param.group_name) throw new Error('Missing group_name parameter');
+        return JSON.stringify(playwrightToolGroups[param.group_name]);
+      },
+    };
+
+    const callToolTool = {
+      name: "call_tool",
+      description: "Call a tool in a group by name.",
+      schema: z.object({
+        group_name: z.string().describe("Tool group name."),
+        tool_name: z.string().describe("Tool name."),
+        args: z.object().describe("Tool parameters."),
+      }),
+      func: async (param: any) => {
+        const { group_name, tool_name, args } = param;
+        if (!group_name || !tool_name) throw new Error('Missing group_name or tool_name parameter');
+        return await this.state.playwrightGroup.callTool(tool_name, args);
+      },
+    };
+
+    const finishTool = {
+      name: "finish_test",
+      description: "Call this tool ONLY when the step goal has been fully achieved or failed. After calling this, no more tools should be called.",
+      schema: z.object({
+        isSuccess: z.boolean().describe("Whether the step goal was successfully achieved."),
+        error: z.string().optional().describe("Reason if the step goal fails to fully achieve, otherwise leave empty."),
+        variables: z.record(z.string(), z.any()).describe(`
+Key-value variables to assign in the test context to use in subsequent steps.
+  - The variable names should be consistent with those in the original test script
+  - Avoid conflicts with existing variable names
+  - Only include variables that have been changed or newly assigned in the fix
+  - The value should be NON-OBJECT types (string, number, boolean, etc.)
+  - If subsequent steps get a variable using 'runner.get', make sure to extract and provide the ACTUAL VALUE from test or page context
+          `),
+        repairedScript: z.string().optional().describe(`
+The complete fixed test script, leave empty if the step fails:
+   - Keep original runner.set, runner.get, runner.reuseTest calls for the steps fixed if possible and ensure no semantic changes
+   - MUST return The COMPLETE fixed test script
+   - ONLY apply fixes relevant to the current step and any necessary previous steps
+   - Do NOT wrap in closures or test body - write direct code using runner API
+   - Add a 'FIXED' comment for the step code being fixed 
+   - Example:
+   \`\`\`typescript
+   // FIXED
+   await runner.runStep('Click button', async () => {
+     await page.click('#button'); 
+   });
+   \`\`\`
+          `),
+      }),
+      func: async (params: any) => {
+        healContext.success = params.isSuccess;
+        healContext.healError = params.error;
+        if (healContext.success) {
+          if (params.variables) {
+            for (const [key, value] of Object.entries(params.variables)) {
+              this.state.variables.set(key, value);
+            }
+          }
+
+          this.state.originalScript = params.repairedScript;
+          this.updateOriginalScript(params.repairedScript);
+        }
+
+        return 'Test step finished.';
+      },
+    };
+
+    return [
+      ...standardTools,
+      new DynamicStructuredTool(getToolGroupsTool),
+      new DynamicStructuredTool(callToolTool),
+      new DynamicStructuredTool(finishTool)
+    ];
+  }
+
+  /**
+   * Update original script and save to database (heal mode)
+   */
+  private updateOriginalScript(script: string): void {
+    this.state.storageService.saveTestScript(this.state.testCaseId, script);
+    this.state.logger.log('Script fixed');
+  }
+
+  async execute(context: HealContext) {
+    const page = this.state.page;
+    if (!page) throw new Error('Missing page object');
+
+    // Check if already aborted
+    if (this.signal.aborted) {
+      throw new Error('Execution aborted');
+    }
+
+    const tools = await this.createPlaywrightTools(context);
+    const agent = createAgent({
+      model: this.state.model!,
+      tools: tools,
+      systemPrompt: this.buildSystemPrompt(),
+    } as any);
+
+    const snapshot = await this.getSnapshotContext();
+    const inputStream = {
+      messages: [{
+        role: 'user' as const,
+        content: this.buildUserMessage(context)
+      },
+      ...snapshot || [],
+      ]
+    };
+
+    const stream = await agent.stream(inputStream, {
+      recursionLimit: 25,
+      streamMode: 'updates',
+    });
+
+    for await (const event of stream) {
+      // Check for abort signal
+      if (this.signal.aborted) {
+        throw new Error('Execution aborted');
+      }
+
+      // Process AI messages (thoughts and tool calls)
+      const messages = event.model_request?.messages || event.messages;
+      if (messages?.[0]?.type === 'ai') {
+        const aiMsg = messages[0] as any;
+
+        const content = this.extractContent(aiMsg.content);
+        // Record thought
+        if (content) {
+          this.state.logger.log(`${content}`);
+        }
+      }
+
+      // Process tool results
+      const toolMsg = event.tools?.messages?.[0];
+      if (toolMsg?.type === 'tool') {
+        const output = this.extractContent(toolMsg.content) as string;
+        this.state.logger.log(`✅ Result [${toolMsg.name}]:`, output.length > 300 ? output.substring(0, 100) + '...' : output);
+      }
+    }
+  }
+
+  private buildSystemPrompt(): string {
+    return `
+You are a Playwright test expert, skilled at analyzing and fixing errors in browser automation tests.
+
+## ** Critial Rules **
+- You MUST run the step using tools first before fixing the script
+- When you step finishes or fails, you MUST call 'finish_test' tool and set the 'isSuccess' parameter correctly.
+- Analyze the current step code, errors and page context and what variables are being set
+- Make sure the fixed code is concise and consistent with the original code in logic and style
+- CRITICAL: When calling 'finish_test', analyze the full test script to identify variables defined in the current step that are used in subsequent steps
+
+## ** Constraints **
+- NEVER set 'isSuccess' to true if the step goal is not fully achieved
+- Fail the step fast and DON'T do unnecessary or unreansonable retries
+- Always check if variables assigned in the current step are referenced in later steps - if so, extract their actual values and include in 'variables'
+`.trim();
+  }
+
+  private async getSnapshotContext(): Promise<BaseMessage[] | undefined> {
+    const snapshot = await this.state.adapter.callTool('browser_snapshot', {});
+    if (snapshot.isError) {
+      this.state.logger.warn(`Failed to get browser snapshot: ${snapshot.error}`);
+      return undefined;
+    }
+
+    const callId = uuid.toString();
+    const aiMessage = new AIMessage({
+      tool_calls: [
+        {
+          name: 'browser_snapshot',
+          id: callId,
+          args: {},
+        },
+      ]
+    });
+    const toolMessage = new ToolMessage({
+      content: snapshot.content,
+      tool_call_id: callId,
+      name: 'browser_snapshot',
+      status: 'success',
+    });
+    return [aiMessage, toolMessage];
+  }
+
+  private buildUserMessage(context: HealContext): string {
+    const variableList = Object.keys(this.state.variables).length > 0
+      ? Object.keys(this.state.variables).map(v => `  - ${v}: ${JSON.stringify(this.state.variables.get(v))}`).join('\n')
+      : undefined;
+    return `
+## Step Description: ${context.stepDescription}
+
+${variableList ? `## Variables in Context\n${variableList}` : ''}
+
+## Stacktrace
+${context.stackTrace}
+
+## Full Test Script
+\`\`\`typescript
+${this.state.originalScript}
+\`\`\`
+`.trim();
+  }
+
+  /**
+     * Extract and normalize content from model response
+     * Handles string, array (ContentBlock[]), and other types
+     * Following LangChain.js best practices
+     */
+  private extractContent(content: unknown): string | Record<string, unknown> {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const asAny = content as any;
+    if (asAny.content) {
+      content = asAny.content;
+    }
+    // If string, return as-is
+    if (typeof content === 'string') {
+      return content;
+    }
+
+    // If array (ContentBlock[]), extract text content
+    if (Array.isArray(content)) {
+      const content0 = content[0];
+      if (typeof content0 === 'string') {
+        return content0;
+      }
+      if (typeof content0 === 'object' && content0 !== null) {
+        // Handle text blocks
+        if ('text' in content0) {
+          return content0.text as Record<string, unknown>;
+        }
+        // Handle other block types by converting to string
+        return JSON.stringify(content0);
+      }
+      return String(content0);
+    }
+
+    // If object, try to extract text field or convert to string
+    if (typeof content === 'object' && content !== null) {
+      if ('text' in content) {
+        return content as Record<string, unknown>;
+      }
+      return content as Record<string, unknown>;
+    }
+
+    // Fallback: convert to string
+    return String(content);
+  }
+}
+
+

package/src/agents/tools/playwright-backend-adapter.ts

@@ -0,0 +1,135 @@
+/**
+ * Playwright Backend 工具适配器 - 新方案
+ *
+ * 基于 BrowserServerBackend 的伪实例实现
+ * 参考：playwright/lib/mcp/test/browserBackend.js
+ *
+ * 这个方案的优势：
+ * 1. 直接使用 Playwright 官方的 BrowserServerBackend 类
+ * 2. 直接导入官方的 identityFactory，完全复用
+ * 3. 获得完整的官方工具功能，包括 Response、Context 等
+ * 4. 无需手动适配 Context 和 Response
+ * 5. 自动跟随 Playwright 更新
+ */
+
+import { DynamicStructuredTool } from '@langchain/core/tools';
+import type { BrowserContext } from 'playwright';
+import * as path from 'path';
+
+// 导入 Playwright MCP 模块的类型定义
+import type { PlaywrightMCPModules, BrowserServerBackendInstance } from './playwright-mcp-types';
+
+export class PlaywrightBackendAdapter {
+
+  private backend: BrowserServerBackendInstance | undefined;
+  private tools: DynamicStructuredTool[] = [];
+
+  static async create(browserContext: BrowserContext): Promise<PlaywrightBackendAdapter> {
+    const adapter = new PlaywrightBackendAdapter();
+    adapter.backend = await adapter.getBackend(browserContext);
+    adapter.tools = await adapter.getPlaywrightBackendToolsFromContext();
+    return adapter;
+  }
+
+  getTools(): DynamicStructuredTool[] {
+    return this.tools;
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  async callTool(name: string, params: any) {
+    return await this.backend!.callTool(name, params);
+  }
+
+  // ============================================================================
+  // 动态加载 Playwright 官方模块
+  // ============================================================================
+
+  /**
+   * 官方的 identityFactory 函数
+   * 参考：playwright/lib/mcp/test/browserBackend.js 第 94-104 行
+   *
+   * 注意：官方文件中定义了这个函数但没有导出，所以我们在这里复制一份
+   */
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  private identityFactory(browserContext: BrowserContext): any {
+    return {
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      createContext: async (clientInfo: any, abortSignal: any) => {
+        void clientInfo;
+        void abortSignal;
+        return {
+          browserContext,
+          close: async () => {
+            // 不关闭 context，因为它是外部的
+          }
+        };
+      }
+    };
+  }
+
+  private loadPlaywrightMCPModules(): PlaywrightMCPModules {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const playwrightRoot = path.dirname((require as any).resolve('playwright'));
+
+    return {
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      BrowserServerBackend: (require as any)(path.join(playwrightRoot, 'lib', 'mcp', 'browser', 'browserServerBackend.js')).BrowserServerBackend,
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      defaultConfig: (require as any)(path.join(playwrightRoot, 'lib', 'mcp', 'browser', 'config.js')).defaultConfig,
+      // 使用本地定义的 identityFactory（因为官方没有导出）
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      identityFactory: this.identityFactory as any,
+    };
+  }
+
+  private async getBackend(browserContext: BrowserContext): Promise<BrowserServerBackendInstance> {
+    // 加载 Playwright 模块（包括官方的 identityFactory）
+    const { BrowserServerBackend, defaultConfig, identityFactory } = this.loadPlaywrightMCPModules();
+
+    // 使用官方的 identityFactory（参考 browserBackend.js 第 34 行）
+    const factory = identityFactory(browserContext);
+
+    // 创建 Backend 实例
+    const backend = new BrowserServerBackend(
+      { ...defaultConfig, capabilities: ['core', 'core-tabs',] },
+      factory
+    );
+
+    // 初始化 Backend
+    await backend.initialize({
+      name: 'ai-runner',
+      version: '1.0.0',
+      roots: [],
+    });
+
+    return backend;
+  }
+  // ============================================================================
+  // 工具转换函数
+  // ============================================================================
+
+  // export type ToolCapability = 'core' | 'core-tabs' | 'core-install' | 'vision' | 'pdf' | 'testing' | 'tracing';
+
+  private async getPlaywrightBackendToolsFromContext(): Promise<DynamicStructuredTool[]> {
+    // 列出所有工具
+    const toolList = await this.backend!.listTools();
+
+    // 转换为 LangChain 工具
+    const tools: DynamicStructuredTool[] = [];
+
+    for (const mcpTool of toolList) {
+      const tool = new DynamicStructuredTool({
+        name: mcpTool.name,
+        description: mcpTool.description,
+        schema: mcpTool.inputSchema,
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        func: async (input: any) => {
+          return await this.backend!.callTool(mcpTool.name, input);
+        }
+      });
+      tools.push(tool);
+    }
+
+    return tools;
+  }
+}

package/src/agents/tools/playwright-mcp-types.d.ts

@@ -0,0 +1,72 @@
+/**
+ * Playwright MCP 模块类型定义
+ *
+ * 为动态加载的 Playwright 内部模块提供 TypeScript 类型支持
+ */
+
+import type { BrowserContext } from 'playwright';
+import type { Root } from '@modelcontextprotocol/sdk/types.js';
+
+export interface BrowserContextFactory {
+  createContext: (
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    clientInfo: any,
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    abortSignal: any,
+    toolName: string
+  ) => Promise<{
+    browserContext: BrowserContext;
+    close: () => Promise<void>;
+  }>;
+}
+
+export interface BackendConfig {
+  capabilities?: string[];
+  saveSession?: boolean;
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  [key: string]: any;
+}
+
+export interface BrowserServerBackendInstance {
+  initialize(clientInfo: { name: string; version: string; roots: Root[]}): Promise<void>;
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  listTools(): Promise<Array<{ name: string; description: string; inputSchema: any }>>;
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  callTool(name: string, params: any): Promise<any>;
+  serverClosed(): void;
+}
+
+export interface BrowserServerBackend {
+  new (config: BackendConfig, factory: BrowserContextFactory): BrowserServerBackendInstance;
+}
+
+export interface DefaultConfig {
+  capabilities?: string[];
+  saveSession?: boolean;
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  [key: string]: any;
+}
+
+/**
+ * 官方的 identityFactory 函数类型
+ * 参考：playwright/lib/mcp/test/browserBackend.js 第 94-104 行
+ */
+export type IdentityFactory = (browserContext: BrowserContext) => BrowserContextFactory;
+
+export interface MCPTool {
+  schema: {
+    name: string;
+    description: string;
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    inputSchema: any;
+  };
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  handle: (context: any, params: any, response: any) => Promise<void>;
+}
+
+// 动态加载的模块类型
+export interface PlaywrightMCPModules {
+  BrowserServerBackend: BrowserServerBackend;
+  defaultConfig: DefaultConfig;
+  identityFactory: IdentityFactory;
+}

package/src/agents/types.ts

@@ -0,0 +1,80 @@
+/**
+ * Playwright Agent Type Definitions
+ */
+
+import type { ModelConfig } from '../types/external';
+import type { ReActStep } from '../types';
+import { BrowserContext, Page } from '@playwright/test';
+import { Logger } from './logger';
+import { PlaywrightGroup } from '../tools/playwright-groups';
+import { PlaywrightBackendAdapter } from './tools/playwright-backend-adapter';
+import { StorageService } from '@cotestdev/core-infra';
+import { BaseLanguageModel } from '@langchain/core/language_models/base';
+
+// ============================================================================
+// Agent Execution Mode
+// ============================================================================
+
+/**
+ * Agent execution mode
+ */
+export type AgentMode = 'heal' | 'execute';
+
+// ============================================================================
+// Agent State
+// ============================================================================
+
+/**
+ * Playwright Agent State
+ */
+export interface PlaywrightAgentState {
+  logger: Logger;
+  /** Playwright Page object */
+  page: Page;
+  /** Playwright BrowserContext object */
+  context: BrowserContext;
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  variables: Map<string, any>;
+  model?: BaseLanguageModel;
+  originalScript: string;
+  playwrightGroup: PlaywrightGroup;
+  adapter: PlaywrightBackendAdapter;
+  storageService: StorageService;
+  testCaseId: string;
+}
+
+// ============================================================================
+// Agent Configuration
+// ============================================================================
+
+/**
+ * Playwright Agent Configuration
+ */
+export interface PlaywrightAgentConfig {
+  /** Model configuration (for LLM Agent) */
+  model: ModelConfig;
+}
+
+// ============================================================================
+// Agent Result
+// ============================================================================
+
+/**
+ * Playwright Agent Execution Result
+ */
+export interface PlaywrightAgentResult {
+  /** Whether the goal was successfully completed */
+  success: boolean;
+
+  /** Agent's reasoning process */
+  reasoning?: string;
+
+  /** If failed, the error reason */
+  errorMessage?: string;
+
+  /** How many steps were executed */
+  steps?: number;
+
+  /** Detailed step records of ReAct loop */
+  reactSteps?: ReActStep[];
+}

package/src/index.ts

@@ -0,0 +1,27 @@
+/**
+ * ai-runner SDK Main Entry Point
+ * Provides AI-driven test execution capabilities
+ */
+
+// Types
+export type {
+  RunnerConfig,
+  ExecutionResult,
+  HealContext,
+  HealSummary,
+  HealingDetail,
+  RunStepOptions,
+} from './types';
+
+
+// Core Classes
+export { Runner } from './runner';
+
+
+// Agent
+export type {
+  PlaywrightAgentState,
+  PlaywrightAgentResult,
+  PlaywrightAgentConfig,
+} from './agents/types';
+