@chen-rmag/ai-runner 0.1.0

package/src/agents/tools/navigate.ts

@@ -0,0 +1,69 @@
+/**
+ * 导航工具 - Navigate
+ */
+
+import { DynamicTool } from '@langchain/core/tools';
+import type { Page } from 'playwright';
+
+/**
+ * 创建导航工具
+ */
+export function createNavigateTool(page: Page): DynamicTool {
+  return new DynamicTool({
+    name: 'browser_navigate',
+    description: 'Navigate to a URL. Input should be the URL as a string.',
+    func: async (input: string) => {
+      try {
+        await page.goto(input, { waitUntil: 'domcontentloaded', timeout: 30000 });
+        return `Navigated to ${input}`;
+      } catch (error) {
+        throw new Error(`Failed to navigate to ${input}: ${(error as Error).message}`);
+      }
+    }
+  });
+}
+
+/**
+ * 创建后退工具
+ */
+export function createGoBackTool(page: Page): DynamicTool {
+  return new DynamicTool({
+    name: 'browser_navigate_back',
+    description: 'Go back to the previous page. No input needed.',
+    func: async () => {
+      try {
+        await page.goBack();
+        return 'Navigated back';
+      } catch (error) {
+        throw new Error(`Failed to go back: ${(error as Error).message}`);
+      }
+    }
+  });
+}
+
+/**
+ * 创建前进工具
+ */
+export function createGoForwardTool(page: Page): DynamicTool {
+  return new DynamicTool({
+    name: 'browser_navigate_forward',
+    description: 'Go forward to the next page. No input needed.',
+    func: async () => {
+      try {
+        await page.goForward();
+        return 'Navigated forward';
+      } catch (error) {
+        throw new Error(`Failed to go forward: ${(error as Error).message}`);
+      }
+    }
+  });
+}
+
+/**
+ * 导航工具集
+ */
+export const navigateTools = (page: Page): DynamicTool[] => [
+  createNavigateTool(page),
+  createGoBackTool(page),
+  createGoForwardTool(page),
+];

package/src/agents/tools/snapshot.ts

@@ -0,0 +1,109 @@
+/**
+ * 快照和交互工具 - Snapshot & Click
+ */
+
+import { DynamicTool } from '@langchain/core/tools';
+import type { Page } from 'playwright';
+
+/**
+ * 创建页面快照工具
+ */
+export function createSnapshotTool(page: Page): DynamicTool {
+  return new DynamicTool({
+    name: 'browser_snapshot',
+    description: 'Get an accessibility snapshot of the current page. This is better than a screenshot for understanding page structure. No input needed.',
+    func: async () => {
+      try {
+        // 使用 aria 快照作为替代
+        const snapshot = await page.locator('body').ariaSnapshot();
+        return `Page accessibility snapshot:\n${snapshot}`;
+      } catch (error) {
+        throw new Error(`Failed to get snapshot: ${(error as Error).message}`);
+      }
+    }
+  });
+}
+
+/**
+ * 创建点击工具
+ */
+export function createClickTool(page: Page): DynamicTool {
+  return new DynamicTool({
+    name: 'browser_click',
+    description: 'Click on an element. Input should be a CSS selector or text that identifies the element. You can also use multiple selectors separated by commas.',
+    func: async (input: string) => {
+      try {
+        // 尝试多个选择器
+        const selectors = input.split(',').map(s => s.trim());
+
+        for (const selector of selectors) {
+          try {
+            // 先尝试作为文本选择器
+            const locator = page.getByText(selector).first();
+            const count = await locator.count();
+            if (count > 0) {
+              await locator.click({ timeout: 5000 });
+              return `Clicked on element with text: "${selector}"`;
+            }
+
+            // 再尝试作为 CSS 选择器
+            await page.click(selector, { timeout: 5000 });
+            return `Clicked on element: "${selector}"`;
+          } catch {
+            // 继续尝试下一个选择器
+            continue;
+          }
+        }
+
+        throw new Error(`No element found matching: ${input}`);
+      } catch (error) {
+        throw new Error(`Failed to click on "${input}": ${(error as Error).message}`);
+      }
+    }
+  });
+}
+
+/**
+ * 创建悬停工具
+ */
+export function createHoverTool(page: Page): DynamicTool {
+  return new DynamicTool({
+    name: 'browser_hover',
+    description: 'Hover over an element. Input should be a CSS selector or text that identifies the element.',
+    func: async (input: string) => {
+      try {
+        // 尝试多个选择器
+        const selectors = input.split(',').map(s => s.trim());
+
+        for (const selector of selectors) {
+          try {
+            const locator = page.getByText(selector).first();
+            const count = await locator.count();
+            if (count > 0) {
+              await locator.hover({ timeout: 5000 });
+              return `Hovered over element with text: "${selector}"`;
+            }
+
+            await page.hover(selector, { timeout: 5000 });
+            return `Hovered over element: "${selector}"`;
+          } catch {
+            continue;
+          }
+        }
+
+        throw new Error(`No element found matching: ${input}`);
+      } catch (error) {
+        throw new Error(`Failed to hover over "${input}": ${(error as Error).message}`);
+      }
+    }
+  });
+}
+
+/**
+ * 快照和交互工具集
+ */
+export const snapshotTools = (page: Page): DynamicTool[] => [
+  createSnapshotTool(page),
+  createClickTool(page),
+  createHoverTool(page),
+];

package/src/agents/tools/verify.ts

@@ -0,0 +1,168 @@
+/**
+ * 验证工具 - Verify
+ */
+
+import { DynamicTool } from '@langchain/core/tools';
+import type { Page } from 'playwright';
+
+/**
+ * 创建获取URL工具
+ */
+export function createGetURLTool(page: Page): DynamicTool {
+  return new DynamicTool({
+    name: 'browser_get_url',
+    description: 'Get the current page URL. No input needed.',
+    func: async () => {
+      try {
+        const url = page.url();
+        return `Current URL: ${url}`;
+      } catch (error) {
+        throw new Error(`Failed to get URL: ${(error as Error).message}`);
+      }
+    }
+  });
+}
+
+/**
+ * 创建获取页面内容工具
+ */
+export function createGetContentTool(page: Page): DynamicTool {
+  return new DynamicTool({
+    name: 'browser_get_content',
+    description: 'Get the page content (HTML). No input needed.',
+    func: async () => {
+      try {
+        const content = await page.content();
+        // 返回前1000个字符以避免过长
+        const preview = content.length > 1000 ? content.substring(0, 1000) + '...' : content;
+        return `Page content:\n${preview}`;
+      } catch (error) {
+        throw new Error(`Failed to get content: ${(error as Error).message}`);
+      }
+    }
+  });
+}
+
+/**
+ * 创建获取文本工具
+ */
+export function createGetTextTool(page: Page): DynamicTool {
+  return new DynamicTool({
+    name: 'browser_get_text',
+    description: 'Get text content from an element. Input should be a CSS selector or text that identifies the element.',
+    func: async (input: string) => {
+      try {
+        // 尝试多个选择器
+        const selectors = input.split(',').map(s => s.trim());
+
+        for (const selector of selectors) {
+          try {
+            // 先尝试作为文本选择器
+            const locator = page.getByText(selector).first();
+            const count = await locator.count();
+            if (count > 0) {
+              const text = await locator.textContent();
+              return `Text from element with text "${selector}": ${text}`;
+            }
+
+            // 再尝试作为 CSS 选择器
+            const element = page.locator(selector).first();
+            const text = await element.textContent();
+            return `Text from element "${selector}": ${text}`;
+          } catch {
+            continue;
+          }
+        }
+
+        throw new Error(`No element found matching: ${input}`);
+      } catch (error) {
+        throw new Error(`Failed to get text: ${(error as Error).message}`);
+      }
+    }
+  });
+}
+
+/**
+ * 创建截图工具
+ */
+export function createScreenshotTool(page: Page): DynamicTool {
+  return new DynamicTool({
+    name: 'browser_screenshot',
+    description: 'Take a screenshot of the current page. No input needed.',
+    func: async () => {
+      try {
+        const timestamp = Date.now();
+        const filename = `screenshot-${timestamp}.png`;
+        await page.screenshot({ path: filename });
+        return `Screenshot saved to ${filename}`;
+      } catch (error) {
+        throw new Error(`Failed to take screenshot: ${(error as Error).message}`);
+      }
+    }
+  });
+}
+
+/**
+ * 创建验证文本可见工具
+ */
+export function createVerifyTextVisibleTool(page: Page): DynamicTool {
+  return new DynamicTool({
+    name: 'browser_verify_text',
+    description: 'Verify that text is visible on the page. Input should be the text to verify.',
+    func: async (input: string) => {
+      try {
+        const locator = page.getByText(input).filter({ visible: true });
+        const count = await locator.count();
+
+        if (count === 0) {
+          throw new Error(`Text "${input}" not found on page`);
+        }
+
+        return `Text "${input}" is visible on the page`;
+      } catch (error) {
+        throw new Error(`Failed to verify text: ${(error as Error).message}`);
+      }
+    }
+  });
+}
+
+/**
+ * 创建验证元素可见工具
+ */
+export function createVerifyElementVisibleTool(page: Page): DynamicTool {
+  return new DynamicTool({
+    name: 'browser_verify_element',
+    description: 'Verify that an element is visible on the page. Input should be a CSS selector.',
+    func: async (input: string) => {
+      try {
+        const locator = page.locator(input).first();
+        const count = await locator.count();
+
+        if (count === 0) {
+          throw new Error(`Element "${input}" not found on page`);
+        }
+
+        const isVisible = await locator.isVisible();
+        if (!isVisible) {
+          throw new Error(`Element "${input}" is not visible`);
+        }
+
+        return `Element "${input}" is visible on the page`;
+      } catch (error) {
+        throw new Error(`Failed to verify element: ${(error as Error).message}`);
+      }
+    }
+  });
+}
+
+/**
+ * 验证工具集
+ */
+export const verifyTools = (page: Page): DynamicTool[] => [
+  createGetURLTool(page),
+  createGetContentTool(page),
+  createGetTextTool(page),
+  createScreenshotTool(page),
+  createVerifyTextVisibleTool(page),
+  createVerifyElementVisibleTool(page),
+];

package/src/agents/tools/wait.ts

@@ -0,0 +1,103 @@
+/**
+ * 等待工具 - Wait
+ */
+
+import { DynamicTool } from '@langchain/core/tools';
+import type { Page } from 'playwright';
+
+/**
+ * 创建等待时间工具
+ */
+export function createWaitForTimeTool(page: Page): DynamicTool {
+  return new DynamicTool({
+    name: 'browser_wait_for_time',
+    description: 'Wait for a specified amount of time. Input should be the time in milliseconds as a number.',
+    func: async (input: string) => {
+      try {
+        const time = parseInt(input, 10);
+        if (isNaN(time)) {
+          throw new Error('Input must be a number representing milliseconds');
+        }
+        await page.waitForTimeout(time);
+        return `Waited for ${time}ms`;
+      } catch (error) {
+        throw new Error(`Failed to wait: ${(error as Error).message}`);
+      }
+    }
+  });
+}
+
+/**
+ * 创建等待元素工具
+ */
+export function createWaitForSelectorTool(page: Page): DynamicTool {
+  return new DynamicTool({
+    name: 'browser_wait_for_selector',
+    description: 'Wait for an element to appear. Input should be a CSS selector or text that identifies the element. You can optionally specify timeout in milliseconds like: "selector, timeout=5000".',
+    func: async (input: string) => {
+      try {
+        // 解析输入（支持可选的 timeout 参数）
+        let selector = input;
+        let timeout = 30000;
+
+        const timeoutMatch = input.match(/timeout=(\d+)/);
+        if (timeoutMatch) {
+          timeout = parseInt(timeoutMatch[1], 10);
+          selector = input.replace(/,?\s*timeout=\d+/, '').trim();
+        }
+
+        // 尝试多个选择器
+        const selectors = selector.split(',').map(s => s.trim());
+
+        for (const sel of selectors) {
+          try {
+            // 先尝试作为文本选择器
+            const locator = page.getByText(sel);
+            const count = await locator.count();
+            if (count > 0) {
+              await locator.waitFor({ state: 'visible', timeout });
+              return `Element with text "${sel}" became visible`;
+            }
+
+            // 再尝试作为 CSS 选择器
+            await page.waitForSelector(sel, { state: 'visible', timeout });
+            return `Element "${sel}" became visible`;
+          } catch {
+            continue;
+          }
+        }
+
+        throw new Error(`No element found matching: ${selector}`);
+      } catch (error) {
+        throw new Error(`Failed to wait for selector: ${(error as Error).message}`);
+      }
+    }
+  });
+}
+
+/**
+ * 创建等待URL工具
+ */
+export function createWaitForURLTool(page: Page): DynamicTool {
+  return new DynamicTool({
+    name: 'browser_wait_for_url',
+    description: 'Wait for the URL to match a pattern. Input should be a URL pattern (string, glob, or regex).',
+    func: async (input: string) => {
+      try {
+        await page.waitForURL(input, { timeout: 30000 });
+        return `URL matched: ${input}`;
+      } catch (error) {
+        throw new Error(`Failed to wait for URL: ${(error as Error).message}`);
+      }
+    }
+  });
+}
+
+/**
+ * 等待工具集
+ */
+export const waitTools = (page: Page): DynamicTool[] => [
+  createWaitForTimeTool(page),
+  createWaitForSelectorTool(page),
+  createWaitForURLTool(page),
+];

package/src/agents/types.ts

@@ -0,0 +1,79 @@
+/**
+ * 自愈 Agent 类型定义
+ */
+
+import type { Page, BrowserContext } from 'playwright';
+import type { ModelConfig } from '../types/external';
+import type { ReActStep } from '../types';
+
+// ============================================================================
+// Agent 状态
+// ============================================================================
+
+/**
+ * 自愈 Agent 状态
+ */
+export interface HealAgentState {
+  // 输入
+  objective: string;              // 步骤描述，如 "点击按钮"
+  error: string;                  // 错误信息
+  stackTrace?: string;            // 错误堆栈
+  originalCode: string;           // 原始代码（用于参考）
+
+  // 执行上下文
+  context: {
+    page: any;                    // Playwright Page 对象
+    context?: any;                // Playwright BrowserContext 对象
+  };
+
+  // 变量绑定
+  variables?: Record<string, any>; // 用户提供的变量列表，Agent 需要回填变量值
+
+  // 执行配置
+  maxSteps?: number;              // 最大执行步数
+  debugMode?: boolean;            // 调试模式
+}
+
+// ============================================================================
+// Agent 配置
+// ============================================================================
+
+/**
+ * Agent 配置
+ */
+export interface HealAgentConfig {
+  model?: ModelConfig;            // 模型配置（用于 LLM Agent）
+  maxSteps?: number;              // 最大执行步数
+  debugMode?: boolean;            // 调试模式
+  setVariable?: (name: string, value: any) => void;  // 变量设置回调
+  getAllVariables?: () => Record<string, any>;  // 获取所有变量回调
+}
+
+// ============================================================================
+// Agent 结果
+// ============================================================================
+
+/**
+ * Agent 执行结果
+ */
+export interface HealAgentResult {
+  success: boolean;                // 是否成功完成目标
+  reasoning?: string;              // Agent 的推理过程
+  errorMessage?: string;           // 如果失败，错误原因
+  steps?: number;                  // 执行了多少步
+  updatedVariables?: Record<string, any>; // Agent 更新的变量值
+  reactSteps?: ReActStep[];        // ReAct 循环的详细步骤记录
+}
+
+// ============================================================================
+// MCP 工具
+// ============================================================================
+
+/**
+ * MCP 工具定义
+ */
+export interface MCPTool {
+  name: string;
+  description: string;
+  inputSchema: Record<string, unknown>;
+}