openclaw-multi-auto 1.4.5 → 1.4.7

package/extensions/page-action-cache/skills/page-action-cache/SKILL.md

@@ -0,0 +1,216 @@
+# 页面操作缓存扩展 - 使用指南
+
+**版本**: 1.0.0
+**更新日期**: 2026-03-13
+
+## 概述
+
+## 概述
+
+本技能提供了页面操作缓存功能，可以缓存 LLM 分析浏览器页面后生成的操作序列。当再次访问相同页面时，直接使用缓存的操作，无需重新分析 DOM，大幅降低 token 消耗和操作延迟。
+
+## 核心优势
+
+- **节省 Token**: 避免每次操作都发送 DOM 给 LLM 分析
+- **加速执行**: 跳过 LLM 分析，直接执行缓存的操作
+- **智能匹配**: 多层场景识别（规则+LLM+历史）
+- **安全可靠**: 自动检测页面变化，缓存失效
+
+## 可用工具
+
+### 1. execute_cached - 执行缓存操作
+
+**用途**: 执行缓存的页面操作序列
+
+**参数**:
+- `cacheKey` (必需): 缓存键（从 cache_info 中获取）
+- `fromIndex` (可选): 从第几个操作开始执行（默认 0）
+- `toIndex` (可选): 执行到第几个操作（不指定则执行全部）
+- `dryRun` (可选): 试运行模式，不实际执行
+- `force` (可选): 强制执行（忽略页面变化检测）
+
+**示例**:
+```
+使用缓存登录 example.com，cacheKey: abc123
+```
+
+### 2. cache_stats - 查看缓存统计
+
+**用途**: 查看缓存命中统计、节省的 token 等
+
+**参数**: 无
+
+**示例**:
+```
+查看缓存统计信息
+```
+
+### 3. cache_list - 列出所有缓存
+
+**用途**: 列出所有缓存条目及其详情
+
+**参数**:
+- `limit` (可选): 最多返回的条目数（默认 20）
+- `filterScenario` (可选): 按场景过滤（如 login, checkout）
+
+**示例**:
+```
+列出所有登录相关的缓存
+```
+
+### 4. cache_clear - 清空缓存
+
+**用途**: 清空缓存条目
+
+**参数**:
+- `scenario` (可选): 只清空指定场景的缓存
+
+**示例**:
+```
+清空所有缓存
+清空登录场景的缓存
+```
+
+### 5. scenario_list - 列出所有场景
+
+**用途**: 列出所有支持的缓存场景及其匹配规则
+
+**参数**: 无
+
+**示例**:
+```
+列出所有支持的缓存场景
+```
+
+### 6. force_refresh - 强制刷新缓存
+
+**用途**: 强制刷新指定页面的缓存
+
+**参数**:
+- `url` (必需): 页面 URL
+- `viewport` (可选): 视口尺寸
+
+**示例**:
+```
+强制刷新 example.com/login 的缓存
+```
+
+## 使用流程
+
+### 场景 1: 首次访问页面
+
+```
+用户: "登录 example.com"
+↓
+LLM: 分析页面 DOM，生成操作序列
+↓
+系统: 自动保存到缓存（L3 级）
+```
+
+### 场景 2: 再次访问相同页面
+
+```
+用户: "登录 example.com"
+↓
+系统: 检测到缓存匹配（置信度 95%）
+↓
+LLM: 收到 cacheInfo，决定使用缓存
+↓
+系统: 使用 execute_cached 工具执行缓存操作
+↓
+结果: 节省 5000 tokens，加速 2 秒
+```
+
+### 场景 3: 页面发生变化
+
+```
+用户: "登录 example.com"
+↓
+系统: 检测页面结构变化
+↓
+系统: 自动失效缓存
+↓
+LLM: 重新分析页面，生成新操作
+↓
+系统: 保存新的缓存
+```
+
+## 支持的场景
+
+| 场景 | 关键词 | 缓存层级 |
+|------|---------|---------|
+| login | 登录、登陆、注册、sign in | L3 |
+| logout | 退出、登出、注销、logout | L3 |
+| search | 搜索、查找、search | L3 |
+| checkout | 结账、结算、支付、checkout | L3 |
+| settings | 设置、配置、修改 | L3 |
+| form_fill | 填写、填表单、提交 | L2 |
+| navigate | 打开、进入、跳转 | L2 |
+| screenshot | 截图、screenshot | L1 |
+
+## 缓存层级
+
+- **L3: 场景级** - 包含变量模板（如 username, password），高置信度时使用
+- **L2: 流程级** - 固定操作序列，中等置信度时使用
+- **L1: 原子级** - 单个操作，低置信度时使用
+
+## 注意事项
+
+1. **页面变化检测**: 系统会自动检测页面结构变化，如果检测到变化会自动失效缓存
+2. **变量替换**: 支持变量模板（如 ${username}），执行时自动从用户输入提取
+3. **敏感数据**: 密码等敏感数据会自动脱敏存储
+4. **试运行**: 使用 dryRun 参数可以预览操作而不执行
+5. **强制执行**: 使用 force 参数可以跳过页面变化检测（谨慎使用）
+
+## 最佳实践
+
+1. **优先使用缓存**: 当工具返回 cacheInfo 时，优先使用 execute_cached 工具
+2. **检查命中统计**: 定期查看 cache_stats，了解缓存效果
+3. **及时清理**: 使用 cache_clear 清理不再需要的缓存
+4. **测试新功能**: 使用 dryRun 模式测试缓存操作
+5. **处理变化**: 当页面变化导致操作失败时，使用 force_refresh 刷新缓存
+
+## 故障排查
+
+### 问题 1: 缓存未命中
+
+**原因**:
+- 首次访问该页面
+- 缓存已过期
+- 页面结构发生变化
+
+**解决**:
+- 使用 scenario_list 查看支持的场景
+- 使用 cache_list 查看现有缓存
+- 使用 force_refresh 强制刷新
+
+### 问题 2: 缓存执行失败
+
+**原因**:
+- 页面结构发生变化
+- 元素选择器失效
+- 操作顺序错误
+
+**解决**:
+- 查看执行结果中的错误信息
+- 使用 force_refresh 刷新缓存
+- 调整操作序列
+
+### 问题 3: 变量未替换
+
+**原因**:
+- 用户输入中不包含变量值
+- 变量名不匹配
+
+**解决**:
+- 确保用户输入包含变量值（如 "用户名 alice"）
+- 使用变量提取测试验证
+
+## 统计信息含义
+
+- **总条目数**: 当前缓存的页面操作条目数量
+- **总命中**: 缓存命中的总次数
+- **命中率**: 缓存命中占总请求的百分比
+- **L3/L2/L1 命中**: 各层级缓存命中的次数
+- **节省 Token**: 估算节省的 token 数量
+- **节省时间**: 估算节省的时间（毫秒）

package/extensions/page-action-cache/src/actions-executor.ts

@@ -0,0 +1,441 @@
+/**
+ * Actions Executor
+ * 操作执行器 - 执行缓存的原子化操作
+ */
+
+import type {
+  AtomicAction,
+  ExecutionResult,
+  PwAi,
+  VariableMap,
+  NavigateAction,
+  ClickAction,
+  TypeAction,
+  PressAction,
+  HoverAction,
+  ScreenshotAction,
+  EvaluateAction,
+} from "./types.js";
+
+// ============================================================================
+// Actions Executor 类
+// ============================================================================
+
+/**
+ * 操作执行器
+ */
+export class ActionsExecutor {
+  private pw: PwAi | null = null;
+  private cdpUrl: string = "";
+
+  constructor(cdpUrl?: string) {
+    if (cdpUrl) {
+      this.cdpUrl = cdpUrl;
+    }
+  }
+
+  /**
+   * 设置 Playwright 客户端
+   */
+  setPlaywrightClient(pw: PwAi): void {
+    this.pw = pw;
+  }
+
+  /**
+   * 设置 CDP URL
+   */
+  setCdpUrl(cdpUrl: string): void {
+    this.cdpUrl = cdpUrl;
+  }
+
+  /**
+   * 执行单个操作
+   */
+  async execute(
+    action: AtomicAction,
+    variables?: VariableMap
+  ): Promise<ExecutionResult> {
+    const startTime = Date.now();
+
+    try {
+      // 应用变量
+      const resolvedAction = this.applyVariables(action, variables);
+
+      // 根据操作类型执行
+      switch (resolvedAction.type) {
+        case "navigate":
+          await this.executeNavigate(resolvedAction as NavigateAction);
+          break;
+
+        case "click":
+          await this.executeClick(resolvedAction as ClickAction);
+          break;
+
+        case "type":
+          await this.executeType(resolvedAction as TypeAction);
+          break;
+
+        case "press":
+          await this.executePress(resolvedAction as PressAction);
+          break;
+
+        case "hover":
+          await this.executeHover(resolvedAction as HoverAction);
+          break;
+
+        case "screenshot":
+          await this.executeScreenshot(resolvedAction as ScreenshotAction);
+          break;
+
+        case "scroll":
+        case "wait":
+        case "select":
+        case "focus":
+        case "drag":
+        case "upload":
+          await this.executeEvaluate(resolvedAction as EvaluateAction);
+          break;
+
+        case "composite":
+          // 复合操作暂不实现
+          throw new Error("Composite actions not yet implemented");
+
+        default:
+          throw new Error(`Unsupported action type: ${action.type}`);
+      }
+
+      return {
+        action: action.type,
+        success: true,
+        duration: Date.now() - startTime,
+      };
+    } catch (error) {
+      return {
+        action: action.type,
+        success: false,
+        error: error instanceof Error ? error.message : String(error),
+        duration: Date.now() - startTime,
+      };
+    }
+  }
+
+  /**
+   * 批量执行操作
+   */
+  async executeBatch(
+    actions: AtomicAction[],
+    variables?: VariableMap,
+    options?: {
+      fromIndex?: number;
+      toIndex?: number;
+      atomic?: boolean; // 是否原子执行（全部成功或全部失败）
+    }
+  ): Promise<ExecutionResult[]> {
+    const from = options?.fromIndex ?? 0;
+    const to =
+      options?.toIndex !== undefined ? options.toIndex + 1 : actions.length;
+    const atomic = options?.atomic ?? false;
+
+    const results: ExecutionResult[] = [];
+    const actionsToExecute = actions.slice(from, to);
+
+    for (const action of actionsToExecute) {
+      const result = await this.execute(action, variables);
+      results.push(result);
+
+      // 原子执行：失败则停止
+      if (atomic && !result.success) {
+        break;
+      }
+    }
+
+    return results;
+  }
+
+  // -------------------------------------------------------------------------
+  // 具体操作实现
+  // -------------------------------------------------------------------------
+
+  private async executeNavigate(action: NavigateAction): Promise<void> {
+    if (!this.pw) {
+      throw new Error("Playwright client not set");
+    }
+
+    await this.pw.navigateViaPlaywright({
+      cdpUrl: action.cdpUrl || this.cdpUrl,
+      targetId: action.targetId,
+      url: action.url,
+      waitUntil: action.navigationPolicy?.waitUntil || "domcontentloaded",
+      timeout: action.navigationPolicy?.timeout,
+    });
+  }
+
+  private async executeClick(action: ClickAction): Promise<void> {
+    if (!this.pw) {
+      throw new Error("Playwright client not set");
+    }
+
+    await this.pw.clickViaPlaywright({
+      cdpUrl: action.cdpUrl || this.cdpUrl,
+      targetId: action.targetId,
+      ref: action.ref,
+      doubleClick: action.doubleClick,
+      button: action.button || "left",
+      modifiers: action.modifiers,
+      timeoutMs: action.timeoutMs,
+    });
+  }
+
+  private async executeType(action: TypeAction): Promise<void> {
+    if (!this.pw) {
+      throw new Error("Playwright client not set");
+    }
+
+    await this.pw.typeViaPlaywright({
+      cdpUrl: action.cdpUrl || this.cdpUrl,
+      targetId: action.targetId,
+      ref: action.ref,
+      text: action.text,
+      submit: action.submit,
+      slowly: action.slowly,
+      timeoutMs: action.timeoutMs,
+    });
+  }
+
+  private async executePress(action: PressAction): Promise<void> {
+    if (!this.pw) {
+      throw new Error("Playwright client not set");
+    }
+
+    await this.pw.pressKeyViaPlaywright({
+      cdpUrl: action.cdpUrl || this.cdpUrl,
+      targetId: action.targetId,
+      key: action.key,
+      delayMs: action.delayMs,
+    });
+  }
+
+  private async executeHover(action: HoverAction): Promise<void> {
+    if (!this.pw) {
+      throw new Error("Playwright client not set");
+    }
+
+    await this.pw.hoverViaPlaywright({
+      cdpUrl: action.cdpUrl || this.cdpUrl,
+      targetId: action.targetId,
+      ref: action.ref,
+      timeoutMs: action.timeoutMs,
+    });
+  }
+
+  private async executeScreenshot(action: ScreenshotAction): Promise<string> {
+    if (!this.pw) {
+      throw new Error("Playwright client not set");
+    }
+
+    return this.pw.screenshotViaPlaywright({
+      cdpUrl: action.cdpUrl || this.cdpUrl,
+      targetId: action.targetId,
+    });
+  }
+
+  private async executeEvaluate(action: EvaluateAction): Promise<any> {
+    if (!this.pw) {
+      throw new Error("Playwright client not set");
+    }
+
+    return this.pw.evaluateViaPlaywright({
+      cdpUrl: action.cdpUrl || this.cdpUrl,
+      targetId: action.targetId,
+      code: action.evaluate.code,
+      args: action.evaluate.args,
+    });
+  }
+
+  // -------------------------------------------------------------------------
+  // 变量替换
+  // -------------------------------------------------------------------------
+
+  /**
+   * 应用变量到操作
+   */
+  private applyVariables(
+    action: AtomicAction,
+    variables?: VariableMap
+  ): AtomicAction {
+    if (!variables) {
+      return action;
+    }
+
+    // 检查是否需要变量替换
+    if (!action.variable) {
+      return action;
+    }
+
+    // 获取变量值
+    const value = variables[action.variable];
+    if (value === undefined) {
+      // 变量未定义，返回原操作
+      return action;
+    }
+
+    // 根据操作类型设置正确的参数
+    const actionType = action.type;
+    if (actionType === "type") {
+      return { ...action, text: value as string };
+    } else if (actionType === "press") {
+      return { ...action, key: value as string };
+    } else if (actionType === "navigate") {
+      return { ...action, url: value as string };
+    } else if (actionType === "scroll" || actionType === "wait" ||
+               actionType === "select" || actionType === "focus" ||
+               actionType === "drag" || actionType === "upload") {
+      // evaluate 操作替换模板变量
+      return this.replaceTemplateVariables(action, variables);
+    } else {
+      // 其他操作替换所有字符串参数中的模板变量
+      return this.replaceTemplateVariables(action, variables);
+    }
+  }
+
+  /**
+   * 替换操作中的模板变量
+   */
+  private replaceTemplateVariables(
+    action: AtomicAction,
+    variables: VariableMap
+  ): AtomicAction {
+    const result: any = { ...action };
+
+    // 替换字符串类型参数中的 ${variable}
+    for (const key of Object.keys(action)) {
+      const value = (action as any)[key];
+      if (typeof value === "string") {
+        result[key] = value.replace(/\$\{([^}]+)\}/g, (match, varName) => {
+          const replacement = variables[varName];
+          return replacement !== undefined ? replacement : match;
+        });
+      }
+    }
+
+    // 特殊处理 evaluate.code（如果存在）
+    if (result.evaluate?.code && typeof result.evaluate.code === "string") {
+      result.evaluate.code = result.evaluate.code.replace(
+        /\$\{([^}]+)\}/g,
+        (match: string, varName: string) => {
+          const replacement = variables[varName];
+          return replacement !== undefined ? replacement : match;
+        }
+      );
+    }
+
+    return result;
+  }
+
+  // -------------------------------------------------------------------------
+  // 格式化输出
+  // -------------------------------------------------------------------------
+
+  /**
+   * 格式化操作列表
+   */
+  formatActions(actions: AtomicAction[]): string {
+    return actions
+      .map((action, i) => this.formatAction(action, i + 1))
+      .join("\n");
+  }
+
+  /**
+   * 格式化单个操作
+   */
+  private formatAction(action: AtomicAction, index: number): string {
+    switch (action.type) {
+      case "navigate":
+        return `${index}. 导航到：${(action as NavigateAction).url}`;
+
+      case "screenshot":
+        return `${index}. 截图`;
+
+      case "click":
+        const btn = (action as ClickAction).button || "left";
+        const dbl = (action as ClickAction).doubleClick ? "双击" : "";
+        const mod = (action as ClickAction).modifiers?.length
+          ? `+${(action as ClickAction).modifiers!.join("+")}`
+          : "";
+        return `${index}. ${dbl}点击（${btn}${mod}）：${action.ref}`;
+
+      case "type":
+        const txt = (action as TypeAction).text
+          ? `="${(action as TypeAction).text}"`
+          : "";
+        const sub = (action as TypeAction).submit ? " [提交]" : "";
+        const slow = (action as TypeAction).slowly ? " [慢速]" : "";
+        return `${index}. 输入：${action.ref}${txt}${sub}${slow}`;
+
+      case "press":
+        const delay = (action as PressAction).delayMs
+          ? `(${(action as PressAction).delayMs}ms)`
+          : "";
+        return `${index}. 按键：${(action as PressAction).key}${delay}${
+          action.ref ? ` @ ${action.ref}` : ""
+        }`;
+
+      case "hover":
+        return `${index}. 悬停：${action.ref}`;
+
+      case "scroll":
+        return `${index}. 滚动：${
+          (action as EvaluateAction).evaluate?.code || "向下滚动"
+        }`;
+
+      case "wait":
+        const waitDelay = action.delay ? `(${action.delay}ms)` : "";
+        return `${index}. 等待：${
+          (action as EvaluateAction).evaluate?.code || "默认等待"
+        }${waitDelay}`;
+
+      case "select":
+        return `${index}. 选择：${action.ref}`;
+
+      case "focus":
+        return `${index}. 聚焦：${action.ref}`;
+
+      case "drag":
+        return `${index}. 拖拽：${action.ref}`;
+
+      case "upload":
+        return `${index}. 上传：${action.ref}`;
+
+      default:
+        return `${index}. ${action.type}：${action.ref || ""}`;
+    }
+  }
+
+  /**
+   * 格式化执行结果
+   */
+  formatExecutionResults(results: ExecutionResult[]): string {
+    return results
+      .map((r, i) => {
+        const icon = r.success ? "✅" : "❌";
+        const time = r.duration ? `(${r.duration}ms)` : "";
+        return `${icon} 操作 ${i + 1}: ${r.action} ${time}${
+          r.error ? ` - ${r.error}` : ""
+        }`;
+      })
+      .join("\n");
+  }
+}
+
+// ============================================================================
+// 单例
+// ============================================================================
+
+let actionsExecutorInstance: ActionsExecutor | null = null;
+
+export function getActionsExecutor(cdpUrl?: string): ActionsExecutor {
+  if (!actionsExecutorInstance) {
+    actionsExecutorInstance = new ActionsExecutor(cdpUrl);
+  }
+  return actionsExecutorInstance;
+}