android-midscene-automation 0.1.6 → 0.1.9

package/README.md

@@ -1,4 +1,4 @@
-# Midscene Playwright Admin
+# Android Midscene Automation
 
 一个最小可用的 Midscene + Playwright 项目，同时提供一个基于 Vue 3 和 Element Plus 的后台，用来把自然语言 prompt 生成结构化自动化脚本。
 
@@ -48,17 +48,16 @@ npm run dev
 PLAYWRIGHT_DOWNLOAD_HOST=https://npmmirror.com/mirrors/playwright/ npx playwright install chromium
 ```
 
-## 局域网访问
+## 本地启动
 
 ```sh
 npm run dev
 ```
 
-启动后访问 Vite 输出的地址。项目默认监听 `0.0.0.0`，支持局域网访问：
+启动后访问 Vite 输出的本机地址。项目默认只监听 `127.0.0.1`：
 
 ```text
 http://127.0.0.1:5173/
-http://你的局域网IP:5173/
 ```
 
 `npm run dev` 会同时启动前端页面和本地后端 API。后端不是独立进程，而是在 `vite.config.ts` 中通过 `server/http-api.ts` 挂载到 Vite dev server 的 Connect middleware。
@@ -126,18 +125,17 @@ http://localhost:5173/
 
 这样后端执行的是当前电脑上的 `adb devices`，页面会识别当前电脑连接的手机。
 
-局域网共享页面只适合查看同一个中央服务，不适合直接识别访问者电脑上的 ADB 设备。远程设备代理方案仅保留在源码仓库的 `demo` 目录中，不随 npm 包发布。
-
 如果要回放 Appium 脚本，本机还需要启动 Appium：
 
 ```sh
 appium
 ```
-如果要使用远程设备代理实验方案，可以在访问者电脑的项目目录中运行：
 
-```sh
-npm run remote-agent -- --server http://中央服务IP:5173 --agent-id 你的名字或机器名
-```
+## 功能文档
+
+- [Appium Recorder 实现说明](docs/appium-recorder/implementation-notes.md)
+- [Appium 录制稳定性增强方案](docs/appium-recorder/robust-recording-design.md)
+- [项目更新记录](docs/appium-recorder/changelog.md)
 
 ## 运行 E2E 测试
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "android-midscene-automation",
-  "version": "0.1.6",
+  "version": "0.1.9",
   "type": "module",
   "bin": {
     "android-midscene-automation": "./bin/android-midscene-automation.js"
@@ -16,10 +16,10 @@
     "README.md"
   ],
   "scripts": {
-    "dev": "vite --host 0.0.0.0",
+    "dev": "vite --host 127.0.0.1",
     "remote-agent": "tsx remote-agent/index.ts",
     "build": "vue-tsc -b && vite build",
-    "preview": "vite preview --host 0.0.0.0",
+    "preview": "vite preview --host 127.0.0.1",
     "pw:install": "playwright install chromium",
     "test:e2e": "playwright test"
   },

package/server/appium-recorder/appium-runner.ts

@@ -1,7 +1,7 @@
 import { execFile } from 'node:child_process';
 import { mkdir, writeFile } from 'node:fs/promises';
 import { join } from 'node:path';
-import type { AppiumRecordedScriptRecord, AppiumRecordedStepRecord } from './repository';
+import { getAppiumRecordedScript, type AppiumRecordedScriptRecord, type AppiumRecordedStepRecord } from './repository';
 import { appDataPath } from '../paths';
 
 type AppiumSessionResponse = {
@@ -53,6 +53,30 @@ function adbTap(deviceId: string, x: number, y: number) {
   });
 }
 
+function adbLaunchApp(deviceId: string, packageName: string) {
+  if (!packageName) return Promise.reject(new Error('启动 APP 缺少包名'));
+  return new Promise<void>((resolve, reject) => {
+    execFile('adb', [
+      '-s',
+      deviceId,
+      'shell',
+      'monkey',
+      '-p',
+      packageName,
+      '-c',
+      'android.intent.category.LAUNCHER',
+      '1',
+    ], { maxBuffer: 4 * 1024 * 1024 }, (error, stdout, stderr) => {
+      const output = `${stdout || ''}\n${stderr || ''}`.trim();
+      if (error || /no activities found|monkey aborted/i.test(output)) {
+        reject(new Error(output || error?.message || `ADB 启动 ${packageName} 失败`));
+        return;
+      }
+      resolve();
+    });
+  });
+}
+
 function adbSwipe(deviceId: string, step: Required<AppiumRecordedStepRecord>['swipe']) {
   return new Promise<void>((resolve, reject) => {
     execFile('adb', [
@@ -117,6 +141,16 @@ async function appendSettingsDiagnostics(lines: string[], deviceId: string) {
   lines.push(`- 服务状态：${serviceInfo || '-'}`);
 }
 
+function appendAndroidSdkDiagnostics(lines: string[]) {
+  lines.push('Android SDK 环境变量未配置：');
+  lines.push('- Windows 常见 SDK 路径：%LOCALAPPDATA%\\Android\\Sdk');
+  lines.push('- PowerShell 设置示例：');
+  lines.push('  [Environment]::SetEnvironmentVariable("ANDROID_HOME", "$env:LOCALAPPDATA\\Android\\Sdk", "User")');
+  lines.push('  [Environment]::SetEnvironmentVariable("ANDROID_SDK_ROOT", "$env:LOCALAPPDATA\\Android\\Sdk", "User")');
+  lines.push('  [Environment]::SetEnvironmentVariable("Path", $env:Path + ";$env:LOCALAPPDATA\\Android\\Sdk\\platform-tools", "User")');
+  lines.push('- 设置后请重新打开终端，并重启 appium 服务。');
+}
+
 async function appiumRequest<T>(path: string, init?: RequestInit) {
   const requestUrl = `${appiumServerUrl()}${path}`;
   let response: Response;
@@ -180,27 +214,63 @@ async function createSession(script: AppiumRecordedScriptRecord, deviceId: strin
   return sessionId;
 }
 
-function toAppiumUsing(step: AppiumRecordedStepRecord) {
-  const selector = step.selector;
-  if (!selector) throw new Error(`${step.label} 缺少 selector`);
+function toAppiumUsing(selector: NonNullable<AppiumRecordedStepRecord['selector']>) {
   if (selector.strategy === 'accessibilityId') return { using: 'accessibility id', value: selector.value || '' };
   if (selector.strategy === 'id') return { using: 'id', value: selector.value || '' };
   if (selector.strategy === 'androidUiAutomator') return { using: '-android uiautomator', value: selector.value || '' };
   if (selector.strategy === 'xpath') return { using: 'xpath', value: selector.value || '' };
-  throw new Error(`${step.label} 需要使用 bounds 坐标执行`);
+  throw new Error('需要使用 bounds 坐标执行');
 }
 
-async function findElement(sessionId: string, step: AppiumRecordedStepRecord) {
-  const using = toAppiumUsing(step);
-  const payload = await appiumRequest<AppiumElementResponse>(`/session/${sessionId}/element`, {
+async function findElementBySelector(
+  sessionId: string,
+  selector: NonNullable<AppiumRecordedStepRecord['selector']>,
+  parentElementId?: string,
+) {
+  const using = toAppiumUsing(selector);
+  const path = parentElementId
+    ? `/session/${sessionId}/element/${parentElementId}/element`
+    : `/session/${sessionId}/element`;
+  const payload = await appiumRequest<AppiumElementResponse>(path, {
     method: 'POST',
     body: JSON.stringify(using),
   });
   const elementId = payload.value?.[ELEMENT_KEY] || payload.value?.ELEMENT || '';
-  if (!elementId) throw new Error(`${step.label} 未找到元素`);
+  if (!elementId) throw new Error('未找到元素');
   return elementId;
 }
 
+async function findElement(sessionId: string, step: AppiumRecordedStepRecord) {
+  if (!step.selector) throw new Error(`${step.label} 缺少 selector`);
+  const attempts: string[] = [];
+
+  if (step.contextSelector) {
+    try {
+      attempts.push(`context ${step.contextSelector.strategy} ${step.contextSelector.value || ''} -> ${step.selector.strategy} ${step.selector.value || ''}`);
+      const parentId = await findElementBySelector(sessionId, step.contextSelector);
+      return await findElementBySelector(sessionId, step.selector, parentId);
+    } catch {
+      // Continue with the fallback chain below.
+    }
+  }
+
+  for (const selector of step.selectorChain || []) {
+    try {
+      attempts.push(`${selector.strategy} ${selector.value || ''}`);
+      return await findElementBySelector(sessionId, selector);
+    } catch {
+      // Continue.
+    }
+  }
+
+  try {
+    attempts.push(`${step.selector.strategy} ${step.selector.value || ''}`);
+    return await findElementBySelector(sessionId, step.selector);
+  } catch (error) {
+    throw new Error(`${step.label} 未找到元素；尝试过：${attempts.filter(Boolean).join('；') || '-'}；${errorDetail(error)}`);
+  }
+}
+
 async function tapFallback(deviceId: string, step: AppiumRecordedStepRecord) {
   if (step.fallback?.strategy !== 'bounds' || !Number.isFinite(step.fallback.centerX) || !Number.isFinite(step.fallback.centerY)) {
     throw new Error(`${step.label} 未找到元素，且没有可用坐标兜底`);
@@ -224,6 +294,19 @@ async function waitForElement(sessionId: string, step: AppiumRecordedStepRecord)
   throw lastError instanceof Error ? lastError : new Error(`${step.label} 等待超时`);
 }
 
+async function findOptionalElement(sessionId: string, step: AppiumRecordedStepRecord) {
+  const startedAt = Date.now();
+  const timeoutMs = step.timeoutMs ?? 2000;
+  while (Date.now() - startedAt <= timeoutMs) {
+    try {
+      return await findElement(sessionId, step);
+    } catch {
+      await wait(250);
+    }
+  }
+  return '';
+}
+
 async function waitForElementGone(sessionId: string, step: AppiumRecordedStepRecord) {
   const startedAt = Date.now();
   const timeoutMs = step.timeoutMs || 10000;
@@ -267,11 +350,9 @@ async function runStep(sessionId: string, deviceId: string, step: AppiumRecorded
   }
 
   if (step.type === 'launchApp') {
-    await appiumRequest(`/session/${sessionId}/appium/device/activate_app`, {
-      method: 'POST',
-      body: JSON.stringify({ appId: step.value }),
-    });
-    return;
+    const packageName = step.value || '';
+    await adbLaunchApp(deviceId, packageName);
+    return `ADB 已启动 ${packageName}`;
   }
 
   if (step.type === 'screenshot') {
@@ -324,6 +405,46 @@ async function runStep(sessionId: string, deviceId: string, step: AppiumRecorded
     return;
   }
 
+  if (step.type === 'tapIfExists') {
+    const elementId = await findOptionalElement(sessionId, step);
+    if (!elementId) return '未出现，已跳过';
+    await appiumRequest(`/session/${sessionId}/element/${elementId}/click`, {
+      method: 'POST',
+      body: JSON.stringify({}),
+    });
+    return '已出现，已点击';
+  }
+
+  if (step.type === 'inputIfExists') {
+    const elementId = await findOptionalElement(sessionId, step);
+    if (!elementId) return '未出现，已跳过';
+    await appiumRequest(`/session/${sessionId}/element/${elementId}/value`, {
+      method: 'POST',
+      body: JSON.stringify({ text: step.value || '', value: [...(step.value || '')] }),
+    });
+    return '已出现，已输入';
+  }
+
+  if (step.type === 'clearIfExists') {
+    const elementId = await findOptionalElement(sessionId, step);
+    if (!elementId) return '未出现，已跳过';
+    await appiumRequest(`/session/${sessionId}/element/${elementId}/clear`, {
+      method: 'POST',
+      body: JSON.stringify({}),
+    });
+    return '已出现，已清空';
+  }
+
+  if (step.type === 'backIfExists') {
+    const elementId = await findOptionalElement(sessionId, step);
+    if (!elementId) return '未出现，已跳过';
+    await appiumRequest(`/session/${sessionId}/appium/device/press_keycode`, {
+      method: 'POST',
+      body: JSON.stringify({ keycode: 4 }),
+    });
+    return '已出现，已返回';
+  }
+
   if (step.type === 'waitFor' || step.type === 'assertExists') {
     await waitForElement(sessionId, step);
     return;
@@ -386,6 +507,225 @@ async function runStep(sessionId: string, deviceId: string, step: AppiumRecorded
   }
 }
 
+async function evaluateCondition(sessionId: string, deviceId: string, step: AppiumRecordedStepRecord) {
+  try {
+    if (step.type === 'waitActivity') {
+      await waitForActivity(deviceId, step);
+      return true;
+    }
+    if (step.type === 'waitDisappear') {
+      await waitForElementGone(sessionId, step);
+      return true;
+    }
+    if (step.type === 'assertText') {
+      const elementId = await findElement(sessionId, step);
+      const payload = await appiumRequest<AppiumValueResponse<string>>(`/session/${sessionId}/element/${elementId}/text`);
+      return (payload.value || '').includes(step.value || '');
+    }
+    await waitForElement(sessionId, step);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+function hasFlowSteps(steps: AppiumRecordedStepRecord[]) {
+  return steps.some((step) => (
+    step.flow?.nodeKind === 'condition'
+    || Boolean(step.flow?.yesTargetId)
+    || Boolean(step.flow?.noTargetId)
+    || Boolean(step.flow?.successTargetId)
+    || Boolean(step.flow?.failureTargetId)
+  ));
+}
+
+async function appendStepDiagnostics(lines: string[], deviceId: string, step: AppiumRecordedStepRecord) {
+  const currentActivity = await getCurrentActivity(deviceId).catch(() => '');
+  lines.push(`诊断 Activity：${currentActivity || '-'}`);
+  lines.push(`诊断 selector：${step.selector ? `${step.selector.strategy} ${step.selector.value || ''}` : '-'}`);
+  if (step.contextSelector) {
+    lines.push(`诊断上下文：${step.contextSelector.strategy} ${step.contextSelector.value || ''}`);
+  }
+  if (step.selectorChain?.length) {
+    lines.push(`诊断备用 selector：${step.selectorChain.map((selector) => `${selector.strategy} ${selector.value || ''}`).join('；')}`);
+  }
+  if (step.pageBefore) {
+    lines.push(`录制前 Activity：${step.pageBefore.activity || '-'}`);
+  }
+  if (step.pageAfter) {
+    lines.push(`录制后 Activity：${step.pageAfter.activity || '-'}`);
+  }
+}
+
+async function replayLinkedScript(
+  sessionId: string,
+  deviceId: string,
+  step: AppiumRecordedStepRecord,
+  lines: string[],
+  stack: string[],
+) {
+  const scriptId = step.value || '';
+  if (!scriptId) throw new Error(`${step.label} 缺少连接脚本 ID`);
+  if (stack.includes(scriptId)) throw new Error(`${step.label} 检测到循环连接脚本`);
+  const linkedScript = getAppiumRecordedScript(scriptId);
+  if (!linkedScript) throw new Error(`${step.label} 指向的脚本不存在`);
+
+  const currentActivity = await getCurrentActivity(deviceId).catch(() => '');
+  if (linkedScript.appActivity && currentActivity !== linkedScript.appActivity) {
+    throw new Error(
+      `${step.label} 无法执行：当前 Activity 为 ${currentActivity || '-'}，`
+      + `目标脚本入口 Activity 为 ${linkedScript.appActivity}`,
+    );
+  }
+
+  lines.push(`连接脚本开始：${linkedScript.name}`);
+  const nextStack = [...stack, linkedScript.id];
+  await replayScriptSteps(sessionId, deviceId, linkedScript.steps, lines, nextStack);
+  lines.push(`连接脚本完成：${linkedScript.name}`);
+}
+
+async function replayLinearSteps(
+  sessionId: string,
+  deviceId: string,
+  steps: AppiumRecordedStepRecord[],
+  lines: string[],
+  stack: string[] = [],
+) {
+  for (const [index, step] of steps.entries()) {
+    lines.push(`[步骤 ${index + 1}] 开始：${step.label}`);
+    try {
+      if (step.type === 'runScript') {
+        await replayLinkedScript(sessionId, deviceId, step, lines, stack);
+      } else {
+        const result = await runStep(sessionId, deviceId, step);
+        if (result) lines.push(`[步骤 ${index + 1}] 结果：${result}`);
+      }
+      lines.push(`[步骤 ${index + 1}] 完成：${step.label}`);
+    } catch (error) {
+      if (step.optional) {
+        lines.push(`[步骤 ${index + 1}] 跳过：${errorDetail(error)}`);
+        continue;
+      }
+      lines.push(`[步骤 ${index + 1}] 失败：${errorDetail(error)}`);
+      await appendStepDiagnostics(lines, deviceId, step);
+      throw error;
+    }
+  }
+}
+
+async function replayFlowSteps(
+  sessionId: string,
+  deviceId: string,
+  steps: AppiumRecordedStepRecord[],
+  lines: string[],
+  stack: string[] = [],
+) {
+  const idToIndex = new Map(steps.map((step, index) => [step.id, index]));
+  const visitedPath: string[] = [];
+  let index: number | undefined = 0;
+  let guard = 0;
+  const maxVisits = Math.max(steps.length * 4, 20);
+
+  while (typeof index === 'number' && index >= 0 && index < steps.length) {
+    guard += 1;
+    if (guard > maxVisits) throw new Error('流程图可能存在循环，已终止回放');
+
+    const step = steps[index];
+    visitedPath.push(step.label);
+    const nodeKind = step.flow?.nodeKind || (step.type === 'assertExists' || step.type === 'assertText' ? 'assertion' : 'action');
+    lines.push(`[节点 ${index + 1}] 开始：${step.label}`);
+
+    if (nodeKind === 'condition') {
+      const matched = await evaluateCondition(sessionId, deviceId, step);
+      const targetId = matched ? step.flow?.yesTargetId : step.flow?.noTargetId;
+      lines.push(`[节点 ${index + 1}] 判断：${matched ? '是' : '否'}${targetId ? `，进入 ${targetId}` : '，流程结束'}`);
+      index = targetId ? idToIndex.get(targetId) : (matched ? index + 1 : undefined);
+      continue;
+    }
+
+    try {
+      if (step.type === 'runScript') {
+        await replayLinkedScript(sessionId, deviceId, step, lines, stack);
+      } else {
+        const result = await runStep(sessionId, deviceId, step);
+        if (result) lines.push(`[节点 ${index + 1}] 结果：${result}`);
+      }
+      lines.push(`[节点 ${index + 1}] 完成：${step.label}`);
+      const targetId = step.flow?.successTargetId;
+      index = targetId ? idToIndex.get(targetId) : index + 1;
+    } catch (error) {
+      if (step.optional) {
+        lines.push(`[节点 ${index + 1}] 跳过：${errorDetail(error)}`);
+        index += 1;
+        continue;
+      }
+      const failureTarget = step.flow?.failureTargetId;
+      if (failureTarget && idToIndex.has(failureTarget)) {
+        lines.push(`[节点 ${index + 1}] 失败分支：${errorDetail(error)}`);
+        index = idToIndex.get(failureTarget);
+        continue;
+      }
+      lines.push(`[节点 ${index + 1}] 失败：${errorDetail(error)}`);
+      lines.push(`执行路径：${visitedPath.join(' -> ')}`);
+      await appendStepDiagnostics(lines, deviceId, step);
+      throw error;
+    }
+  }
+}
+
+async function replayStepGroup(
+  sessionId: string,
+  deviceId: string,
+  steps: AppiumRecordedStepRecord[],
+  lines: string[],
+  stack: string[],
+) {
+  if (!steps.length) return;
+  if (hasFlowSteps(steps)) {
+    await replayFlowSteps(sessionId, deviceId, steps, lines, stack);
+  } else {
+    await replayLinearSteps(sessionId, deviceId, steps, lines, stack);
+  }
+}
+
+async function replayScriptSteps(
+  sessionId: string,
+  deviceId: string,
+  steps: AppiumRecordedStepRecord[],
+  lines: string[],
+  stack: string[],
+) {
+  const hasScopedSteps = steps.some((step) => step.flow?.scope === 'pre' || step.flow?.scope === 'post');
+  if (!hasScopedSteps) {
+    await replayStepGroup(sessionId, deviceId, steps, lines, stack);
+    return;
+  }
+
+  const preSteps = steps.filter((step) => step.flow?.scope === 'pre');
+  const mainSteps = steps.filter((step) => !step.flow?.scope || step.flow.scope === 'main');
+  const postSteps = steps.filter((step) => step.flow?.scope === 'post');
+  let primaryError: unknown;
+
+  try {
+    if (preSteps.length) lines.push('执行前置操作...');
+    await replayStepGroup(sessionId, deviceId, preSteps, lines, stack);
+    if (mainSteps.length) lines.push('执行主流程...');
+    await replayStepGroup(sessionId, deviceId, mainSteps, lines, stack);
+  } catch (error) {
+    primaryError = error;
+  }
+
+  try {
+    if (postSteps.length) lines.push('执行后置操作...');
+    await replayStepGroup(sessionId, deviceId, postSteps, lines, stack);
+  } catch (error) {
+    if (!primaryError) throw error;
+    lines.push(`后置操作失败：${errorDetail(error)}`);
+  }
+
+  if (primaryError) throw primaryError;
+}
+
 export async function replayAppiumScript(script: AppiumRecordedScriptRecord, deviceId: string) {
   const targetDeviceId = deviceId || script.deviceId;
   if (!targetDeviceId) throw new Error('未检测到可用设备');
@@ -402,23 +742,19 @@ export async function replayAppiumScript(script: AppiumRecordedScriptRecord, dev
     lines.push('正在创建 Appium session...');
     sessionId = await createSession(script, targetDeviceId);
     lines.push(`Appium session 已创建：${sessionId}`);
-    for (const [index, step] of script.steps.entries()) {
-      lines.push(`[步骤 ${index + 1}] 开始：${step.label}`);
-      try {
-        await runStep(sessionId, targetDeviceId, step);
-        lines.push(`[步骤 ${index + 1}] 完成：${step.label}`);
-      } catch (error) {
-        lines.push(`[步骤 ${index + 1}] 失败：${errorDetail(error)}`);
-        throw error;
-      }
-    }
+    if (hasFlowSteps(script.steps)) lines.push('按流程图路径回放...');
+    await replayScriptSteps(sessionId, targetDeviceId, script.steps, lines, [script.id]);
     lines.push('回放完成');
     return { success: true, output: lines.join('\n') };
   } catch (error) {
-    if (errorDetail(error).includes('Appium Settings app is not running')) {
+    const detail = errorDetail(error);
+    if (detail.includes('Appium Settings app is not running')) {
       await appendSettingsDiagnostics(lines, targetDeviceId);
     }
-    lines.push(`回放终止：${errorDetail(error)}`);
+    if (detail.includes('Neither ANDROID_HOME nor ANDROID_SDK_ROOT')) {
+      appendAndroidSdkDiagnostics(lines);
+    }
+    lines.push(`回放终止：${detail}`);
     return { success: false, output: lines.join('\n') };
   } finally {
     if (sessionId) {