android-midscene-automation 1.1.1 → 1.1.3

package/CHANGELOG.md

@@ -1,6 +1,12 @@
 # 更新记录
+## v1.1.3
+- [Appium]修复登录等点击操作已经触发 Activity 跳转后，UiAutomator2 仍等待点击确认、最终误报超时并占用命令队列的问题；元素点击先读取实时区域，再通过 ADB 点击中心坐标，并限制界面空闲等待时间。
+
+## v1.1.2
+- [Appium]修复登录等点击操作已经触发 Activity 跳转后，持续动画或频繁界面事件仍让 UiAutomator2 长时间等待、最终误报点击超时的问题；限制界面空闲等待时间，避免命令队列被长期占用。
+
 ## v1.1.1
-更新README.md
+- [文档]更新 README.md。
 
 ## v1.1.0
 - [Artemis]修复自定义模型返回 `<think>` 思考标签导致 Hopper 等节点 JSON 解析失败的问题：非 Google 模型的 Pydantic 结构化输出接入统一清理、提取和校验流程，显式提供 JSON Schema，格式错误时限次要求模型纠正。

package/README.md

@@ -1502,8 +1502,14 @@ Linux: ~/.local/share/android-midscene-automation
 
 # 项目更新记录
 
+## v1.1.3
+- [Appium]修复登录等点击操作已经触发 Activity 跳转后，UiAutomator2 仍等待点击确认、最终误报超时并占用命令队列的问题；元素点击先读取实时区域，再通过 ADB 点击中心坐标，并限制界面空闲等待时间。
+
+## v1.1.2
+- [Appium]修复登录等点击操作已经触发 Activity 跳转后，持续动画或频繁界面事件仍让 UiAutomator2 长时间等待、最终误报点击超时的问题；限制界面空闲等待时间，避免命令队列被长期占用。
+
 ## v1.1.1
-更新README.md
+- [文档]更新 README.md。
 
 ## v1.1.0
 - [Artemis]修复自定义模型返回 `<think>` 思考标签导致 Hopper 等节点 JSON 解析失败的问题：非 Google 模型的 Pydantic 结构化输出接入统一清理、提取和校验流程，显式提供 JSON Schema，格式错误时限次要求模型纠正。

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "android-midscene-automation",
-  "version": "1.1.1",
+  "version": "1.1.3",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/oooooooko/android-midscene-automation.git"

package/server/appium-recorder/appium-runner.ts

@@ -50,6 +50,7 @@ type AppiumValueResponse<T> = {
 };
 
 const ELEMENT_KEY = 'element-6066-11e4-a52e-4f735466cecf';
+const UIAUTOMATOR_IDLE_TIMEOUT_MS = 1000;
 const configuredAppiumServerUrl = () => (process.env.APPIUM_SERVER_URL || 'http://127.0.0.1:4723').replace(/\/+$/, '');
 
 class ReplayStoppedError extends Error {
@@ -420,6 +421,9 @@ async function createSession(deviceId: string) {
           'appium:udid': deviceId,
           'appium:autoLaunch': false,
           'appium:noReset': true,
+          // 实时画面、动画和频繁无障碍事件可能让 UiAutomator2 长时间等不到界面空闲。
+          // 节点已有显式等待，因此这里只保留很短的空闲窗口，避免点击已生效后请求仍占住命令队列。
+          'appium:settings': { waitForIdleTimeout: UIAUTOMATOR_IDLE_TIMEOUT_MS },
           // 托管进程由 finally 释放；外部服务保留回收期限，避免创建请求取消后遗留永久会话。
           'appium:newCommandTimeout': usesManagedAppiumServer() ? 0 : 60,
           'appium:skipDeviceInitialization': true,
@@ -532,11 +536,36 @@ async function imageCheckRegion(sessionId: string, step: AppiumRecordedStepRecor
 
 async function tapFallback(deviceId: string, step: AppiumRecordedStepRecord) {
   if (step.fallback?.strategy !== 'bounds' || !Number.isFinite(step.fallback.centerX) || !Number.isFinite(step.fallback.centerY)) {
-    throw new Error(`${step.label} 未找到元素，且没有可用坐标兜底`);
+    throw new Error(`${step.label} 没有可用点击坐标`);
   }
   await adbTap(deviceId, Number(step.fallback.centerX), Number(step.fallback.centerY));
 }
 
+async function tapLocatedElement(
+  sessionId: string,
+  deviceId: string,
+  elementId: string,
+  step?: AppiumRecordedStepRecord,
+) {
+  try {
+    const payload = await appiumRequest<AppiumValueResponse<VisualChangeRegion>>(`/session/${sessionId}/element/${elementId}/rect`);
+    const rect = payload.value;
+    if (rect && [rect.x, rect.y, rect.width, rect.height].every(Number.isFinite) && rect.width > 0 && rect.height > 0) {
+      await adbTap(deviceId, rect.x + rect.width / 2, rect.y + rect.height / 2);
+      return;
+    }
+  } catch (error) {
+    throwIfReplayStopped(error);
+    // 普通点击的外层会重新定位一次，避免列表刷新后使用已失效元素的旧位置。
+    if (isStaleElementError(error)) throw error;
+  }
+  if (step) {
+    await tapFallback(deviceId, step);
+    return;
+  }
+  throw new Error('无法读取目标元素的实时点击区域');
+}
+
 async function waitForElement(sessionId: string, step: AppiumRecordedStepRecord) {
   const startedAt = Date.now();
   const timeoutMs = step.timeoutMs ?? DEFAULT_NODE_TIMEOUT_MS;
@@ -904,10 +933,7 @@ async function runStep(
   if (step.type === 'tapIfExists') {
     const elementId = await findOptionalElement(sessionId, step);
     if (!elementId) return '未出现，已跳过';
-    await appiumRequest(`/session/${sessionId}/element/${elementId}/click`, {
-      method: 'POST',
-      body: JSON.stringify({}),
-    });
+    await tapLocatedElement(sessionId, deviceId, elementId, step);
     return '已出现，已点击';
   }
 
@@ -958,10 +984,7 @@ async function runStep(
     for (let attempt = 0; attempt < 2; attempt += 1) {
       try {
         const elementId = await findElement(sessionId, step);
-        await appiumRequest(`/session/${sessionId}/element/${elementId}/click`, {
-          method: 'POST',
-          body: JSON.stringify({}),
-        });
+        await tapLocatedElement(sessionId, deviceId, elementId, step);
         return;
       } catch (error) {
         throwIfReplayStopped(error);
@@ -1048,7 +1071,7 @@ async function evaluateCondition(sessionId: string, deviceId: string, step: Appi
         const element = payload.value[0]!;
         const id = element[ELEMENT_KEY] || element.ELEMENT;
         if (!id) throw new Error('文字点击查询缺少元素 ID');
-        await appiumRequest(`/session/${sessionId}/element/${id}/click`, { method: 'POST', body: '{}' });
+        await tapLocatedElement(sessionId, deviceId, id);
         return true;
       }
       if (Date.now() >= deadline) {

package/server/appium-recorder/element-tap.test.ts

@@ -0,0 +1,86 @@
+import assert from 'node:assert/strict';
+import { test } from 'node:test';
+import childProcess from 'node:child_process';
+import { syncBuiltinESMExports } from 'node:module';
+import { mkdtemp, mkdir, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import type { AppiumRecordedScriptRecord } from './repository';
+
+test('element taps use the live element center through ADB without waiting for Appium click acknowledgement', async () => {
+  const root = await mkdtemp(join(tmpdir(), 'element-tap-'));
+  const env = { ...process.env };
+  const original = { execFile: childProcess.execFile, fetch: globalThis.fetch };
+  const appiumPaths: string[] = [];
+  const adbCalls: string[][] = [];
+  let elementRequestCount = 0;
+  try {
+    const sdk = join(root, 'sdk');
+    await mkdir(join(sdk, 'platform-tools'), { recursive: true });
+    await writeFile(join(sdk, 'platform-tools', 'adb'), '');
+    Object.assign(process.env, {
+      ANDROID_MIDSCENE_DATA_ROOT: root,
+      ANDROID_HOME: sdk,
+      ANDROID_SDK_ROOT: sdk,
+      APPIUM_SERVER_URL: 'http://external.invalid',
+    });
+    childProcess.execFile = ((_command, args: string[], _options, callback) => {
+      adbCalls.push(args);
+      queueMicrotask(() => callback(null, '', ''));
+      return {};
+    }) as typeof childProcess.execFile;
+    syncBuiltinESMExports();
+    globalThis.fetch = async (input, init) => {
+      const path = new URL(String(input)).pathname;
+      appiumPaths.push(`${init?.method || 'GET'} ${path}`);
+      if (path === '/session' && init?.method === 'POST') {
+        return Response.json({ value: { sessionId: 'test-session' } });
+      }
+      if (path === '/session/test-session/element' && init?.method === 'POST') {
+        elementRequestCount += 1;
+        return Response.json({ value: { 'element-6066-11e4-a52e-4f735466cecf': `button-${elementRequestCount}` } });
+      }
+      if (path === '/session/test-session/element/button-2/rect') {
+        return Response.json({ value: { error: 'stale element reference', message: 'element changed' } }, { status: 404 });
+      }
+      if (/\/element\/button-[13]\/rect$/.test(path)) {
+        return Response.json({ value: { x: 10, y: 20, width: 100, height: 60 } });
+      }
+      return Response.json({ value: null });
+    };
+    const { replayAppiumScript } = await import('./appium-runner');
+    const script: AppiumRecordedScriptRecord = {
+      id: 'tap', name: 'tap', deviceId: 'fake-device', appPackage: '', appActivity: '', createdAt: '', updatedAt: '',
+      steps: [{
+        id: 'login', type: 'tap', label: '点击登录按钮',
+        selector: { strategy: 'id', value: 'example:id/login', unique: true },
+        fallback: { strategy: 'bounds', centerX: 999, centerY: 999 },
+      }, {
+        id: 'refreshing', type: 'tap', label: '点击刷新列表中的按钮',
+        selector: { strategy: 'id', value: 'example:id/refreshing', unique: true },
+        fallback: { strategy: 'bounds', centerX: 999, centerY: 999 },
+      }],
+    };
+    const result = await replayAppiumScript(script, 'fake-device', undefined, undefined, {
+      screenshotReport: false,
+      reportSummaryEnabled: false,
+    });
+    assert.equal(result.success, true, result.output);
+    assert.ok(appiumPaths.includes('GET /session/test-session/element/button-1/rect'));
+    assert.ok(appiumPaths.includes('GET /session/test-session/element/button-3/rect'));
+    assert.equal(elementRequestCount, 3, 're-locate once after a stale element rect');
+    assert.equal(appiumPaths.some(path => path.endsWith('/click')), false, 'must not call the blocking Appium click endpoint');
+    assert.deepEqual(adbCalls.filter(args => args.includes('tap')).map(args => args.slice(-4)), [
+      ['input', 'tap', '60', '50'],
+      ['input', 'tap', '60', '50'],
+    ]);
+  } finally {
+    childProcess.execFile = original.execFile;
+    globalThis.fetch = original.fetch;
+    syncBuiltinESMExports();
+    for (const key of ['ANDROID_MIDSCENE_DATA_ROOT', 'ANDROID_HOME', 'ANDROID_SDK_ROOT', 'APPIUM_SERVER_URL']) {
+      if (env[key] === undefined) delete process.env[key]; else process.env[key] = env[key];
+    }
+    await rm(root, { recursive: true, force: true });
+  }
+});

package/server/appium-recorder/managed-cleanup.test.ts

@@ -47,6 +47,7 @@ test('replay survives command idle periods and cleans sessions after success, ab
       if (path === '/session' && init?.method === 'POST') {
         if (mode === 'creation-failure') return Response.json({ value: { error: 'session not created', message: 'creation failed' } }, { status: 500 });
         const capabilities = JSON.parse(String(init.body)).capabilities.alwaysMatch;
+        assert.deepEqual(capabilities['appium:settings'], { waitForIdleTimeout: 1000 }, 'bound UiAutomator idle waits for animated screens');
         commandTimeoutMs = (capabilities['appium:newCommandTimeout'] ?? 60) * 1000;
         lastCommandMs = elapsedMs;
         return Response.json({ value: { sessionId: 'test-session' } });