android-midscene-automation 1.1.2 → 1.1.3

package/CHANGELOG.md

@@ -1,4 +1,7 @@
 # 更新记录
+## v1.1.3
+- [Appium]修复登录等点击操作已经触发 Activity 跳转后，UiAutomator2 仍等待点击确认、最终误报超时并占用命令队列的问题；元素点击先读取实时区域，再通过 ADB 点击中心坐标，并限制界面空闲等待时间。
+
 ## v1.1.2
 - [Appium]修复登录等点击操作已经触发 Activity 跳转后，持续动画或频繁界面事件仍让 UiAutomator2 长时间等待、最终误报点击超时的问题；限制界面空闲等待时间，避免命令队列被长期占用。
 

package/README.md

@@ -1502,6 +1502,9 @@ Linux: ~/.local/share/android-midscene-automation
 
 # 项目更新记录
 
+## v1.1.3
+- [Appium]修复登录等点击操作已经触发 Activity 跳转后，UiAutomator2 仍等待点击确认、最终误报超时并占用命令队列的问题；元素点击先读取实时区域，再通过 ADB 点击中心坐标，并限制界面空闲等待时间。
+
 ## v1.1.2
 - [Appium]修复登录等点击操作已经触发 Activity 跳转后，持续动画或频繁界面事件仍让 UiAutomator2 长时间等待、最终误报点击超时的问题；限制界面空闲等待时间，避免命令队列被长期占用。
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "android-midscene-automation",
-  "version": "1.1.2",
+  "version": "1.1.3",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/oooooooko/android-midscene-automation.git"

package/server/appium-recorder/appium-runner.ts

@@ -536,11 +536,36 @@ async function imageCheckRegion(sessionId: string, step: AppiumRecordedStepRecor
 
 async function tapFallback(deviceId: string, step: AppiumRecordedStepRecord) {
   if (step.fallback?.strategy !== 'bounds' || !Number.isFinite(step.fallback.centerX) || !Number.isFinite(step.fallback.centerY)) {
-    throw new Error(`${step.label} 未找到元素，且没有可用坐标兜底`);
+    throw new Error(`${step.label} 没有可用点击坐标`);
   }
   await adbTap(deviceId, Number(step.fallback.centerX), Number(step.fallback.centerY));
 }
 
+async function tapLocatedElement(
+  sessionId: string,
+  deviceId: string,
+  elementId: string,
+  step?: AppiumRecordedStepRecord,
+) {
+  try {
+    const payload = await appiumRequest<AppiumValueResponse<VisualChangeRegion>>(`/session/${sessionId}/element/${elementId}/rect`);
+    const rect = payload.value;
+    if (rect && [rect.x, rect.y, rect.width, rect.height].every(Number.isFinite) && rect.width > 0 && rect.height > 0) {
+      await adbTap(deviceId, rect.x + rect.width / 2, rect.y + rect.height / 2);
+      return;
+    }
+  } catch (error) {
+    throwIfReplayStopped(error);
+    // 普通点击的外层会重新定位一次，避免列表刷新后使用已失效元素的旧位置。
+    if (isStaleElementError(error)) throw error;
+  }
+  if (step) {
+    await tapFallback(deviceId, step);
+    return;
+  }
+  throw new Error('无法读取目标元素的实时点击区域');
+}
+
 async function waitForElement(sessionId: string, step: AppiumRecordedStepRecord) {
   const startedAt = Date.now();
   const timeoutMs = step.timeoutMs ?? DEFAULT_NODE_TIMEOUT_MS;
@@ -908,10 +933,7 @@ async function runStep(
   if (step.type === 'tapIfExists') {
     const elementId = await findOptionalElement(sessionId, step);
     if (!elementId) return '未出现，已跳过';
-    await appiumRequest(`/session/${sessionId}/element/${elementId}/click`, {
-      method: 'POST',
-      body: JSON.stringify({}),
-    });
+    await tapLocatedElement(sessionId, deviceId, elementId, step);
     return '已出现，已点击';
   }
 
@@ -962,10 +984,7 @@ async function runStep(
     for (let attempt = 0; attempt < 2; attempt += 1) {
       try {
         const elementId = await findElement(sessionId, step);
-        await appiumRequest(`/session/${sessionId}/element/${elementId}/click`, {
-          method: 'POST',
-          body: JSON.stringify({}),
-        });
+        await tapLocatedElement(sessionId, deviceId, elementId, step);
         return;
       } catch (error) {
         throwIfReplayStopped(error);
@@ -1052,7 +1071,7 @@ async function evaluateCondition(sessionId: string, deviceId: string, step: Appi
         const element = payload.value[0]!;
         const id = element[ELEMENT_KEY] || element.ELEMENT;
         if (!id) throw new Error('文字点击查询缺少元素 ID');
-        await appiumRequest(`/session/${sessionId}/element/${id}/click`, { method: 'POST', body: '{}' });
+        await tapLocatedElement(sessionId, deviceId, id);
         return true;
       }
       if (Date.now() >= deadline) {

package/server/appium-recorder/element-tap.test.ts

@@ -0,0 +1,86 @@
+import assert from 'node:assert/strict';
+import { test } from 'node:test';
+import childProcess from 'node:child_process';
+import { syncBuiltinESMExports } from 'node:module';
+import { mkdtemp, mkdir, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import type { AppiumRecordedScriptRecord } from './repository';
+
+test('element taps use the live element center through ADB without waiting for Appium click acknowledgement', async () => {
+  const root = await mkdtemp(join(tmpdir(), 'element-tap-'));
+  const env = { ...process.env };
+  const original = { execFile: childProcess.execFile, fetch: globalThis.fetch };
+  const appiumPaths: string[] = [];
+  const adbCalls: string[][] = [];
+  let elementRequestCount = 0;
+  try {
+    const sdk = join(root, 'sdk');
+    await mkdir(join(sdk, 'platform-tools'), { recursive: true });
+    await writeFile(join(sdk, 'platform-tools', 'adb'), '');
+    Object.assign(process.env, {
+      ANDROID_MIDSCENE_DATA_ROOT: root,
+      ANDROID_HOME: sdk,
+      ANDROID_SDK_ROOT: sdk,
+      APPIUM_SERVER_URL: 'http://external.invalid',
+    });
+    childProcess.execFile = ((_command, args: string[], _options, callback) => {
+      adbCalls.push(args);
+      queueMicrotask(() => callback(null, '', ''));
+      return {};
+    }) as typeof childProcess.execFile;
+    syncBuiltinESMExports();
+    globalThis.fetch = async (input, init) => {
+      const path = new URL(String(input)).pathname;
+      appiumPaths.push(`${init?.method || 'GET'} ${path}`);
+      if (path === '/session' && init?.method === 'POST') {
+        return Response.json({ value: { sessionId: 'test-session' } });
+      }
+      if (path === '/session/test-session/element' && init?.method === 'POST') {
+        elementRequestCount += 1;
+        return Response.json({ value: { 'element-6066-11e4-a52e-4f735466cecf': `button-${elementRequestCount}` } });
+      }
+      if (path === '/session/test-session/element/button-2/rect') {
+        return Response.json({ value: { error: 'stale element reference', message: 'element changed' } }, { status: 404 });
+      }
+      if (/\/element\/button-[13]\/rect$/.test(path)) {
+        return Response.json({ value: { x: 10, y: 20, width: 100, height: 60 } });
+      }
+      return Response.json({ value: null });
+    };
+    const { replayAppiumScript } = await import('./appium-runner');
+    const script: AppiumRecordedScriptRecord = {
+      id: 'tap', name: 'tap', deviceId: 'fake-device', appPackage: '', appActivity: '', createdAt: '', updatedAt: '',
+      steps: [{
+        id: 'login', type: 'tap', label: '点击登录按钮',
+        selector: { strategy: 'id', value: 'example:id/login', unique: true },
+        fallback: { strategy: 'bounds', centerX: 999, centerY: 999 },
+      }, {
+        id: 'refreshing', type: 'tap', label: '点击刷新列表中的按钮',
+        selector: { strategy: 'id', value: 'example:id/refreshing', unique: true },
+        fallback: { strategy: 'bounds', centerX: 999, centerY: 999 },
+      }],
+    };
+    const result = await replayAppiumScript(script, 'fake-device', undefined, undefined, {
+      screenshotReport: false,
+      reportSummaryEnabled: false,
+    });
+    assert.equal(result.success, true, result.output);
+    assert.ok(appiumPaths.includes('GET /session/test-session/element/button-1/rect'));
+    assert.ok(appiumPaths.includes('GET /session/test-session/element/button-3/rect'));
+    assert.equal(elementRequestCount, 3, 're-locate once after a stale element rect');
+    assert.equal(appiumPaths.some(path => path.endsWith('/click')), false, 'must not call the blocking Appium click endpoint');
+    assert.deepEqual(adbCalls.filter(args => args.includes('tap')).map(args => args.slice(-4)), [
+      ['input', 'tap', '60', '50'],
+      ['input', 'tap', '60', '50'],
+    ]);
+  } finally {
+    childProcess.execFile = original.execFile;
+    globalThis.fetch = original.fetch;
+    syncBuiltinESMExports();
+    for (const key of ['ANDROID_MIDSCENE_DATA_ROOT', 'ANDROID_HOME', 'ANDROID_SDK_ROOT', 'APPIUM_SERVER_URL']) {
+      if (env[key] === undefined) delete process.env[key]; else process.env[key] = env[key];
+    }
+    await rm(root, { recursive: true, force: true });
+  }
+});