android-midscene-automation 0.1.0

package/server/appium-recorder/appium-runner.ts

@@ -0,0 +1,427 @@
+import { execFile } from 'node:child_process';
+import { mkdir, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import type { AppiumRecordedScriptRecord, AppiumRecordedStepRecord } from './repository';
+
+type AppiumSessionResponse = {
+  value?: {
+    sessionId?: string;
+    capabilities?: unknown;
+  };
+  sessionId?: string;
+};
+
+type AppiumElementResponse = {
+  value?: Record<string, string>;
+};
+
+type AppiumValueResponse<T> = {
+  value?: T;
+};
+
+const ELEMENT_KEY = 'element-6066-11e4-a52e-4f735466cecf';
+const appiumServerUrl = () => (process.env.APPIUM_SERVER_URL || 'http://127.0.0.1:4723').replace(/\/+$/, '');
+
+function errorDetail(error: unknown) {
+  if (!(error instanceof Error)) return String(error || '未知错误');
+  const cause = error.cause;
+  if (cause instanceof Error && cause.message && cause.message !== error.message) {
+    return `${error.message}：${cause.message}`;
+  }
+  return error.message;
+}
+
+function isStaleElementError(error: unknown) {
+  const detail = errorDetail(error);
+  return detail.includes('stale element reference') || detail.includes('does not exist in DOM anymore');
+}
+
+function wait(ms: number) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function adbTap(deviceId: string, x: number, y: number) {
+  return new Promise<void>((resolve, reject) => {
+    execFile('adb', ['-s', deviceId, 'shell', 'input', 'tap', String(Math.round(x)), String(Math.round(y))], (error, _stdout, stderr) => {
+      if (error) {
+        reject(new Error(stderr || error.message));
+        return;
+      }
+      resolve();
+    });
+  });
+}
+
+function adbSwipe(deviceId: string, step: Required<AppiumRecordedStepRecord>['swipe']) {
+  return new Promise<void>((resolve, reject) => {
+    execFile('adb', [
+      '-s',
+      deviceId,
+      'shell',
+      'input',
+      'swipe',
+      String(Math.round(step.startX)),
+      String(Math.round(step.startY)),
+      String(Math.round(step.endX)),
+      String(Math.round(step.endY)),
+      String(Math.round(step.duration)),
+    ], (error, _stdout, stderr) => {
+      if (error) {
+        reject(new Error(stderr || error.message));
+        return;
+      }
+      resolve();
+    });
+  });
+}
+
+function adbText(deviceId: string, args: string[]) {
+  return new Promise<string>((resolve) => {
+    execFile('adb', ['-s', deviceId, ...args], { maxBuffer: 4 * 1024 * 1024 }, (_error, stdout, stderr) => {
+      resolve((stdout || stderr || '').trim());
+    });
+  });
+}
+
+async function getCurrentActivity(deviceId: string) {
+  const output = await adbText(deviceId, ['shell', 'dumpsys', 'activity', 'activities']);
+  const resumedLine = output
+    .split(/\r?\n/)
+    .find((line) => /(?:topResumedActivity|ResumedActivity|mResumedActivity)/.test(line));
+  return resumedLine?.match(/\s([\w.$]+\/[\w.$]+)\s/)?.[1] || '';
+}
+
+async function waitForActivity(deviceId: string, step: AppiumRecordedStepRecord) {
+  const expectedActivity = step.value || '';
+  if (!expectedActivity) throw new Error(`${step.label} 缺少目标 Activity`);
+  const startedAt = Date.now();
+  const timeoutMs = step.timeoutMs || 10000;
+  let currentActivity = '';
+  while (Date.now() - startedAt <= timeoutMs) {
+    currentActivity = await getCurrentActivity(deviceId);
+    if (currentActivity === expectedActivity) return;
+    await wait(500);
+  }
+  throw new Error(`${step.label} 等待超时，当前 Activity：${currentActivity || '-'}`);
+}
+
+async function appendSettingsDiagnostics(lines: string[], deviceId: string) {
+  const packageInfo = await adbText(deviceId, ['shell', 'dumpsys', 'package', 'io.appium.settings']);
+  const launchInfo = await adbText(deviceId, ['shell', 'am', 'start-activity', '-n', 'io.appium.settings/.Settings', '-a', 'android.intent.action.MAIN', '-c', 'android.intent.category.LAUNCHER']);
+  await new Promise((resolve) => setTimeout(resolve, 1000));
+  const serviceInfo = await adbText(deviceId, ['shell', 'dumpsys', 'activity', 'services', 'io.appium.settings']);
+  lines.push('Appium Settings 诊断：');
+  lines.push(`- 安装状态：${packageInfo.includes('Package [io.appium.settings]') ? '已安装' : '未检测到 io.appium.settings'}`);
+  lines.push(`- 手动启动：${launchInfo || '-'}`);
+  lines.push(`- 服务状态：${serviceInfo || '-'}`);
+}
+
+async function appiumRequest<T>(path: string, init?: RequestInit) {
+  const requestUrl = `${appiumServerUrl()}${path}`;
+  let response: Response;
+  try {
+    response = await fetch(requestUrl, {
+      ...init,
+      headers: {
+        'Content-Type': 'application/json',
+        ...(init?.headers || {}),
+      },
+    });
+  } catch (error) {
+    throw new Error(`无法连接 Appium 服务 ${requestUrl}，${errorDetail(error)}`);
+  }
+
+  const responseText = await response.text();
+  const payload = (() => {
+    try {
+      return responseText ? JSON.parse(responseText) : {};
+    } catch {
+      return {};
+    }
+  })() as T & {
+    value?: { error?: string; message?: string; stacktrace?: string };
+    error?: string;
+    message?: string;
+    stacktrace?: string;
+  };
+  if (!response.ok) {
+    const detail = [
+      payload.value?.error || payload.error ? `错误类型：${payload.value?.error || payload.error}` : '',
+      payload.value?.message || payload.message ? `错误信息：${payload.value?.message || payload.message}` : '',
+      payload.value?.stacktrace || payload.stacktrace ? `堆栈信息：\n${payload.value?.stacktrace || payload.stacktrace}` : '',
+      responseText ? `原始响应：\n${responseText}` : '',
+    ].filter(Boolean).join('\n');
+    throw new Error(`Appium ${init?.method || 'GET'} ${path} 失败（HTTP ${response.status}）：\n${detail || '未返回错误详情'}`);
+  }
+  return payload;
+}
+
+async function createSession(script: AppiumRecordedScriptRecord, deviceId: string) {
+  const payload = await appiumRequest<AppiumSessionResponse>('/session', {
+    method: 'POST',
+    body: JSON.stringify({
+      capabilities: {
+        alwaysMatch: {
+          platformName: 'Android',
+          'appium:automationName': 'UiAutomator2',
+          'appium:udid': deviceId,
+          'appium:appPackage': script.appPackage,
+          ...(script.appActivity ? { 'appium:appActivity': script.appActivity } : {}),
+          'appium:noReset': true,
+          'appium:skipDeviceInitialization': true,
+          'appium:ignoreHiddenApiPolicyError': true,
+        },
+      },
+    }),
+  });
+  const sessionId = payload.value?.sessionId || payload.sessionId || '';
+  if (!sessionId) throw new Error('Appium 未返回 sessionId');
+  return sessionId;
+}
+
+function toAppiumUsing(step: AppiumRecordedStepRecord) {
+  const selector = step.selector;
+  if (!selector) throw new Error(`${step.label} 缺少 selector`);
+  if (selector.strategy === 'accessibilityId') return { using: 'accessibility id', value: selector.value || '' };
+  if (selector.strategy === 'id') return { using: 'id', value: selector.value || '' };
+  if (selector.strategy === 'androidUiAutomator') return { using: '-android uiautomator', value: selector.value || '' };
+  if (selector.strategy === 'xpath') return { using: 'xpath', value: selector.value || '' };
+  throw new Error(`${step.label} 需要使用 bounds 坐标执行`);
+}
+
+async function findElement(sessionId: string, step: AppiumRecordedStepRecord) {
+  const using = toAppiumUsing(step);
+  const payload = await appiumRequest<AppiumElementResponse>(`/session/${sessionId}/element`, {
+    method: 'POST',
+    body: JSON.stringify(using),
+  });
+  const elementId = payload.value?.[ELEMENT_KEY] || payload.value?.ELEMENT || '';
+  if (!elementId) throw new Error(`${step.label} 未找到元素`);
+  return elementId;
+}
+
+async function tapFallback(deviceId: string, step: AppiumRecordedStepRecord) {
+  if (step.fallback?.strategy !== 'bounds' || !Number.isFinite(step.fallback.centerX) || !Number.isFinite(step.fallback.centerY)) {
+    throw new Error(`${step.label} 未找到元素，且没有可用坐标兜底`);
+  }
+  await adbTap(deviceId, Number(step.fallback.centerX), Number(step.fallback.centerY));
+}
+
+async function waitForElement(sessionId: string, step: AppiumRecordedStepRecord) {
+  const startedAt = Date.now();
+  const timeoutMs = step.timeoutMs || 10000;
+  let lastError: unknown;
+  while (Date.now() - startedAt <= timeoutMs) {
+    try {
+      await findElement(sessionId, step);
+      return;
+    } catch (error) {
+      lastError = error;
+      await wait(500);
+    }
+  }
+  throw lastError instanceof Error ? lastError : new Error(`${step.label} 等待超时`);
+}
+
+async function waitForElementGone(sessionId: string, step: AppiumRecordedStepRecord) {
+  const startedAt = Date.now();
+  const timeoutMs = step.timeoutMs || 10000;
+  while (Date.now() - startedAt <= timeoutMs) {
+    try {
+      await findElement(sessionId, step);
+    } catch {
+      return;
+    }
+    await wait(500);
+  }
+  throw new Error(`${step.label} 等待消失超时`);
+}
+
+async function saveScreenshot(sessionId: string) {
+  const payload = await appiumRequest<AppiumValueResponse<string>>(`/session/${sessionId}/screenshot`);
+  if (!payload.value) throw new Error('Appium 未返回截图数据');
+  const dir = join(process.cwd(), '.midscene-app', 'screenshots');
+  await mkdir(dir, { recursive: true });
+  const file = join(dir, `appium-${Date.now()}.png`);
+  await writeFile(file, Buffer.from(payload.value, 'base64'));
+}
+
+async function runStep(sessionId: string, deviceId: string, step: AppiumRecordedStepRecord) {
+  if (step.type === 'delay') {
+    await wait(Math.max(0, step.timeoutMs || 1000));
+    return;
+  }
+
+  if (step.type === 'waitActivity') {
+    await waitForActivity(deviceId, step);
+    return;
+  }
+
+  if (step.type === 'key') {
+    await appiumRequest(`/session/${sessionId}/appium/device/press_keycode`, {
+      method: 'POST',
+      body: JSON.stringify({ keycode: step.keyCode || 4 }),
+    });
+    return;
+  }
+
+  if (step.type === 'launchApp') {
+    await appiumRequest(`/session/${sessionId}/appium/device/activate_app`, {
+      method: 'POST',
+      body: JSON.stringify({ appId: step.value }),
+    });
+    return;
+  }
+
+  if (step.type === 'screenshot') {
+    await saveScreenshot(sessionId);
+    return;
+  }
+
+  if (step.type === 'coordinateTap') {
+    await tapFallback(deviceId, step);
+    return;
+  }
+
+  if (step.type === 'swipe') {
+    if (!step.swipe) throw new Error(`${step.label} 缺少滑动坐标`);
+    await adbSwipe(deviceId, step.swipe);
+    return;
+  }
+
+  if (step.type === 'longPress') {
+    if (step.fallback?.strategy !== 'bounds' || !Number.isFinite(step.fallback.centerX) || !Number.isFinite(step.fallback.centerY)) {
+      throw new Error(`${step.label} 缺少长按坐标`);
+    }
+    const x = Number(step.fallback.centerX);
+    const y = Number(step.fallback.centerY);
+    await adbSwipe(deviceId, { startX: x, startY: y, endX: x, endY: y, duration: step.timeoutMs || 800 });
+    return;
+  }
+
+  if (step.type === 'pinch') {
+    if (!step.pinch) throw new Error(`${step.label} 缺少缩放参数`);
+    const size = Math.round(Math.min(500, Math.max(120, step.pinch.centerX, step.pinch.centerY)));
+    await appiumRequest(`/session/${sessionId}/execute/sync`, {
+      method: 'POST',
+      body: JSON.stringify({
+        script: step.pinch.direction === 'out' ? 'mobile: pinchOpenGesture' : 'mobile: pinchCloseGesture',
+        args: [{
+          left: Math.max(0, Math.round(step.pinch.centerX - size / 2)),
+          top: Math.max(0, Math.round(step.pinch.centerY - size / 2)),
+          width: size,
+          height: size,
+          percent: step.pinch.percent,
+        }],
+      }),
+    });
+    return;
+  }
+
+  if (step.type === 'waitDisappear') {
+    await waitForElementGone(sessionId, step);
+    return;
+  }
+
+  if (step.type === 'waitFor' || step.type === 'assertExists') {
+    await waitForElement(sessionId, step);
+    return;
+  }
+
+  if (step.type === 'assertText') {
+    const elementId = await findElement(sessionId, step);
+    const payload = await appiumRequest<AppiumValueResponse<string>>(`/session/${sessionId}/element/${elementId}/text`);
+    const actual = payload.value || '';
+    if (!actual.includes(step.value || '')) throw new Error(`${step.label} 不匹配，实际文本：${actual || '-'}`);
+    return;
+  }
+
+  if (step.type === 'tap') {
+    for (let attempt = 0; attempt < 2; attempt += 1) {
+      try {
+        const elementId = await findElement(sessionId, step);
+        await appiumRequest(`/session/${sessionId}/element/${elementId}/click`, {
+          method: 'POST',
+          body: JSON.stringify({}),
+        });
+        return;
+      } catch (error) {
+        if (attempt === 0 && isStaleElementError(error)) {
+          await wait(300);
+          continue;
+        }
+        await tapFallback(deviceId, step);
+        return;
+      }
+    }
+    return;
+  }
+
+  if (step.type === 'input') {
+    for (let attempt = 0; attempt < 2; attempt += 1) {
+      try {
+        const elementId = await findElement(sessionId, step);
+        await appiumRequest(`/session/${sessionId}/element/${elementId}/value`, {
+          method: 'POST',
+          body: JSON.stringify({ text: step.value || '', value: [...(step.value || '')] }),
+        });
+        return;
+      } catch (error) {
+        if (attempt === 0 && isStaleElementError(error)) {
+          await wait(300);
+          continue;
+        }
+        throw error;
+      }
+    }
+  }
+
+  if (step.type === 'clearInput') {
+    const elementId = await findElement(sessionId, step);
+    await appiumRequest(`/session/${sessionId}/element/${elementId}/clear`, {
+      method: 'POST',
+      body: JSON.stringify({}),
+    });
+  }
+}
+
+export async function replayAppiumScript(script: AppiumRecordedScriptRecord, deviceId: string) {
+  const targetDeviceId = deviceId || script.deviceId;
+  if (!targetDeviceId) throw new Error('未检测到可用设备');
+  if (!script.steps.length) throw new Error('脚本没有可回放步骤');
+
+  const lines: string[] = [
+    `Appium 服务：${appiumServerUrl()}`,
+    `目标设备：${targetDeviceId}`,
+    `App 包名：${script.appPackage}`,
+    `录制步骤：${script.steps.length}`,
+  ];
+  let sessionId = '';
+  try {
+    lines.push('正在创建 Appium session...');
+    sessionId = await createSession(script, targetDeviceId);
+    lines.push(`Appium session 已创建：${sessionId}`);
+    for (const [index, step] of script.steps.entries()) {
+      lines.push(`[步骤 ${index + 1}] 开始：${step.label}`);
+      try {
+        await runStep(sessionId, targetDeviceId, step);
+        lines.push(`[步骤 ${index + 1}] 完成：${step.label}`);
+      } catch (error) {
+        lines.push(`[步骤 ${index + 1}] 失败：${errorDetail(error)}`);
+        throw error;
+      }
+    }
+    lines.push('回放完成');
+    return { success: true, output: lines.join('\n') };
+  } catch (error) {
+    if (errorDetail(error).includes('Appium Settings app is not running')) {
+      await appendSettingsDiagnostics(lines, targetDeviceId);
+    }
+    lines.push(`回放终止：${errorDetail(error)}`);
+    return { success: false, output: lines.join('\n') };
+  } finally {
+    if (sessionId) {
+      await appiumRequest(`/session/${sessionId}`, { method: 'DELETE' }).catch(() => undefined);
+    }
+  }
+}

package/server/appium-recorder/repository.ts

@@ -0,0 +1,228 @@
+import {
+  createId,
+  querySql,
+  runSql,
+  sqlJson,
+  sqlNullableString,
+  sqlString,
+} from '../storage/sqlite';
+
+export type AppiumRecordedStepRecord = {
+  id: string;
+  type:
+    | 'tap'
+    | 'input'
+    | 'waitFor'
+    | 'assertExists'
+    | 'key'
+    | 'waitActivity'
+    | 'delay'
+    | 'clearInput'
+    | 'coordinateTap'
+    | 'swipe'
+    | 'screenshot'
+    | 'launchApp'
+    | 'waitDisappear'
+    | 'assertText'
+    | 'longPress'
+    | 'pinch';
+  label: string;
+  selector?: {
+    strategy: 'accessibilityId' | 'id' | 'androidUiAutomator' | 'xpath' | 'bounds';
+    value?: string;
+    centerX?: number;
+    centerY?: number;
+  };
+  fallback?: {
+    strategy: 'accessibilityId' | 'id' | 'androidUiAutomator' | 'xpath' | 'bounds';
+    value?: string;
+    centerX?: number;
+    centerY?: number;
+  };
+  value?: string;
+  keyCode?: number;
+  timeoutMs?: number;
+  swipe?: {
+    startX: number;
+    startY: number;
+    endX: number;
+    endY: number;
+    duration: number;
+  };
+  pinch?: {
+    direction: 'in' | 'out';
+    centerX: number;
+    centerY: number;
+    percent: number;
+  };
+  snapshot?: {
+    text: string;
+    resourceId: string;
+    contentDesc: string;
+    className: string;
+  };
+};
+
+export type AppiumRecordedScriptRecord = {
+  id: string;
+  name: string;
+  appPackage: string;
+  appActivity: string;
+  deviceId: string;
+  steps: AppiumRecordedStepRecord[];
+  createdAt: string;
+  updatedAt: string;
+};
+
+type AppiumRecordedScriptRow = {
+  id: string;
+  name: string;
+  app_package: string;
+  app_activity: string | null;
+  device_id: string | null;
+  steps_json: string;
+  created_at: string;
+  updated_at: string;
+};
+
+let initialized = false;
+
+function initDb() {
+  if (initialized) return;
+  initialized = true;
+  runSql(`
+    CREATE TABLE IF NOT EXISTS appium_recorded_scripts (
+      id TEXT PRIMARY KEY,
+      name TEXT NOT NULL UNIQUE,
+      app_package TEXT NOT NULL,
+      app_activity TEXT,
+      device_id TEXT,
+      steps_json TEXT NOT NULL DEFAULT '[]',
+      created_at TEXT NOT NULL,
+      updated_at TEXT NOT NULL
+    );
+    CREATE INDEX IF NOT EXISTS idx_appium_recorded_scripts_updated_at
+      ON appium_recorded_scripts(updated_at DESC);
+  `);
+}
+
+function rowToRecord(row: AppiumRecordedScriptRow): AppiumRecordedScriptRecord {
+  return {
+    id: row.id,
+    name: row.name,
+    appPackage: row.app_package,
+    appActivity: row.app_activity || '',
+    deviceId: row.device_id || '',
+    steps: JSON.parse(row.steps_json || '[]') as AppiumRecordedStepRecord[],
+    createdAt: row.created_at,
+    updatedAt: row.updated_at,
+  };
+}
+
+export function listAppiumRecordedScripts() {
+  initDb();
+  return querySql<AppiumRecordedScriptRow>(`
+    SELECT
+      id,
+      name,
+      app_package,
+      app_activity,
+      device_id,
+      steps_json,
+      created_at,
+      updated_at
+    FROM appium_recorded_scripts
+    ORDER BY updated_at DESC;
+  `).map(rowToRecord);
+}
+
+export function getAppiumRecordedScript(id: string) {
+  initDb();
+  const row = querySql<AppiumRecordedScriptRow>(`
+    SELECT
+      id,
+      name,
+      app_package,
+      app_activity,
+      device_id,
+      steps_json,
+      created_at,
+      updated_at
+    FROM appium_recorded_scripts
+    WHERE id = ${sqlString(id)}
+    LIMIT 1;
+  `)[0];
+  return row ? rowToRecord(row) : null;
+}
+
+export function saveAppiumRecordedScript(input: {
+  id?: string;
+  name: string;
+  appPackage: string;
+  appActivity?: string;
+  deviceId?: string;
+  steps: AppiumRecordedStepRecord[];
+}) {
+  initDb();
+  const now = new Date().toISOString();
+  const id = input.id || createId('appium_script');
+  const name = input.name.trim();
+  const appPackage = input.appPackage.trim();
+
+  if (!name) throw new Error('脚本名称不能为空');
+  if (!appPackage) throw new Error('App 包名不能为空');
+
+  if (input.id && getAppiumRecordedScript(input.id)) {
+    runSql(`
+      UPDATE appium_recorded_scripts
+      SET
+        name = ${sqlString(name)},
+        app_package = ${sqlString(appPackage)},
+        app_activity = ${sqlNullableString(input.appActivity || '')},
+        device_id = ${sqlNullableString(input.deviceId || '')},
+        steps_json = ${sqlJson(input.steps || [])},
+        updated_at = ${sqlString(now)}
+      WHERE id = ${sqlString(input.id)};
+    `);
+    return getAppiumRecordedScript(input.id);
+  }
+
+  runSql(`
+    INSERT INTO appium_recorded_scripts (
+      id,
+      name,
+      app_package,
+      app_activity,
+      device_id,
+      steps_json,
+      created_at,
+      updated_at
+    )
+    VALUES (
+      ${sqlString(id)},
+      ${sqlString(name)},
+      ${sqlString(appPackage)},
+      ${sqlNullableString(input.appActivity || '')},
+      ${sqlNullableString(input.deviceId || '')},
+      ${sqlJson(input.steps || [])},
+      ${sqlString(now)},
+      ${sqlString(now)}
+    )
+    ON CONFLICT(name) DO UPDATE SET
+      app_package = excluded.app_package,
+      app_activity = excluded.app_activity,
+      device_id = excluded.device_id,
+      steps_json = excluded.steps_json,
+      updated_at = excluded.updated_at;
+  `);
+
+  return listAppiumRecordedScripts().find((script) => script.name === name) || getAppiumRecordedScript(id);
+}
+
+export function deleteAppiumRecordedScript(id: string) {
+  initDb();
+  runSql(`
+    DELETE FROM appium_recorded_scripts
+    WHERE id = ${sqlString(id)};
+  `);
+}