npm - android-midscene-automation - Versions diffs - 0.1.0 - Mend

android-midscene-automation 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (78) hide show

package/README.md +160 -0
package/bin/android-midscene-automation.js +27 -0
package/index.html +12 -0
package/package.json +49 -0
package/remote-agent/index.ts +206 -0
package/server/appium-recorder/appium-runner.ts +427 -0
package/server/appium-recorder/repository.ts +228 -0
package/server/appium-recorder/routes.ts +219 -0
package/server/config-store.ts +167 -0
package/server/config.ts +130 -0
package/server/device-locks/repository.ts +147 -0
package/server/device-locks/service.ts +72 -0
package/server/device-locks/types.ts +22 -0
package/server/device-sessions/repository.ts +169 -0
package/server/device-sessions/service.ts +59 -0
package/server/device-sessions/types.ts +24 -0
package/server/http-api.ts +1389 -0
package/server/model-call-usage-importer.ts +108 -0
package/server/model-tester.ts +104 -0
package/server/model-usage-repository.ts +131 -0
package/server/operations/repository.ts +218 -0
package/server/operations/service.ts +84 -0
package/server/operations/types.ts +27 -0
package/server/paths.ts +27 -0
package/server/remote-agents/protocol.ts +38 -0
package/server/remote-agents/registry.ts +136 -0
package/server/remote-agents/routes.ts +89 -0
package/server/script-agent.ts +284 -0
package/server/script-db.ts +281 -0
package/server/script-runner.ts +551 -0
package/server/storage/sqlite.ts +49 -0
package/server/test-case-import/formatter.ts +28 -0
package/server/test-case-import/parsers/excel.ts +69 -0
package/server/test-case-import/parsers/txt.ts +11 -0
package/server/test-case-import/parsers/word.ts +9 -0
package/server/test-case-import/service.ts +58 -0
package/server/test-case-import/text-normalizer.ts +98 -0
package/server/test-case-import/types.ts +24 -0
package/server/test-case-import/validator.ts +34 -0
package/src/App.vue +1450 -0
package/src/api.ts +290 -0
package/src/appium-recorder/AppiumPage.vue +894 -0
package/src/appium-recorder/api.ts +64 -0
package/src/appium-recorder/components/ComponentTree.vue +44 -0
package/src/appium-recorder/components/NodeDetail.vue +152 -0
package/src/appium-recorder/components/RecordedSteps.vue +79 -0
package/src/appium-recorder/tree.ts +129 -0
package/src/appium-recorder/types.ts +88 -0
package/src/assets/device-actions/back.svg +5 -0
package/src/assets/device-actions/home.svg +3 -0
package/src/assets/device-actions/power.svg +5 -0
package/src/assets/device-actions/tasks.svg +3 -0
package/src/assets/device-actions/volume-down.svg +3 -0
package/src/assets/device-actions/volume-up.svg +3 -0
package/src/components/config/ModelUsageChart.vue +188 -0
package/src/components/device/DevicePreviewPanel.vue +266 -0
package/src/components/generator/GeneratedCodePanel.vue +70 -0
package/src/components/generator/TestCaseFileUpload.vue +97 -0
package/src/config/midscene-model-presets.ts +75 -0
package/src/config/prompt-example.ts +6 -0
package/src/main.ts +7 -0
package/src/pages/AiGeneratorPage.vue +90 -0
package/src/pages/AutomationPage.vue +161 -0
package/src/pages/ConfigPage.vue +273 -0
package/src/pages/GeneratorPage.vue +97 -0
package/src/pages/ManualStepsPage.vue +179 -0
package/src/script-generator/codegen.ts +126 -0
package/src/script-generator/index.ts +4 -0
package/src/script-generator/presets.ts +37 -0
package/src/script-generator/step-options.ts +52 -0
package/src/script-generator/types.ts +27 -0
package/src/style.css +1983 -0
package/src/types.ts +157 -0
package/src/vite-env.d.ts +1 -0
package/tsconfig.app.json +8 -0
package/tsconfig.json +11 -0
package/tsconfig.node.json +16 -0
package/vite.config.ts +28 -0

package/README.md ADDED Viewed

@@ -0,0 +1,160 @@
+# Midscene Playwright Admin
+一个最小可用的 Midscene + Playwright 项目，同时提供一个基于 Vue 3 和 Element Plus 的后台，用来把自然语言 prompt 生成结构化自动化脚本。
+## 环境要求
+- Node.js 18+
+- npm
+- Android SDK / ADB，移动端自动化测试需要
+- Playwright Chromium，Web 示例脚本需要
+## 一行启动
+```sh
+npx android-midscene-automation
+```
+启动后访问：
+```text
+http://127.0.0.1:5173/
+```
+如果还没有发布到 npm，也可以用 GitHub 地址运行：
+```sh
+npx github:你的组织/你的仓库
+```
+每个测试人员在自己的电脑运行这条命令，后端执行的就是当前电脑上的 `adb devices`，页面会识别当前电脑连接的手机。
+如果 `5173` 端口被占用，可以指定端口：
+```sh
+npx android-midscene-automation --port 5174
+```
+## 源码开发
+```sh
+npm install
+npm run dev
+```
+`Playwright` 浏览器下载不走 npm registry。单独安装浏览器时，可临时指定镜像：
+```sh
+PLAYWRIGHT_DOWNLOAD_HOST=https://npmmirror.com/mirrors/playwright/ npx playwright install chromium
+```
+## 局域网访问
+```sh
+npm run dev
+```
+启动后访问 Vite 输出的地址。项目默认监听 `0.0.0.0`，支持局域网访问：
+```text
+http://127.0.0.1:5173/
+http://你的局域网IP:5173/
+```
+`npm run dev` 会同时启动前端页面和本地后端 API。后端不是独立进程，而是在 `vite.config.ts` 中通过 `server/http-api.ts` 挂载到 Vite dev server 的 Connect middleware。
+## 模型配置
+首次启动后进入页面的“参数配置”：
+- `Midscene模型`：执行测试脚本时使用，支持“自定义提供方”和“使用 Codex”。
+- `AI生成脚本模型`：根据测试需求生成脚本时使用。
+配置会保存到本地 SQLite：
+```text
+.midscene-app/script-cache.sqlite
+```
+旧版 `config.yaml` 或 `config.json` 会在读取后迁移到数据库。当前运行时会优先读取数据库中的模型配置。
+## 后端说明
+本项目的本地后端位于 `server/http-api.ts`，通过 `/api/*` 暴露能力，主要包括：
+- 脚本生成：`POST /api/generate`
+- 脚本保存和列表：`POST /api/save-script`、`GET /api/scripts`
+- 脚本执行和停止：`POST /api/run-script`、`POST /api/stop-script`
+- 模型配置：`GET /api/config`、`POST /api/config`
+- 模型测试和消耗统计：`POST /api/test-model`、`GET /api/model-usage-records`
+- Android 设备：`GET /api/android-devices`、`POST /api/android-device`
+- Android 预览和操作：`GET /api/android-preview`、`POST /api/android-tap`、`POST /api/android-swipe`、`POST /api/android-keyevent`
+- App 预设：`GET /api/app-presets`、`POST /api/save-app-preset`、`POST /api/delete-app-preset`
+后端运行态文件默认都在项目根目录下：
+```text
+.midscene-app/                 # SQLite 数据库
+.midscene-generated/           # 执行前生成的临时脚本
+scripts-output/                # 保存的脚本输出
+midscene_run/                  # Midscene 执行报告和运行产物
+```
+## Android 自动化
+执行移动端脚本前确认设备已连接：
+```sh
+adb devices -l
+```
+页面会通过 `/api/android-devices` 获取设备列表。执行脚本时会把当前选择的设备 ID 传给后端，由 `@midscene/android` 连接设备并运行生成脚本。
+如果要走不依赖大模型的 Appium 组件树录制方案，源码仓库里的 `demo` 目录保留了方案文档；npm 包不会发布 `demo` 目录。
+如果测试人员需要识别自己电脑上的手机，推荐每个人在自己的电脑本地运行：
+```sh
+npx android-midscene-automation
+```
+然后访问自己本机的：
+```text
+http://localhost:5173/
+```
+这样后端执行的是当前电脑上的 `adb devices`，页面会识别当前电脑连接的手机。
+局域网共享页面只适合查看同一个中央服务，不适合直接识别访问者电脑上的 ADB 设备。远程设备代理方案仅保留在源码仓库的 `demo` 目录中，不随 npm 包发布。
+如果要回放 Appium 脚本，本机还需要启动 Appium：
+```sh
+appium
+```
+如果要使用远程设备代理实验方案，可以在访问者电脑的项目目录中运行：
+```sh
+npm run remote-agent -- --server http://中央服务IP:5173 --agent-id 你的名字或机器名
+```
+## 运行 E2E 测试
+```sh
+npm run test:e2e
+```
+## 构建
+```sh
+npm run build
+```
+## 本地预览构建结果
+```sh
+npm run preview
+```
+`npm run preview` 同样会挂载 `/api/*` 后端 middleware。

package/bin/android-midscene-automation.js ADDED Viewed

@@ -0,0 +1,27 @@
+#!/usr/bin/env node
+import { spawn } from 'node:child_process';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+const packageRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
+const viteBin = path.join(packageRoot, 'node_modules', 'vite', 'bin', 'vite.js');
+const args = process.argv.slice(2);
+if (!args.includes('--host')) {
+  args.unshift('127.0.0.1');
+  args.unshift('--host');
+}
+const child = spawn(process.execPath, [viteBin, ...args], {
+  cwd: packageRoot,
+  env: process.env,
+  stdio: 'inherit',
+});
+child.on('exit', (code, signal) => {
+  if (signal) {
+    process.kill(process.pid, signal);
+    return;
+  }
+  process.exit(code ?? 0);
+});

package/index.html ADDED Viewed

@@ -0,0 +1,12 @@
+<!doctype html>
+<html lang="zh-CN">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Midscene 脚本后台</title>
+  </head>
+  <body>
+    <div id="app"></div>
+    <script type="module" src="/src/main.ts"></script>
+  </body>
+  </html>

package/package.json ADDED Viewed

@@ -0,0 +1,49 @@
+{
+  "name": "android-midscene-automation",
+  "version": "0.1.0",
+  "type": "module",
+  "bin": {
+    "android-midscene-automation": "./bin/android-midscene-automation.js"
+  },
+  "files": [
+    "bin",
+    "index.html",
+    "remote-agent",
+    "server",
+    "src",
+    "tsconfig*.json",
+    "vite.config.ts",
+    "README.md"
+  ],
+  "scripts": {
+    "dev": "vite --host 0.0.0.0",
+    "remote-agent": "tsx remote-agent/index.ts",
+    "build": "vue-tsc -b && vite build",
+    "preview": "vite preview --host 0.0.0.0",
+    "pw:install": "playwright install chromium",
+    "test:e2e": "playwright test"
+  },
+  "dependencies": {
+    "@element-plus/icons-vue": "^2.3.2",
+    "@midscene/android": "1.10.12",
+    "@midscene/web": "1.10.12",
+    "@vitejs/plugin-vue": "^6.0.1",
+    "element-plus": "^2.11.1",
+    "openai": "^5.12.2",
+    "playwright": "^1.54.1",
+    "tsx": "^4.20.3",
+    "typescript": "^5.8.3",
+    "vite": "^7.0.4",
+    "vue": "^3.5.18",
+    "word-extractor": "^1.0.4",
+    "xlsx": "^0.18.5",
+    "yaml": "^2.8.1"
+  },
+  "devDependencies": {
+    "@playwright/test": "^1.54.1",
+    "@types/node": "^24.0.13",
+    "@types/word-extractor": "^1.0.6",
+    "@vue/tsconfig": "^0.7.0",
+    "vue-tsc": "^3.0.3"
+  }
+}

package/remote-agent/index.ts ADDED Viewed

@@ -0,0 +1,206 @@
+import { execFile } from 'node:child_process';
+import fs from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+import type { RemoteAgentDevice, RemoteCommand } from '../server/remote-agents/protocol';
+import { replayAppiumScript } from '../server/appium-recorder/appium-runner';
+import type { AppiumRecordedScriptRecord } from '../server/appium-recorder/repository';
+const VERSION = '0.1.0';
+function argValue(name: string, fallback = '') {
+  const index = process.argv.indexOf(`--${name}`);
+  return index >= 0 ? process.argv[index + 1] || fallback : fallback;
+}
+const server = argValue('server').replace(/\/+$/, '');
+const agentId = argValue('agent-id', os.hostname()).replace(/:/g, '-');
+const agentName = argValue('agent-name', agentId);
+const token = argValue('token', process.env.REMOTE_AGENT_TOKEN || '');
+if (!server) {
+  console.error('缺少 --server，例如：npm run remote-agent -- --server http://172.16.20.116:5173');
+  process.exit(1);
+}
+function execText(command: string, args: string[] = []) {
+  return new Promise<string>((resolve, reject) => {
+    execFile(command, args, { maxBuffer: 20 * 1024 * 1024 }, (error, stdout, stderr) => {
+      if (error) {
+        reject(new Error(stderr || error.message));
+        return;
+      }
+      resolve(stdout);
+    });
+  });
+}
+function execBuffer(command: string, args: string[] = []) {
+  return new Promise<Buffer>((resolve, reject) => {
+    execFile(command, args, { encoding: 'buffer', maxBuffer: 20 * 1024 * 1024 }, (error, stdout, stderr) => {
+      if (error) {
+        reject(new Error(Buffer.isBuffer(stderr) ? stderr.toString() : stderr || error.message));
+        return;
+      }
+      resolve(Buffer.isBuffer(stdout) ? stdout : Buffer.from(stdout));
+    });
+  });
+}
+async function postJson<T>(url: string, body: unknown) {
+  const response = await fetch(url, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify(body),
+  });
+  const payload = await response.json().catch(() => ({})) as T & { message?: string };
+  if (!response.ok) throw new Error(payload.message || `HTTP ${response.status}`);
+  return payload;
+}
+async function listDevices(): Promise<RemoteAgentDevice[]> {
+  const output = await execText('adb', ['devices', '-l']);
+  return output
+    .split('\n')
+    .map((line) => line.trim())
+    .filter((line) => line && !line.startsWith('List of devices attached'))
+    .map((line) => {
+      const parts = line.split(/\s+/);
+      const id = parts[0] || '';
+      const status = parts[1] || 'unknown';
+      const model = parts.find((part) => part.startsWith('model:'))?.slice(6) || '';
+      const device = parts.find((part) => part.startsWith('device:'))?.slice(7) || '';
+      return { id, status, model, description: device || status };
+    })
+    .filter((device) => device.id);
+}
+async function getDisplayInfo(deviceId: string) {
+  const output = await execText('adb', ['-s', deviceId, 'shell', 'wm', 'size']);
+  const match = output.match(/Override size:\s*(\d+)x(\d+)/i) || output.match(/Physical size:\s*(\d+)x(\d+)/i);
+  if (!match) throw new Error('读取设备分辨率失败');
+  return { width: Number(match[1]), height: Number(match[2]) };
+}
+async function dumpWindowHierarchy(deviceId: string) {
+  const localPath = path.join(os.tmpdir(), `midscene-remote-agent-${deviceId.replace(/[^\w.-]/g, '_')}-${Date.now()}.xml`);
+  const remotePath = '/data/local/tmp/midscene_remote_agent_uidump.xml';
+  await execText('adb', ['-s', deviceId, 'shell', 'uiautomator', 'dump', remotePath]);
+  await execText('adb', ['-s', deviceId, 'pull', remotePath, localPath]);
+  try {
+    return await fs.readFile(localPath, 'utf8');
+  } finally {
+    await fs.unlink(localPath).catch(() => undefined);
+  }
+}
+async function getCurrentActivity(deviceId: string) {
+  const output = await execText('adb', ['-s', deviceId, 'shell', 'dumpsys', 'activity', 'activities']);
+  const resumedLine = output
+    .split(/\r?\n/)
+    .find((line) => /(?:topResumedActivity|ResumedActivity|mResumedActivity)/.test(line));
+  return resumedLine?.match(/\s([\w.$]+\/[\w.$]+)\s/)?.[1] || '';
+}
+async function handleCommand(command: RemoteCommand) {
+  const payload = command.payload || {};
+  if (command.type === 'screenshot') {
+    const image = await execBuffer('adb', ['-s', command.deviceId, 'exec-out', 'screencap', '-p']);
+    return { base64: image.toString('base64') };
+  }
+  if (command.type === 'displayInfo') return await getDisplayInfo(command.deviceId);
+  if (command.type === 'tree') {
+    const [xml, activity] = await Promise.all([
+      dumpWindowHierarchy(command.deviceId),
+      getCurrentActivity(command.deviceId).catch(() => ''),
+    ]);
+    return { xml, activity, dumpedAt: new Date().toISOString() };
+  }
+  if (command.type === 'tap') {
+    await execText('adb', ['-s', command.deviceId, 'shell', 'input', 'tap', String(Math.round(Number(payload.x))), String(Math.round(Number(payload.y)))]);
+    return { success: true };
+  }
+  if (command.type === 'swipe') {
+    await execText('adb', [
+      '-s',
+      command.deviceId,
+      'shell',
+      'input',
+      'swipe',
+      String(Math.round(Number(payload.startX))),
+      String(Math.round(Number(payload.startY))),
+      String(Math.round(Number(payload.endX))),
+      String(Math.round(Number(payload.endY))),
+      String(Math.max(80, Math.round(Number(payload.duration) || 160))),
+    ]);
+    return { success: true };
+  }
+  if (command.type === 'key') {
+    await execText('adb', ['-s', command.deviceId, 'shell', 'input', 'keyevent', String(Math.round(Number(payload.keyCode)))]);
+    return { success: true };
+  }
+  if (command.type === 'replay') {
+    const script = payload.script as AppiumRecordedScriptRecord | undefined;
+    if (!script) throw new Error('远程回放缺少脚本内容');
+    return await replayAppiumScript(script, command.deviceId);
+  }
+  throw new Error(`不支持的命令：${command.type}`);
+}
+async function heartbeat() {
+  const devices = await listDevices().catch((error) => {
+    console.warn(`扫描设备失败：${error instanceof Error ? error.message : String(error)}`);
+    return [];
+  });
+  await postJson(`${server}/api/remote-agents/heartbeat`, {
+    agentId,
+    agentName,
+    version: VERSION,
+    token,
+    devices,
+  });
+  console.log(`已上报 ${devices.length} 台设备到 ${server}`);
+}
+async function reportResult(command: RemoteCommand, ok: boolean, data?: unknown, error?: string) {
+  await postJson(`${server}/api/remote-agents/result`, {
+    agentId,
+    token,
+    commandId: command.id,
+    ok,
+    data,
+    error,
+  });
+}
+async function pollOnce() {
+  const url = new URL(`${server}/api/remote-agents/poll`);
+  url.searchParams.set('agentId', agentId);
+  if (token) url.searchParams.set('token', token);
+  const response = await fetch(url);
+  const payload = await response.json().catch(() => ({})) as { command?: RemoteCommand | null; message?: string };
+  if (!response.ok) throw new Error(payload.message || `HTTP ${response.status}`);
+  if (!payload.command) return;
+  try {
+    const result = await handleCommand(payload.command);
+    await reportResult(payload.command, true, result);
+  } catch (error) {
+    await reportResult(payload.command, false, undefined, error instanceof Error ? error.message : String(error));
+  }
+}
+async function main() {
+  console.log(`Remote Agent 启动：${agentId}`);
+  console.log(`中央服务：${server}`);
+  await heartbeat();
+  setInterval(() => void heartbeat().catch((error) => console.warn(`心跳失败：${error.message}`)), 3000);
+  for (;;) {
+    await pollOnce().catch((error) => console.warn(`轮询失败：${error.message}`));
+    await new Promise((resolve) => setTimeout(resolve, 300));
+  }
+}
+main().catch((error) => {
+  console.error(error);
+  process.exitCode = 1;
+});