android-midscene-automation 0.2.1 → 0.2.3

package/server/appium-recorder/managed-cleanup.test.ts

@@ -8,12 +8,14 @@ import { mkdtemp, mkdir, writeFile, rm } from 'node:fs/promises';
 import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 
-test('owned device instrumentation is cleaned after success, abort and failed session creation; external servers are untouched', async () => {
+test('replay survives command idle periods and cleans sessions after success, abort and creation failure', async () => {
   const root = await mkdtemp(join(tmpdir(), 'managed-cleanup-'));
   const env = { ...process.env };
   const original = { execFile: childProcess.execFile, spawn: childProcess.spawn, fetch: globalThis.fetch };
   const calls: string[][] = [];
   let mode = 'success', stopped = false;
+  let elapsedMs = 0, lastCommandMs = 0, commandTimeoutMs = 60_000;
+  let keyCommands = 0, deletedSessions = 0;
   let controller = new AbortController();
   try {
     const sdk = join(root, 'sdk');
@@ -44,8 +46,20 @@ test('owned device instrumentation is cleaned after success, abort and failed se
       const path = new URL(String(input)).pathname;
       if (path === '/session' && init?.method === 'POST') {
         if (mode === 'creation-failure') return Response.json({ value: { error: 'session not created', message: 'creation failed' } }, { status: 500 });
+        const capabilities = JSON.parse(String(init.body)).capabilities.alwaysMatch;
+        commandTimeoutMs = (capabilities['appium:newCommandTimeout'] ?? 60) * 1000;
+        lastCommandMs = elapsedMs;
         return Response.json({ value: { sessionId: 'test-session' } });
       }
+      if (path.endsWith('/press_keycode')) {
+        keyCommands++;
+        // Model Appium's idle expiry without waiting a real minute or touching a device.
+        if (commandTimeoutMs && elapsedMs - lastCommandMs > commandTimeoutMs) {
+          return Response.json({ value: { error: 'invalid session id', message: 'New Command Timeout expired' } }, { status: 404 });
+        }
+        lastCommandMs = elapsedMs;
+      }
+      if (path === '/session/test-session' && init?.method === 'DELETE') deletedSessions++;
       return Response.json({ value: {} });
     };
     const { replayAppiumScript } = await import('./appium-runner');
@@ -53,19 +67,35 @@ test('owned device instrumentation is cleaned after success, abort and failed se
       id: 'cleanup', name: 'cleanup', deviceId: 'test-device', appPackage: '', appActivity: '', createdAt: '', updatedAt: '',
       steps: [{ id: 'log', type: 'log' as const, label: 'log', value: 'ok' }],
     };
-    for (mode of ['success', 'abort', 'creation-failure', 'cleanup-failure', 'external']) {
+    for (mode of ['success', 'long-idle', 'abort', 'creation-failure', 'cleanup-failure', 'external', 'external-abort']) {
       calls.length = 0;
+      elapsedMs = lastCommandMs = keyCommands = deletedSessions = 0;
       controller = new AbortController();
-      if (mode === 'external') process.env.APPIUM_SERVER_URL = 'http://external.invalid';
-      const result = await replayAppiumScript(script, 'test-device', line => {
-        if (mode === 'abort' && line.includes('Appium session 已创建')) controller.abort();
+      const external = mode.startsWith('external');
+      const aborted = mode.endsWith('abort');
+      const longIdle = mode.endsWith('long-idle');
+      if (external) process.env.APPIUM_SERVER_URL = 'http://external.invalid';
+      else delete process.env.APPIUM_SERVER_URL;
+      const replayScript = longIdle ? { ...script, steps: [
+        ...script.steps,
+        { id: 'home', type: 'key' as const, label: 'Home', keyCode: 3 },
+      ] } : script;
+      const result = await replayAppiumScript(replayScript, 'test-device', line => {
+        // ADB operations and model inference do not reset Appium's command timer.
+        if (longIdle && line.includes('[节点 1] 完成')) elapsedMs += 120_000;
+        if (aborted && line.includes('Appium session 已创建')) controller.abort();
       }, controller.signal);
-      assert.equal(calls.length, mode === 'external' ? 0 : 2);
-      if (mode !== 'external') {
+      assert.equal(calls.length, external ? 0 : 2);
+      assert.equal(deletedSessions, mode === 'creation-failure' || (aborted && !external) ? 0 : 1, 'release the replay session even when idle expiry is disabled');
+      if (!external) {
         assert.deepEqual(calls.map(args => args.slice(0, 5)), Array(2).fill(['-s', 'test-device', 'shell', 'am', 'force-stop']));
         assert.deepEqual(calls.map(args => args.at(-1)).sort(), ['io.appium.uiautomator2.server', 'io.appium.uiautomator2.server.test']);
       }
-      if (mode === 'abort') assert.equal(result.stopped, true);
+      if (longIdle) {
+        assert.equal(keyCommands, 1, 'execute Home after the long interval without Appium commands');
+        assert.equal(result.success, true, result.output);
+      }
+      if (aborted) assert.equal(result.stopped, true);
       if (mode === 'creation-failure') assert.equal(result.success, false);
       if (mode === 'cleanup-failure') assert.match(result.output, /UiAutomator2 清理失败/);
     }

package/server/appium-recorder/replay-lifecycle.test.ts

@@ -0,0 +1,123 @@
+import assert from 'node:assert/strict';
+import { test } from 'node:test';
+import childProcess from 'node:child_process';
+import { syncBuiltinESMExports } from 'node:module';
+import { mkdtemp, mkdir, writeFile, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { createServer } from 'node:http';
+import type { AppiumRecordedScriptRecord, AppiumRecordedStepRecord } from './repository';
+
+test('ADB cancellation and deadlines cover actions; external sessions survive idle work and expire after creation cancellation', async () => {
+  const root = await mkdtemp(join(tmpdir(), 'replay-lifecycle-'));
+  const env = { ...process.env };
+  const original = { execFile: childProcess.execFile, fetch: globalThis.fetch, setInterval: globalThis.setInterval };
+  let controller = new AbortController();
+  let mode = '', deletes = 0, launchedAfterAbort = false;
+  let requestCount = 0;
+  try {
+    const sdk = join(root, 'sdk');
+    await mkdir(join(sdk, 'platform-tools'), { recursive: true });
+    await writeFile(join(sdk, 'platform-tools', 'adb'), '');
+    Object.assign(process.env, { ANDROID_MIDSCENE_DATA_ROOT: root, ANDROID_HOME: sdk, ANDROID_SDK_ROOT: sdk, APPIUM_SERVER_URL: 'http://external.invalid' });
+    childProcess.execFile = ((_cmd, args: string[], options, callback) => {
+      requestCount++;
+      if (mode === 'foreground-abort' && args.includes('activities')) {
+        queueMicrotask(() => { controller.abort(); callback(new Error('aborted'), '', ''); });
+      } else if (args.includes('monkey') || args.includes('clear') || args.includes('input')) {
+        launchedAfterAbort ||= controller.signal.aborted;
+        assert.ok(options.timeout > 0, 'every device action has a deadline');
+        assert.equal(options.signal, controller.signal);
+        if (mode === 'timeout') {
+          queueMicrotask(() => callback(new Error('ADB command timed out'), '', ''));
+        } else {
+          options.signal.addEventListener('abort', () => callback(new Error('aborted'), '', ''), { once: true });
+          queueMicrotask(() => controller.abort());
+        }
+      } else queueMicrotask(() => callback(null, '', ''));
+      return {};
+    }) as typeof childProcess.execFile;
+    syncBuiltinESMExports();
+    globalThis.fetch = async (input, init) => {
+      if (init?.method === 'DELETE') deletes++;
+      return Response.json({ value: new URL(String(input)).pathname === '/session' ? { sessionId: 'test-session' } : {} });
+    };
+    const { replayAppiumScript, launchAppOnDevice, clearAppDataOnDevice } = await import('./appium-runner');
+    const script: AppiumRecordedScriptRecord = { id: 'audit', name: 'audit', deviceId: 'fake-device', appPackage: '', appActivity: '', createdAt: '', updatedAt: '', steps: [] };
+    const run = (steps: AppiumRecordedStepRecord[]) => replayAppiumScript({ ...script, steps }, 'fake-device', undefined, controller.signal, { screenshotReport: false, reportSummaryEnabled: false });
+    const actions: AppiumRecordedStepRecord[] = [
+      { id: 'launch', type: 'launchApp', label: 'launch', value: 'example.test' },
+      { id: 'clear', type: 'clearAppData', label: 'clear', value: 'example.test' },
+      { id: 'tap', type: 'coordinateTap', label: 'tap', fallback: { strategy: 'bounds', centerX: 10, centerY: 10 } },
+      { id: 'swipe', type: 'swipe', label: 'swipe', swipe: { startX: 10, startY: 10, endX: 50, endY: 50, duration: 100 } },
+    ];
+    for (const action of actions) {
+      mode = 'abort'; controller = new AbortController(); deletes = 0;
+      assert.equal((await run([action])).stopped, true, action.type);
+      assert.equal(deletes, 1, 'abort releases the known external session');
+    }
+    mode = 'foreground-abort'; controller = new AbortController(); launchedAfterAbort = false;
+    assert.equal((await run([actions[0]!])).stopped, true);
+    assert.equal(launchedAfterAbort, false, 'foreground cancellation must not launch monkey');
+    const count = requestCount;
+    assert.throws(() => launchAppOnDevice('fake', 'example.test', controller.signal));
+    assert.throws(() => clearAppDataOnDevice('fake', 'example.test', controller.signal));
+    assert.equal(requestCount, count, 'already-cancelled actions never spawn');
+    mode = 'timeout'; controller = new AbortController(); deletes = 0;
+    const failed = await run([actions[0]!]);
+    assert.equal(failed.success, false); assert.match(failed.output, /timed out/); assert.equal(deletes, 1);
+
+    // A local HTTP server models the idle lease at accelerated time, without a real device.
+    globalThis.fetch = original.fetch;
+    globalThis.setInterval = ((fn, ms, ...args) => original.setInterval(fn, ms === 15000 ? 15 : ms, ...args)) as typeof setInterval;
+    let active = false, timeoutSeconds = 0, heartbeats = 0, homeKeys = 0, deleted = 0;
+    let expiry: ReturnType<typeof setTimeout> | undefined;
+    let createdResolve: () => void = () => {};
+    const renew = () => { clearTimeout(expiry); expiry = setTimeout(() => { active = false; }, timeoutSeconds * 3); };
+    const server = createServer(async (req, res) => {
+      res.setHeader('Content-Type', 'application/json');
+      if (req.url === '/session' && req.method === 'POST') {
+        let body = ''; for await (const chunk of req) body += chunk;
+        timeoutSeconds = JSON.parse(body).capabilities.alwaysMatch['appium:newCommandTimeout'];
+        if (mode === 'creation-abort') { controller.abort(); await new Promise(resolve => setTimeout(resolve, 20)); }
+        active = true; renew(); createdResolve();
+        res.end(JSON.stringify({ value: { sessionId: 'delayed-session' } }));
+      } else if (req.method === 'DELETE') {
+        deleted++; active = false; clearTimeout(expiry); res.end('{"value":null}');
+      } else if (!active) {
+        res.statusCode = 404; res.end('{"value":{"error":"invalid session id"}}');
+      } else {
+        renew();
+        if (req.url?.endsWith('/timeouts')) heartbeats++;
+        if (req.url?.endsWith('/press_keycode')) homeKeys++;
+        res.end('{"value":{}}');
+      }
+    });
+    await new Promise<void>(resolve => server.listen(0, '127.0.0.1', resolve));
+    process.env.APPIUM_SERVER_URL = `http://127.0.0.1:${(server.address() as { port: number }).port}`;
+    try {
+      mode = 'long-idle'; controller = new AbortController();
+      const result = await run([{ id: 'wait', type: 'delay', label: 'idle', timeoutMs: 450 }, { id: 'home', type: 'key', label: 'Home', keyCode: 3 }]);
+      assert.equal(result.success, true, result.output);
+      assert.equal(timeoutSeconds, 60); assert.ok(heartbeats > 3); assert.equal(homeKeys, 1); assert.equal(deleted, 1);
+      const heartbeatCount = heartbeats;
+      await new Promise(resolve => setTimeout(resolve, 40));
+      assert.equal(heartbeats, heartbeatCount, 'heartbeat stops after cleanup');
+      mode = 'creation-abort'; controller = new AbortController();
+      const created = new Promise<void>(resolve => { createdResolve = resolve; });
+      assert.equal((await run([actions[0]!])).stopped, true);
+      await created;
+      assert.equal(active, true, 'server can create a session after the client cancelled');
+      await new Promise(resolve => setTimeout(resolve, 220));
+      assert.equal(active, false, 'unclaimed external session has a finite idle lease');
+    } finally {
+      clearTimeout(expiry); server.closeAllConnections(); await new Promise<void>(resolve => server.close(() => resolve()));
+    }
+  } finally {
+    childProcess.execFile = original.execFile; globalThis.fetch = original.fetch; globalThis.setInterval = original.setInterval; syncBuiltinESMExports();
+    for (const key of ['ANDROID_MIDSCENE_DATA_ROOT', 'ANDROID_HOME', 'ANDROID_SDK_ROOT', 'APPIUM_SERVER_URL']) {
+      if (env[key] === undefined) delete process.env[key]; else process.env[key] = env[key];
+    }
+    await rm(root, { recursive: true, force: true });
+  }
+});

package/server/appium-recorder/routes.ts

@@ -1,3 +1,4 @@
+import { readBody, requestErrorStatus } from '../request-body';
 import type { AiObservationConfig } from '../../src/appium-recorder/ai-recognition';
 import { execFile } from 'node:child_process';
 import { sendReplayVideo } from './video-response';
@@ -23,7 +24,7 @@ import {
   type AppiumRecordedStepRecord,
 } from './repository';
 import { clearAppDataOnDevice, launchAppOnDevice, replayAppiumScript } from './appium-runner';
-import { isRemoteDeviceId, sendRemoteCommand } from '../remote-agents/registry';
+import { isRemoteDeviceId, sendRemoteCommand, stopRemoteReplay } from '../remote-agents/registry';
 import { getAdbCommand } from '../android-sdk';
 import { createAppiumScriptExport } from './script-export';
 import { resolveReportSummary } from '../../src/appium-recorder/report-summary';
@@ -43,21 +44,10 @@ function sendStreamEvent(res: ServerResponse, payload: unknown) {
   res.write(`${JSON.stringify(payload)}\n`);
 }
 
-async function readBody<T>(req: IncomingMessage) {
-  let body = '';
-  req.on('data', (chunk) => {
-    body += chunk;
-  });
-  return await new Promise<T>((resolve) => {
-    req.on('end', () => {
-      resolve(JSON.parse(body || '{}') as T);
-    });
-  });
-}
 
 function execFileText(command: string, args: string[] = []) {
   return new Promise<string>((resolve, reject) => {
-    execFile(command, args, { maxBuffer: 20 * 1024 * 1024 }, (error, stdout, stderr) => {
+    execFile(command, args, { maxBuffer: 20 * 1024 * 1024, timeout: 10000 }, (error, stdout, stderr) => {
       if (error) {
         reject(new Error(stderr || error.message));
         return;
@@ -386,6 +376,10 @@ export async function handleAppiumRecorderRequest(
       const parsed = await readBody<{ deviceId?: string }>(req);
       const deviceId = parsed.deviceId || selectedDeviceId;
       assertDeviceAllowed(deviceId);
+      if (isRemoteDeviceId(deviceId)) {
+        sendJson(res, { success: true, stopped: stopRemoteReplay(deviceId) });
+        return true;
+      }
       const controller = replayAbortControllers.get(deviceId);
       controller?.abort();
       sendJson(res, { success: true, stopped: Boolean(controller) });
@@ -444,10 +438,6 @@ export async function handleAppiumRecorderRequest(
           globalVariables: getPresetVariables(),
           linkedScripts: linkedScriptSnapshot(script),
         }) as Awaited<ReturnType<typeof replayAppiumScript>>;
-        if (result.history) {
-          try { saveRunHistory({ ...result.history, scriptId: script.id, deviceId }); }
-          catch { result.output += '\n历史记录保存失败'; }
-        }
         const { history: _history, ...response } = result;
         if (streamOutput) {
           result.output?.split(/\r?\n/).forEach((line) => sendStreamEvent(res, { type: 'log', line }));
@@ -504,7 +494,7 @@ export async function handleAppiumRecorderRequest(
       sendStreamEvent(res, { type: 'error', message });
       res.end();
     } else {
-      sendJson(res, { message }, 500);
+      sendJson(res, { message }, requestErrorStatus(error));
     }
     return true;
   }

package/server/appium-recorder/run-history.ts

@@ -40,3 +40,14 @@ export function deleteRunHistory(scriptId: string, id: string) {
   }
   runSql(`PRAGMA secure_delete=ON; DELETE FROM appium_run_history WHERE script_id=${sqlString(scriptId)} AND id=${sqlString(id)};`);
 }
+
+export function listRunAnalytics() {
+  ensureTable();
+  return querySql<{ scriptId: string; scriptName: string; startedAt: string; durationMs: number; status: RunSummary['status'] }>(`
+    SELECT script_id AS scriptId, started_at AS startedAt,
+      json_extract(summary, '$.scriptName') AS scriptName,
+      json_extract(summary, '$.durationMs') AS durationMs,
+      json_extract(summary, '$.status') AS status
+    FROM appium_run_history;
+  `);
+}

package/server/config-access.test.ts

@@ -0,0 +1,27 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import type { IncomingMessage } from 'node:http';
+import { canManageConfig, publicConfig, resolveSavedModelKey } from './config-access';
+import { SAVED_API_KEY } from '../src/config/credentials';
+import type { AppConfig } from './config';
+
+test('configuration access requires a loopback client and matching browser origin', () => {
+  const request = (address: string, origin?: string) => ({ socket: { remoteAddress: address }, headers: { host: 'localhost:5173', origin } }) as IncomingMessage;
+  assert.equal(canManageConfig(request('192.0.2.10')), false);
+  assert.equal(canManageConfig(request('127.0.0.1', 'https://foreign.example')), false);
+  assert.equal(canManageConfig(request('::1', 'http://localhost:5173')), true);
+  assert.equal(canManageConfig(request('::ffff:127.0.0.1')), true);
+});
+
+test('public config masks every key and server env without modifying stored values; masked keys cannot move to another endpoint', () => {
+  const model = { baseUrl: 'https://model.example/v1', apiKey: 'private-secret', name: 'model' };
+  const stored = { runtime: { androidSdkPath: '', reportOutputPath: '' }, midscene: { model: { ...model, provider: 'custom', family: 'gpt-5' }, env: { SECRET: 'secret-env' } }, scriptOptimizer: { model }, appium: { model, promptOptimizer: { model } } } as AppConfig;
+  const output = publicConfig(stored);
+  assert.equal(JSON.stringify(output).includes('private-secret'), false);
+  assert.deepEqual(output.midscene.env, {});
+  assert.equal(stored.midscene.model.apiKey, 'private-secret');
+  assert.equal(resolveSavedModelKey({ ...model, apiKey: SAVED_API_KEY }, model), 'private-secret');
+  assert.throws(() => resolveSavedModelKey({ ...model, baseUrl: 'https://other.example', apiKey: SAVED_API_KEY }, model), /重新输入/);
+  assert.throws(() => resolveSavedModelKey({ ...model, provider: 'codex', apiKey: SAVED_API_KEY }, model), /重新输入/);
+  assert.equal(resolveSavedModelKey({ ...model, apiKey: 'replacement' }, model), 'replacement');
+});

package/server/config-access.ts

@@ -0,0 +1,27 @@
+import type { IncomingMessage } from 'node:http';
+import type { AppConfig } from './config';
+import { SAVED_API_KEY } from '../src/config/credentials';
+
+export function canManageConfig(req: IncomingMessage) {
+  if (!['127.0.0.1', '::1', '::ffff:127.0.0.1'].includes(req.socket.remoteAddress || '')) return false;
+  if (!req.headers.origin) return true;
+  try { return new URL(req.headers.origin).host === req.headers.host; } catch { return false; }
+}
+
+export function publicConfig(config: AppConfig): AppConfig {
+  const result = structuredClone(config);
+  for (const model of [result.midscene.model, result.scriptOptimizer.model, result.appium.model, result.appium.promptOptimizer?.model]) {
+    if (model?.apiKey) model.apiKey = SAVED_API_KEY;
+  }
+  // Runtime environment overrides are only consumed by the server.
+  result.midscene.env = {};
+  return result;
+}
+
+export function resolveSavedModelKey(model: { apiKey?: string; baseUrl?: string; provider?: string }, stored?: { apiKey: string; baseUrl: string; provider?: string }) {
+  if (model.apiKey !== SAVED_API_KEY) return model.apiKey || '';
+  if (!stored || model.baseUrl?.trim().replace(/\/+$/, '') !== stored.baseUrl.trim().replace(/\/+$/, '') || model.provider !== stored.provider) {
+    throw new Error('模型连接地址或提供方已改变，请重新输入 API Key');
+  }
+  return stored.apiKey;
+}