android-midscene-automation 0.1.5 → 0.1.9

package/server/appium-recorder/repository.ts

@@ -12,6 +12,10 @@ export type AppiumRecordedStepRecord = {
   type:
     | 'tap'
     | 'input'
+    | 'tapIfExists'
+    | 'inputIfExists'
+    | 'clearIfExists'
+    | 'backIfExists'
     | 'waitFor'
     | 'assertExists'
     | 'key'
@@ -25,23 +29,72 @@ export type AppiumRecordedStepRecord = {
     | 'waitDisappear'
     | 'assertText'
     | 'longPress'
-    | 'pinch';
+    | 'pinch'
+    | 'runScript';
   label: string;
   selector?: {
     strategy: 'accessibilityId' | 'id' | 'androidUiAutomator' | 'xpath' | 'bounds';
     value?: string;
     centerX?: number;
     centerY?: number;
+    unique?: boolean;
+    matchCount?: number;
+    source?: 'node' | 'ancestor' | 'fallback';
   };
+  contextSelector?: {
+    strategy: 'accessibilityId' | 'id' | 'androidUiAutomator' | 'xpath' | 'bounds';
+    value?: string;
+    centerX?: number;
+    centerY?: number;
+    unique?: boolean;
+    matchCount?: number;
+    source?: 'node' | 'ancestor' | 'fallback';
+  };
+  selectorChain?: Array<{
+    strategy: 'accessibilityId' | 'id' | 'androidUiAutomator' | 'xpath' | 'bounds';
+    value?: string;
+    centerX?: number;
+    centerY?: number;
+    unique?: boolean;
+    matchCount?: number;
+    source?: 'node' | 'ancestor' | 'fallback';
+  }>;
   fallback?: {
     strategy: 'accessibilityId' | 'id' | 'androidUiAutomator' | 'xpath' | 'bounds';
     value?: string;
     centerX?: number;
     centerY?: number;
+    unique?: boolean;
+    matchCount?: number;
+    source?: 'node' | 'ancestor' | 'fallback';
   };
   value?: string;
+  optional?: boolean;
   keyCode?: number;
   timeoutMs?: number;
+  flow?: {
+    scope?: 'pre' | 'main' | 'post';
+    nodeKind?: 'action' | 'condition' | 'assertion';
+    yesTargetId?: string;
+    noTargetId?: string;
+    parentConditionId?: string;
+    parentBranch?: 'yes' | 'no';
+    successTargetId?: string;
+    failureTargetId?: string;
+    collapsed?: boolean;
+  };
+  pageBefore?: {
+    activity?: string;
+    packageName?: string;
+    treeSignature?: string;
+    selectedNodePath?: string;
+  };
+  pageAfter?: {
+    activity?: string;
+    packageName?: string;
+    treeSignature?: string;
+    selectedNodePath?: string;
+  };
   swipe?: {
     startX: number;
     startY: number;
@@ -80,16 +133,14 @@ type AppiumRecordedScriptRow = {
   app_package: string;
   app_activity: string | null;
   device_id: string | null;
-  steps_json: string;
+  flow_json: string | null;
   created_at: string;
   updated_at: string;
 };
 
 let initialized = false;
 
-function initDb() {
-  if (initialized) return;
-  initialized = true;
+function createAppiumRecordedScriptsTable() {
   runSql(`
     CREATE TABLE IF NOT EXISTS appium_recorded_scripts (
       id TEXT PRIMARY KEY,
@@ -97,7 +148,7 @@ function initDb() {
       app_package TEXT NOT NULL,
       app_activity TEXT,
       device_id TEXT,
-      steps_json TEXT NOT NULL DEFAULT '[]',
+      flow_json TEXT NOT NULL DEFAULT '[]',
       created_at TEXT NOT NULL,
       updated_at TEXT NOT NULL
     );
@@ -106,6 +157,32 @@ function initDb() {
   `);
 }
 
+function initDb() {
+  if (initialized) return;
+  initialized = true;
+  createAppiumRecordedScriptsTable();
+  const columns = querySql<{ name: string }>('PRAGMA table_info(appium_recorded_scripts);').map((column) => column.name);
+  if (columns.includes('steps_json') || !columns.includes('flow_json')) {
+    runSql('DROP TABLE appium_recorded_scripts;');
+    createAppiumRecordedScriptsTable();
+  }
+}
+
+function parseStepList(value?: string | null) {
+  try {
+    const parsed = JSON.parse(value || '[]');
+    if (!Array.isArray(parsed)) return [];
+    return parsed.filter((step) => (
+      step
+      && typeof step === 'object'
+      && typeof step.id === 'string'
+      && typeof step.type === 'string'
+    )) as AppiumRecordedStepRecord[];
+  } catch {
+    return [];
+  }
+}
+
 function rowToRecord(row: AppiumRecordedScriptRow): AppiumRecordedScriptRecord {
   return {
     id: row.id,
@@ -113,7 +190,7 @@ function rowToRecord(row: AppiumRecordedScriptRow): AppiumRecordedScriptRecord {
     appPackage: row.app_package,
     appActivity: row.app_activity || '',
     deviceId: row.device_id || '',
-    steps: JSON.parse(row.steps_json || '[]') as AppiumRecordedStepRecord[],
+    steps: parseStepList(row.flow_json),
     createdAt: row.created_at,
     updatedAt: row.updated_at,
   };
@@ -128,7 +205,7 @@ export function listAppiumRecordedScripts() {
       app_package,
       app_activity,
       device_id,
-      steps_json,
+      flow_json,
       created_at,
       updated_at
     FROM appium_recorded_scripts
@@ -145,7 +222,7 @@ export function getAppiumRecordedScript(id: string) {
       app_package,
       app_activity,
       device_id,
-      steps_json,
+      flow_json,
       created_at,
       updated_at
     FROM appium_recorded_scripts
@@ -180,7 +257,7 @@ export function saveAppiumRecordedScript(input: {
         app_package = ${sqlString(appPackage)},
         app_activity = ${sqlNullableString(input.appActivity || '')},
         device_id = ${sqlNullableString(input.deviceId || '')},
-        steps_json = ${sqlJson(input.steps || [])},
+        flow_json = ${sqlJson(input.steps || [])},
         updated_at = ${sqlString(now)}
       WHERE id = ${sqlString(input.id)};
     `);
@@ -194,7 +271,7 @@ export function saveAppiumRecordedScript(input: {
       app_package,
       app_activity,
       device_id,
-      steps_json,
+      flow_json,
       created_at,
       updated_at
     )
@@ -212,7 +289,7 @@ export function saveAppiumRecordedScript(input: {
       app_package = excluded.app_package,
       app_activity = excluded.app_activity,
       device_id = excluded.device_id,
-      steps_json = excluded.steps_json,
+      flow_json = excluded.flow_json,
       updated_at = excluded.updated_at;
   `);
 

package/src/App.vue

@@ -72,6 +72,7 @@ const playgroundFrameUrl = ref('');
 const playgroundFrameSignature = ref('');
 const devicePreviewUrl = ref('');
 const devicePreviewMode = ref<'stream' | 'screenshot' | ''>('');
+const backendOffline = shallowRef(false);
 const androidDevices = ref<AndroidDevice[]>([]);
 const deviceInterfaceSize = reactive({ width: 0, height: 0 });
 const deviceDebug = reactive({
@@ -116,6 +117,17 @@ let appiumPreviewTimer: number | null = null;
 let executionProgressTimer: number | null = null;
 let scriptRunAbortController: AbortController | null = null;
 
+const isBackendFetchError = (error: unknown) => {
+  const message = error instanceof Error ? error.message : String(error || '');
+  return error instanceof TypeError || /Failed to fetch|NetworkError|Load failed/i.test(message);
+};
+
+const stopPlaygroundPollTimer = () => {
+  if (!playgroundPollTimer) return;
+  window.clearInterval(playgroundPollTimer);
+  playgroundPollTimer = null;
+};
+
 const form = reactive({
   promptTitle: '',
   testName: '',
@@ -739,10 +751,20 @@ const testModel = async (key: 'midscene' | 'scriptOptimizer') => {
 };
 
 const loadAndroidDevices = async () => {
-  const payload = await api.getAndroidDevices();
-  androidDevices.value = payload.devices || [];
-  playgroundDeviceId.value = payload.currentDeviceId || '';
-  playgroundAvailable.value = androidDevices.value.some((item) => item.status === 'device');
+  if (backendOffline.value) return;
+  try {
+    const payload = await api.getAndroidDevices();
+    backendOffline.value = false;
+    androidDevices.value = payload.devices || [];
+    playgroundDeviceId.value = payload.currentDeviceId || '';
+    playgroundAvailable.value = androidDevices.value.some((item) => item.status === 'device');
+  } catch (error) {
+    if (isBackendFetchError(error)) {
+      markBackendOffline();
+      return;
+    }
+    throw error;
+  }
 };
 
 const loadDeviceInterface = async () => {
@@ -757,6 +779,7 @@ const loadDeviceInterface = async () => {
 };
 
 const loadAdbPreview = () => {
+  if (backendOffline.value) return;
   if (!playgroundDeviceId.value) {
     devicePreviewUrl.value = '';
     devicePreviewMode.value = '';
@@ -786,6 +809,7 @@ const refreshDevicePreview = () => {
 };
 
 const startAppiumPreviewTimer = () => {
+  if (backendOffline.value) return;
   if (appiumPreviewTimer) return;
   loadAdbPreview();
   appiumPreviewTimer = window.setInterval(loadAdbPreview, 1000);
@@ -797,7 +821,22 @@ const stopAppiumPreviewTimer = () => {
   appiumPreviewTimer = null;
 };
 
+const markBackendOffline = () => {
+  if (backendOffline.value) return;
+  backendOffline.value = true;
+  playgroundAvailable.value = false;
+  playgroundFrameUrl.value = '';
+  playgroundFrameSignature.value = '';
+  devicePreviewUrl.value = '';
+  devicePreviewMode.value = '';
+  playgroundPreviewError.value = '本地服务已断开，请重新运行启动命令后刷新页面。';
+  errorMessage.value = playgroundPreviewError.value;
+  stopPlaygroundPollTimer();
+  stopAppiumPreviewTimer();
+};
+
 const loadPlaygroundStatus = async () => {
+  if (backendOffline.value) return;
   if (!playgroundDeviceId.value) {
     playgroundAvailable.value = false;
     playgroundPreviewError.value = '';
@@ -844,6 +883,10 @@ const loadPlaygroundStatus = async () => {
       loadAdbPreview();
     }
   } catch (error) {
+    if (isBackendFetchError(error)) {
+      markBackendOffline();
+      return;
+    }
     playgroundAvailable.value = false;
     playgroundFrameUrl.value = '';
     playgroundFrameSignature.value = '';
@@ -1207,15 +1250,17 @@ onMounted(async () => {
   await Promise.allSettled([loadConfig(), loadAppPresets(), loadAndroidDevices(), loadSavedScripts(), loadModelUsageRecords()]);
   await loadPlaygroundStatus();
   playgroundPollTimer = window.setInterval(() => {
+    if (backendOffline.value) {
+      stopPlaygroundPollTimer();
+      return;
+    }
     void loadAndroidDevices();
     void loadPlaygroundStatus();
   }, 2500);
 });
 
 onUnmounted(() => {
-  if (playgroundPollTimer) {
-    window.clearInterval(playgroundPollTimer);
-  }
+  stopPlaygroundPollTimer();
   stopAppiumPreviewTimer();
   stopExecutionProgressTimer();
   scriptRunAbortController?.abort();