android-midscene-automation 0.1.26 → 0.1.28

package/src/components/config/AppiumConfigPanel.vue

@@ -0,0 +1,48 @@
+<script setup lang="ts">
+import type { ConfigForm } from '../../types';
+
+defineProps<{
+  configForm: ConfigForm;
+  testingModelKey: string;
+  isSavingModelConfig: boolean;
+  testStatus?: string;
+}>();
+defineEmits<{ testModel: []; saveModelConfig: [] }>();
+</script>
+
+<template>
+  <el-card shadow="never" class="config-module-card config-appium-card">
+    <template #header>
+      <div class="panel-header">
+        <span>节点模型配置</span>
+        <el-button type="primary" :loading="isSavingModelConfig" @click="$emit('saveModelConfig')">
+          保存模型配置
+        </el-button>
+      </div>
+    </template>
+    <section class="appium-model-form">
+      <div class="panel-header panel-header--sub">
+        <span>AI 识别模型</span>
+        <el-button :loading="testingModelKey === 'appium'" @click="$emit('testModel')">测试模型</el-button>
+      </div>
+      <el-form label-position="top">
+        <el-form-item label="Base URL">
+          <el-input v-model="configForm.appium.model.baseUrl" />
+        </el-form-item>
+        <el-form-item label="API Key">
+          <el-input v-model="configForm.appium.model.apiKey" show-password />
+        </el-form-item>
+        <el-form-item label="Model Name">
+          <el-input v-model="configForm.appium.model.name" placeholder="支持图片输入的模型" />
+        </el-form-item>
+        <el-form-item v-if="testStatus" label="测试结果">
+          <el-input :model-value="testStatus" readonly />
+        </el-form-item>
+      </el-form>
+    </section>
+  </el-card>
+</template>
+
+<style scoped>
+.appium-model-form { max-width: 640px; }
+</style>

package/src/components/config/BaseConfigPanel.vue

@@ -0,0 +1,90 @@
+<script setup lang="ts">
+import { Delete, Edit } from '@element-plus/icons-vue';
+import type { AppPreset, ConfigForm } from '../../types';
+
+defineProps<{
+  configForm: ConfigForm;
+  appPresets: AppPreset[];
+  appPresetForm: Pick<AppPreset, 'id' | 'name' | 'packageName'>;
+  isSavingModelConfig: boolean;
+  isSavingAppPreset: boolean;
+}>();
+
+defineEmits<{
+  saveModelConfig: [];
+  saveAppPreset: [];
+  editAppPreset: [app: AppPreset];
+  deleteAppPreset: [id: string];
+}>();
+</script>
+
+<template>
+  <div class="config-grid">
+    <el-card shadow="never" class="config-module-card">
+      <template #header>
+        <div class="panel-header">
+          <span>运行配置</span>
+          <el-button type="primary" :loading="isSavingModelConfig" @click="$emit('saveModelConfig')">
+            保存运行配置
+          </el-button>
+        </div>
+      </template>
+
+      <el-form label-position="top">
+        <el-form-item label="Android SDK 路径">
+          <el-input
+            v-model="configForm.runtime.androidSdkPath"
+            clearable
+            placeholder="留空时读取 ANDROID_SDK_ROOT、ANDROID_HOME 或系统默认路径"
+          />
+        </el-form-item>
+        <el-form-item label="回放报告目录">
+          <el-input
+            v-model="configForm.runtime.reportOutputPath"
+            clearable
+            placeholder="留空时保存到启动目录下的 output"
+          />
+        </el-form-item>
+      </el-form>
+    </el-card>
+
+    <el-card shadow="never" class="config-module-card">
+      <template #header>
+        <div class="panel-header">
+          <div class="panel-header__title">
+            <span>预设 App 参数</span>
+            <el-tag v-if="appPresetForm.id" size="small" type="warning">编辑中</el-tag>
+          </div>
+          <div class="panel-header__actions">
+            <el-button type="primary" :loading="isSavingAppPreset" @click="$emit('saveAppPreset')">
+              {{ appPresetForm.id ? '保存修改' : '保存 App' }}
+            </el-button>
+          </div>
+        </div>
+      </template>
+
+      <el-form label-position="top">
+        <el-form-item label="App 名称">
+          <el-input v-model="appPresetForm.name" placeholder="例如：示例 App" />
+        </el-form-item>
+        <el-form-item label="App 包名">
+          <el-input v-model="appPresetForm.packageName" placeholder="例如：com.example.app" />
+        </el-form-item>
+      </el-form>
+
+      <div class="app-preset-list">
+        <div v-for="app in appPresets" :key="app.id" class="app-preset-row">
+          <div class="app-preset-row__main">
+            <strong>{{ app.name }}</strong>
+            <span>{{ app.packageName }}</span>
+          </div>
+          <div class="script-row__actions">
+            <el-button text size="small" :icon="Edit" @click="$emit('editAppPreset', app)" />
+            <el-button text size="small" :icon="Delete" @click="$emit('deleteAppPreset', app.id)" />
+          </div>
+        </div>
+        <el-empty v-if="!appPresets.length" description="暂无预设 App" />
+      </div>
+    </el-card>
+  </div>
+</template>

package/src/components/config/MidsceneConfigPanel.vue

@@ -0,0 +1,204 @@
+<script setup lang="ts">
+import { computed } from 'vue';
+import { QuestionFilled } from '@element-plus/icons-vue';
+import {
+  codexMidsceneModelOptions,
+  midsceneModelPresets,
+  type MidsceneModelProvider,
+} from '../../config/midscene-model-presets';
+import type { ConfigForm } from '../../types';
+
+const props = defineProps<{
+  configForm: ConfigForm;
+  testingModelKey: string;
+  isSavingModelConfig: boolean;
+  modelTestStatus: { midscene: string; scriptOptimizer: string };
+}>();
+
+const emit = defineEmits<{
+  testModel: [key: 'midscene' | 'scriptOptimizer'];
+  saveModelConfig: [];
+  updateMidsceneModelProvider: [provider: MidsceneModelProvider];
+}>();
+
+const activeMidsceneProvider = computed<MidsceneModelProvider>(() =>
+  props.configForm.midscene.model.provider === 'codex' ? 'codex' : 'custom',
+);
+const modelConfigGuideUrl = 'https://midscenejs.com/zh/model-common-config.html';
+
+const normalizeBaseUrl = (value: string) => value.trim().replace(/\/+$/, '');
+
+const updateMidsceneModelName = (value: string) => {
+  props.configForm.midscene.model.name = value;
+  const option = codexMidsceneModelOptions.find((item) => item.value === value);
+  if (option) {
+    props.configForm.midscene.model.family = option.family;
+  }
+};
+
+const updateCustomMidsceneBaseUrl = (value: string) => {
+  props.configForm.midscene.model.baseUrl = value;
+  const preset = midsceneModelPresets.find((item) => normalizeBaseUrl(item.baseUrl) === normalizeBaseUrl(value));
+  if (preset) {
+    props.configForm.midscene.model.name = preset.modelName;
+    props.configForm.midscene.model.family = preset.modelFamily;
+  }
+};
+
+const updateMidsceneProvider = (value: string) => {
+  if (value === 'custom' || value === 'codex') {
+    emit('updateMidsceneModelProvider', value);
+  }
+};
+
+const openModelConfigGuide = () => {
+  window.open(modelConfigGuideUrl, '_blank', 'noopener,noreferrer');
+};
+</script>
+
+<template>
+  <el-card shadow="never" class="config-module-card">
+    <template #header>
+      <div class="panel-header">
+        <span>模型配置</span>
+        <el-button type="primary" :loading="isSavingModelConfig" @click="$emit('saveModelConfig')">
+          保存模型配置
+        </el-button>
+      </div>
+    </template>
+
+    <div class="config-model-grid">
+      <section>
+        <div class="panel-header panel-header--sub">
+          <span>Midscene 模型</span>
+          <el-button
+            :loading="testingModelKey === 'midscene'"
+            @click="$emit('testModel', 'midscene')"
+          >
+            测试模型
+          </el-button>
+        </div>
+        <el-form label-position="top">
+          <el-form-item label="接入方式">
+            <el-select
+              :model-value="activeMidsceneProvider"
+              @change="updateMidsceneProvider"
+            >
+              <el-option label="自定义提供方" value="custom" />
+              <el-option label="使用 Codex" value="codex" />
+            </el-select>
+          </el-form-item>
+
+          <template v-if="activeMidsceneProvider === 'custom'">
+            <el-form-item label="Base URL">
+              <el-input
+                :model-value="configForm.midscene.model.baseUrl"
+                placeholder="输入 Base URL，匹配已知提供方时自动填充模型"
+                @input="updateCustomMidsceneBaseUrl"
+              />
+            </el-form-item>
+            <el-form-item label="API Key">
+              <el-input v-model="configForm.midscene.model.apiKey" show-password />
+            </el-form-item>
+            <el-form-item>
+              <template #label>
+                <span class="config-field-label">
+                  <span>Model Name</span>
+                  <el-tooltip content="查看模型填写参考" placement="top">
+                    <el-button
+                      class="config-field-help"
+                      text
+                      size="small"
+                      :icon="QuestionFilled"
+                      aria-label="查看 Model Name 填写参考"
+                      @click.stop="openModelConfigGuide"
+                    />
+                  </el-tooltip>
+                </span>
+              </template>
+              <el-input v-model="configForm.midscene.model.name" placeholder="例如：gpt-5.5" />
+            </el-form-item>
+            <el-form-item>
+              <template #label>
+                <span class="config-field-label">
+                  <span>Model Family</span>
+                  <el-tooltip content="查看模型填写参考" placement="top">
+                    <el-button
+                      class="config-field-help"
+                      text
+                      size="small"
+                      :icon="QuestionFilled"
+                      aria-label="查看 Model Family 填写参考"
+                      @click.stop="openModelConfigGuide"
+                    />
+                  </el-tooltip>
+                </span>
+              </template>
+              <el-input v-model="configForm.midscene.model.family" placeholder="例如：gpt-5" />
+            </el-form-item>
+          </template>
+
+          <template v-else>
+            <el-alert
+              class="config-form-alert"
+              type="info"
+              :closable="false"
+              title="复用 Codex 登录态，无需 API Key。请确保 codex 在 PATH 中可用，并已完成 codex login。"
+            />
+            <el-form-item label="Base URL">
+              <el-input :model-value="configForm.midscene.model.baseUrl" readonly />
+            </el-form-item>
+            <el-form-item label="Model Name">
+              <el-select
+                :model-value="configForm.midscene.model.name"
+                @change="updateMidsceneModelName"
+              >
+                <el-option
+                  v-for="option in codexMidsceneModelOptions"
+                  :key="option.value"
+                  :label="option.label"
+                  :value="option.value"
+                />
+              </el-select>
+            </el-form-item>
+            <el-form-item label="Model Family">
+              <el-select v-model="configForm.midscene.model.family" disabled>
+                <el-option label="gpt-5" value="gpt-5" />
+              </el-select>
+            </el-form-item>
+          </template>
+          <el-form-item v-if="modelTestStatus.midscene" label="测试结果">
+            <el-input :model-value="modelTestStatus.midscene" readonly />
+          </el-form-item>
+        </el-form>
+      </section>
+
+      <section>
+        <div class="panel-header panel-header--sub">
+          <span>脚本优化模型</span>
+          <el-button
+            :loading="testingModelKey === 'scriptOptimizer'"
+            @click="$emit('testModel', 'scriptOptimizer')"
+          >
+            测试模型
+          </el-button>
+        </div>
+        <el-form label-position="top">
+          <el-form-item label="Base URL">
+            <el-input v-model="configForm.scriptOptimizer.model.baseUrl" />
+          </el-form-item>
+          <el-form-item label="API Key">
+            <el-input v-model="configForm.scriptOptimizer.model.apiKey" show-password />
+          </el-form-item>
+          <el-form-item label="Model Name">
+            <el-input v-model="configForm.scriptOptimizer.model.name" />
+          </el-form-item>
+          <el-form-item v-if="modelTestStatus.scriptOptimizer" label="测试结果">
+            <el-input :model-value="modelTestStatus.scriptOptimizer" readonly />
+          </el-form-item>
+        </el-form>
+      </section>
+    </div>
+
+  </el-card>
+</template>

package/src/components/device/DevicePreviewPanel.vue

@@ -1,5 +1,5 @@
 <script setup lang="ts">
-import { computed, shallowRef } from 'vue';
+import { computed, shallowRef, watch } from 'vue';
 import { Cpu, Refresh } from '@element-plus/icons-vue';
 import type { AndroidDevice, DeviceAction } from '../../types';
 
@@ -11,6 +11,13 @@ type DeviceOverlayBounds = {
   bottom: number;
 };
 
+type DeviceRegionSelection = {
+  x: number;
+  y: number;
+  width: number;
+  height: number;
+};
+
 const props = defineProps<{
   available: boolean;
   devices: AndroidDevice[];
@@ -21,6 +28,9 @@ const props = defineProps<{
   actions?: readonly DeviceAction[];
   overlayBounds?: readonly DeviceOverlayBounds[];
   selectedBounds?: DeviceOverlayBounds;
+  selectedRegion?: DeviceOverlayBounds;
+  regionSelection?: boolean;
+  regionDrawMode?: boolean;
   deviceWidth?: number;
   deviceHeight?: number;
 }>();
@@ -32,6 +42,7 @@ const emit = defineEmits<{
   previewError: [];
   refreshPreview: [];
   tap: [point: { x: number; y: number }];
+  regionSelect: [region: DeviceRegionSelection];
   swipe: [gesture: {
     startX: number;
     startY: number;
@@ -43,11 +54,14 @@ const emit = defineEmits<{
 
 const imageSize = shallowRef({ width: 0, height: 0 });
 let pointerSession: {
+  mode: 'gesture' | 'region-draw' | 'region-move';
   pointerId: number;
   start: { x: number; y: number };
   startClientX: number;
   startClientY: number;
+  origin?: DeviceOverlayBounds;
 } | null = null;
+const draftRegion = shallowRef<DeviceOverlayBounds | null>(null);
 const hasPreview = computed(() => Boolean(props.frameUrl || props.imageUrl));
 const imageBoxStyle = computed(() => {
   if (props.frameUrl) return {};
@@ -61,6 +75,12 @@ const overlayViewBox = computed(() => {
   return width && height ? `0 0 ${width} ${height}` : '';
 });
 
+watch(() => props.regionSelection, (enabled) => {
+  if (enabled) return;
+  pointerSession = null;
+  draftRegion.value = null;
+});
+
 function updateImageSize(event: Event) {
   const image = event.target as HTMLImageElement;
   imageSize.value = { width: image.naturalWidth, height: image.naturalHeight };
@@ -97,18 +117,109 @@ function getDevicePoint(target: HTMLElement, clientX: number, clientY: number) {
   };
 }
 
+function boundsFromPoints(
+  id: string,
+  start: { x: number; y: number },
+  end: { x: number; y: number },
+) {
+  return {
+    id,
+    left: Math.min(start.x, end.x),
+    top: Math.min(start.y, end.y),
+    right: Math.max(start.x, end.x),
+    bottom: Math.max(start.y, end.y),
+  };
+}
+
+function pointInBounds(point: { x: number; y: number }, bounds?: DeviceOverlayBounds) {
+  if (!bounds) return false;
+  return point.x >= bounds.left
+    && point.x <= bounds.right
+    && point.y >= bounds.top
+    && point.y <= bounds.bottom;
+}
+
+function clampRegion(bounds: DeviceOverlayBounds) {
+  const width = props.deviceWidth || imageSize.value.width;
+  const height = props.deviceHeight || imageSize.value.height;
+  const regionWidth = Math.max(1, bounds.right - bounds.left);
+  const regionHeight = Math.max(1, bounds.bottom - bounds.top);
+  const left = Math.max(0, Math.min(bounds.left, Math.max(0, width - regionWidth)));
+  const top = Math.max(0, Math.min(bounds.top, Math.max(0, height - regionHeight)));
+  return {
+    id: bounds.id,
+    left,
+    top,
+    right: left + regionWidth,
+    bottom: top + regionHeight,
+  };
+}
+
+function moveBounds(
+  origin: DeviceOverlayBounds,
+  start: { x: number; y: number },
+  end: { x: number; y: number },
+) {
+  const dx = end.x - start.x;
+  const dy = end.y - start.y;
+  return clampRegion({
+    id: origin.id,
+    left: origin.left + dx,
+    top: origin.top + dy,
+    right: origin.right + dx,
+    bottom: origin.bottom + dy,
+  });
+}
+
+function emitRegion(bounds: DeviceOverlayBounds) {
+  const width = bounds.right - bounds.left;
+  const height = bounds.bottom - bounds.top;
+  if (width < 2 || height < 2) return;
+  emit('regionSelect', {
+    x: bounds.left,
+    y: bounds.top,
+    width,
+    height,
+  });
+}
+
 function handlePointerDown(event: PointerEvent) {
   if (event.pointerType === 'mouse' && event.button !== 0) return;
   const target = event.currentTarget as HTMLElement;
   const point = getDevicePoint(target, event.clientX, event.clientY);
   if (!point) return;
+  if (props.regionSelection) {
+    event.preventDefault();
+    event.stopPropagation();
+  }
   target.setPointerCapture(event.pointerId);
+  const isMovingRegion = props.regionSelection
+    && !props.regionDrawMode
+    && pointInBounds(point, props.selectedRegion);
   pointerSession = {
+    mode: props.regionSelection ? (isMovingRegion ? 'region-move' : 'region-draw') : 'gesture',
     pointerId: event.pointerId,
     start: point,
     startClientX: event.clientX,
     startClientY: event.clientY,
+    origin: isMovingRegion && props.selectedRegion ? { ...props.selectedRegion } : undefined,
   };
+  if (props.regionSelection) {
+    draftRegion.value = pointerSession.origin || boundsFromPoints('draft-region', point, point);
+  }
+}
+
+function handlePointerMove(event: PointerEvent) {
+  if (!pointerSession || pointerSession.pointerId !== event.pointerId || pointerSession.mode === 'gesture') return;
+  const target = event.currentTarget as HTMLElement;
+  const point = getDevicePoint(target, event.clientX, event.clientY);
+  if (!point) return;
+  event.preventDefault();
+  if (pointerSession.mode === 'region-move' && pointerSession.origin) {
+    draftRegion.value = moveBounds(pointerSession.origin, pointerSession.start, point);
+    return;
+  }
+  draftRegion.value = boundsFromPoints('draft-region', pointerSession.start, point);
 }
 
 function handlePointerUp(event: PointerEvent) {
@@ -118,7 +229,17 @@ function handlePointerUp(event: PointerEvent) {
   const session = pointerSession;
   pointerSession = null;
   if (target.hasPointerCapture(event.pointerId)) target.releasePointerCapture(event.pointerId);
-  if (!point) return;
+  if (!point) {
+    if (session.mode !== 'gesture') draftRegion.value = null;
+    return;
+  }
+
+  if (session.mode !== 'gesture') {
+    const bounds = draftRegion.value || boundsFromPoints('selected-region', session.start, point);
+    draftRegion.value = null;
+    emitRegion(bounds);
+    return;
+  }
 
   const moved = Math.hypot(
     event.clientX - session.startClientX,
@@ -139,7 +260,10 @@ function handlePointerUp(event: PointerEvent) {
 }
 
 function handlePointerCancel(event: PointerEvent) {
-  if (pointerSession?.pointerId === event.pointerId) pointerSession = null;
+  if (pointerSession?.pointerId === event.pointerId) {
+    pointerSession = null;
+    draftRegion.value = null;
+  }
 }
 </script>
 
@@ -208,7 +332,9 @@ function handlePointerCancel(event: PointerEvent) {
       >
         <div
           class="device-preview__surface"
+          :class="{ 'device-preview__surface--selecting': regionSelection }"
           @pointerdown="handlePointerDown"
+          @pointermove="handlePointerMove"
           @pointerup="handlePointerUp"
           @pointercancel="handlePointerCancel"
         />
@@ -229,7 +355,7 @@ function handlePointerCancel(event: PointerEvent) {
           @error="emit('previewError')"
         />
         <svg
-          v-if="overlayViewBox && (overlayBounds?.length || selectedBounds)"
+          v-if="overlayViewBox && (overlayBounds?.length || selectedBounds || selectedRegion || draftRegion)"
           class="device-preview__overlay"
           :viewBox="overlayViewBox"
           preserveAspectRatio="xMidYMid meet"
@@ -254,6 +380,24 @@ function handlePointerCancel(event: PointerEvent) {
             :height="Math.max(0, selectedBounds.bottom - selectedBounds.top)"
             vector-effect="non-scaling-stroke"
           />
+          <rect
+            v-if="selectedRegion && !draftRegion"
+            class="device-preview__selected-region"
+            :x="selectedRegion.left"
+            :y="selectedRegion.top"
+            :width="Math.max(0, selectedRegion.right - selectedRegion.left)"
+            :height="Math.max(0, selectedRegion.bottom - selectedRegion.top)"
+            vector-effect="non-scaling-stroke"
+          />
+          <rect
+            v-if="draftRegion"
+            class="device-preview__draft-region"
+            :x="draftRegion.left"
+            :y="draftRegion.top"
+            :width="Math.max(0, draftRegion.right - draftRegion.left)"
+            :height="Math.max(0, draftRegion.bottom - draftRegion.top)"
+            vector-effect="non-scaling-stroke"
+          />
         </svg>
       </div>
       <div v-else class="device-preview__empty">