android-midscene-automation 0.1.0

package/src/pages/ManualStepsPage.vue

@@ -0,0 +1,179 @@
+<script setup lang="ts">
+import { ref } from 'vue';
+import { Delete, Plus } from '@element-plus/icons-vue';
+import {
+  createPromptPreset,
+  stepTypeDescriptions,
+  stepTypeOptions,
+  type ScriptStep,
+} from '../script-generator';
+import type { AppPreset, GeneratorForm } from '../types';
+
+defineProps<{
+  form: GeneratorForm;
+  appPresets: AppPreset[];
+  steps: ScriptStep[];
+}>();
+
+defineEmits<{
+  addStep: [];
+  removeStep: [id: string];
+}>();
+
+const stepsDialogVisible = ref(false);
+const exampleSteps = createPromptPreset();
+
+const formatRepeatValue = (value: number | string | undefined) => {
+  const repeat = Number(value);
+  return Number.isFinite(repeat) && repeat > 0 ? String(Math.floor(repeat)) : '';
+};
+
+const parseRepeatValue = (value: string) => {
+  const repeat = Number(value);
+  return Number.isFinite(repeat) && repeat > 0 ? Math.floor(repeat) : 0;
+};
+</script>
+
+<template>
+  <section class="page-grid page-grid--single">
+    <el-card shadow="never">
+      <template #header>基础配置</template>
+      <el-form label-position="top">
+        <el-form-item label="脚本名称">
+          <el-input v-model="form.testName" placeholder="请输入脚本名称" />
+        </el-form-item>
+        <el-form-item label="场景标题">
+          <el-input v-model="form.promptTitle" placeholder="请输入场景标题" />
+        </el-form-item>
+        <el-form-item label="选择 App">
+          <el-select v-model="form.appPresetId" placeholder="选择预设 App">
+            <el-option
+              v-for="app in appPresets"
+              :key="app.id"
+              :label="`${app.name} · ${app.packageName}`"
+              :value="app.id"
+            />
+          </el-select>
+        </el-form-item>
+      </el-form>
+    </el-card>
+  </section>
+
+  <el-card shadow="never" class="manual-steps-card">
+    <template #header>
+      <span>步骤编排</span>
+    </template>
+
+    <details class="method-guide">
+      <summary class="method-guide__summary">方法说明</summary>
+      <div class="method-guide__content">
+        <div v-for="option in stepTypeOptions" :key="option.value" class="method-guide__row">
+          <strong>{{ option.label }}</strong>
+          <div class="method-guide__text">
+            <span>{{ stepTypeDescriptions[option.value].description }}</span>
+            <code>{{ stepTypeDescriptions[option.value].example }}</code>
+          </div>
+        </div>
+      </div>
+    </details>
+
+    <div class="manual-steps-toolbar">
+      <el-button type="primary" plain @click="stepsDialogVisible = true">步骤示例</el-button>
+      <el-button type="primary" :icon="Plus" @click="$emit('addStep')">新增步骤</el-button>
+    </div>
+
+    <div class="step-list">
+      <div v-for="(step, index) in steps" :key="step.id" class="step-item">
+        <div class="step-item__top">
+          <span class="step-item__index">步骤 {{ index + 1 }}</span>
+          <div class="step-item__actions">
+            <el-switch v-model="step.enabled" inline-prompt active-text="开" inactive-text="关" />
+            <el-button circle :icon="Delete" @click="$emit('removeStep', step.id)" />
+          </div>
+        </div>
+        <el-form label-position="top" class="step-form">
+          <div class="step-grid">
+            <el-form-item label="方法">
+              <el-select v-model="step.type">
+                <el-option
+                  v-for="option in stepTypeOptions"
+                  :key="option.value"
+                  :label="option.label"
+                  :value="option.value"
+                />
+              </el-select>
+            </el-form-item>
+            <el-form-item label="步骤标题">
+              <el-input v-model="step.label" />
+            </el-form-item>
+            <el-form-item label="Prompt" class="step-grid__wide">
+              <el-input v-model="step.prompt" type="textarea" :rows="3" />
+            </el-form-item>
+            <el-form-item v-if="['act', 'tap'].includes(step.type)" label="重复次数">
+              <el-input-number
+                v-model="step.repeat"
+                :min="0"
+                :max="10"
+                :formatter="formatRepeatValue"
+                :parser="parseRepeatValue"
+                placeholder="不重复"
+              />
+            </el-form-item>
+            <el-form-item
+              v-if="['query', 'boolean', 'string', 'number'].includes(step.type)"
+              label="输出变量"
+            >
+              <el-input v-model="step.outputVar" />
+            </el-form-item>
+            <el-form-item v-if="step.type === 'input'" label="输入值">
+              <el-input v-model="step.value" />
+            </el-form-item>
+          </div>
+        </el-form>
+      </div>
+      <el-empty v-if="!steps.length" description="暂无步骤，点击新增步骤开始编排" />
+    </div>
+
+    <el-dialog v-model="stepsDialogVisible" title="步骤示例" width="920px" class="steps-dialog">
+      <div class="step-list step-list--dialog">
+        <div v-for="(step, index) in exampleSteps" :key="step.id" class="step-item step-item--readonly">
+          <div class="step-item__top">
+            <span class="step-item__index">步骤 {{ index + 1 }}</span>
+          </div>
+          <el-form label-position="top" class="step-form">
+            <div class="step-grid">
+              <el-form-item label="方法">
+                <el-input
+                  :model-value="stepTypeOptions.find((option) => option.value === step.type)?.label || step.type"
+                  readonly
+                />
+              </el-form-item>
+              <el-form-item label="步骤标题">
+                <el-input :model-value="step.label" readonly />
+              </el-form-item>
+              <el-form-item label="Prompt" class="step-grid__wide">
+                <el-input :model-value="step.prompt" type="textarea" :rows="3" readonly />
+              </el-form-item>
+              <el-form-item v-if="['act', 'tap'].includes(step.type)" label="重复次数">
+                <el-input :model-value="step.repeat && step.repeat > 1 ? String(step.repeat) : '不重复'" readonly />
+              </el-form-item>
+              <el-form-item
+                v-if="['query', 'boolean', 'string', 'number'].includes(step.type)"
+                label="输出变量"
+              >
+                <el-input :model-value="step.outputVar || ''" readonly />
+              </el-form-item>
+              <el-form-item v-if="step.type === 'input'" label="输入值">
+                <el-input :model-value="step.value || ''" readonly />
+              </el-form-item>
+            </div>
+          </el-form>
+        </div>
+      </div>
+
+      <template #footer>
+        <el-button @click="stepsDialogVisible = false">关闭</el-button>
+      </template>
+    </el-dialog>
+  </el-card>
+</template>

package/src/script-generator/codegen.ts

@@ -0,0 +1,126 @@
+import type { ScriptForm, ScriptStep } from './types';
+
+const safeVar = (value: string, fallback: string) => {
+  const normalized = value.trim().replace(/[^a-zA-Z0-9_$]+/g, '_').replace(/^(\d)/, '_$1');
+  return normalized || fallback;
+};
+
+const escapeTemplate = (value: string) => value.replace(/`/g, '\\`');
+
+function getRepeatCount(step: ScriptStep) {
+  const repeat = Number(step.repeat || 1);
+  if (!Number.isFinite(repeat)) return 1;
+  return Math.max(1, Math.min(10, Math.floor(repeat)));
+}
+
+function withRepeat(step: ScriptStep, body: string) {
+  const repeat = getRepeatCount(step);
+  if (repeat <= 1) return body;
+
+  const inner = body
+    .split('\n')
+    .map((line) => `    ${line.replace(/^  /, '')}`)
+    .join('\n');
+
+  return `  for (let attempt = 1; attempt <= ${repeat}; attempt += 1) {
+    console.log(${JSON.stringify(`执行${step.label || step.type}`)}, attempt);
+${inner}
+    await new Promise((resolve) => setTimeout(resolve, 1000));
+  }`;
+}
+
+// 瞬时提示可能在下一次单帧截图前消失，因此只对明确标记的动作开启观察窗口。
+function withObservation(step: ScriptStep, index: number, body: string) {
+  const assertion = step.observePrompt?.trim();
+  if (!assertion) return body;
+
+  const observerVar = `observer${index + 1}`;
+  const nestedBody = body
+    .split('\n')
+    .map((line) => `  ${line}`)
+    .join('\n');
+
+  return `  const ${observerVar} = await agent.startObserving({ intervalMs: 500, maxFrames: 20 });
+  try {
+${nestedBody}
+  } finally {
+    await ${observerVar}.stop();
+  }
+  await ${observerVar}.aiAssert(\`${escapeTemplate(assertion)}\`);`;
+}
+
+function getBooleanFailureRule(step: ScriptStep, outputVar: string) {
+  const signalText = `${outputVar} ${step.label}`;
+  const shouldFailWhenTrue =
+    /has.*(modal|popup|dialog|blocking)|hasBlocking|blockingModal/i.test(signalText) ||
+    step.label.includes('确认无干扰弹窗') ||
+    step.label.includes('遮挡弹窗');
+  const shouldFailWhenFalse =
+    /is.*(ready|loaded|success)|isLoginReady|loginReady/i.test(signalText) ||
+    step.label.includes('验证登录页就绪') ||
+    step.label.includes('登录页就绪');
+  return { shouldFailWhenTrue, shouldFailWhenFalse };
+}
+
+function buildStep(step: ScriptStep, index: number) {
+  const prompt = escapeTemplate(step.prompt);
+  switch (step.type) {
+    case 'comment':
+      return `  // ${step.label || step.prompt}`;
+    case 'waitFor':
+      return `  await agent.aiWaitFor(\`${prompt}\`);`;
+    case 'act':
+      return withObservation(step, index, withRepeat(step, `  await agent.aiAct(\`${prompt}\`);`));
+    case 'tap':
+      return withObservation(step, index, withRepeat(step, `  await agent.aiTap(\`${prompt}\`);`));
+    case 'input':
+      return withObservation(
+        step,
+        index,
+        `  await agent.aiInput(\`${prompt}\`, { value: ${JSON.stringify(step.value || '')} });`,
+      );
+    case 'query':
+      return `  const ${safeVar(step.outputVar || 'result', 'result')} = await agent.aiQuery(${JSON.stringify(step.prompt)});\n  console.log(${JSON.stringify(step.label || step.outputVar || 'query result')}, ${safeVar(step.outputVar || 'result', 'result')});`;
+    case 'boolean':
+      {
+        const outputVar = safeVar(step.outputVar || 'flag', 'flag');
+        const { shouldFailWhenTrue, shouldFailWhenFalse } = getBooleanFailureRule(step, outputVar);
+        return `  const ${outputVar} = await agent.aiBoolean(\`${prompt}\`);\n  console.log(${JSON.stringify(step.label || step.outputVar || 'boolean result')}, ${outputVar});${
+          shouldFailWhenTrue
+            ? `\n  if (${outputVar}) {\n    throw new Error(${JSON.stringify(`${step.label || outputVar} 未通过：仍存在遮挡登录操作的弹窗`)});\n  }`
+            : ''
+        }${
+          shouldFailWhenFalse
+            ? `\n  if (!${outputVar}) {\n    throw new Error(${JSON.stringify(`${step.label || outputVar} 未通过：页面未达到预期状态`)});\n  }`
+            : ''
+        }`;
+      }
+    case 'string':
+      return `  const ${safeVar(step.outputVar || 'text', 'text')} = await agent.aiString(\`${prompt}\`);\n  console.log(${JSON.stringify(step.label || step.outputVar || 'string result')}, ${safeVar(step.outputVar || 'text', 'text')});`;
+    case 'number':
+      return `  const ${safeVar(step.outputVar || 'count', 'count')} = await agent.aiNumber(\`${prompt}\`);\n  console.log(${JSON.stringify(step.label || step.outputVar || 'number result')}, ${safeVar(step.outputVar || 'count', 'count')});`;
+  }
+}
+
+export function buildScript(form: ScriptForm, steps: ScriptStep[]) {
+  const activeSteps = steps.filter((step) => step.enabled);
+
+  return `import { agentFromAdbDevice } from '@midscene/android';
+
+async function main() {
+  const agent = await agentFromAdbDevice(process.env.ANDROID_SERIAL);
+
+  try {
+    // ${form.promptTitle}
+${activeSteps.map((step, index) => buildStep(step, index)).join('\n')}
+  } finally {
+    await agent.destroy?.();
+  }
+}
+
+main().catch((error) => {
+  console.error(error);
+  process.exitCode = 1;
+});
+`;
+}

package/src/script-generator/index.ts

@@ -0,0 +1,4 @@
+export { buildScript } from './codegen';
+export { createPromptPreset } from './presets';
+export { stepTypeDescriptions, stepTypeOptions } from './step-options';
+export type { ScriptForm, ScriptStep, StepType } from './types';

package/src/script-generator/presets.ts

@@ -0,0 +1,37 @@
+import type { ScriptStep } from './types';
+
+export function createPromptPreset(): ScriptStep[] {
+  return [
+    {
+      id: crypto.randomUUID(),
+      type: 'comment',
+      label: '进入目标 App',
+      prompt: '确保进入目标 App，再处理启动阶段弹窗',
+      enabled: true,
+    },
+    {
+      id: crypto.randomUUID(),
+      type: 'act',
+      label: '进入目标 App',
+      prompt:
+        '先观察当前界面：如果已经在目标 App 内，保持当前状态；如果不在目标 App 内，使用包名打开目标 App 并等待首屏加载完成。不要查找桌面图标或从最近任务中选择 App。',
+      enabled: true,
+    },
+    {
+      id: crypto.randomUUID(),
+      type: 'act',
+      label: '循环处理启动弹窗',
+      prompt:
+        '检查当前页面：如果仍有启动阶段的声明与条款、协议、隐私政策、权限、广告、活动、更新或通知引导弹窗遮挡业务操作，点击同意、允许、关闭、跳过或稍后再说。如果没有遮挡弹窗，不要点击业务内容、表单字段、复选框或提交按钮。',
+      repeat: 1,
+      enabled: true,
+    },
+    {
+      id: crypto.randomUUID(),
+      type: 'waitFor',
+      label: '等待起始页面稳定',
+      prompt: '页面已经稳定显示为本次测试的起始页面，并且没有协议、权限、广告、更新、活动或通知引导弹窗遮挡业务操作',
+      enabled: true,
+    },
+  ];
+}

package/src/script-generator/step-options.ts

@@ -0,0 +1,52 @@
+import type { StepType } from './types';
+
+export const stepTypeOptions: Array<{ label: string; value: StepType }> = [
+  { label: '注释', value: 'comment' },
+  { label: 'aiAct', value: 'act' },
+  { label: 'aiWaitFor', value: 'waitFor' },
+  { label: 'aiTap', value: 'tap' },
+  { label: 'aiInput', value: 'input' },
+  { label: 'aiQuery', value: 'query' },
+  { label: 'aiBoolean', value: 'boolean' },
+  { label: 'aiString', value: 'string' },
+  { label: 'aiNumber', value: 'number' },
+];
+
+export const stepTypeDescriptions: Record<StepType, { description: string; example: string }> = {
+  comment: {
+    description: '只写注释，不执行动作。适合说明意图、标记阶段或补充上下文。',
+    example: '示例：处理登录前弹窗',
+  },
+  act: {
+    description: '适合一段自然语言动作，比如连续点击、关闭弹窗、完成一组界面操作。',
+    example: '示例：若出现权限弹窗则点击允许，若出现广告弹窗则关闭',
+  },
+  waitFor: {
+    description: '等待某个界面条件成立后再继续，适合加载完成、页面稳定、元素出现。',
+    example: '示例：登录页面已经稳定显示，并且可以开始输入账号密码',
+  },
+  tap: {
+    description: '点击一个明确目标，适合按钮、列表项、返回键这类单步操作。',
+    example: '示例：登录按钮',
+  },
+  input: {
+    description: '向输入框填写内容。Prompt 写定位目标，输入值单独填在“输入值”。',
+    example: '示例：Prompt=手机号输入框；输入值=13800138000',
+  },
+  query: {
+    description: '提取结构化数据，适合列表、表格、对象数组，输出到变量供后续使用。',
+    example: '示例：{ title: string, price: number }[], 当前页面商品列表',
+  },
+  boolean: {
+    description: '判断真假，适合判断弹窗是否存在、状态是否满足、按钮是否可见。',
+    example: '示例：当前页面上是否还存在会遮挡登录操作的弹窗',
+  },
+  string: {
+    description: '提取单条文本，适合标题、用户名、标签内容。',
+    example: '示例：当前页面最显眼的标题文本',
+  },
+  number: {
+    description: '提取数字，适合计数、金额、数量、页码。',
+    example: '示例：购物车角标上的商品数量',
+  },
+};

package/src/script-generator/types.ts

@@ -0,0 +1,27 @@
+export type StepType =
+  | 'comment'
+  | 'waitFor'
+  | 'act'
+  | 'tap'
+  | 'input'
+  | 'query'
+  | 'boolean'
+  | 'string'
+  | 'number';
+
+export interface ScriptStep {
+  id: string;
+  type: StepType;
+  label: string;
+  prompt: string;
+  outputVar?: string;
+  value?: string;
+  observePrompt?: string;
+  repeat?: number;
+  enabled: boolean;
+}
+
+export interface ScriptForm {
+  promptTitle: string;
+  testName: string;
+}