android-midscene-automation 0.1.34 → 0.1.36

package/src/appium-recorder/flow-unmerge.ts

@@ -0,0 +1,145 @@
+import type { AppiumRecordedStep } from './types';
+import { branchSteps } from './flow-merge';
+import { defaultFlowKind } from './flow-labels';
+import { createStepId } from './visual-change';
+
+const edges = ['yesTargetId', 'noTargetId', 'successTargetId', 'failureTargetId'] as const;
+const structure = [...edges, 'parentConditionId', 'parentBranch'] as const;
+export type AddedNodeSide = 'yes' | 'no' | 'both';
+
+export function unmergeBranches(steps: AppiumRecordedStep[], conditionId: string, addedSide?: AddedNodeSide) {
+  const condition = steps.find(step => step.id === conditionId);
+  const record = condition?.mergeUndo;
+  if (!record || record.version !== 1 || record.conditionId !== conditionId) {
+    throw new Error('该合并没有可用的还原记录，无法安全取消；仅新执行的合并支持还原');
+  }
+  const original = new Set(record.originalIds);
+  const common = new Set(record.commonIds);
+  const affected = new Set(record.flows.map(item => item.id));
+  const byId = new Map(steps.map(step => [step.id, step]));
+  if (record.flows.some(item => !byId.has(item.id)) || record.removed.some(step => byId.has(step.id))) {
+    throw new Error('合并涉及的节点已删除或 ID 冲突，无法安全取消');
+  }
+  const boundary = record.boundaryId ? steps.findIndex(step => step.id === record.boundaryId) : steps.length;
+  if (boundary < 0) throw new Error('公共流程边界已删除，无法安全取消');
+  const added = steps.slice(steps.indexOf(condition!) + 1, boundary).filter(step => !original.has(step.id)
+    && step.flow?.parentConditionId === condition!.flow?.parentConditionId
+    && step.flow?.parentBranch === condition!.flow?.parentBranch);
+  const shared = new Set([...common, ...added.map(step => step.id)]);
+  // 普通新增操作可分配到任一侧；控制节点和跨节点配对不做猜测性复制。
+  if (added.some(step => defaultFlowKind(step) === 'condition'
+    || ['visualChange', 'breakLoop', 'endFlow'].includes(step.type) || step.mergeUndo)) {
+    throw new Error('公共流程新增了判断、循环、终止或配对节点，请先移出这些节点再取消合并');
+  }
+  if (added.some(step => edges.some(key => step.flow?.[key]
+    && !shared.has(step.flow[key]!) && step.flow[key] !== record.boundaryId))) {
+    throw new Error('公共流程新增操作存在外部连接，请先处理该连接再取消合并');
+  }
+  for (const item of record.flows) {
+    const current = byId.get(item.id)!;
+    for (const key of structure) {
+      if (current.flow?.[key] === item.after?.[key]) continue;
+      const target = edges.includes(key as typeof edges[number]) ? byId.get(current.flow?.[key] || '') : undefined;
+      if (!target || original.has(target.id) || defaultFlowKind(target) === 'condition') {
+        throw new Error('合并后的连接或分支归属已改变，请先处理结构冲突再取消合并');
+      }
+    }
+  }
+  for (const step of steps) {
+    if (!affected.has(step.id) && original.has(step.id) && edges.some(key => shared.has(step.flow?.[key] || ''))) {
+      throw new Error('其他流程引用了公共节点，无法安全取消合并');
+    }
+  }
+  const side = addedSide || record.source;
+  const next: AppiumRecordedStep[] = JSON.parse(JSON.stringify(steps));
+  const nextById = new Map(next.map(step => [step.id, step]));
+  // 只撤回合并写入的结构字段，保留用户修改的名称、参数、备注等。
+  for (const item of record.flows) {
+    const step = nextById.get(item.id)!;
+    const flow = step.flow ||= {};
+    for (const key of structure) {
+      if (item.before?.[key] === item.after?.[key]) continue;
+      const current = flow[key];
+      if (current !== item.after?.[key] && !shared.has(current || '')) continue;
+      if (item.before?.[key] === undefined) delete flow[key];
+      else Object.assign(flow, { [key]: item.before[key] });
+    }
+  }
+  // 恢复被合并删除的另一侧节点，按原始邻接顺序插回，而非覆盖现有节点。
+  for (const backup of record.removed) {
+    const restored: AppiumRecordedStep = JSON.parse(JSON.stringify(backup));
+    const later = record.originalIds.slice(record.originalIds.indexOf(backup.id) + 1);
+    const anchor = later.map(id => next.findIndex(step => step.id === id)).find(index => index >= 0);
+    next.splice(anchor ?? next.length, 0, restored);
+    nextById.set(restored.id, restored);
+  }
+  const assigned = side === 'both' ? record.source : side;
+  for (const item of added) {
+    const step = nextById.get(item.id)!;
+    step.flow = { ...step.flow, parentConditionId: conditionId, parentBranch: assigned };
+    for (const key of edges) delete step.flow[key];
+  }
+  if (assigned !== record.source) {
+    for (const item of added) {
+      const index = next.findIndex(step => step.id === item.id);
+      next.push(...next.splice(index, 1));
+    }
+  }
+  if (side === 'both') {
+    for (const item of added) {
+      const copy: AppiumRecordedStep = JSON.parse(JSON.stringify(nextById.get(item.id)));
+      copy.id = createStepId();
+      copy.flow!.parentBranch = record.source === 'yes' ? 'no' : 'yes';
+      next.push(copy);
+    }
+  }
+  const other = record.source === 'yes' ? 'no' : 'yes';
+  if (added.length && (side === 'both' || side === other)) {
+    const siblings = branchSteps(next, conditionId, other);
+    const appended = siblings.slice(-added.length);
+    const previous = siblings[siblings.length - appended.length - 1];
+    if (previous && (defaultFlowKind(previous) === 'condition' || ['endFlow', 'breakLoop'].includes(previous.type))) {
+      throw new Error('另一侧末尾是判断、循环或终止操作，新增节点不能安全追加，请仅保留在原来源分支');
+    }
+    if (previous?.flow?.successTargetId) {
+      appended[appended.length - 1]!.flow!.successTargetId = previous.flow.successTargetId;
+      previous.flow.successTargetId = appended[0]!.id;
+    }
+  }
+  const restoredCondition = nextById.get(conditionId)!;
+  delete restoredCondition.mergeUndo;
+  const ownerBranch = (step: AppiumRecordedStep): 'yes' | 'no' | undefined => {
+    const visited = new Set<string>();
+    let current: AppiumRecordedStep | undefined = step;
+    while (current?.flow?.parentConditionId && !visited.has(current.id)) {
+      visited.add(current.id);
+      if (current.flow.parentConditionId === conditionId) return current.flow.parentBranch;
+      current = nextById.get(current.flow.parentConditionId);
+    }
+  };
+  // 新增的分支尾节点可能仍指向公共入口，改接自己一侧的后续节点。
+  for (const step of next) {
+    const branch = ownerBranch(step);
+    if (!branch) continue;
+    const siblings = next.filter(item => item.flow?.parentConditionId === step.flow?.parentConditionId && item.flow?.parentBranch === step.flow?.parentBranch);
+    const sibling = siblings[siblings.indexOf(step) + 1];
+    for (const key of edges) {
+      const target = nextById.get(step.flow?.[key] || '');
+      if (!target || !shared.has(target.id)) continue;
+      if (ownerBranch(target) === branch && !added.some(item => item.id === target.id)) continue;
+      if (sibling) step.flow![key] = sibling.id;
+      else delete step.flow![key];
+    }
+  }
+  for (const branch of ['yes', 'no'] as const) {
+    const first = branchSteps(next, conditionId, branch)[0];
+    const key = branch === 'yes' ? 'yesTargetId' : 'noTargetId';
+    if (first) restoredCondition.flow![key] = first.id;
+    else delete restoredCondition.flow![key];
+  }
+  // 将两个分支的完整子树重新放到判断之后，公共后继保持原位。
+  const owned = next.filter(step => ownerBranch(step));
+  const result = next.filter(step => !owned.includes(step));
+  result.splice(result.indexOf(restoredCondition) + 1, 0, ...owned);
+  return { steps: result, addedCount: added.length };
+}

package/src/appium-recorder/image-check.ts

@@ -0,0 +1,117 @@
+import type { AppiumVisualChangeRegion } from './types';
+
+export const IMAGE_CHECK_MODES = { template: '模板匹配', state: '图片状态判断', black: '黑屏检测', color: '区域颜色判断', change: '多帧变化检测' } as const;
+export type ImageCheckConfig = {
+  mode: keyof typeof IMAGE_CHECK_MODES;
+  target: 'region' | 'element';
+  region: AppiumVisualChangeRegion;
+  screenWidth: number;
+  screenHeight: number;
+  template?: string;
+  negativeTemplate?: string;
+  expectation: 'present' | 'absent';
+  threshold: number;
+  minScoreGap: number;
+  color: string;
+  tolerance: number;
+  ratio: number;
+  durationMs: number;
+  intervalMs: number;
+  consecutive: number;
+};
+export type ImageCheckResult = {
+  result: boolean | null;
+  mode: ImageCheckConfig['mode'];
+  region: AppiumVisualChangeRegion;
+  message: string;
+  durationMs: number;
+  sampleCount: number;
+  metrics: Record<string, number>;
+  images: { label: string; base64: string }[];
+};
+
+export function createImageCheckConfig(): ImageCheckConfig {
+  return { mode: 'template', target: 'region', region: { x: 0, y: 0, width: 100, height: 100 },
+    screenWidth: 0, screenHeight: 0, expectation: 'present', threshold: 0.9, minScoreGap: 0.1,
+    color: '#000000', tolerance: 30, ratio: 95, durationMs: 5000, intervalMs: 1000, consecutive: 1 };
+}
+
+// 只读取 PNG 头部尺寸，避免为了表单校验解码整张模板。
+export function imageTemplateSize(base64?: string) {
+  if (!base64) return undefined;
+  try {
+    const header = atob(base64.slice(0, 32));
+    if (header.length < 24 || header.slice(0, 8) !== '\x89PNG\r\n\x1a\n') return undefined;
+    const uint = (offset: number) => (header.charCodeAt(offset) * 0x1000000
+      + header.charCodeAt(offset + 1) * 0x10000 + header.charCodeAt(offset + 2) * 0x100 + header.charCodeAt(offset + 3));
+    const width = uint(16), height = uint(20);
+    return width && height ? { width, height } : undefined;
+  } catch { return undefined; }
+}
+
+export function imageTemplateRegionIssue(config: ImageCheckConfig) {
+  if (!['template', 'state'].includes(config.mode)) return '';
+  const templates = [{ value: config.template, label: config.mode === 'state' ? '选中模板' : '模板' },
+    ...(config.mode === 'state' ? [{ value: config.negativeTemplate, label: '未选中模板' }] : [])];
+  for (const template of templates) {
+    const size = imageTemplateSize(template.value);
+    if (size && (size.width > config.region.width || size.height > config.region.height)) {
+      return `${template.label} ${size.width}×${size.height} 超过检测区域 ${config.region.width}×${config.region.height}，${config.target === 'region' ? '请扩大区域或重新采集模板' : '请重新选择组件或采集对应模板'}`;
+    }
+  }
+  return '';
+}
+
+export function expandedImageTemplateRegion(config: ImageCheckConfig) {
+  if (config.target !== 'region' || !['template', 'state'].includes(config.mode)) return undefined;
+  const sizes = [config.template, ...(config.mode === 'state' ? [config.negativeTemplate] : [])].map(imageTemplateSize);
+  const width = Math.max(config.region.width, ...sizes.map(size => size?.width || 0));
+  const height = Math.max(config.region.height, ...sizes.map(size => size?.height || 0));
+  if (width > config.screenWidth || height > config.screenHeight || !Number.isFinite(width + height)) return undefined;
+  // 用户明确选择扩区时以原区域中心扩展，并保证仍在屏幕内，不改变模板像素。
+  const x = Math.max(0, Math.min(config.screenWidth - width, Math.floor(config.region.x + (config.region.width - width) / 2)));
+  const y = Math.max(0, Math.min(config.screenHeight - height, Math.floor(config.region.y + (config.region.height - height) / 2)));
+  return { x, y, width, height };
+}
+
+export function validateImageCheck(config: ImageCheckConfig | undefined) {
+  if (!config || !Object.prototype.hasOwnProperty.call(IMAGE_CHECK_MODES, config.mode)) throw new Error('请选择图像判断模式');
+  const range = (value: number, min: number, max: number, name: string, integer = false) => {
+    if (!Number.isFinite(value) || value < min || value > max || (integer && !Number.isInteger(value))) throw new Error(`${name}必须在 ${min} 至 ${max} 之间${integer ? '，且为整数' : ''}`);
+  };
+  if (!['region', 'element'].includes(config.target)) throw new Error('检测目标无效');
+  if (!['present', 'absent'].includes(config.expectation)) throw new Error('判断条件无效');
+  range(config.region?.x, 0, 16000, '区域 X', true);
+  range(config.region?.y, 0, 16000, '区域 Y', true);
+  range(config.region?.width, 1, 16000, '区域宽度', true);
+  range(config.region?.height, 1, 16000, '区域高度', true);
+  if (config.target === 'region') {
+    range(config.screenWidth, 1, 16000, '采集屏幕宽度', true);
+    range(config.screenHeight, 1, 16000, '采集屏幕高度', true);
+  }
+  range(config.threshold, 0.01, 1, '模板阈值');
+  range(config.minScoreGap, 0.001, 1, '最小得分差');
+  range(config.tolerance, 0, 255, '像素容差', true);
+  range(config.ratio, 0.01, 100, '像素占比');
+  range(config.durationMs, 0, 60000, '观察时长', true);
+  range(config.intervalMs, 200, 10000, '采样间隔', true);
+  range(config.consecutive, 1, 100, '连续满足帧数', true);
+  if (config.consecutive > Math.floor(config.durationMs / config.intervalMs) + 1) throw new Error('观察时间不足以采集连续满足帧数');
+  if (config.mode === 'change' && config.durationMs < config.intervalMs) throw new Error('多帧变化的观察时长不能小于采样间隔');
+  if (!/^#[0-9a-f]{6}$/i.test(config.color)) throw new Error('目标颜色必须为 #RRGGBB 格式');
+  for (const value of [config.template, config.negativeTemplate]) {
+    if (value && (value.length > 4 * 1024 * 1024 || !/^[A-Za-z0-9+/]+={0,2}$/.test(value))) throw new Error('模板必须是小于 3MB 的 PNG 图片');
+  }
+  if (['template', 'state'].includes(config.mode) && !config.template) throw new Error('请采集或上传模板');
+  if (config.mode === 'state' && !config.negativeTemplate) throw new Error('请采集或上传未选中模板');
+  if (config.target === 'region') {
+    const issue = imageTemplateRegionIssue(config);
+    if (issue) throw new Error(issue);
+  }
+  return config;
+}
+
+export function imageCheckSummary(config?: ImageCheckConfig) {
+  if (!config) return '未配置图像判断';
+  return `${IMAGE_CHECK_MODES[config.mode]} · ${config.target === 'element' ? '组件区域' : '框选区域'} · ${config.durationMs}ms`;
+}

package/src/appium-recorder/types.ts

@@ -1,4 +1,5 @@
 import type { TestVariable, VariableExtraction, ScriptReturn } from './variables';
+import type { ImageCheckConfig } from './image-check';
 
 export type AppiumBounds = {
   left: number;
@@ -67,6 +68,7 @@ export type AppiumNode = {
 };
 
 export type AppiumRecordedStep = {
+  mergeUndo?: BranchMergeUndo;
   id: string;
   type:
     | 'tap'
@@ -81,6 +83,7 @@ export type AppiumRecordedStep = {
     | 'checkedState'
     | 'radioButtonState'
     | 'aiRecognition'
+    | 'imageCheck'
     | 'textClick'
     | 'key'
     | 'waitActivity'
@@ -150,6 +153,7 @@ export type AppiumRecordedStep = {
     percent: number;
   };
   visualChange?: AppiumVisualChangeConfig;
+  imageCheck?: ImageCheckConfig;
   snapshot?: {
     text: string;
     resourceId: string;
@@ -158,6 +162,17 @@ export type AppiumRecordedStep = {
   };
 };
 
+export type BranchMergeUndo = {
+  version: 1;
+  conditionId: string;
+  source: 'yes' | 'no';
+  commonIds: string[];
+  originalIds: string[];
+  boundaryId?: string;
+  flows: { id: string; before: AppiumRecordedStep['flow']; after: AppiumRecordedStep['flow'] }[];
+  removed: AppiumRecordedStep[];
+};
+
 export type AppiumRecordedScript = {
   id: string;
   name: string;

package/src/appium-recorder/visual-change-pairs.ts

@@ -1,6 +1,7 @@
 import type { AppiumRecordedStep } from './types';
 
 export type VisualChangeBranchTarget = {
+  beforeStepId?: string;
   stepId: string;
   branch: 'yes' | 'no';
 };
@@ -85,6 +86,7 @@ export function branchPathForInsert(
   branchTarget?: VisualChangeBranchTarget,
 ) {
   if (branchTarget) {
+    if (branchTarget.beforeStepId) return branchPathForStep(steps, steps.find(step => step.id === branchTarget.beforeStepId));
     const condition = steps.find((step) => step.id === branchTarget.stepId);
     return [...branchPathForStep(steps, condition), `${branchTarget.stepId}:${branchTarget.branch}`];
   }

package/src/appium-recorder/visual-change.ts

@@ -11,6 +11,18 @@ export function createStepId() {
   return `step_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 8)}`;
 }
 
+export function syncVisualChangeThreshold(steps: AppiumRecordedStep[], source: AppiumRecordedStep) {
+  const config = source.visualChange;
+  if (source.type !== 'visualChange' || config?.role !== 'end' || !config.pairId) return steps;
+  const { changeRatioThreshold } = normalizeVisualChangeConfig(config);
+  // 回放以开始节点配置为准，结束节点修改阈值时同步整组；不同配对互不影响。
+  return steps.map(step => (
+    step.type === 'visualChange' && step.visualChange && step.visualChange.pairId === config.pairId
+      ? { ...step, visualChange: { ...step.visualChange, changeRatioThreshold } }
+      : step
+  ));
+}
+
 export function boundsToVisualRegion(bounds: AppiumBounds) {
   return {
     x: bounds.left,
@@ -47,7 +59,7 @@ export function normalizeVisualChangeConfig(
     changeRatioThreshold: Math.max(0.01, Number(config?.changeRatioThreshold) || VISUAL_CHANGE_DEFAULTS.changeRatioThreshold),
     pixelmatchThreshold: Math.min(
       1,
-      Math.max(0, Number(config?.pixelmatchThreshold) || VISUAL_CHANGE_DEFAULTS.pixelmatchThreshold),
+      Math.max(0, Number.isFinite(config?.pixelmatchThreshold) ? Number(config?.pixelmatchThreshold) : VISUAL_CHANGE_DEFAULTS.pixelmatchThreshold),
     ),
   };
 }