android-midscene-automation 0.1.39 → 0.1.40

package/CHANGELOG.md

@@ -1,8 +1,11 @@
 # 更新记录
+## v0.1.40
+- 回放录屏支持按连接脚本分段保存 MP4，返回父脚本继续执行时另存片段；历史结果可选择片段播放，HTML 报告按节点时间自动切换视频，删除历史结果时一并清理全部片段。
+
 ## v0.1.39
 - Midscene 相关模块统一升级至 1.12.7，
 - scrcpy 视频帧完整性修复：可靠传输视频包、修正关键帧标记，并在缓冲溢出后等待完整关键帧恢复，降低设备预览花屏风险。
-- 回放按钮右侧下拉设置新增默认关闭的“录制回放视频”，使用内置服务端后台接收视频流并封装 MP4，无需安装 scrcpy 或 FFmpeg；远程设备暂不支持。
+- 回放按钮右侧下拉设置新增默认关闭的“录制回放视频”，远程设备暂不支持。
 - 图像模板判断改为“预期匹配结果”，匹配严格度以百分比配置，100% 时提示细微像素差异风险；节点显示预期结果，日志和 HTML 报告分别展示实际匹配、预期匹配、得分与条件结论，保留原检测算法及已有阈值。
 - 移除 AI 识别操作、测试入口及 Appium 参数中的 AI 模型配置，保留流程背景色设置；旧脚本中的 AI 节点提示替换，回放前阻止执行，不自动删除或改变分支。
 

package/README.md

@@ -1229,10 +1229,13 @@ Linux: ~/.local/share/android-midscene-automation
 
 # 项目更新记录
 
+## v0.1.40
+- 回放录屏支持按连接脚本分段保存 MP4，返回父脚本继续执行时另存片段；历史结果可选择片段播放，HTML 报告按节点时间自动切换视频，删除历史结果时一并清理全部片段。
+
 ## v0.1.39
 - Midscene 相关模块统一升级至 1.12.7，
 - scrcpy 视频帧完整性修复：可靠传输视频包、修正关键帧标记，并在缓冲溢出后等待完整关键帧恢复，降低设备预览花屏风险。
-- 回放按钮右侧下拉设置新增默认关闭的“录制回放视频”，使用内置服务端后台接收视频流并封装 MP4，无需安装 scrcpy 或 FFmpeg；远程设备暂不支持。
+- 回放按钮右侧下拉设置新增默认关闭的“录制回放视频”，远程设备暂不支持。
 - 图像模板判断改为“预期匹配结果”，匹配严格度以百分比配置，100% 时提示细微像素差异风险；节点显示预期结果，日志和 HTML 报告分别展示实际匹配、预期匹配、得分与条件结论，保留原检测算法及已有阈值。
 - 移除 AI 识别操作、测试入口及 Appium 参数中的 AI 模型配置，保留流程背景色设置；旧脚本中的 AI 节点提示替换，回放前阻止执行，不自动删除或改变分支。
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "android-midscene-automation",
-  "version": "0.1.39",
+  "version": "0.1.40",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/oooooooko/android-midscene-automation.git"

package/server/appium-recorder/appium-runner.ts

@@ -77,6 +77,7 @@ const replayContext = new AsyncLocalStorage<{
   visualStartOffsets: Map<string, number>;
   softFailureCount: number;
   flowEnded?: boolean;
+  selectVideoScript?: (id: string, name: string, force?: boolean) => Promise<void>;
 }>();
 
 const appiumServerUrl = () => replayContext.getStore()?.serverUrl || configuredAppiumServerUrl();
@@ -1140,6 +1141,8 @@ async function replayLinkedScript(
     }
   }
 
+  // 每次调用独立分段，即使连续两次连接的是同一个脚本。
+  await replayContext.getStore()?.selectVideoScript?.(linkedScript.id, linkedScript.name, true);
   lines.push(`连接脚本开始：${linkedScript.name}`);
   const nextStack = [...stack, linkedScript.id];
   const parent = variableContext.getStore()!;
@@ -1180,6 +1183,7 @@ async function replayLinearSteps(
       lines.push(`[节点 ${index + 1}] 跳过：${step.label}（${linkedScriptInitSkipReason(step)}）`);
       continue;
     }
+    await replayContext.getStore()?.selectVideoScript?.(stack.at(-1) || '', options.scriptName || '');
     lines.push(`[节点 ${index + 1}] 开始：${step.label}`);
     await captureReplayFrame(sessionId, step, index + 1, options.scriptName || '', 'before', '执行前');
     try {
@@ -1237,6 +1241,7 @@ async function replayFlowSteps(
       index = nextIndexAfterStep(index, step);
       continue;
     }
+    await replayContext.getStore()?.selectVideoScript?.(stack.at(-1) || '', options.scriptName || '');
     visitedPath.push(step.label);
     const nodeKind = defaultFlowKind(step);
     lines.push(`[节点 ${index + 1}] 开始：${frameStep.label}`);
@@ -1532,6 +1537,8 @@ export async function replayAppiumScript(
         if (scope.privacy.enabled) throw new Error('本次运行包含敏感变量，为避免泄露已禁止录屏，请关闭录制回放视频');
         lines.push('正在启动后台录屏...');
         recording = await startReplayVideo(targetDeviceId, signal);
+        replayContext.getStore()!.selectVideoScript = recording.selectScript;
+        await recording.selectScript(script.id, script.name);
         lines.push('后台录屏已启动（MP4 / H.264 / 15fps / 2Mbps，无音频）');
       }
       if (hasFlowSteps(script.steps)) lines.push('按流程图路径回放...');
@@ -1565,11 +1572,15 @@ export async function replayAppiumScript(
         try {
           video = await recording.stop();
           if (scope.privacy.enabled) {
-            await import('node:fs/promises').then(fs => fs.rm(video!.filePath, { force: true }));
+            for (const segment of video.segments || [video]) {
+              await import('node:fs/promises').then(fs => fs.rm(segment.filePath, { force: true }));
+            }
             video = undefined;
             lines.push('敏感变量保护：已删除回放视频');
           } else {
-            lines.push(`回放视频已保存：${video.filePath}`);
+            for (const [index, segment] of (video.segments || [video]).entries()) {
+              lines.push(`回放视频 ${index + 1}（${segment.scriptName || script.name}）已保存：${segment.filePath}`);
+            }
             if (video.warning) lines.push(`录屏警告：${video.warning}`);
           }
         } catch (error) { lines.push(`回放视频保存失败：${errorDetail(error)}`); }

package/server/appium-recorder/replay-mp4.ts

@@ -14,7 +14,7 @@ export class ReplayMp4Writer {
   private startedMonotonic = 0;
   private endedMonotonic?: number;
 
-  constructor(path: string) {
+  constructor(path: string, private anchor?: number) {
     this.output = new Output({ target: new FilePathTarget(path, { chunkSize: 1024 * 1024 }), format: new Mp4OutputFormat({ fastStart: 'fragmented' }) });
     this.output.addVideoTrack(this.source);
   }
@@ -37,8 +37,8 @@ export class ReplayMp4Writer {
     if (this.origin === undefined) {
       if (!packet.keyframe) return;
       this.origin = packet.pts;
-      this.startedAt = Date.now();
-      this.startedMonotonic = performance.now();
+      this.startedAt = this.anchor ?? Date.now();
+      this.startedMonotonic = performance.now() - (Date.now() - this.startedAt);
     }
     const timestamp = Number(packet.pts - this.origin) / 1000000;
     if (!Number.isFinite(timestamp) || timestamp < 0 || (this.pending && timestamp < this.pending.timestamp)) throw new Error('录屏时间戳异常');

package/server/appium-recorder/replay-video-segments.ts

@@ -0,0 +1,77 @@
+import { randomUUID } from 'node:crypto';
+import { dirname, join } from 'node:path';
+import { rm } from 'node:fs/promises';
+import type { ScrcpyMediaStreamPacket } from '@yume-chan/scrcpy';
+import { ReplayMp4Writer } from './replay-mp4';
+import type { ReplayVideo } from './replay-video';
+
+// 缓存最近一个 GOP，使切换脚本时无需重启编码器，也不会丢失首个操作。
+export class ReplayVideoSegments {
+  private writer: ReplayMp4Writer;
+  private current: ReplayVideo;
+  readonly segments: ReplayVideo[] = [];
+  private config?: ScrcpyMediaStreamPacket;
+  private gop: ScrcpyMediaStreamPacket[] = [];
+  private bytes = 0;
+  private origin?: bigint;
+  private epoch = 0;
+  private keyTime = 0;
+  private queue: Promise<unknown> = Promise.resolve();
+  constructor(filePath: string, fileName: string) {
+    this.current = { filePath, fileName, startedAt: '' };
+    this.writer = new ReplayMp4Writer(filePath);
+  }
+  get startedAt() { return this.writer.startedAt; }
+  private serialize<T>(task: () => Promise<T>) {
+    const result = this.queue.then(task);
+    this.queue = result.catch(() => undefined);
+    return result;
+  }
+  start() { return this.writer.start(); }
+  add(packet: ScrcpyMediaStreamPacket) {
+    return this.serialize(async () => {
+      await this.writer.add(packet);
+      if (packet.type === 'configuration') { this.config = packet; return; }
+      if (packet.keyframe && packet.pts !== undefined) {
+        if (this.origin === undefined) { this.origin = packet.pts; this.epoch = this.writer.startedAt; }
+        this.keyTime = this.epoch + Number(packet.pts - this.origin) / 1000;
+        this.gop = []; this.bytes = 0;
+      }
+      if (this.keyTime) {
+        this.bytes += packet.data.byteLength;
+        // 编码器长期不发关键帧时停止缓存，避免无限占用内存。
+        if (this.bytes > 32 * 1024 * 1024) { this.gop = []; this.keyTime = 0; }
+        else this.gop.push(packet);
+      }
+    });
+  }
+  selectScript(scriptId: string, scriptName: string, force = false) {
+    return this.serialize(async () => {
+      if (!force && this.current.scriptId === scriptId) return;
+      const boundary = new Date().toISOString();
+      if (this.current.scriptId !== undefined) {
+        if (!this.config || !this.gop.length) throw new Error('录屏分段缺少完整关键帧，请重新运行');
+        await this.finishCurrent();
+        const fileName = `replay-${Date.now()}-${randomUUID()}.mp4`;
+        this.current = { filePath: join(dirname(this.current.filePath), fileName), fileName, startedAt: '' };
+        this.writer = new ReplayMp4Writer(this.current.filePath, this.keyTime);
+        await this.writer.start();
+        await this.writer.add(this.config);
+        for (const packet of this.gop) await this.writer.add(packet);
+      }
+      Object.assign(this.current, { scriptId, scriptName, boundaryAt: boundary });
+    });
+  }
+  end() { this.writer.end(); }
+  private async finishCurrent() {
+    this.writer.end();
+    await this.writer.finish();
+    this.current.startedAt = new Date(this.writer.startedAt).toISOString();
+    this.segments.push(this.current);
+  }
+  finish() { return this.serialize(() => this.finishCurrent()); }
+  async cancel() {
+    await this.writer.cancel().catch(() => undefined);
+    await rm(this.current.filePath, { force: true });
+  }
+}

package/server/appium-recorder/replay-video.ts

@@ -4,9 +4,9 @@ import { randomUUID } from 'node:crypto';
 import { loadConfig } from '../config';
 import { appDataPath } from '../paths';
 import { openRecordingSource, waitForVideoTask } from './scrcpy-recording-source';
-import { ReplayMp4Writer } from './replay-mp4';
+import { ReplayVideoSegments } from './replay-video-segments';
 
-export type ReplayVideo = { filePath: string; fileName: string; startedAt: string; warning?: string };
+export type ReplayVideo = { filePath: string; fileName: string; startedAt: string; warning?: string; scriptId?: string; scriptName?: string; boundaryAt?: string; segments?: ReplayVideo[] };
 
 // 检查 MP4 的索引和媒体块，不将大视频整体读入内存。
 export async function isFinalizedMp4(path: string) {
@@ -46,7 +46,7 @@ export async function startReplayVideo(deviceId: string, signal?: AbortSignal) {
 
 // 独立于连接层，便于用真实 H.264 数据验证封装、断流和终止收尾。
 export async function recordVideoStream(source: Awaited<ReturnType<typeof openRecordingSource>>, filePath: string, fileName: string, startupSignal: AbortSignal) {
-  const writer = new ReplayMp4Writer(filePath);
+  const writer = new ReplayVideoSegments(filePath, fileName);
   const reader = source.stream.getReader();
   let stopping = false;
   let warning: string | undefined;
@@ -62,11 +62,13 @@ export async function recordVideoStream(source: Awaited<ReturnType<typeof openRe
     await pump;
     try {
       await writer.finish();
-      if (!await isFinalizedMp4(filePath)) throw new Error('录屏未生成完整 MP4');
-      return { filePath, fileName, startedAt: new Date(writer.startedAt).toISOString(), warning };
+      for (const segment of writer.segments) {
+        if (!await isFinalizedMp4(segment.filePath)) throw new Error('录屏未生成完整 MP4');
+      }
+      return { ...writer.segments[0]!, segments: writer.segments, warning };
     } catch (error) {
       await writer.cancel().catch(() => undefined);
-      await rm(filePath, { force: true });
+      if (writer.segments.length) return { ...writer.segments[0]!, segments: writer.segments, warning: `部分录像保存失败：${String(error)}` };
       throw error;
     }
   })();
@@ -90,7 +92,13 @@ export async function recordVideoStream(source: Awaited<ReturnType<typeof openRe
       } finally { writer.end(); reader.releaseLock(); }
     })();
     await waitForVideoTask(ready, startupSignal);
-    return { startedAt: new Date(writer.startedAt).toISOString(), stop };
+    return {
+      startedAt: new Date(writer.startedAt).toISOString(), stop,
+      selectScript: (id: string, name: string, force = false) => {
+        if (stopping || warning) return Promise.reject(new Error(warning || '录屏已停止'));
+        return writer.selectScript(id, name, force);
+      },
+    };
   } catch (error) {
     await stop().catch(() => undefined);
     await rm(filePath, { force: true });

package/server/appium-recorder/report.ts

@@ -241,7 +241,7 @@ function stepSection(
 }
 
 function createReplayHtml(input: {
-  video?: { url: string; startedAt: string };
+  video?: { url: string; startedAt: string; segments?: Array<{ url: string; startedAt: string; boundaryAt?: string; scriptName?: string }> };
   script: AppiumRecordedScriptRecord;
   deviceId: string;
   resultText: string;
@@ -418,6 +418,7 @@ function createReplayHtml(input: {
         <div id="caption" class="caption">暂无截图</div>
         <div class="controls">
           <button id="toggle-recording" hidden>查看节点截图</button>
+          <select id="video-segment" aria-label="视频片段" hidden style="max-width:100%;min-width:0"></select>
           <button id="previous" type="button" title="上一帧">上一帧</button>
           <button id="play" type="button">播放</button>
           <button id="next" type="button" title="下一帧">下一帧</button>
@@ -441,7 +442,16 @@ function createReplayHtml(input: {
     const reportStartedAt = Date.parse(data.startedAt) || 0;
     const recording = document.querySelector('#recording');
     const toggleRecording = document.querySelector('#toggle-recording');
-    const videoOffset = Math.max(0, (Date.parse(data.video?.startedAt) || reportStartedAt) - reportStartedAt);
+    const videos = data.video ? (data.video.segments || [data.video]) : [];
+    const segmentSelect = document.querySelector('#video-segment');
+    let activeVideo = 0;
+    let videoOffset = Math.max(0, (Date.parse(videos[0]?.startedAt) || reportStartedAt) - reportStartedAt);
+    videos.forEach((video, index) => {
+      const option = document.createElement('option');
+      option.value = String(index); option.textContent = (index + 1) + '. ' + (video.scriptName || data.scriptName || '回放视频');
+      segmentSelect.appendChild(option);
+    });
+    segmentSelect.hidden = videos.length < 2;
     let videoMode = Boolean(data.video);
     let pendingVideoTime = 0;
     if (data.video) { recording.src = data.video.url; toggleRecording.hidden = false; }
@@ -539,11 +549,32 @@ function createReplayHtml(input: {
     };
     const seekTo = (milliseconds, scrollStep = true) => {
       currentMs = Math.max(0, Math.min(durationMs, milliseconds));
+      let index = 0;
+      for (let i = 1; i < videos.length; i++) {
+        if (Date.parse(videos[i].boundaryAt || videos[i].startedAt) - reportStartedAt <= currentMs) index = i;
+      }
+      if (index !== activeVideo) {
+        activeVideo = index; segmentSelect.value = String(index);
+        videoOffset = Math.max(0, Date.parse(videos[index].startedAt) - reportStartedAt);
+        recording.src = videos[index].url;
+      }
       pendingVideoTime = Math.max(0, (currentMs - videoOffset) / 1000);
       if (data.video && recording.readyState >= 1) recording.currentTime = Math.min(pendingVideoTime, recording.duration || pendingVideoTime);
       render(scrollStep);
     };
     recording.addEventListener('loadedmetadata', () => { recording.currentTime = Math.min(pendingVideoTime, recording.duration); });
+    segmentSelect.addEventListener('change', () => {
+      pause();
+      const video = videos[Number(segmentSelect.value)];
+      seekTo(Date.parse(video.boundaryAt || video.startedAt) - reportStartedAt);
+    });
+    recording.addEventListener('ended', () => {
+      if (videoMode && activeVideo + 1 < videos.length) {
+        const next = videos[activeVideo + 1];
+        seekTo(Date.parse(next.boundaryAt || next.startedAt) - reportStartedAt);
+        recording.play().catch(() => undefined);
+      }
+    });
     recording.addEventListener('timeupdate', () => {
       if (!videoMode || recording.paused) return;
       currentMs = recording.currentTime * 1000 + videoOffset;
@@ -806,7 +837,14 @@ export async function createAppiumReplayReport(input: {
   const visualCheckByStepId = new Map(visualChecks.map((check) => [check.nodeId, check]));
   const visualFailureCount = visualChecks.filter((check) => check.status === 'failed').length;
   const html = createReplayHtml({
-    video: input.video ? { startedAt: input.video.startedAt, url: relative(outputDir, input.video.filePath).split(/[/\\\\]/).map(encodeURIComponent).join('/') } : undefined,
+    video: input.video ? {
+      startedAt: input.video.startedAt,
+      url: relative(outputDir, input.video.filePath).split(/[/\\\\]/).map(encodeURIComponent).join('/'),
+      segments: (input.video.segments || [input.video]).map(segment => ({
+        startedAt: segment.startedAt, boundaryAt: segment.boundaryAt, scriptName: segment.scriptName,
+        url: relative(outputDir, segment.filePath).split(/[/\\\\]/).map(encodeURIComponent).join('/'),
+      })),
+    } : undefined,
     script: input.script,
     deviceId: input.deviceId,
     resultText,

package/server/appium-recorder/routes.ts

@@ -337,7 +337,10 @@ export async function handleAppiumRecorderRequest(
       const run = getRunHistory(decodeURIComponent(videoMatch[1]), decodeURIComponent(videoMatch[2]));
       if (!run?.video) { sendJson(res, { error: '本次运行没有视频' }, 404); return true; }
       assertDeviceAllowed(run.deviceId);
-      await sendReplayVideo(req, res, run.video.filePath);
+      const index = Number(new URL(req.url || '', 'http://localhost').searchParams.get('segment') || 0);
+      const segment = Number.isSafeInteger(index) && index >= 0 ? (run.video.segments || [run.video])[index] : undefined;
+      if (!segment) { sendJson(res, { error: '视频片段不存在' }, 404); return true; }
+      await sendReplayVideo(req, res, segment.filePath);
       return true;
     }
     const historyMatch = pathname.match(/^\/api\/appium-recorder\/scripts\/([^/]+)\/history(?:\/([^/]+))?$/);

package/server/appium-recorder/run-history.ts

@@ -33,8 +33,10 @@ export function deleteRunHistory(scriptId: string, id: string) {
   ensureTable();
   const video = getRunHistory(scriptId, id)?.video;
   // 只清理服务端生成并关联到该记录的录像，不接受客户端文件路径。
-  if (video && basename(video.filePath) === video.fileName && /^replay-[\d]+-[\da-f-]+\.mp4$/.test(video.fileName)) {
-    rmSync(video.filePath, { force: true });
+  for (const segment of video?.segments || (video ? [video] : [])) {
+    if (basename(segment.filePath) === segment.fileName && /^replay-[\d]+-[\da-f-]+\.mp4$/.test(segment.fileName)) {
+      rmSync(segment.filePath, { force: true });
+    }
   }
   runSql(`PRAGMA secure_delete=ON; DELETE FROM appium_run_history WHERE script_id=${sqlString(scriptId)} AND id=${sqlString(id)};`);
 }

package/src/appium-recorder/RunHistoryDialog.vue

@@ -1,5 +1,5 @@
 <script setup lang="ts">
-import { computed, onMounted, ref } from 'vue';
+import { computed, onMounted, ref, watch } from 'vue';
 import { Delete, Refresh, VideoPlay } from '@element-plus/icons-vue';
 import { ElMessage, ElMessageBox } from 'element-plus';
 import { historyStatistics, type RunSummary, type RunDetail } from './run-history';
@@ -17,6 +17,8 @@ const error = ref('');
 const nodeKey = ref('');
 const logQuery = ref('');
 const videoRun = ref<RunSummary>();
+const videoSegment = ref(0);
+watch(videoRun, () => { videoSegment.value = 0; }, { flush: 'sync' });
 const base = `/api/appium-recorder/scripts/${encodeURIComponent(props.scriptId)}/history`;
 const labels = { passed: '通过', failed: '失败', stopped: '已终止' };
 const versions = computed(() => [...new Set(runs.value.map(run => run.appVersion || '未知'))]);
@@ -132,7 +134,10 @@ onMounted(load);
       </section>
     </div>
     <el-dialog :model-value="Boolean(videoRun)" title="回放视频" width="min(800px, 90vw)" align-center append-to-body destroy-on-close @close="videoRun = undefined">
-      <video v-if="videoRun" :src="`${base}/${encodeURIComponent(videoRun.id)}/video`" controls preload="metadata" style="width:100%;max-height:70vh" />
+      <el-select v-if="videoRun?.video?.segments?.length" v-model="videoSegment" aria-label="视频片段" style="width:100%;margin-bottom:12px">
+        <el-option v-for="(segment, index) in videoRun.video.segments" :key="segment.fileName" :label="`${index + 1}. ${segment.scriptName || videoRun.scriptName}`" :value="index" />
+      </el-select>
+      <video v-if="videoRun" :src="`${base}/${encodeURIComponent(videoRun.id)}/video?segment=${videoSegment}`" controls preload="metadata" style="width:100%;max-height:70vh" />
     </el-dialog>
   </el-dialog>
 </template>

package/src/appium-recorder/run-history.ts

@@ -5,8 +5,10 @@ export type HistoryNode = {
   durationMs: number | null;
 };
 
+export type RunVideo = { filePath: string; fileName: string; startedAt: string; warning?: string; scriptId?: string; scriptName?: string; boundaryAt?: string; segments?: RunVideo[] };
+
 export type RunSummary = {
-  video?: { filePath: string; fileName: string; startedAt: string; warning?: string };
+  video?: RunVideo;
   id: string;
   scriptId: string;
   scriptName: string;