@autobest-ui/agent 1.0.12 → 1.0.14

package/mcp/trace-mcp-recorder/README.md

@@ -1,6 +1,6 @@
 # trace-mcp-recorder
 
-`trace-mcp-recorder` 是基于 `@modelcontextprotocol/sdk` 和 `StdioServerTransport` 的 Node.js MCP Server。它打开非无头 Chromium 供测试人员手动操作，停止后上传 `trace.zip` 和浏览器元数据，并返回私有后端的 Trace Viewer 链接。同一进程同一时间只允许一个录制任务。
+`trace-mcp-recorder` 是基于 `@modelcontextprotocol/sdk` 和 `StdioServerTransport` 的 Node.js MCP Server。它打开非无头 Chromium 供测试人员手动操作，使用 Chromium CDP 在内存中探测画面差异，仅在画面发生明显变化时调用 Playwright 截图写入原生 `trace.zip`，然后上传 trace 和浏览器元数据，并返回私有后端的 Trace Viewer 链接。同一进程同一时间只允许一个录制任务。
 
 ## 安装与启动
 
@@ -43,6 +43,8 @@ npm start
 
 录制超时后服务自动停止并自动上传（无需人工确认），随后第一次人工调用 `stop_trace_recording` 会取回自动停止的结果；再调用则返回没有活动会话。
 
+画面探测默认每 500ms 执行一次，像素差异比例达到 0.5% 后进入防抖冷却；画面连续稳定 800ms 才写入一个轻量 `waitForTimeout(1)` Trace Action，持续变化超过 3 秒时强制写入一个。Trace 自带的 screenshots/snapshots 会记录这些 Action 前后的页面状态，停止录制时会补一个最终 Action。探测截图不会写入 Trace，因此静止页面和连续滚动不会产生大量重复图片。
+
 ## 后端接口契约
 
 服务期望后端提供以下接口：

package/mcp/trace-mcp-recorder/index.js

@@ -8,11 +8,16 @@ import { fileURLToPath } from 'node:url';
 import axios from 'axios';
 import FormData from 'form-data';
 import { chromium } from 'playwright';
+import { PNG } from 'pngjs';
 import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
 import { z } from 'zod';
 
 const DEFAULT_MAX_RECORD_DURATION = 1_800_000;
+const VISUAL_PROBE_INTERVAL = 500;
+const VISUAL_DIFF_THRESHOLD = 0.005;
+const VISUAL_DEBOUNCE_DELAY = 800;
+const VISUAL_MAX_PENDING_DELAY = 3_000;
 const directory = path.dirname(fileURLToPath(import.meta.url));
 
 function readConfiguration(environment = process.env) {
@@ -99,6 +104,32 @@ function toolError(error) {
   };
 }
 
+// Compare decoded PNG pixels. A small tolerance avoids frames caused only by
+// anti-aliasing noise being written to the Trace.
+export function pixelDifference(previousBuffer, currentBuffer, channelTolerance = 12) {
+  if (!previousBuffer || !currentBuffer) return 1;
+  try {
+    const previous = PNG.sync.read(previousBuffer);
+    const current = PNG.sync.read(currentBuffer);
+    if (previous.width !== current.width || previous.height !== current.height) return 1;
+    const pixelCount = previous.width * previous.height;
+    let changedPixels = 0;
+    for (let offset = 0; offset < previous.data.length; offset += 4) {
+      if (
+        Math.abs(previous.data[offset] - current.data[offset]) > channelTolerance ||
+        Math.abs(previous.data[offset + 1] - current.data[offset + 1]) > channelTolerance ||
+        Math.abs(previous.data[offset + 2] - current.data[offset + 2]) > channelTolerance ||
+        Math.abs(previous.data[offset + 3] - current.data[offset + 3]) > channelTolerance
+      ) {
+        changedPixels += 1;
+      }
+    }
+    return changedPixels / pixelCount;
+  } catch {
+    return 1;
+  }
+}
+
 export class TraceRecorder {
   constructor(options = {}) {
     this.environment = options.environment ?? process.env;
@@ -120,6 +151,12 @@ export class TraceRecorder {
       options.removeTemporaryDirectory ?? (temporaryDirectory => rm(temporaryDirectory, { recursive: true, force: true }));
     this.setTimer = options.setTimer ?? setTimeout;
     this.clearTimer = options.clearTimer ?? clearTimeout;
+    this.setInterval = options.setInterval ?? setInterval;
+    this.clearInterval = options.clearInterval ?? clearInterval;
+    this.visualProbeInterval = options.visualProbeInterval ?? VISUAL_PROBE_INTERVAL;
+    this.visualDiffThreshold = options.visualDiffThreshold ?? VISUAL_DIFF_THRESHOLD;
+    this.visualDebounceDelay = options.visualDebounceDelay ?? VISUAL_DEBOUNCE_DELAY;
+    this.visualMaxPendingDelay = options.visualMaxPendingDelay ?? VISUAL_MAX_PENDING_DELAY;
     this.platformInfo = options.platformInfo ?? (() => ({
       platform: os.platform(),
       type: os.type(),
@@ -184,9 +221,18 @@ export class TraceRecorder {
         state: 'recording',
         stopPromise: undefined,
         uploadPromise: undefined,
-        timer: undefined
+        timer: undefined,
+        visualProbeTimer: undefined,
+        visualProbePromise: undefined,
+        visualDebounceTimer: undefined,
+        visualRecordPromise: undefined,
+        visualPendingSince: undefined,
+        cdpSession: undefined,
+        previousProbeFrame: undefined,
+        lastRecordedFrameAt: Date.now()
       };
       this.activeSession = session;
+      await this.startVisualCapture(session);
 
       const failOnBrowserExit = () => {
         if (session.state === 'recording') {
@@ -264,6 +310,7 @@ export class TraceRecorder {
 
   async prepareSession(session) {
     try {
+      await this.stopVisualCapture(session, true);
       await session.context.tracing.stop({ path: session.tracePath });
       await this.closeResources({ page: session.page, context: session.context, browser: session.browser });
       session.state = 'awaiting_upload';
@@ -279,6 +326,86 @@ export class TraceRecorder {
     }
   }
 
+  async startVisualCapture(session) {
+    if (typeof session.context.newCDPSession !== 'function') return;
+    session.cdpSession = await session.context.newCDPSession(session.page);
+    await session.cdpSession.send('Page.enable');
+    session.previousProbeFrame = await this.captureProbeFrame(session);
+    session.visualProbeTimer = this.setInterval(() => {
+      if (session.visualProbePromise) return;
+      session.visualProbePromise = this.probeAndRecordVisualChange(session).catch(error => {
+        if (session.state === 'recording') {
+          console.error('[trace-mcp-recorder] 画面差异探测失败:', errorMessage(error));
+        }
+      }).finally(() => {
+        session.visualProbePromise = undefined;
+      });
+    }, this.visualProbeInterval);
+    session.visualProbeTimer?.unref?.();
+  }
+
+  async captureProbeFrame(session) {
+    const result = await session.cdpSession.send('Page.captureScreenshot', {
+      format: 'png',
+      fromSurface: true
+    });
+    return Buffer.from(result.data, 'base64');
+  }
+
+  async probeAndRecordVisualChange(session) {
+    const currentFrame = await this.captureProbeFrame(session);
+    const difference = pixelDifference(session.previousProbeFrame, currentFrame);
+    session.previousProbeFrame = currentFrame;
+    if (difference >= this.visualDiffThreshold) {
+      this.scheduleVisualRecord(session);
+    }
+  }
+
+  scheduleVisualRecord(session) {
+    const now = Date.now();
+    session.visualPendingSince ??= now;
+    if (session.visualDebounceTimer) this.clearTimer(session.visualDebounceTimer);
+    const pendingFor = now - session.visualPendingSince;
+    const delay = pendingFor >= this.visualMaxPendingDelay ? 0 : this.visualDebounceDelay;
+    session.visualDebounceTimer = this.setTimer(() => {
+      session.visualDebounceTimer = undefined;
+      session.visualRecordPromise = this.recordPendingVisual(session).finally(() => {
+        session.visualRecordPromise = undefined;
+      });
+    }, delay);
+    session.visualDebounceTimer?.unref?.();
+  }
+
+  async recordPendingVisual(session) {
+    if (session.state !== 'recording' || !session.visualPendingSince) return;
+    // waitForTimeout is a traceable Playwright action. The trace screenshots and
+    // snapshots captured around it represent the changed page state, while the
+    // native Trace Viewer can advance through this action during playback.
+    await session.page.waitForTimeout(1);
+    session.lastRecordedFrameAt = Date.now();
+    session.visualPendingSince = undefined;
+  }
+
+  async stopVisualCapture(session, recordFinalFrame = false) {
+    if (!session.cdpSession) return;
+    if (session.visualProbeTimer) {
+      this.clearInterval(session.visualProbeTimer);
+      session.visualProbeTimer = undefined;
+    }
+    await session.visualProbePromise?.catch(() => {});
+    if (session.visualDebounceTimer) {
+      this.clearTimer(session.visualDebounceTimer);
+      session.visualDebounceTimer = undefined;
+    }
+    await session.visualRecordPromise?.catch(() => {});
+    session.visualPendingSince = undefined;
+    if (recordFinalFrame && session.page) {
+      await session.page.waitForTimeout(1).catch(() => {});
+    }
+    await session.cdpSession?.detach().catch(() => {});
+    session.cdpSession = undefined;
+  }
+
   async uploadSession(session, trigger) {
     if (session.uploadPromise) return session.uploadPromise;
     session.state = 'uploading';
@@ -349,6 +476,7 @@ export class TraceRecorder {
     if (session.state !== 'recording') return;
     session.state = 'failed';
     this.clearTimer(session.timer);
+    await this.stopVisualCapture(session);
     await this.closeResources(session);
     await this.removeTemporaryDirectory(session.temporaryDirectory).catch(() => {});
     if (this.activeSession === session) this.activeSession = undefined;

package/mcp/trace-mcp-recorder/index.test.js

@@ -5,11 +5,21 @@ import test from 'node:test';
 import { fileURLToPath } from 'node:url';
 import { Client } from '@modelcontextprotocol/sdk/client/index.js';
 import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js';
-import { TraceRecorder } from './index.js';
+import { PNG } from 'pngjs';
+import { pixelDifference, TraceRecorder } from './index.js';
 
 const directory = path.dirname(fileURLToPath(import.meta.url));
 const sessionUuid = '123e4567-e89b-42d3-a456-426614174000';
 
+test('calculates pixel differences for visual probes', () => {
+  const first = new PNG({ width: 2, height: 1 });
+  const second = new PNG({ width: 2, height: 1 });
+  first.data.fill(255);
+  second.data.fill(255);
+  second.data[0] = 0;
+  assert.equal(pixelDifference(PNG.sync.write(first), PNG.sync.write(second)), 0.5);
+});
+
 function createFixture(options = {}) {
   const calls = [];
   const browser = new EventEmitter();