@midscene/visualizer 1.5.2 → 1.5.3-beta-20260305031559.0

package/dist/es/component/player/remotion/frame-calculator.mjs

@@ -0,0 +1,149 @@
+const FPS = 30;
+function calculateFrameMap(scripts) {
+    let baseImageWidth = 1920;
+    let baseImageHeight = 1080;
+    for (const s of scripts)if (('img' === s.type || 'insight' === s.type) && s.img) {
+        baseImageWidth = s.imageWidth || 1920;
+        baseImageHeight = s.imageHeight || 1080;
+        break;
+    }
+    const scriptFrames = [];
+    let currentFrame = 0;
+    for (const script of scripts){
+        const durationMs = script.duration;
+        switch(script.type){
+            case 'sleep':
+                {
+                    const frames = Math.ceil(durationMs / 1000 * FPS);
+                    scriptFrames.push({
+                        type: 'sleep',
+                        startFrame: currentFrame,
+                        durationInFrames: frames,
+                        title: script.title,
+                        subTitle: script.subTitle,
+                        taskId: script.taskId
+                    });
+                    currentFrame += frames;
+                    break;
+                }
+            case 'img':
+                {
+                    const frames = Math.max(Math.ceil(durationMs / 1000 * FPS), 1);
+                    const camera = script.camera;
+                    const iw = script.imageWidth || baseImageWidth;
+                    const ih = script.imageHeight || baseImageHeight;
+                    const sf = {
+                        type: 'img',
+                        startFrame: currentFrame,
+                        durationInFrames: frames,
+                        img: script.img,
+                        imageWidth: iw,
+                        imageHeight: ih,
+                        title: script.title,
+                        subTitle: script.subTitle,
+                        taskId: script.taskId
+                    };
+                    if (camera) {
+                        var _camera_pointerLeft, _camera_pointerTop;
+                        sf.cameraTarget = {
+                            left: camera.left,
+                            top: camera.top,
+                            width: camera.width,
+                            pointerLeft: null != (_camera_pointerLeft = camera.pointerLeft) ? _camera_pointerLeft : Math.round(iw / 2),
+                            pointerTop: null != (_camera_pointerTop = camera.pointerTop) ? _camera_pointerTop : Math.round(ih / 2)
+                        };
+                    }
+                    scriptFrames.push(sf);
+                    currentFrame += frames;
+                    break;
+                }
+            case 'insight':
+                {
+                    const insightPhaseFrames = Math.max(Math.ceil(durationMs / 1000 * FPS), 1);
+                    const cameraDurationMs = script.insightCameraDuration || 0;
+                    const cameraPhaseFrames = Math.ceil(cameraDurationMs / 1000 * FPS);
+                    const totalFrames = insightPhaseFrames + cameraPhaseFrames;
+                    const iw = script.imageWidth || baseImageWidth;
+                    const ih = script.imageHeight || baseImageHeight;
+                    const camera = script.camera;
+                    const sf = {
+                        type: 'insight',
+                        startFrame: currentFrame,
+                        durationInFrames: totalFrames,
+                        img: script.img,
+                        imageWidth: iw,
+                        imageHeight: ih,
+                        insightPhaseFrames,
+                        cameraPhaseFrames,
+                        highlightElement: script.highlightElement,
+                        searchArea: script.searchArea,
+                        title: script.title,
+                        subTitle: script.subTitle,
+                        taskId: script.taskId
+                    };
+                    if (camera) {
+                        var _camera_pointerLeft1, _camera_pointerTop1;
+                        sf.cameraTarget = {
+                            left: camera.left,
+                            top: camera.top,
+                            width: camera.width,
+                            pointerLeft: null != (_camera_pointerLeft1 = camera.pointerLeft) ? _camera_pointerLeft1 : Math.round(iw / 2),
+                            pointerTop: null != (_camera_pointerTop1 = camera.pointerTop) ? _camera_pointerTop1 : Math.round(ih / 2)
+                        };
+                    }
+                    scriptFrames.push(sf);
+                    currentFrame += totalFrames;
+                    break;
+                }
+            case 'clear-insight':
+                {
+                    const frames = Math.max(Math.ceil(durationMs / 1000 * FPS), 1);
+                    scriptFrames.push({
+                        type: 'clear-insight',
+                        startFrame: currentFrame,
+                        durationInFrames: frames,
+                        title: script.title,
+                        subTitle: script.subTitle,
+                        taskId: script.taskId
+                    });
+                    currentFrame += frames;
+                    break;
+                }
+            case 'spinning-pointer':
+                {
+                    const frames = Math.max(Math.ceil(durationMs / 1000 * FPS), 1);
+                    scriptFrames.push({
+                        type: 'spinning-pointer',
+                        startFrame: currentFrame,
+                        durationInFrames: frames,
+                        title: script.title,
+                        subTitle: script.subTitle,
+                        taskId: script.taskId
+                    });
+                    currentFrame += frames;
+                    break;
+                }
+            case 'pointer':
+                scriptFrames.push({
+                    type: 'pointer',
+                    startFrame: currentFrame,
+                    durationInFrames: 0,
+                    pointerImg: script.img,
+                    title: script.title,
+                    subTitle: script.subTitle,
+                    taskId: script.taskId
+                });
+                break;
+        }
+    }
+    const stepsDurationInFrames = Math.max(currentFrame, 1);
+    return {
+        scriptFrames,
+        totalDurationInFrames: stepsDurationInFrames,
+        fps: FPS,
+        stepsDurationInFrames,
+        imageWidth: baseImageWidth,
+        imageHeight: baseImageHeight
+    };
+}
+export { FPS, calculateFrameMap };

package/dist/es/component/player/use-frame-player.mjs

@@ -0,0 +1,88 @@
+import { useCallback, useEffect, useRef, useState } from "react";
+function useFramePlayer(options) {
+    const { durationInFrames, fps, autoPlay = false, loop = false } = options;
+    const [currentFrame, setCurrentFrame] = useState(0);
+    const [playing, setPlaying] = useState(autoPlay);
+    const playingRef = useRef(playing);
+    const frameRef = useRef(currentFrame);
+    var _options_playbackRate;
+    const rateRef = useRef(null != (_options_playbackRate = options.playbackRate) ? _options_playbackRate : 1);
+    const durationRef = useRef(durationInFrames);
+    const fpsRef = useRef(fps);
+    const loopRef = useRef(loop);
+    playingRef.current = playing;
+    frameRef.current = currentFrame;
+    var _options_playbackRate1;
+    rateRef.current = null != (_options_playbackRate1 = options.playbackRate) ? _options_playbackRate1 : 1;
+    durationRef.current = durationInFrames;
+    fpsRef.current = fps;
+    loopRef.current = loop;
+    useEffect(()=>{
+        if (!playing) return;
+        let rafId;
+        let lastTime = null;
+        let accumulated = 0;
+        const tick = (now)=>{
+            if (null !== lastTime) {
+                const delta = (now - lastTime) * rateRef.current;
+                accumulated += delta;
+                const frameDuration = 1000 / fpsRef.current;
+                while(accumulated >= frameDuration){
+                    accumulated -= frameDuration;
+                    const next = frameRef.current + 1;
+                    if (next >= durationRef.current) if (loopRef.current) {
+                        frameRef.current = 0;
+                        setCurrentFrame(0);
+                    } else {
+                        frameRef.current = durationRef.current - 1;
+                        setCurrentFrame(durationRef.current - 1);
+                        setPlaying(false);
+                        return;
+                    }
+                    else {
+                        frameRef.current = next;
+                        setCurrentFrame(next);
+                    }
+                }
+            }
+            lastTime = now;
+            rafId = requestAnimationFrame(tick);
+        };
+        rafId = requestAnimationFrame(tick);
+        return ()=>cancelAnimationFrame(rafId);
+    }, [
+        playing
+    ]);
+    const play = useCallback(()=>{
+        if (frameRef.current >= durationRef.current - 1) {
+            frameRef.current = 0;
+            setCurrentFrame(0);
+        }
+        setPlaying(true);
+    }, []);
+    const pause = useCallback(()=>setPlaying(false), []);
+    const toggle = useCallback(()=>{
+        if (playingRef.current) setPlaying(false);
+        else {
+            if (frameRef.current >= durationRef.current - 1) {
+                frameRef.current = 0;
+                setCurrentFrame(0);
+            }
+            setPlaying(true);
+        }
+    }, []);
+    const seekTo = useCallback((frame)=>{
+        const clamped = Math.max(0, Math.min(frame, durationRef.current - 1));
+        frameRef.current = clamped;
+        setCurrentFrame(clamped);
+    }, []);
+    return {
+        currentFrame,
+        playing,
+        play,
+        pause,
+        toggle,
+        seekTo
+    };
+}
+export { useFramePlayer };

package/dist/es/component/universal-playground/index.mjs

@@ -100,7 +100,20 @@ function UniversalPlayground({ playgroundSDK, storage, contextProvider, config:
         playgroundSDK
     ]);
     const { loading, setLoading, infoList, setInfoList, actionSpace, actionSpaceLoading, uiContextPreview, setUiContextPreview, showScrollToBottomButton, verticalMode, replayCounter, setReplayCounter, infoListRef, currentRunningIdRef, interruptedFlagRef, clearInfoList, handleScrollToBottom } = usePlaygroundState(playgroundSDK, effectiveStorage, contextProvider, branding.targetName);
-    const { handleRun: executeAction, handleStop, canStop } = usePlaygroundExecution(playgroundSDK, effectiveStorage, actionSpace, loading, setLoading, setInfoList, replayCounter, setReplayCounter, verticalMode, currentRunningIdRef, interruptedFlagRef);
+    const { handleRun: executeAction, handleStop, canStop } = usePlaygroundExecution({
+        playgroundSDK,
+        storage: effectiveStorage,
+        actionSpace,
+        loading,
+        setLoading,
+        setInfoList,
+        replayCounter,
+        setReplayCounter,
+        verticalMode,
+        currentRunningIdRef,
+        interruptedFlagRef,
+        deviceType: componentConfig.deviceType
+    });
     useEffect(()=>{
         if ((null == playgroundSDK ? void 0 : playgroundSDK.overrideConfig) && config) playgroundSDK.overrideConfig(config).catch((error)=>{
             console.error('Failed to override SDK config:', error);

package/dist/es/hooks/usePlaygroundExecution.mjs

@@ -88,7 +88,7 @@ function buildProgressContent(task) {
     const description = paramStr(task);
     return description ? `${action} - ${description}` : action;
 }
-function wrapExecutionDumpForReplay(dump) {
+function wrapExecutionDumpForReplay(dump, deviceType) {
     const modelBriefsSet = new Set();
     if ((null == dump ? void 0 : dump.tasks) && Array.isArray(dump.tasks)) dump.tasks.forEach((task)=>{
         if (task.usage) {
@@ -106,10 +106,12 @@ function wrapExecutionDumpForReplay(dump) {
         modelBriefs,
         executions: [
             dump
-        ]
+        ],
+        deviceType
     };
 }
-function usePlaygroundExecution(playgroundSDK, storage, actionSpace, loading, setLoading, setInfoList, replayCounter, setReplayCounter, verticalMode, currentRunningIdRef, interruptedFlagRef) {
+function usePlaygroundExecution(options) {
+    const { playgroundSDK, storage, actionSpace, loading, setLoading, setInfoList, replayCounter, setReplayCounter, verticalMode, currentRunningIdRef, interruptedFlagRef, deviceType } = options;
     const { deepLocate, deepThink, screenshotIncluded, domIncluded } = useEnvConfig();
     const handleRun = useCallback((value)=>_async_to_generator(function*() {
             if (!playgroundSDK) return void console.warn('PlaygroundSDK is not available');
@@ -206,7 +208,7 @@ function usePlaygroundExecution(playgroundSDK, storage, actionSpace, loading, se
             let counter = replayCounter;
             if (null == result ? void 0 : result.dump) {
                 if (result.dump.tasks && Array.isArray(result.dump.tasks)) {
-                    const groupedDump = wrapExecutionDumpForReplay(result.dump);
+                    const groupedDump = wrapExecutionDumpForReplay(result.dump, deviceType);
                     const info = allScriptsFromDump(groupedDump);
                     setReplayCounter((c)=>c + 1);
                     replayInfo = info;
@@ -264,7 +266,8 @@ function usePlaygroundExecution(playgroundSDK, storage, actionSpace, loading, se
         deepLocate,
         deepThink,
         screenshotIncluded,
-        domIncluded
+        domIncluded,
+        deviceType
     ]);
     const handleStop = useCallback(()=>_async_to_generator(function*() {
             const thisRunningId = currentRunningIdRef.current;
@@ -291,7 +294,7 @@ function usePlaygroundExecution(playgroundSDK, storage, actionSpace, loading, se
                     let replayInfo = null;
                     let counter = replayCounter;
                     if ((null == (_executionData_dump = executionData.dump) ? void 0 : _executionData_dump.tasks) && Array.isArray(executionData.dump.tasks)) {
-                        const groupedDump = wrapExecutionDumpForReplay(executionData.dump);
+                        const groupedDump = wrapExecutionDumpForReplay(executionData.dump, deviceType);
                         replayInfo = allScriptsFromDump(groupedDump);
                         setReplayCounter((c)=>c + 1);
                         counter = replayCounter + 1;
@@ -349,7 +352,8 @@ function usePlaygroundExecution(playgroundSDK, storage, actionSpace, loading, se
         setLoading,
         setInfoList,
         verticalMode,
-        replayCounter
+        replayCounter,
+        deviceType
     ]);
     const canStop = loading && !!currentRunningIdRef.current && !!playgroundSDK && !!playgroundSDK.cancelExecution;
     return {

package/dist/es/store/store.mjs

@@ -33,6 +33,7 @@ const ELEMENTS_VISIBLE_KEY = 'midscene-elements-visible';
 const MODEL_CALL_DETAILS_KEY = 'midscene-model-call-details';
 const DARK_MODE_KEY = 'midscene-dark-mode';
 const PLAYBACK_SPEED_KEY = 'midscene-playback-speed';
+const SUBTITLE_ENABLED_KEY = 'midscene-subtitle-enabled';
 const parseBooleanParam = (value)=>{
     if (null === value) return;
     const normalized = value.trim().toLowerCase();
@@ -62,6 +63,7 @@ const useGlobalPreference = store_create((set)=>{
     const savedDarkMode = 'true' === localStorage.getItem(DARK_MODE_KEY);
     const parsedPlaybackSpeed = Number.parseFloat(localStorage.getItem(PLAYBACK_SPEED_KEY) || '1');
     const savedPlaybackSpeed = Number.isNaN(parsedPlaybackSpeed) ? 1 : parsedPlaybackSpeed;
+    const savedSubtitleEnabled = 'false' !== localStorage.getItem(SUBTITLE_ENABLED_KEY);
     const autoZoomFromQuery = getQueryPreference('focusOnCursor');
     const elementsVisibleFromQuery = getQueryPreference('showElementMarkers');
     const darkModeFromQuery = getQueryPreference('darkMode');
@@ -114,6 +116,13 @@ const useGlobalPreference = store_create((set)=>{
                 playbackSpeed: speed
             });
             localStorage.setItem(PLAYBACK_SPEED_KEY, speed.toString());
+        },
+        subtitleEnabled: savedSubtitleEnabled,
+        setSubtitleEnabled: (enabled)=>{
+            set({
+                subtitleEnabled: enabled
+            });
+            localStorage.setItem(SUBTITLE_ENABLED_KEY, enabled.toString());
         }
     };
 });

package/dist/es/utils/replay-scripts.mjs

@@ -158,7 +158,8 @@ const allScriptsFromDump = (dump)=>{
         width: firstWidth,
         height: firstHeight,
         sdkVersion,
-        modelBriefs
+        modelBriefs,
+        deviceType: normalizedDump.deviceType
     };
 };
 const generateAnimationScripts = (execution, task, imageWidth, imageHeight, executionIndex = 0)=>{

package/dist/lib/component/blackboard/index.css

@@ -21,21 +21,99 @@
   max-width: 500px;
 }
 
-.blackboard-filter {
-  margin: 10px 0;
+.blackboard-main-content {
+  position: relative;
+  overflow: hidden;
 }
 
-.blackboard-main-content canvas {
+.blackboard-screenshot {
   box-sizing: border-box;
   border: 1px solid #888;
   width: 100%;
+  display: block;
+}
+
+.blackboard-overlay {
+  pointer-events: none;
+  width: 100%;
+  position: absolute;
+  top: 0;
+  left: 0;
+}
+
+.blackboard-rect {
+  box-sizing: border-box;
+  pointer-events: none;
+  position: absolute;
+}
+
+.blackboard-rect-label {
+  white-space: nowrap;
+  padding: 1px 4px;
+  font-size: 14px;
+  font-weight: 600;
+  line-height: 1.4;
+  position: absolute;
+  bottom: 100%;
+  left: 0;
+}
+
+.blackboard-rect-search {
+  background: rgba(2, 131, 145, .4);
+  border: 1px solid #028391;
+  box-shadow: 4px 4px 2px rgba(51, 51, 51, .4);
+}
+
+.blackboard-rect-search .blackboard-rect-label {
+  color: #028391;
+}
+
+.blackboard-rect-highlight {
+  background: rgba(253, 89, 7, .4);
+  border: 1px solid #fd5907;
+  animation: 1.2s ease-in-out infinite blackboard-pulse;
+  box-shadow: 4px 4px 2px rgba(51, 51, 51, .4);
+}
+
+.blackboard-rect-highlight .blackboard-rect-label {
+  color: #000;
+}
+
+.blackboard-point {
+  pointer-events: none;
+  background: rgba(253, 89, 7, .4);
+  border: 1px solid #fd5907;
+  border-radius: 50%;
+  width: 20px;
+  height: 20px;
+  margin-top: -10px;
+  margin-left: -10px;
+  animation: 1.2s ease-in-out infinite blackboard-pulse;
+  position: absolute;
+  box-shadow: 0 0 8px rgba(253, 89, 7, .5);
+}
+
+@keyframes blackboard-pulse {
+  0%, 100% {
+    opacity: .4;
+    box-shadow: 4px 4px 2px rgba(51, 51, 51, .4);
+  }
+
+  50% {
+    opacity: 1;
+    box-shadow: 4px 4px 2px rgba(51, 51, 51, .4), 0 0 16px rgba(253, 89, 7, .5);
+  }
 }
 
 [data-theme="dark"] .blackboard .footer, [data-theme="dark"] .blackboard .bottom-tip-item {
   color: rgba(255, 255, 255, .45);
 }
 
-[data-theme="dark"] .blackboard-main-content canvas {
+[data-theme="dark"] .blackboard-screenshot {
   border-color: rgba(255, 255, 255, .12);
 }
 
+[data-theme="dark"] .blackboard-rect-highlight .blackboard-rect-label {
+  color: #fff;
+}
+