@midscene/visualizer 1.0.1-beta-20251027074226.0 → 1.0.1-beta-20251028065320.0

package/dist/es/components/screenshot-viewer/index.css

@@ -0,0 +1,170 @@
+.screenshot-viewer {
+  flex-direction: column;
+  height: 100%;
+  display: flex;
+}
+
+.screenshot-viewer.offline, .screenshot-viewer.loading, .screenshot-viewer.error {
+  text-align: center;
+  color: #666;
+  justify-content: center;
+  align-items: center;
+}
+
+.screenshot-viewer.offline .screenshot-placeholder h3, .screenshot-viewer.loading .screenshot-placeholder h3, .screenshot-viewer.error .screenshot-placeholder h3 {
+  color: #1890ff;
+  text-transform: capitalize;
+  margin-bottom: 12px;
+  font-size: 18px;
+}
+
+.screenshot-viewer.offline .screenshot-placeholder p, .screenshot-viewer.loading .screenshot-placeholder p, .screenshot-viewer.error .screenshot-placeholder p {
+  color: #666;
+  margin: 0;
+}
+
+.screenshot-viewer.offline .screenshot-placeholder p.error-message, .screenshot-viewer.loading .screenshot-placeholder p.error-message, .screenshot-viewer.error .screenshot-placeholder p.error-message {
+  color: #ff4d4f;
+}
+
+.screenshot-viewer .screenshot-header {
+  justify-content: space-between;
+  align-items: center;
+  height: 56px;
+  display: flex;
+}
+
+.screenshot-viewer .screenshot-header .screenshot-title {
+  flex-direction: column;
+  gap: 4px;
+  display: flex;
+}
+
+.screenshot-viewer .screenshot-header .screenshot-title h3 {
+  color: #000;
+  text-transform: capitalize;
+  align-items: center;
+  gap: 8px;
+  margin: 0;
+  font-size: 14px;
+  font-weight: 600;
+  display: flex;
+}
+
+.screenshot-viewer .screenshot-header .screenshot-title .screenshot-subtitle {
+  color: #999;
+  margin: 0;
+  font-size: 12px;
+}
+
+.screenshot-viewer .screenshot-container {
+  background: #f2f4f7;
+  border-radius: 16px;
+  flex-direction: column;
+  flex: 1;
+  padding: 0 15px;
+  display: flex;
+  overflow: hidden;
+}
+
+.screenshot-viewer .screenshot-container .screenshot-overlay {
+  z-index: 10;
+  justify-content: space-between;
+  align-items: flex-start;
+  padding: 12px 5px 8px;
+  display: flex;
+}
+
+.screenshot-viewer .screenshot-container .screenshot-overlay .device-name-overlay {
+  color: rgba(0, 0, 0, .85);
+  text-transform: capitalize;
+  align-items: center;
+  gap: 8px;
+  font-size: 12px;
+  font-weight: 500;
+  display: flex;
+}
+
+.screenshot-viewer .screenshot-container .screenshot-overlay .device-name-overlay .info-icon {
+  opacity: .8;
+  cursor: pointer;
+  font-size: 12px;
+}
+
+.screenshot-viewer .screenshot-container .screenshot-overlay .device-name-overlay .info-icon:hover {
+  opacity: 1;
+}
+
+.screenshot-viewer .screenshot-container .screenshot-overlay .screenshot-controls {
+  opacity: 1;
+  box-sizing: border-box;
+  border-radius: 4px;
+  align-items: center;
+  gap: 10px;
+  height: 18px;
+  padding: 4px 8px;
+  display: flex;
+}
+
+.screenshot-viewer .screenshot-container .screenshot-overlay .screenshot-controls .last-update-time {
+  color: #666;
+  white-space: nowrap;
+  text-overflow: ellipsis;
+  font-size: 12px;
+  overflow: hidden;
+}
+
+.screenshot-viewer .screenshot-container .screenshot-overlay .screenshot-controls .operation-indicator {
+  align-items: center;
+  gap: 4px;
+  font-size: 12px;
+  display: flex;
+}
+
+.screenshot-viewer .screenshot-container .screenshot-overlay .screenshot-controls .ant-btn {
+  box-shadow: none;
+  background: none;
+  border: none;
+  justify-content: center;
+  align-items: center;
+  width: 16px;
+  min-width: 16px;
+  height: 16px;
+  padding: 0;
+  display: flex;
+}
+
+.screenshot-viewer .screenshot-container .screenshot-overlay .screenshot-controls .ant-btn:hover {
+  background-color: rgba(0, 0, 0, .06);
+  border-radius: 2px;
+  transition: all .2s;
+  transform: scale(1.1);
+}
+
+.screenshot-viewer .screenshot-container .screenshot-overlay .screenshot-controls .ant-btn .anticon {
+  color: #666;
+  font-size: 12px;
+  transition: color .2s;
+}
+
+.screenshot-viewer .screenshot-container .screenshot-content {
+  flex: 1;
+  justify-content: center;
+  align-items: center;
+  min-height: 0;
+  display: flex;
+}
+
+.screenshot-viewer .screenshot-container .screenshot-content .screenshot-image {
+  object-fit: contain;
+  border-radius: 12px;
+  max-width: 100%;
+  height: auto;
+  max-height: 100%;
+}
+
+.screenshot-viewer .screenshot-container .screenshot-content .screenshot-placeholder {
+  text-align: center;
+  color: #999;
+}
+

package/dist/es/components/screenshot-viewer/index.mjs

@@ -0,0 +1,258 @@
+import { jsx, jsxs } from "react/jsx-runtime";
+import { InfoCircleOutlined, ReloadOutlined } from "@ant-design/icons";
+import { Button, Spin, Tooltip } from "antd";
+import { useCallback, useEffect, useRef, useState } from "react";
+import "./index.css";
+function ScreenshotViewer(param) {
+    let { getScreenshot, getInterfaceInfo, serverOnline, isUserOperating = false } = param;
+    const [screenshot, setScreenshot] = useState(null);
+    const [loading, setLoading] = useState(false);
+    const [error, setError] = useState(null);
+    const [lastUpdateTime, setLastUpdateTime] = useState(0);
+    const [interfaceInfo, setInterfaceInfo] = useState(null);
+    const pollingIntervalRef = useRef(null);
+    const isPollingPausedRef = useRef(false);
+    const fetchScreenshot = useCallback(async function() {
+        let isManual = arguments.length > 0 && void 0 !== arguments[0] ? arguments[0] : false;
+        if (!serverOnline) return;
+        setLoading(true);
+        if (isManual) setError(null);
+        try {
+            const result = await getScreenshot();
+            console.log('Screenshot API response:', result);
+            if (null == result ? void 0 : result.screenshot) {
+                const screenshotData = result.screenshot.toString().trim();
+                if (screenshotData) {
+                    setScreenshot(screenshotData);
+                    setError(null);
+                    setLastUpdateTime(Date.now());
+                } else setError('Empty screenshot data received');
+            } else setError('No screenshot data in response');
+        } catch (err) {
+            console.error('Screenshot fetch error:', err);
+            setError(err instanceof Error ? err.message : 'Failed to fetch screenshot');
+        } finally{
+            setLoading(false);
+        }
+    }, [
+        getScreenshot,
+        serverOnline
+    ]);
+    const fetchInterfaceInfo = useCallback(async ()=>{
+        if (!serverOnline || !getInterfaceInfo) return;
+        try {
+            const info = await getInterfaceInfo();
+            if (info) setInterfaceInfo(info);
+        } catch (err) {
+            console.error('Interface info fetch error:', err);
+        }
+    }, [
+        getInterfaceInfo,
+        serverOnline
+    ]);
+    const startPolling = useCallback(()=>{
+        if (pollingIntervalRef.current) clearInterval(pollingIntervalRef.current);
+        console.log('Starting screenshot polling (5s interval)');
+        pollingIntervalRef.current = setInterval(()=>{
+            if (!isPollingPausedRef.current && serverOnline) fetchScreenshot(false);
+        }, 5000);
+    }, [
+        fetchScreenshot,
+        serverOnline
+    ]);
+    const stopPolling = useCallback(()=>{
+        if (pollingIntervalRef.current) {
+            console.log('Stopping screenshot polling');
+            clearInterval(pollingIntervalRef.current);
+            pollingIntervalRef.current = null;
+        }
+    }, []);
+    const pausePolling = useCallback(()=>{
+        console.log('Pausing screenshot polling');
+        isPollingPausedRef.current = true;
+    }, []);
+    const resumePolling = useCallback(()=>{
+        console.log('Resuming screenshot polling');
+        isPollingPausedRef.current = false;
+    }, []);
+    const handleManualRefresh = useCallback(()=>{
+        fetchScreenshot(true);
+    }, [
+        fetchScreenshot
+    ]);
+    useEffect(()=>{
+        if (!serverOnline) {
+            setScreenshot(null);
+            setError(null);
+            setInterfaceInfo(null);
+            stopPolling();
+            return;
+        }
+        fetchScreenshot(false);
+        fetchInterfaceInfo();
+        startPolling();
+        return ()=>{
+            stopPolling();
+        };
+    }, [
+        serverOnline,
+        startPolling,
+        stopPolling,
+        fetchScreenshot,
+        fetchInterfaceInfo
+    ]);
+    useEffect(()=>{
+        if (!serverOnline) return;
+        if (isUserOperating) pausePolling();
+        else {
+            resumePolling();
+            fetchScreenshot(false);
+        }
+    }, [
+        isUserOperating,
+        pausePolling,
+        resumePolling,
+        fetchScreenshot,
+        serverOnline
+    ]);
+    useEffect(()=>()=>{
+            stopPolling();
+        }, [
+        stopPolling
+    ]);
+    if (!serverOnline) return /*#__PURE__*/ jsx("div", {
+        className: "screenshot-viewer offline",
+        children: /*#__PURE__*/ jsxs("div", {
+            className: "screenshot-placeholder",
+            children: [
+                /*#__PURE__*/ jsx("h3", {
+                    children: "\uD83D\uDCF1 Screen Preview"
+                }),
+                /*#__PURE__*/ jsx("p", {
+                    children: "Start the playground server to see real-time screenshots"
+                })
+            ]
+        })
+    });
+    if (loading && !screenshot) return /*#__PURE__*/ jsxs("div", {
+        className: "screenshot-viewer loading",
+        children: [
+            /*#__PURE__*/ jsx(Spin, {
+                size: "large"
+            }),
+            /*#__PURE__*/ jsx("p", {
+                children: "Loading screenshot..."
+            })
+        ]
+    });
+    if (error && !screenshot) return /*#__PURE__*/ jsx("div", {
+        className: "screenshot-viewer error",
+        children: /*#__PURE__*/ jsxs("div", {
+            className: "screenshot-placeholder",
+            children: [
+                /*#__PURE__*/ jsx("h3", {
+                    children: "\uD83D\uDCF1 Screen Preview"
+                }),
+                /*#__PURE__*/ jsx("p", {
+                    className: "error-message",
+                    children: error
+                })
+            ]
+        })
+    });
+    const formatLastUpdateTime = (timestamp)=>{
+        if (!timestamp) return '';
+        const now = Date.now();
+        const diff = Math.floor((now - timestamp) / 1000);
+        if (diff < 60) return `${diff}s ago`;
+        if (diff < 3600) return `${Math.floor(diff / 60)}m ago`;
+        return new Date(timestamp).toLocaleTimeString();
+    };
+    return /*#__PURE__*/ jsxs("div", {
+        className: "screenshot-viewer",
+        children: [
+            /*#__PURE__*/ jsx("div", {
+                className: "screenshot-header",
+                children: /*#__PURE__*/ jsx("div", {
+                    className: "screenshot-title",
+                    children: /*#__PURE__*/ jsx("h3", {
+                        children: (null == interfaceInfo ? void 0 : interfaceInfo.type) ? interfaceInfo.type : 'Device Name'
+                    })
+                })
+            }),
+            /*#__PURE__*/ jsxs("div", {
+                className: "screenshot-container",
+                children: [
+                    /*#__PURE__*/ jsxs("div", {
+                        className: "screenshot-overlay",
+                        children: [
+                            /*#__PURE__*/ jsxs("div", {
+                                className: "device-name-overlay",
+                                children: [
+                                    "Device Name",
+                                    /*#__PURE__*/ jsx(Tooltip, {
+                                        title: null == interfaceInfo ? void 0 : interfaceInfo.description,
+                                        children: /*#__PURE__*/ jsx(InfoCircleOutlined, {
+                                            size: 16,
+                                            className: "info-icon"
+                                        })
+                                    })
+                                ]
+                            }),
+                            /*#__PURE__*/ jsxs("div", {
+                                className: "screenshot-controls",
+                                children: [
+                                    lastUpdateTime > 0 && /*#__PURE__*/ jsxs("span", {
+                                        className: "last-update-time",
+                                        children: [
+                                            "Last updated ",
+                                            formatLastUpdateTime(lastUpdateTime)
+                                        ]
+                                    }),
+                                    /*#__PURE__*/ jsx(Tooltip, {
+                                        title: "Refresh screenshot",
+                                        children: /*#__PURE__*/ jsx(Button, {
+                                            icon: /*#__PURE__*/ jsx(ReloadOutlined, {}),
+                                            onClick: handleManualRefresh,
+                                            loading: loading,
+                                            size: "small"
+                                        })
+                                    }),
+                                    isUserOperating && /*#__PURE__*/ jsxs("span", {
+                                        className: "operation-indicator",
+                                        children: [
+                                            /*#__PURE__*/ jsx(Spin, {
+                                                size: "small"
+                                            }),
+                                            " Operating..."
+                                        ]
+                                    })
+                                ]
+                            })
+                        ]
+                    }),
+                    /*#__PURE__*/ jsx("div", {
+                        className: "screenshot-content",
+                        children: screenshot ? /*#__PURE__*/ jsx("img", {
+                            src: screenshot.startsWith('data:image/') ? screenshot : `data:image/png;base64,${screenshot}`,
+                            alt: "Device Screenshot",
+                            className: "screenshot-image",
+                            onLoad: ()=>console.log('Screenshot image loaded successfully'),
+                            onError: (e)=>{
+                                console.error('Screenshot image load error:', e);
+                                console.error('Screenshot data preview:', screenshot.substring(0, 100));
+                                setError('Failed to load screenshot image');
+                            }
+                        }) : /*#__PURE__*/ jsx("div", {
+                            className: "screenshot-placeholder",
+                            children: /*#__PURE__*/ jsx("p", {
+                                children: "No screenshot available"
+                            })
+                        })
+                    })
+                ]
+            })
+        ]
+    });
+}
+export { ScreenshotViewer as default };

package/dist/es/index.mjs

@@ -1,5 +1,6 @@
 import "./component/playground/index.css";
 import "./component/universal-playground/index.css";
+import "./components/screenshot-viewer/index.css";
 import { allScriptsFromDump, generateAnimationScripts } from "./utils/replay-scripts.mjs";
 import { useEnvConfig } from "./store/store.mjs";
 import { colorForName, globalThemeConfig, highlightColorForType } from "./utils/color.mjs";
@@ -16,10 +17,11 @@ import { ContextPreview } from "./component/context-preview/index.mjs";
 import { PromptInput } from "./component/prompt-input/index.mjs";
 import { Player } from "./component/player/index.mjs";
 import { Blackboard } from "./component/blackboard/index.mjs";
+import screenshot_viewer from "./components/screenshot-viewer/index.mjs";
 import { actionNameForType, getPlaceholderForType, staticAgentFromContext } from "./utils/playground-utils.mjs";
 import { filterBase64Value, timeStr } from "./utils/index.mjs";
 import shiny_text from "./component/shiny-text/index.mjs";
 import universal_playground, { UniversalPlayground } from "./component/universal-playground/index.mjs";
 import { IndexedDBStorageProvider, LocalStorageProvider, MemoryStorageProvider, NoOpStorageProvider, StorageType, createStorageProvider, detectBestStorageType } from "./component/universal-playground/providers/storage-provider.mjs";
 import { AgentContextProvider, BaseContextProvider, NoOpContextProvider, StaticContextProvider } from "./component/universal-playground/providers/context-provider.mjs";
-export { AgentContextProvider, BaseContextProvider, Blackboard, ContextPreview, EnvConfig, EnvConfigReminder, IndexedDBStorageProvider, LocalStorageProvider, Logo, MemoryStorageProvider, NavActions, NoOpContextProvider, NoOpStorageProvider, Player, PlaygroundResultView, PromptInput, ServiceModeControl, shiny_text as ShinyText, StaticContextProvider, StorageType, UniversalPlayground, universal_playground as UniversalPlaygroundDefault, actionNameForType, allScriptsFromDump, colorForName, createStorageProvider, detectBestStorageType, filterBase64Value, generateAnimationScripts, getPlaceholderForType, globalThemeConfig, highlightColorForType, iconForStatus, safeOverrideAIConfig, staticAgentFromContext, timeCostStrElement, timeStr, useEnvConfig, useSafeOverrideAIConfig, useServerValid };
+export { AgentContextProvider, BaseContextProvider, Blackboard, ContextPreview, EnvConfig, EnvConfigReminder, IndexedDBStorageProvider, LocalStorageProvider, Logo, MemoryStorageProvider, NavActions, NoOpContextProvider, NoOpStorageProvider, Player, PlaygroundResultView, PromptInput, screenshot_viewer as ScreenshotViewer, ServiceModeControl, shiny_text as ShinyText, StaticContextProvider, StorageType, UniversalPlayground, universal_playground as UniversalPlaygroundDefault, actionNameForType, allScriptsFromDump, colorForName, createStorageProvider, detectBestStorageType, filterBase64Value, generateAnimationScripts, getPlaceholderForType, globalThemeConfig, highlightColorForType, iconForStatus, safeOverrideAIConfig, staticAgentFromContext, timeCostStrElement, timeStr, useEnvConfig, useSafeOverrideAIConfig, useServerValid };

package/dist/es/utils/replay-scripts.mjs

@@ -160,9 +160,9 @@ const generateAnimationScripts = (execution, task, imageWidth, imageHeight)=>{
                 });
             }
         } else if ('Insight' === task.type && 'Locate' === task.subType) {
-            var _insightTask_output;
-            const insightTask = task;
-            const resultElement = null == (_insightTask_output = insightTask.output) ? void 0 : _insightTask_output.element;
+            var _serviceTask_output;
+            const serviceTask = task;
+            const resultElement = null == (_serviceTask_output = serviceTask.output) ? void 0 : _serviceTask_output.element;
             const title = typeStr(task);
             const subTitle = paramStr(task);
             if (null == resultElement ? void 0 : resultElement.rect) insightCameraState = {
@@ -170,10 +170,10 @@ const generateAnimationScripts = (execution, task, imageWidth, imageHeight)=>{
                 pointerLeft: resultElement.center[0],
                 pointerTop: resultElement.center[1]
             };
-            const context = insightTask.uiContext;
+            const context = serviceTask.uiContext;
             if (null == context ? void 0 : context.screenshotBase64) {
-                var _insightTask_output1, _insightDump_taskInfo, _context_size, _context_size1;
-                const insightDump = insightTask.log;
+                var _serviceTask_output1, _insightDump_taskInfo, _context_size, _context_size1;
+                const insightDump = serviceTask.log;
                 const insightContentLength = 0;
                 if (context.screenshotBase64) {
                     var _context_size2, _context_size3;
@@ -194,7 +194,7 @@ const generateAnimationScripts = (execution, task, imageWidth, imageHeight)=>{
                     img: context.screenshotBase64,
                     context: context,
                     camera: cameraState,
-                    highlightElement: (null == (_insightTask_output1 = insightTask.output) ? void 0 : _insightTask_output1.element) || void 0,
+                    highlightElement: (null == (_serviceTask_output1 = serviceTask.output) ? void 0 : _serviceTask_output1.element) || void 0,
                     searchArea: null == insightDump ? void 0 : null == (_insightDump_taskInfo = insightDump.taskInfo) ? void 0 : _insightDump_taskInfo.searchArea,
                     duration: insightContentLength > 20 ? locateDuration : 0.5 * locateDuration,
                     insightCameraDuration: locateDuration,

package/dist/lib/components/screenshot-viewer/index.css

@@ -0,0 +1,170 @@
+.screenshot-viewer {
+  flex-direction: column;
+  height: 100%;
+  display: flex;
+}
+
+.screenshot-viewer.offline, .screenshot-viewer.loading, .screenshot-viewer.error {
+  text-align: center;
+  color: #666;
+  justify-content: center;
+  align-items: center;
+}
+
+.screenshot-viewer.offline .screenshot-placeholder h3, .screenshot-viewer.loading .screenshot-placeholder h3, .screenshot-viewer.error .screenshot-placeholder h3 {
+  color: #1890ff;
+  text-transform: capitalize;
+  margin-bottom: 12px;
+  font-size: 18px;
+}
+
+.screenshot-viewer.offline .screenshot-placeholder p, .screenshot-viewer.loading .screenshot-placeholder p, .screenshot-viewer.error .screenshot-placeholder p {
+  color: #666;
+  margin: 0;
+}
+
+.screenshot-viewer.offline .screenshot-placeholder p.error-message, .screenshot-viewer.loading .screenshot-placeholder p.error-message, .screenshot-viewer.error .screenshot-placeholder p.error-message {
+  color: #ff4d4f;
+}
+
+.screenshot-viewer .screenshot-header {
+  justify-content: space-between;
+  align-items: center;
+  height: 56px;
+  display: flex;
+}
+
+.screenshot-viewer .screenshot-header .screenshot-title {
+  flex-direction: column;
+  gap: 4px;
+  display: flex;
+}
+
+.screenshot-viewer .screenshot-header .screenshot-title h3 {
+  color: #000;
+  text-transform: capitalize;
+  align-items: center;
+  gap: 8px;
+  margin: 0;
+  font-size: 14px;
+  font-weight: 600;
+  display: flex;
+}
+
+.screenshot-viewer .screenshot-header .screenshot-title .screenshot-subtitle {
+  color: #999;
+  margin: 0;
+  font-size: 12px;
+}
+
+.screenshot-viewer .screenshot-container {
+  background: #f2f4f7;
+  border-radius: 16px;
+  flex-direction: column;
+  flex: 1;
+  padding: 0 15px;
+  display: flex;
+  overflow: hidden;
+}
+
+.screenshot-viewer .screenshot-container .screenshot-overlay {
+  z-index: 10;
+  justify-content: space-between;
+  align-items: flex-start;
+  padding: 12px 5px 8px;
+  display: flex;
+}
+
+.screenshot-viewer .screenshot-container .screenshot-overlay .device-name-overlay {
+  color: rgba(0, 0, 0, .85);
+  text-transform: capitalize;
+  align-items: center;
+  gap: 8px;
+  font-size: 12px;
+  font-weight: 500;
+  display: flex;
+}
+
+.screenshot-viewer .screenshot-container .screenshot-overlay .device-name-overlay .info-icon {
+  opacity: .8;
+  cursor: pointer;
+  font-size: 12px;
+}
+
+.screenshot-viewer .screenshot-container .screenshot-overlay .device-name-overlay .info-icon:hover {
+  opacity: 1;
+}
+
+.screenshot-viewer .screenshot-container .screenshot-overlay .screenshot-controls {
+  opacity: 1;
+  box-sizing: border-box;
+  border-radius: 4px;
+  align-items: center;
+  gap: 10px;
+  height: 18px;
+  padding: 4px 8px;
+  display: flex;
+}
+
+.screenshot-viewer .screenshot-container .screenshot-overlay .screenshot-controls .last-update-time {
+  color: #666;
+  white-space: nowrap;
+  text-overflow: ellipsis;
+  font-size: 12px;
+  overflow: hidden;
+}
+
+.screenshot-viewer .screenshot-container .screenshot-overlay .screenshot-controls .operation-indicator {
+  align-items: center;
+  gap: 4px;
+  font-size: 12px;
+  display: flex;
+}
+
+.screenshot-viewer .screenshot-container .screenshot-overlay .screenshot-controls .ant-btn {
+  box-shadow: none;
+  background: none;
+  border: none;
+  justify-content: center;
+  align-items: center;
+  width: 16px;
+  min-width: 16px;
+  height: 16px;
+  padding: 0;
+  display: flex;
+}
+
+.screenshot-viewer .screenshot-container .screenshot-overlay .screenshot-controls .ant-btn:hover {
+  background-color: rgba(0, 0, 0, .06);
+  border-radius: 2px;
+  transition: all .2s;
+  transform: scale(1.1);
+}
+
+.screenshot-viewer .screenshot-container .screenshot-overlay .screenshot-controls .ant-btn .anticon {
+  color: #666;
+  font-size: 12px;
+  transition: color .2s;
+}
+
+.screenshot-viewer .screenshot-container .screenshot-content {
+  flex: 1;
+  justify-content: center;
+  align-items: center;
+  min-height: 0;
+  display: flex;
+}
+
+.screenshot-viewer .screenshot-container .screenshot-content .screenshot-image {
+  object-fit: contain;
+  border-radius: 12px;
+  max-width: 100%;
+  height: auto;
+  max-height: 100%;
+}
+
+.screenshot-viewer .screenshot-container .screenshot-content .screenshot-placeholder {
+  text-align: center;
+  color: #999;
+}
+

package/dist/lib/components/screenshot-viewer/index.js

@@ -0,0 +1,292 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    default: ()=>ScreenshotViewer
+});
+const jsx_runtime_namespaceObject = require("react/jsx-runtime");
+const icons_namespaceObject = require("@ant-design/icons");
+const external_antd_namespaceObject = require("antd");
+const external_react_namespaceObject = require("react");
+require("./index.css");
+function ScreenshotViewer(param) {
+    let { getScreenshot, getInterfaceInfo, serverOnline, isUserOperating = false } = param;
+    const [screenshot, setScreenshot] = (0, external_react_namespaceObject.useState)(null);
+    const [loading, setLoading] = (0, external_react_namespaceObject.useState)(false);
+    const [error, setError] = (0, external_react_namespaceObject.useState)(null);
+    const [lastUpdateTime, setLastUpdateTime] = (0, external_react_namespaceObject.useState)(0);
+    const [interfaceInfo, setInterfaceInfo] = (0, external_react_namespaceObject.useState)(null);
+    const pollingIntervalRef = (0, external_react_namespaceObject.useRef)(null);
+    const isPollingPausedRef = (0, external_react_namespaceObject.useRef)(false);
+    const fetchScreenshot = (0, external_react_namespaceObject.useCallback)(async function() {
+        let isManual = arguments.length > 0 && void 0 !== arguments[0] ? arguments[0] : false;
+        if (!serverOnline) return;
+        setLoading(true);
+        if (isManual) setError(null);
+        try {
+            const result = await getScreenshot();
+            console.log('Screenshot API response:', result);
+            if (null == result ? void 0 : result.screenshot) {
+                const screenshotData = result.screenshot.toString().trim();
+                if (screenshotData) {
+                    setScreenshot(screenshotData);
+                    setError(null);
+                    setLastUpdateTime(Date.now());
+                } else setError('Empty screenshot data received');
+            } else setError('No screenshot data in response');
+        } catch (err) {
+            console.error('Screenshot fetch error:', err);
+            setError(err instanceof Error ? err.message : 'Failed to fetch screenshot');
+        } finally{
+            setLoading(false);
+        }
+    }, [
+        getScreenshot,
+        serverOnline
+    ]);
+    const fetchInterfaceInfo = (0, external_react_namespaceObject.useCallback)(async ()=>{
+        if (!serverOnline || !getInterfaceInfo) return;
+        try {
+            const info = await getInterfaceInfo();
+            if (info) setInterfaceInfo(info);
+        } catch (err) {
+            console.error('Interface info fetch error:', err);
+        }
+    }, [
+        getInterfaceInfo,
+        serverOnline
+    ]);
+    const startPolling = (0, external_react_namespaceObject.useCallback)(()=>{
+        if (pollingIntervalRef.current) clearInterval(pollingIntervalRef.current);
+        console.log('Starting screenshot polling (5s interval)');
+        pollingIntervalRef.current = setInterval(()=>{
+            if (!isPollingPausedRef.current && serverOnline) fetchScreenshot(false);
+        }, 5000);
+    }, [
+        fetchScreenshot,
+        serverOnline
+    ]);
+    const stopPolling = (0, external_react_namespaceObject.useCallback)(()=>{
+        if (pollingIntervalRef.current) {
+            console.log('Stopping screenshot polling');
+            clearInterval(pollingIntervalRef.current);
+            pollingIntervalRef.current = null;
+        }
+    }, []);
+    const pausePolling = (0, external_react_namespaceObject.useCallback)(()=>{
+        console.log('Pausing screenshot polling');
+        isPollingPausedRef.current = true;
+    }, []);
+    const resumePolling = (0, external_react_namespaceObject.useCallback)(()=>{
+        console.log('Resuming screenshot polling');
+        isPollingPausedRef.current = false;
+    }, []);
+    const handleManualRefresh = (0, external_react_namespaceObject.useCallback)(()=>{
+        fetchScreenshot(true);
+    }, [
+        fetchScreenshot
+    ]);
+    (0, external_react_namespaceObject.useEffect)(()=>{
+        if (!serverOnline) {
+            setScreenshot(null);
+            setError(null);
+            setInterfaceInfo(null);
+            stopPolling();
+            return;
+        }
+        fetchScreenshot(false);
+        fetchInterfaceInfo();
+        startPolling();
+        return ()=>{
+            stopPolling();
+        };
+    }, [
+        serverOnline,
+        startPolling,
+        stopPolling,
+        fetchScreenshot,
+        fetchInterfaceInfo
+    ]);
+    (0, external_react_namespaceObject.useEffect)(()=>{
+        if (!serverOnline) return;
+        if (isUserOperating) pausePolling();
+        else {
+            resumePolling();
+            fetchScreenshot(false);
+        }
+    }, [
+        isUserOperating,
+        pausePolling,
+        resumePolling,
+        fetchScreenshot,
+        serverOnline
+    ]);
+    (0, external_react_namespaceObject.useEffect)(()=>()=>{
+            stopPolling();
+        }, [
+        stopPolling
+    ]);
+    if (!serverOnline) return /*#__PURE__*/ (0, jsx_runtime_namespaceObject.jsx)("div", {
+        className: "screenshot-viewer offline",
+        children: /*#__PURE__*/ (0, jsx_runtime_namespaceObject.jsxs)("div", {
+            className: "screenshot-placeholder",
+            children: [
+                /*#__PURE__*/ (0, jsx_runtime_namespaceObject.jsx)("h3", {
+                    children: "\uD83D\uDCF1 Screen Preview"
+                }),
+                /*#__PURE__*/ (0, jsx_runtime_namespaceObject.jsx)("p", {
+                    children: "Start the playground server to see real-time screenshots"
+                })
+            ]
+        })
+    });
+    if (loading && !screenshot) return /*#__PURE__*/ (0, jsx_runtime_namespaceObject.jsxs)("div", {
+        className: "screenshot-viewer loading",
+        children: [
+            /*#__PURE__*/ (0, jsx_runtime_namespaceObject.jsx)(external_antd_namespaceObject.Spin, {
+                size: "large"
+            }),
+            /*#__PURE__*/ (0, jsx_runtime_namespaceObject.jsx)("p", {
+                children: "Loading screenshot..."
+            })
+        ]
+    });
+    if (error && !screenshot) return /*#__PURE__*/ (0, jsx_runtime_namespaceObject.jsx)("div", {
+        className: "screenshot-viewer error",
+        children: /*#__PURE__*/ (0, jsx_runtime_namespaceObject.jsxs)("div", {
+            className: "screenshot-placeholder",
+            children: [
+                /*#__PURE__*/ (0, jsx_runtime_namespaceObject.jsx)("h3", {
+                    children: "\uD83D\uDCF1 Screen Preview"
+                }),
+                /*#__PURE__*/ (0, jsx_runtime_namespaceObject.jsx)("p", {
+                    className: "error-message",
+                    children: error
+                })
+            ]
+        })
+    });
+    const formatLastUpdateTime = (timestamp)=>{
+        if (!timestamp) return '';
+        const now = Date.now();
+        const diff = Math.floor((now - timestamp) / 1000);
+        if (diff < 60) return `${diff}s ago`;
+        if (diff < 3600) return `${Math.floor(diff / 60)}m ago`;
+        return new Date(timestamp).toLocaleTimeString();
+    };
+    return /*#__PURE__*/ (0, jsx_runtime_namespaceObject.jsxs)("div", {
+        className: "screenshot-viewer",
+        children: [
+            /*#__PURE__*/ (0, jsx_runtime_namespaceObject.jsx)("div", {
+                className: "screenshot-header",
+                children: /*#__PURE__*/ (0, jsx_runtime_namespaceObject.jsx)("div", {
+                    className: "screenshot-title",
+                    children: /*#__PURE__*/ (0, jsx_runtime_namespaceObject.jsx)("h3", {
+                        children: (null == interfaceInfo ? void 0 : interfaceInfo.type) ? interfaceInfo.type : 'Device Name'
+                    })
+                })
+            }),
+            /*#__PURE__*/ (0, jsx_runtime_namespaceObject.jsxs)("div", {
+                className: "screenshot-container",
+                children: [
+                    /*#__PURE__*/ (0, jsx_runtime_namespaceObject.jsxs)("div", {
+                        className: "screenshot-overlay",
+                        children: [
+                            /*#__PURE__*/ (0, jsx_runtime_namespaceObject.jsxs)("div", {
+                                className: "device-name-overlay",
+                                children: [
+                                    "Device Name",
+                                    /*#__PURE__*/ (0, jsx_runtime_namespaceObject.jsx)(external_antd_namespaceObject.Tooltip, {
+                                        title: null == interfaceInfo ? void 0 : interfaceInfo.description,
+                                        children: /*#__PURE__*/ (0, jsx_runtime_namespaceObject.jsx)(icons_namespaceObject.InfoCircleOutlined, {
+                                            size: 16,
+                                            className: "info-icon"
+                                        })
+                                    })
+                                ]
+                            }),
+                            /*#__PURE__*/ (0, jsx_runtime_namespaceObject.jsxs)("div", {
+                                className: "screenshot-controls",
+                                children: [
+                                    lastUpdateTime > 0 && /*#__PURE__*/ (0, jsx_runtime_namespaceObject.jsxs)("span", {
+                                        className: "last-update-time",
+                                        children: [
+                                            "Last updated ",
+                                            formatLastUpdateTime(lastUpdateTime)
+                                        ]
+                                    }),
+                                    /*#__PURE__*/ (0, jsx_runtime_namespaceObject.jsx)(external_antd_namespaceObject.Tooltip, {
+                                        title: "Refresh screenshot",
+                                        children: /*#__PURE__*/ (0, jsx_runtime_namespaceObject.jsx)(external_antd_namespaceObject.Button, {
+                                            icon: /*#__PURE__*/ (0, jsx_runtime_namespaceObject.jsx)(icons_namespaceObject.ReloadOutlined, {}),
+                                            onClick: handleManualRefresh,
+                                            loading: loading,
+                                            size: "small"
+                                        })
+                                    }),
+                                    isUserOperating && /*#__PURE__*/ (0, jsx_runtime_namespaceObject.jsxs)("span", {
+                                        className: "operation-indicator",
+                                        children: [
+                                            /*#__PURE__*/ (0, jsx_runtime_namespaceObject.jsx)(external_antd_namespaceObject.Spin, {
+                                                size: "small"
+                                            }),
+                                            " Operating..."
+                                        ]
+                                    })
+                                ]
+                            })
+                        ]
+                    }),
+                    /*#__PURE__*/ (0, jsx_runtime_namespaceObject.jsx)("div", {
+                        className: "screenshot-content",
+                        children: screenshot ? /*#__PURE__*/ (0, jsx_runtime_namespaceObject.jsx)("img", {
+                            src: screenshot.startsWith('data:image/') ? screenshot : `data:image/png;base64,${screenshot}`,
+                            alt: "Device Screenshot",
+                            className: "screenshot-image",
+                            onLoad: ()=>console.log('Screenshot image loaded successfully'),
+                            onError: (e)=>{
+                                console.error('Screenshot image load error:', e);
+                                console.error('Screenshot data preview:', screenshot.substring(0, 100));
+                                setError('Failed to load screenshot image');
+                            }
+                        }) : /*#__PURE__*/ (0, jsx_runtime_namespaceObject.jsx)("div", {
+                            className: "screenshot-placeholder",
+                            children: /*#__PURE__*/ (0, jsx_runtime_namespaceObject.jsx)("p", {
+                                children: "No screenshot available"
+                            })
+                        })
+                    })
+                ]
+            })
+        ]
+    });
+}
+exports["default"] = __webpack_exports__["default"];
+for(var __webpack_i__ in __webpack_exports__)if (-1 === [
+    "default"
+].indexOf(__webpack_i__)) exports[__webpack_i__] = __webpack_exports__[__webpack_i__];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});

package/dist/lib/index.js

@@ -63,6 +63,7 @@ __webpack_require__.d(__webpack_exports__, {
     PromptInput: ()=>prompt_input_index_js_namespaceObject.PromptInput,
     StaticContextProvider: ()=>context_provider_js_namespaceObject.StaticContextProvider,
     Blackboard: ()=>blackboard_index_js_namespaceObject.Blackboard,
+    ScreenshotViewer: ()=>screenshot_viewer_index_js_default(),
     Player: ()=>player_index_js_namespaceObject.Player,
     AgentContextProvider: ()=>context_provider_js_namespaceObject.AgentContextProvider,
     EnvConfigReminder: ()=>env_config_reminder_index_js_namespaceObject.EnvConfigReminder,
@@ -76,6 +77,7 @@ __webpack_require__.d(__webpack_exports__, {
 });
 require("./component/playground/index.css");
 require("./component/universal-playground/index.css");
+require("./components/screenshot-viewer/index.css");
 const replay_scripts_js_namespaceObject = require("./utils/replay-scripts.js");
 const store_js_namespaceObject = require("./store/store.js");
 const color_js_namespaceObject = require("./utils/color.js");
@@ -92,6 +94,8 @@ const context_preview_index_js_namespaceObject = require("./component/context-pr
 const prompt_input_index_js_namespaceObject = require("./component/prompt-input/index.js");
 const player_index_js_namespaceObject = require("./component/player/index.js");
 const blackboard_index_js_namespaceObject = require("./component/blackboard/index.js");
+const screenshot_viewer_index_js_namespaceObject = require("./components/screenshot-viewer/index.js");
+var screenshot_viewer_index_js_default = /*#__PURE__*/ __webpack_require__.n(screenshot_viewer_index_js_namespaceObject);
 const playground_utils_js_namespaceObject = require("./utils/playground-utils.js");
 const external_utils_index_js_namespaceObject = require("./utils/index.js");
 const shiny_text_index_js_namespaceObject = require("./component/shiny-text/index.js");
@@ -116,6 +120,7 @@ exports.NoOpStorageProvider = __webpack_exports__.NoOpStorageProvider;
 exports.Player = __webpack_exports__.Player;
 exports.PlaygroundResultView = __webpack_exports__.PlaygroundResultView;
 exports.PromptInput = __webpack_exports__.PromptInput;
+exports.ScreenshotViewer = __webpack_exports__.ScreenshotViewer;
 exports.ServiceModeControl = __webpack_exports__.ServiceModeControl;
 exports.ShinyText = __webpack_exports__.ShinyText;
 exports.StaticContextProvider = __webpack_exports__.StaticContextProvider;
@@ -157,6 +162,7 @@ for(var __webpack_i__ in __webpack_exports__)if (-1 === [
     "Player",
     "PlaygroundResultView",
     "PromptInput",
+    "ScreenshotViewer",
     "ServiceModeControl",
     "ShinyText",
     "StaticContextProvider",

package/dist/lib/utils/replay-scripts.js

@@ -191,9 +191,9 @@ const generateAnimationScripts = (execution, task, imageWidth, imageHeight)=>{
                 });
             }
         } else if ('Insight' === task.type && 'Locate' === task.subType) {
-            var _insightTask_output;
-            const insightTask = task;
-            const resultElement = null == (_insightTask_output = insightTask.output) ? void 0 : _insightTask_output.element;
+            var _serviceTask_output;
+            const serviceTask = task;
+            const resultElement = null == (_serviceTask_output = serviceTask.output) ? void 0 : _serviceTask_output.element;
             const title = (0, agent_namespaceObject.typeStr)(task);
             const subTitle = (0, agent_namespaceObject.paramStr)(task);
             if (null == resultElement ? void 0 : resultElement.rect) insightCameraState = {
@@ -201,10 +201,10 @@ const generateAnimationScripts = (execution, task, imageWidth, imageHeight)=>{
                 pointerLeft: resultElement.center[0],
                 pointerTop: resultElement.center[1]
             };
-            const context = insightTask.uiContext;
+            const context = serviceTask.uiContext;
             if (null == context ? void 0 : context.screenshotBase64) {
-                var _insightTask_output1, _insightDump_taskInfo, _context_size, _context_size1;
-                const insightDump = insightTask.log;
+                var _serviceTask_output1, _insightDump_taskInfo, _context_size, _context_size1;
+                const insightDump = serviceTask.log;
                 const insightContentLength = 0;
                 if (context.screenshotBase64) {
                     var _context_size2, _context_size3;
@@ -225,7 +225,7 @@ const generateAnimationScripts = (execution, task, imageWidth, imageHeight)=>{
                     img: context.screenshotBase64,
                     context: context,
                     camera: cameraState,
-                    highlightElement: (null == (_insightTask_output1 = insightTask.output) ? void 0 : _insightTask_output1.element) || void 0,
+                    highlightElement: (null == (_serviceTask_output1 = serviceTask.output) ? void 0 : _serviceTask_output1.element) || void 0,
                     searchArea: null == insightDump ? void 0 : null == (_insightDump_taskInfo = insightDump.taskInfo) ? void 0 : _insightDump_taskInfo.searchArea,
                     duration: insightContentLength > 20 ? locateDuration : 0.5 * locateDuration,
                     insightCameraDuration: locateDuration,

package/dist/types/components/screenshot-viewer/index.d.ts

@@ -0,0 +1,15 @@
+import './index.less';
+interface ScreenshotViewerProps {
+    getScreenshot: () => Promise<{
+        screenshot: string;
+        timestamp: number;
+    } | null>;
+    getInterfaceInfo?: () => Promise<{
+        type: string;
+        description?: string;
+    } | null>;
+    serverOnline: boolean;
+    isUserOperating?: boolean;
+}
+export default function ScreenshotViewer({ getScreenshot, getInterfaceInfo, serverOnline, isUserOperating, }: ScreenshotViewerProps): import("react").JSX.Element;
+export {};

package/dist/types/index.d.ts

@@ -1,5 +1,6 @@
 import './component/playground/index.less';
 import './component/universal-playground/index.less';
+import './components/screenshot-viewer/index.less';
 export { type AnimationScript, type ReplayScriptsInfo, allScriptsFromDump, generateAnimationScripts, } from './utils/replay-scripts';
 export { useEnvConfig } from './store/store';
 export { colorForName, highlightColorForType, globalThemeConfig, } from './utils/color';
@@ -18,6 +19,7 @@ export { ContextPreview } from './component/context-preview';
 export { PromptInput } from './component/prompt-input';
 export { Player } from './component/player';
 export { Blackboard } from './component/blackboard';
+export { default as ScreenshotViewer } from './components/screenshot-viewer';
 export { actionNameForType, staticAgentFromContext, getPlaceholderForType, } from './utils/playground-utils';
 export { timeStr, filterBase64Value } from './utils';
 export { default as ShinyText } from './component/shiny-text';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@midscene/visualizer",
-  "version": "1.0.1-beta-20251027074226.0",
+  "version": "1.0.1-beta-20251028065320.0",
   "repository": "https://github.com/web-infra-dev/midscene",
   "homepage": "https://midscenejs.com/",
   "types": "./dist/types/index.d.ts",
@@ -60,10 +60,10 @@
     "antd": "^5.21.6",
     "buffer": "6.0.3",
     "dayjs": "^1.11.11",
-    "@midscene/core": "1.0.1-beta-20251027074226.0",
-    "@midscene/web": "1.0.1-beta-20251027074226.0",
-    "@midscene/playground": "1.0.1-beta-20251027074226.0",
-    "@midscene/shared": "1.0.1-beta-20251027074226.0"
+    "@midscene/playground": "1.0.1-beta-20251028065320.0",
+    "@midscene/web": "1.0.1-beta-20251028065320.0",
+    "@midscene/core": "1.0.1-beta-20251028065320.0",
+    "@midscene/shared": "1.0.1-beta-20251028065320.0"
   },
   "license": "MIT",
   "scripts": {