@midscene/visualizer 1.9.6 → 1.9.7

package/dist/es/component/config-selector/index.mjs

@@ -27,6 +27,15 @@ const ConfigSelector = ({ showDeepLocateOption = false, showDeepThinkOption = fa
     const hasDeviceOptions = hasDeviceSpecificConfig(deviceType);
     if (!enableTracking && !showDeepLocateOption && !showDeepThinkOption && !showDataExtractionOptions && !hasDeviceOptions) return null;
     const configItems = buildConfigItems();
+    const dropdownTrigger = null != trigger ? trigger : /*#__PURE__*/ jsx("button", {
+        "aria-label": "Open run configuration",
+        className: "selector-trigger-button",
+        type: "button",
+        children: /*#__PURE__*/ jsx(setting, {
+            width: 24,
+            height: 24
+        })
+    });
     return /*#__PURE__*/ jsx("div", {
         className: "selector-trigger",
         children: /*#__PURE__*/ jsx(Dropdown, {
@@ -42,10 +51,7 @@ const ConfigSelector = ({ showDeepLocateOption = false, showDeepThinkOption = fa
             trigger: [
                 'click'
             ],
-            children: null != trigger ? trigger : /*#__PURE__*/ jsx(setting, {
-                width: 24,
-                height: 24
-            })
+            children: dropdownTrigger
         })
     });
     function buildConfigItems() {

package/dist/es/component/env-config/index.mjs

@@ -1,6 +1,6 @@
 import { jsx, jsxs } from "react/jsx-runtime";
 import { SettingOutlined } from "@ant-design/icons";
-import { Alert, Button, Input, Modal, Tooltip, message } from "antd";
+import { Alert, App, Button, Input, Modal, Tooltip } from "antd";
 import { useEffect, useRef, useState } from "react";
 import { useEnvConfig } from "../../store/store.mjs";
 import { notifyError } from "../../utils/index.mjs";
@@ -31,6 +31,7 @@ function _async_to_generator(fn) {
     };
 }
 function EnvConfig({ showTooltipWhenEmpty = true, showModelName = true, tooltipPlacement = 'bottom', mode = 'icon', playgroundSDK }) {
+    const { message } = App.useApp();
     const { config, configString, loadConfig, syncFromStorage } = useEnvConfig();
     const [isModalOpen, setIsModalOpen] = useState(false);
     const [tempConfigString, setTempConfigString] = useState(configString);

package/dist/es/component/player/index.mjs

@@ -3,7 +3,7 @@ import { jsx, jsxs } from "react/jsx-runtime";
 import { useCallback, useEffect, useMemo, useRef, useState } from "react";
 import "./index.css";
 import { CaretRightOutlined, CompressOutlined, DownloadOutlined, ExpandOutlined, ExportOutlined, FontSizeOutlined, PauseOutlined, ThunderboltOutlined } from "@ant-design/icons";
-import { Button, Dropdown, Progress, Switch, Tooltip, message } from "antd";
+import { App, Button, Dropdown, Progress, Switch, Tooltip } from "antd";
 import global_perspective from "../../icons/global-perspective.mjs";
 import player_setting from "../../icons/player-setting.mjs";
 import { useGlobalPreference } from "../../store/store.mjs";
@@ -64,6 +64,7 @@ function formatTime(frame, fps) {
     return `${String(m).padStart(2, '0')}:${String(s).padStart(2, '0')}`;
 }
 function Player(props) {
+    const { message } = App.useApp();
     const { autoZoom, setAutoZoom, playbackSpeed, setPlaybackSpeed, subtitleEnabled, setSubtitleEnabled } = useGlobalPreference();
     useEffect(()=>{
         if ((null == props ? void 0 : props.autoZoom) !== void 0) setAutoZoom(props.autoZoom);

package/dist/es/component/playground/index.css

@@ -551,6 +551,19 @@
   transition: all .2s;
 }
 
+.selector-trigger .selector-trigger-button {
+  cursor: pointer;
+  width: 24px;
+  height: 24px;
+  color: inherit;
+  background: none;
+  border: none;
+  justify-content: center;
+  align-items: center;
+  padding: 0;
+  display: inline-flex;
+}
+
 .selector-trigger .action-icon {
   color: rgba(0, 0, 0, .85);
   font-size: 14px;

package/dist/es/component/prompt-input/index.css

@@ -459,6 +459,19 @@
   transition: all .2s;
 }
 
+.selector-trigger .selector-trigger-button {
+  cursor: pointer;
+  width: 24px;
+  height: 24px;
+  color: inherit;
+  background: none;
+  border: none;
+  justify-content: center;
+  align-items: center;
+  padding: 0;
+  display: inline-flex;
+}
+
 .selector-trigger .action-icon {
   color: rgba(0, 0, 0, .85);
   font-size: 14px;

package/dist/es/component/universal-playground/index.mjs

@@ -46,6 +46,29 @@ function _async_to_generator(fn) {
         });
     };
 }
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+function _object_spread(target) {
+    for(var i = 1; i < arguments.length; i++){
+        var source = null != arguments[i] ? arguments[i] : {};
+        var ownKeys = Object.keys(source);
+        if ("function" == typeof Object.getOwnPropertySymbols) ownKeys = ownKeys.concat(Object.getOwnPropertySymbols(source).filter(function(sym) {
+            return Object.getOwnPropertyDescriptor(source, sym).enumerable;
+        }));
+        ownKeys.forEach(function(key) {
+            _define_property(target, key, source[key]);
+        });
+    }
+    return target;
+}
 const { Text } = Typography;
 const handledExternalRunRequestIds = new Set();
 const MAX_HANDLED_EXTERNAL_RUN_REQUEST_IDS = 100;
@@ -73,13 +96,6 @@ function UniversalPlayground({ playgroundSDK, storage, contextProvider, config:
     const { config } = useEnvConfig();
     const [sdkReady, setSdkReady] = useState(false);
     const lastExternalRunRequestIdRef = useRef(null);
-    useEffect(()=>{
-        form.setFieldsValue({
-            type: defaultMainButtons["0"]
-        });
-    }, [
-        form
-    ]);
     useEffect(()=>{
         const initializeSDK = ()=>_async_to_generator(function*() {
                 if (playgroundSDK && 'function' == typeof playgroundSDK.checkStatus) try {
@@ -164,9 +180,11 @@ function UniversalPlayground({ playgroundSDK, storage, contextProvider, config:
             const oldestRequestId = handledExternalRunRequestIds.values().next().value;
             if (oldestRequestId) handledExternalRunRequestIds.delete(oldestRequestId);
         }
-        executeAction(request.value, {
+        executeAction(request.value, _object_spread({
             displayContent: request.displayContent
-        }).catch((error)=>{
+        }, request.reportDisplay ? {
+            reportDisplay: request.reportDisplay
+        } : {})).catch((error)=>{
             notifyError(error, {
                 title: 'Execution failed'
             });
@@ -180,7 +198,7 @@ function UniversalPlayground({ playgroundSDK, storage, contextProvider, config:
     const configAlreadySet = Object.keys(config || {}).length >= 1;
     const runButtonEnabled = componentConfig.serverMode || !dryMode && !actionSpaceLoading && configAlreadySet;
     const watchedType = Form.useWatch('type', form);
-    const selectedType = watchedType || form.getFieldValue('type');
+    const selectedType = watchedType || defaultMainButtons["0"];
     const serviceMode = useMemo(()=>{
         if (!playgroundSDK || 'function' != typeof playgroundSDK.getServiceMode) return 'Server';
         return playgroundSDK.getServiceMode();
@@ -289,6 +307,9 @@ function UniversalPlayground({ playgroundSDK, storage, contextProvider, config:
             form: form,
             onFinish: handleFormRun,
             className: "command-form",
+            initialValues: {
+                type: defaultMainButtons["0"]
+            },
             children: [
                 finalShowContextPreview && /*#__PURE__*/ jsx("div", {
                     className: "context-preview-section",

package/dist/es/component/universal-playground/providers/indexeddb-storage-provider.mjs

@@ -67,6 +67,9 @@ function _object_spread_props(target, source) {
     });
     return target;
 }
+function isReportActionDump(dump) {
+    return Array.isArray(null == dump ? void 0 : dump.executions);
+}
 const DB_NAME = 'midscene_playground';
 const DB_VERSION = 1;
 const MESSAGES_STORE = 'playground_messages';
@@ -152,9 +155,29 @@ class IndexedDBStorageProvider {
         }).call(this);
     }
     compressResultForStorage(result) {
-        var _result_result_dump, _result_result;
-        if (!(null == (_result_result = result.result) ? void 0 : null == (_result_result_dump = _result_result.dump) ? void 0 : _result_result_dump.tasks)) return result;
-        const compressedTasks = result.result.dump.tasks.map((task)=>{
+        const playgroundResult = result.result;
+        const dump = null == playgroundResult ? void 0 : playgroundResult.dump;
+        if (!playgroundResult || !dump) return result;
+        if (isReportActionDump(dump)) return _object_spread_props(_object_spread({}, result), {
+            result: _object_spread_props(_object_spread({}, playgroundResult), {
+                dump: _object_spread_props(_object_spread({}, dump), {
+                    executions: dump.executions.map((execution)=>_object_spread_props(_object_spread({}, execution), {
+                            tasks: this.compressExecutionTasks(execution.tasks)
+                        }))
+                })
+            })
+        });
+        if (!dump.tasks) return result;
+        return _object_spread_props(_object_spread({}, result), {
+            result: _object_spread_props(_object_spread({}, playgroundResult), {
+                dump: _object_spread_props(_object_spread({}, dump), {
+                    tasks: this.compressExecutionTasks(dump.tasks)
+                })
+            })
+        });
+    }
+    compressExecutionTasks(tasks) {
+        return tasks.map((task)=>{
             var _task_recorder;
             return _object_spread_props(_object_spread({}, task), {
                 uiContext: task.uiContext ? _object_spread_props(_object_spread({}, task.uiContext), {
@@ -165,13 +188,6 @@ class IndexedDBStorageProvider {
                     }))
             });
         });
-        return _object_spread_props(_object_spread({}, result), {
-            result: _object_spread_props(_object_spread({}, result.result), {
-                dump: _object_spread_props(_object_spread({}, result.result.dump), {
-                    tasks: compressedTasks
-                })
-            })
-        });
     }
     compressScreenshotIfNeeded(screenshot) {
         if (!screenshot) return screenshot;

package/dist/es/hooks/usePlaygroundExecution.mjs

@@ -99,6 +99,15 @@ function wrapExecutionDumpForReplay(dump, deviceType) {
         deviceType
     };
 }
+function isReportActionDump(dump) {
+    return Array.isArray(null == dump ? void 0 : dump.executions);
+}
+function replayInfoFromDump(dump, deviceType) {
+    if (!dump) return null;
+    if (isReportActionDump(dump)) return allScriptsFromDump(dump);
+    if (dump.tasks && Array.isArray(dump.tasks)) return allScriptsFromDump(wrapExecutionDumpForReplay(dump, deviceType));
+    return null;
+}
 function shouldForwardDeepThink(actionType) {
     return 'aiAct' === actionType || 'runMarkdown' === actionType;
 }
@@ -170,7 +179,7 @@ function usePlaygroundExecution(options) {
                     requestId: thisRunningId.toString()
                 });
                 const resolvedDeepThink = 'unset' === deepThink ? void 0 : deepThink;
-                const executionOptions = _object_spread_props(_object_spread({
+                const executionOptions = _object_spread(_object_spread_props(_object_spread({
                     requestId: thisRunningId.toString(),
                     deepLocate
                 }, shouldForwardDeepThink(actionType) && void 0 !== resolvedDeepThink ? {
@@ -184,7 +193,9 @@ function usePlaygroundExecution(options) {
                         keyboardDismissStrategy,
                         alwaysRefreshScreenInfo
                     }
-                });
+                }), runOptions.reportDisplay ? {
+                    reportDisplay: runOptions.reportDisplay
+                } : {});
                 result.result = yield playgroundSDK.executeAction(actionType, value, executionOptions);
                 if ('object' == typeof result.result && null !== result.result) {
                     const resultObj = result.result;
@@ -207,9 +218,8 @@ function usePlaygroundExecution(options) {
             let replayInfo = null;
             let counter = replayCounter;
             if (null == result ? void 0 : result.dump) {
-                if (result.dump.tasks && Array.isArray(result.dump.tasks)) {
-                    const groupedDump = wrapExecutionDumpForReplay(result.dump, deviceType);
-                    const info = allScriptsFromDump(groupedDump);
+                const info = replayInfoFromDump(result.dump, deviceType);
+                if (info) {
                     setReplayCounter((c)=>c + 1);
                     replayInfo = info;
                     counter = replayCounter + 1;
@@ -294,12 +304,10 @@ function usePlaygroundExecution(options) {
                             loadingProgressText: ''
                         }) : item));
                 if (executionData && (executionData.dump || executionData.reportHTML)) {
-                    var _executionData_dump;
                     let replayInfo = null;
                     let counter = replayCounter;
-                    if ((null == (_executionData_dump = executionData.dump) ? void 0 : _executionData_dump.tasks) && Array.isArray(executionData.dump.tasks)) {
-                        const groupedDump = wrapExecutionDumpForReplay(executionData.dump, deviceType);
-                        replayInfo = allScriptsFromDump(groupedDump);
+                    replayInfo = replayInfoFromDump(executionData.dump, deviceType);
+                    if (replayInfo) {
                         setReplayCounter((c)=>c + 1);
                         counter = replayCounter + 1;
                     }

package/dist/es/utils/replay-scripts.mjs

@@ -110,13 +110,21 @@ const normalizeDump = (dump)=>{
         ]
     };
 };
+const sortExecutionsByLogTime = (executions)=>executions.map((execution, index)=>({
+            execution,
+            index
+        })).sort((left, right)=>{
+        const leftTime = Number.isFinite(left.execution.logTime) ? left.execution.logTime : 1 / 0;
+        const rightTime = Number.isFinite(right.execution.logTime) ? right.execution.logTime : 1 / 0;
+        return leftTime - rightTime || left.index - right.index;
+    }).map(({ execution })=>execution);
 const extractMetaFromNormalized = (normalizedDump)=>{
     var _normalizedDump_executions;
     let firstWidth;
     let firstHeight;
     const sdkVersion = normalizedDump.sdkVersion;
     const modelBriefs = [];
-    null == (_normalizedDump_executions = normalizedDump.executions) || _normalizedDump_executions.filter(Boolean).forEach((execution)=>{
+    sortExecutionsByLogTime((null == (_normalizedDump_executions = normalizedDump.executions) ? void 0 : _normalizedDump_executions.filter(Boolean)) || []).forEach((execution)=>{
         execution.tasks.forEach((task)=>{
             var _task_uiContext;
             const shotSize = null == (_task_uiContext = task.uiContext) ? void 0 : _task_uiContext.shotSize;
@@ -174,7 +182,7 @@ const allScriptsFromDump = (dump)=>{
     };
     const { width: firstWidth, height: firstHeight } = metaInfo;
     const allScripts = [];
-    const executions = (null == (_normalizedDump_executions = normalizedDump.executions) ? void 0 : _normalizedDump_executions.filter(Boolean)) || [];
+    const executions = sortExecutionsByLogTime((null == (_normalizedDump_executions = normalizedDump.executions) ? void 0 : _normalizedDump_executions.filter(Boolean)) || []);
     for(let execIndex = 0; execIndex < executions.length; execIndex++){
         const execution = executions[execIndex];
         const scripts = generateAnimationScripts(execution, -1, firstWidth, firstHeight, execIndex);

package/dist/lib/component/config-selector/index.js

@@ -65,6 +65,15 @@ const ConfigSelector = ({ showDeepLocateOption = false, showDeepThinkOption = fa
     const hasDeviceOptions = (0, device_capabilities_js_namespaceObject.hasDeviceSpecificConfig)(deviceType);
     if (!enableTracking && !showDeepLocateOption && !showDeepThinkOption && !showDataExtractionOptions && !hasDeviceOptions) return null;
     const configItems = buildConfigItems();
+    const dropdownTrigger = null != trigger ? trigger : /*#__PURE__*/ (0, jsx_runtime_namespaceObject.jsx)("button", {
+        "aria-label": "Open run configuration",
+        className: "selector-trigger-button",
+        type: "button",
+        children: /*#__PURE__*/ (0, jsx_runtime_namespaceObject.jsx)(setting_js_default(), {
+            width: 24,
+            height: 24
+        })
+    });
     return /*#__PURE__*/ (0, jsx_runtime_namespaceObject.jsx)("div", {
         className: "selector-trigger",
         children: /*#__PURE__*/ (0, jsx_runtime_namespaceObject.jsx)(external_antd_namespaceObject.Dropdown, {
@@ -80,10 +89,7 @@ const ConfigSelector = ({ showDeepLocateOption = false, showDeepThinkOption = fa
             trigger: [
                 'click'
             ],
-            children: null != trigger ? trigger : /*#__PURE__*/ (0, jsx_runtime_namespaceObject.jsx)(setting_js_default(), {
-                width: 24,
-                height: 24
-            })
+            children: dropdownTrigger
         })
     });
     function buildConfigItems() {

package/dist/lib/component/env-config/index.js

@@ -59,6 +59,7 @@ function _async_to_generator(fn) {
     };
 }
 function EnvConfig({ showTooltipWhenEmpty = true, showModelName = true, tooltipPlacement = 'bottom', mode = 'icon', playgroundSDK }) {
+    const { message } = external_antd_namespaceObject.App.useApp();
     const { config, configString, loadConfig, syncFromStorage } = (0, store_js_namespaceObject.useEnvConfig)();
     const [isModalOpen, setIsModalOpen] = (0, external_react_namespaceObject.useState)(false);
     const [tempConfigString, setTempConfigString] = (0, external_react_namespaceObject.useState)(configString);
@@ -98,13 +99,13 @@ function EnvConfig({ showTooltipWhenEmpty = true, showModelName = true, tooltipP
                 const result = yield sdk.runConnectivityTest();
                 setConnectivityResult(result);
                 if (result.passed) {
-                    external_antd_namespaceObject.message.success('Model verification passed');
+                    message.success('Model verification passed');
                     clearCloseTimer();
                     closeTimerRef.current = window.setTimeout(()=>{
                         setIsModalOpen(false);
                         closeTimerRef.current = null;
                     }, 2000);
-                } else external_antd_namespaceObject.message.warning('Model verification found issues');
+                } else message.warning('Model verification found issues');
             } catch (error) {
                 const errorMessage = error instanceof Error ? error.message : String(error);
                 (0, index_js_namespaceObject.notifyError)(error, {

package/dist/lib/component/player/index.js

@@ -103,6 +103,7 @@ function formatTime(frame, fps) {
     return `${String(m).padStart(2, '0')}:${String(s).padStart(2, '0')}`;
 }
 function Player(props) {
+    const { message } = external_antd_namespaceObject.App.useApp();
     const { autoZoom, setAutoZoom, playbackSpeed, setPlaybackSpeed, subtitleEnabled, setSubtitleEnabled } = (0, store_js_namespaceObject.useGlobalPreference)();
     (0, external_react_namespaceObject.useEffect)(()=>{
         if ((null == props ? void 0 : props.autoZoom) !== void 0) setAutoZoom(props.autoZoom);
@@ -331,7 +332,7 @@ function Player(props) {
                 yield (0, export_branded_video_js_namespaceObject.exportBrandedVideo)(frameMap, {
                     autoZoom
                 }, (pct)=>setExportProgress(Math.round(100 * pct)));
-                external_antd_namespaceObject.message.success('Video exported');
+                message.success('Video exported');
             } catch (e) {
                 console.error('Export failed:', e);
                 (0, index_js_namespaceObject.notifyError)(e, {

package/dist/lib/component/playground/index.css

@@ -551,6 +551,19 @@
   transition: all .2s;
 }
 
+.selector-trigger .selector-trigger-button {
+  cursor: pointer;
+  width: 24px;
+  height: 24px;
+  color: inherit;
+  background: none;
+  border: none;
+  justify-content: center;
+  align-items: center;
+  padding: 0;
+  display: inline-flex;
+}
+
 .selector-trigger .action-icon {
   color: rgba(0, 0, 0, .85);
   font-size: 14px;

package/dist/lib/component/prompt-input/index.css

@@ -459,6 +459,19 @@
   transition: all .2s;
 }
 
+.selector-trigger .selector-trigger-button {
+  cursor: pointer;
+  width: 24px;
+  height: 24px;
+  color: inherit;
+  background: none;
+  border: none;
+  justify-content: center;
+  align-items: center;
+  padding: 0;
+  display: inline-flex;
+}
+
 .selector-trigger .action-icon {
   color: rgba(0, 0, 0, .85);
   font-size: 14px;

package/dist/lib/component/universal-playground/index.js

@@ -87,6 +87,29 @@ function _async_to_generator(fn) {
         });
     };
 }
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+function _object_spread(target) {
+    for(var i = 1; i < arguments.length; i++){
+        var source = null != arguments[i] ? arguments[i] : {};
+        var ownKeys = Object.keys(source);
+        if ("function" == typeof Object.getOwnPropertySymbols) ownKeys = ownKeys.concat(Object.getOwnPropertySymbols(source).filter(function(sym) {
+            return Object.getOwnPropertyDescriptor(source, sym).enumerable;
+        }));
+        ownKeys.forEach(function(key) {
+            _define_property(target, key, source[key]);
+        });
+    }
+    return target;
+}
 const { Text } = external_antd_namespaceObject.Typography;
 const handledExternalRunRequestIds = new Set();
 const MAX_HANDLED_EXTERNAL_RUN_REQUEST_IDS = 100;
@@ -114,13 +137,6 @@ function UniversalPlayground({ playgroundSDK, storage, contextProvider, config:
     const { config } = (0, store_js_namespaceObject.useEnvConfig)();
     const [sdkReady, setSdkReady] = (0, external_react_namespaceObject.useState)(false);
     const lastExternalRunRequestIdRef = (0, external_react_namespaceObject.useRef)(null);
-    (0, external_react_namespaceObject.useEffect)(()=>{
-        form.setFieldsValue({
-            type: constants_js_namespaceObject.defaultMainButtons["0"]
-        });
-    }, [
-        form
-    ]);
     (0, external_react_namespaceObject.useEffect)(()=>{
         const initializeSDK = ()=>_async_to_generator(function*() {
                 if (playgroundSDK && 'function' == typeof playgroundSDK.checkStatus) try {
@@ -205,9 +221,11 @@ function UniversalPlayground({ playgroundSDK, storage, contextProvider, config:
             const oldestRequestId = handledExternalRunRequestIds.values().next().value;
             if (oldestRequestId) handledExternalRunRequestIds.delete(oldestRequestId);
         }
-        executeAction(request.value, {
+        executeAction(request.value, _object_spread({
             displayContent: request.displayContent
-        }).catch((error)=>{
+        }, request.reportDisplay ? {
+            reportDisplay: request.reportDisplay
+        } : {})).catch((error)=>{
             (0, index_js_namespaceObject.notifyError)(error, {
                 title: 'Execution failed'
             });
@@ -221,7 +239,7 @@ function UniversalPlayground({ playgroundSDK, storage, contextProvider, config:
     const configAlreadySet = Object.keys(config || {}).length >= 1;
     const runButtonEnabled = componentConfig.serverMode || !dryMode && !actionSpaceLoading && configAlreadySet;
     const watchedType = external_antd_namespaceObject.Form.useWatch('type', form);
-    const selectedType = watchedType || form.getFieldValue('type');
+    const selectedType = watchedType || constants_js_namespaceObject.defaultMainButtons["0"];
     const serviceMode = (0, external_react_namespaceObject.useMemo)(()=>{
         if (!playgroundSDK || 'function' != typeof playgroundSDK.getServiceMode) return 'Server';
         return playgroundSDK.getServiceMode();
@@ -330,6 +348,9 @@ function UniversalPlayground({ playgroundSDK, storage, contextProvider, config:
             form: form,
             onFinish: handleFormRun,
             className: "command-form",
+            initialValues: {
+                type: constants_js_namespaceObject.defaultMainButtons["0"]
+            },
             children: [
                 finalShowContextPreview && /*#__PURE__*/ (0, jsx_runtime_namespaceObject.jsx)("div", {
                     className: "context-preview-section",

package/dist/lib/component/universal-playground/providers/indexeddb-storage-provider.js

@@ -97,6 +97,9 @@ function _object_spread_props(target, source) {
     });
     return target;
 }
+function isReportActionDump(dump) {
+    return Array.isArray(null == dump ? void 0 : dump.executions);
+}
 const DB_NAME = 'midscene_playground';
 const DB_VERSION = 1;
 const MESSAGES_STORE = 'playground_messages';
@@ -182,9 +185,29 @@ class IndexedDBStorageProvider {
         }).call(this);
     }
     compressResultForStorage(result) {
-        var _result_result_dump, _result_result;
-        if (!(null == (_result_result = result.result) ? void 0 : null == (_result_result_dump = _result_result.dump) ? void 0 : _result_result_dump.tasks)) return result;
-        const compressedTasks = result.result.dump.tasks.map((task)=>{
+        const playgroundResult = result.result;
+        const dump = null == playgroundResult ? void 0 : playgroundResult.dump;
+        if (!playgroundResult || !dump) return result;
+        if (isReportActionDump(dump)) return _object_spread_props(_object_spread({}, result), {
+            result: _object_spread_props(_object_spread({}, playgroundResult), {
+                dump: _object_spread_props(_object_spread({}, dump), {
+                    executions: dump.executions.map((execution)=>_object_spread_props(_object_spread({}, execution), {
+                            tasks: this.compressExecutionTasks(execution.tasks)
+                        }))
+                })
+            })
+        });
+        if (!dump.tasks) return result;
+        return _object_spread_props(_object_spread({}, result), {
+            result: _object_spread_props(_object_spread({}, playgroundResult), {
+                dump: _object_spread_props(_object_spread({}, dump), {
+                    tasks: this.compressExecutionTasks(dump.tasks)
+                })
+            })
+        });
+    }
+    compressExecutionTasks(tasks) {
+        return tasks.map((task)=>{
             var _task_recorder;
             return _object_spread_props(_object_spread({}, task), {
                 uiContext: task.uiContext ? _object_spread_props(_object_spread({}, task.uiContext), {
@@ -195,13 +218,6 @@ class IndexedDBStorageProvider {
                     }))
             });
         });
-        return _object_spread_props(_object_spread({}, result), {
-            result: _object_spread_props(_object_spread({}, result.result), {
-                dump: _object_spread_props(_object_spread({}, result.result.dump), {
-                    tasks: compressedTasks
-                })
-            })
-        });
     }
     compressScreenshotIfNeeded(screenshot) {
         if (!screenshot) return screenshot;

package/dist/lib/hooks/usePlaygroundExecution.js

@@ -127,6 +127,15 @@ function wrapExecutionDumpForReplay(dump, deviceType) {
         deviceType
     };
 }
+function isReportActionDump(dump) {
+    return Array.isArray(null == dump ? void 0 : dump.executions);
+}
+function replayInfoFromDump(dump, deviceType) {
+    if (!dump) return null;
+    if (isReportActionDump(dump)) return (0, replay_scripts_js_namespaceObject.allScriptsFromDump)(dump);
+    if (dump.tasks && Array.isArray(dump.tasks)) return (0, replay_scripts_js_namespaceObject.allScriptsFromDump)(wrapExecutionDumpForReplay(dump, deviceType));
+    return null;
+}
 function shouldForwardDeepThink(actionType) {
     return 'aiAct' === actionType || 'runMarkdown' === actionType;
 }
@@ -198,7 +207,7 @@ function usePlaygroundExecution(options) {
                     requestId: thisRunningId.toString()
                 });
                 const resolvedDeepThink = 'unset' === deepThink ? void 0 : deepThink;
-                const executionOptions = _object_spread_props(_object_spread({
+                const executionOptions = _object_spread(_object_spread_props(_object_spread({
                     requestId: thisRunningId.toString(),
                     deepLocate
                 }, shouldForwardDeepThink(actionType) && void 0 !== resolvedDeepThink ? {
@@ -212,7 +221,9 @@ function usePlaygroundExecution(options) {
                         keyboardDismissStrategy,
                         alwaysRefreshScreenInfo
                     }
-                });
+                }), runOptions.reportDisplay ? {
+                    reportDisplay: runOptions.reportDisplay
+                } : {});
                 result.result = yield playgroundSDK.executeAction(actionType, value, executionOptions);
                 if ('object' == typeof result.result && null !== result.result) {
                     const resultObj = result.result;
@@ -235,9 +246,8 @@ function usePlaygroundExecution(options) {
             let replayInfo = null;
             let counter = replayCounter;
             if (null == result ? void 0 : result.dump) {
-                if (result.dump.tasks && Array.isArray(result.dump.tasks)) {
-                    const groupedDump = wrapExecutionDumpForReplay(result.dump, deviceType);
-                    const info = (0, replay_scripts_js_namespaceObject.allScriptsFromDump)(groupedDump);
+                const info = replayInfoFromDump(result.dump, deviceType);
+                if (info) {
                     setReplayCounter((c)=>c + 1);
                     replayInfo = info;
                     counter = replayCounter + 1;
@@ -322,12 +332,10 @@ function usePlaygroundExecution(options) {
                             loadingProgressText: ''
                         }) : item));
                 if (executionData && (executionData.dump || executionData.reportHTML)) {
-                    var _executionData_dump;
                     let replayInfo = null;
                     let counter = replayCounter;
-                    if ((null == (_executionData_dump = executionData.dump) ? void 0 : _executionData_dump.tasks) && Array.isArray(executionData.dump.tasks)) {
-                        const groupedDump = wrapExecutionDumpForReplay(executionData.dump, deviceType);
-                        replayInfo = (0, replay_scripts_js_namespaceObject.allScriptsFromDump)(groupedDump);
+                    replayInfo = replayInfoFromDump(executionData.dump, deviceType);
+                    if (replayInfo) {
                         setReplayCounter((c)=>c + 1);
                         counter = replayCounter + 1;
                     }

package/dist/lib/utils/replay-scripts.js

@@ -142,13 +142,21 @@ const normalizeDump = (dump)=>{
         ]
     };
 };
+const sortExecutionsByLogTime = (executions)=>executions.map((execution, index)=>({
+            execution,
+            index
+        })).sort((left, right)=>{
+        const leftTime = Number.isFinite(left.execution.logTime) ? left.execution.logTime : 1 / 0;
+        const rightTime = Number.isFinite(right.execution.logTime) ? right.execution.logTime : 1 / 0;
+        return leftTime - rightTime || left.index - right.index;
+    }).map(({ execution })=>execution);
 const extractMetaFromNormalized = (normalizedDump)=>{
     var _normalizedDump_executions;
     let firstWidth;
     let firstHeight;
     const sdkVersion = normalizedDump.sdkVersion;
     const modelBriefs = [];
-    null == (_normalizedDump_executions = normalizedDump.executions) || _normalizedDump_executions.filter(Boolean).forEach((execution)=>{
+    sortExecutionsByLogTime((null == (_normalizedDump_executions = normalizedDump.executions) ? void 0 : _normalizedDump_executions.filter(Boolean)) || []).forEach((execution)=>{
         execution.tasks.forEach((task)=>{
             var _task_uiContext;
             const shotSize = null == (_task_uiContext = task.uiContext) ? void 0 : _task_uiContext.shotSize;
@@ -206,7 +214,7 @@ const allScriptsFromDump = (dump)=>{
     };
     const { width: firstWidth, height: firstHeight } = metaInfo;
     const allScripts = [];
-    const executions = (null == (_normalizedDump_executions = normalizedDump.executions) ? void 0 : _normalizedDump_executions.filter(Boolean)) || [];
+    const executions = sortExecutionsByLogTime((null == (_normalizedDump_executions = normalizedDump.executions) ? void 0 : _normalizedDump_executions.filter(Boolean)) || []);
     for(let execIndex = 0; execIndex < executions.length; execIndex++){
         const execution = executions[execIndex];
         const scripts = generateAnimationScripts(execution, -1, firstWidth, firstHeight, execIndex);

package/dist/types/component/universal-playground/providers/indexeddb-storage-provider.d.ts

@@ -33,6 +33,7 @@ export declare class IndexedDBStorageProvider implements StorageProvider {
      * Compress result data for storage while preserving playback functionality
      */
     private compressResultForStorage;
+    private compressExecutionTasks;
     /**
      * Compress screenshot if it exceeds size threshold
      */

package/dist/types/hooks/usePlaygroundExecution.d.ts

@@ -1,5 +1,5 @@
 import type { DeviceAction } from '@midscene/core';
-import type { FormValue, InfoListItem, PlaygroundSDKLike, StorageProvider } from '../types';
+import type { ExecutionReportDisplay, FormValue, InfoListItem, PlaygroundSDKLike, StorageProvider } from '../types';
 export interface UsePlaygroundExecutionOptions {
     playgroundSDK: PlaygroundSDKLike | null;
     storage: StorageProvider | undefined | null;
@@ -16,6 +16,7 @@ export interface UsePlaygroundExecutionOptions {
 }
 export interface RunActionOptions {
     displayContent?: string;
+    reportDisplay?: ExecutionReportDisplay;
 }
 /**
  * Hook for handling playground execution logic

package/dist/types/types.d.ts

@@ -72,11 +72,11 @@ export declare const unwrapZodType: (field: ZodType) => {
     hasDefault: boolean;
 };
 export declare const extractDefaultValue: (field: ZodType) => unknown;
-import type { ExecutionDump, IExecutionDump } from '@midscene/core';
+import type { ExecutionDump, IExecutionDump, IReportActionDump } from '@midscene/core';
 import type { BeforeActionHook, ExecutionOptions, PlaygroundAgent, PlaygroundRuntimeInfo } from '@midscene/playground';
 export interface PlaygroundResult {
     result: any;
-    dump?: ExecutionDump | IExecutionDump | null;
+    dump?: ExecutionDump | IExecutionDump | IReportActionDump | null;
     reportHTML?: string | null;
     error: string | null;
 }
@@ -105,10 +105,15 @@ export interface FormValue {
     prompt?: string;
     params?: Record<string, unknown>;
 }
+export interface ExecutionReportDisplay {
+    type?: string;
+    prompt?: string;
+}
 export interface ExternalRunRequest {
     id: string;
     value: FormValue;
     displayContent?: string;
+    reportDisplay?: ExecutionReportDisplay;
 }
 export type { ExecutionOptions };
 export type ProgressCallback = (step: string, status?: 'loading' | 'completed' | 'error') => void;
@@ -118,11 +123,11 @@ export interface PlaygroundSDKLike {
     onProgressUpdate?: (callback: ProgressCallback) => void;
     onDumpUpdate?: (callback: (dump: string, executionDump?: ExecutionDump) => void) => void;
     cancelExecution?(requestId: string): Promise<{
-        dump: ExecutionDump | null;
+        dump: ExecutionDump | IExecutionDump | IReportActionDump | null;
         reportHTML: string | null;
     } | null>;
     getCurrentExecutionData?(): Promise<{
-        dump: ExecutionDump | null;
+        dump: ExecutionDump | IExecutionDump | IReportActionDump | null;
         reportHTML: string | null;
     }>;
     overrideConfig?(config: any): Promise<void>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@midscene/visualizer",
-  "version": "1.9.6",
+  "version": "1.9.7",
   "repository": "https://github.com/web-infra-dev/midscene",
   "homepage": "https://midscenejs.com/",
   "types": "./dist/types/index.d.ts",
@@ -73,10 +73,10 @@
     "antd": "^5.21.6",
     "buffer": "6.0.3",
     "dayjs": "^1.11.11",
-    "@midscene/playground": "1.9.6",
-    "@midscene/shared": "1.9.6",
-    "@midscene/web": "1.9.6",
-    "@midscene/core": "1.9.6"
+    "@midscene/core": "1.9.7",
+    "@midscene/playground": "1.9.7",
+    "@midscene/shared": "1.9.7",
+    "@midscene/web": "1.9.7"
   },
   "license": "MIT",
   "scripts": {