@midscene/visualizer 0.30.10 → 1.0.0

package/dist/lib/utils/replay-scripts.js

@@ -1,5 +1,5 @@
-'use client';
 "use strict";
+"use client";
 var __webpack_require__ = {};
 (()=>{
     __webpack_require__.d = (exports1, definition)=>{
@@ -25,20 +25,62 @@ var __webpack_require__ = {};
 var __webpack_exports__ = {};
 __webpack_require__.r(__webpack_exports__);
 __webpack_require__.d(__webpack_exports__, {
-    cameraStateForRect: ()=>cameraStateForRect,
-    mergeTwoCameraState: ()=>mergeTwoCameraState,
     generateAnimationScripts: ()=>generateAnimationScripts,
+    mergeTwoCameraState: ()=>mergeTwoCameraState,
+    cameraStateForRect: ()=>cameraStateForRect,
     allScriptsFromDump: ()=>allScriptsFromDump
 });
 const external_index_js_namespaceObject = require("./index.js");
 const agent_namespaceObject = require("@midscene/core/agent");
-const extractor_namespaceObject = require("@midscene/shared/extractor");
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+function _object_spread(target) {
+    for(var i = 1; i < arguments.length; i++){
+        var source = null != arguments[i] ? arguments[i] : {};
+        var ownKeys = Object.keys(source);
+        if ("function" == typeof Object.getOwnPropertySymbols) ownKeys = ownKeys.concat(Object.getOwnPropertySymbols(source).filter(function(sym) {
+            return Object.getOwnPropertyDescriptor(source, sym).enumerable;
+        }));
+        ownKeys.forEach(function(key) {
+            _define_property(target, key, source[key]);
+        });
+    }
+    return target;
+}
+function replay_scripts_ownKeys(object, enumerableOnly) {
+    var keys = Object.keys(object);
+    if (Object.getOwnPropertySymbols) {
+        var symbols = Object.getOwnPropertySymbols(object);
+        if (enumerableOnly) symbols = symbols.filter(function(sym) {
+            return Object.getOwnPropertyDescriptor(object, sym).enumerable;
+        });
+        keys.push.apply(keys, symbols);
+    }
+    return keys;
+}
+function _object_spread_props(target, source) {
+    source = null != source ? source : {};
+    if (Object.getOwnPropertyDescriptors) Object.defineProperties(target, Object.getOwnPropertyDescriptors(source));
+    else replay_scripts_ownKeys(Object(source)).forEach(function(key) {
+        Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key));
+    });
+    return target;
+}
 const stillDuration = 900;
 const actionSpinningPointerDuration = 300;
 const stillAfterInsightDuration = 300;
 const locateDuration = 800;
-const actionDuration = 1000;
+const actionDuration = 500;
 const clearInsightDuration = 200;
+const lastFrameDuration = 200;
 const cameraStateForRect = (rect, imageWidth, imageHeight)=>{
     const canvasRatio = imageWidth / imageHeight;
     const rectRatio = rect.width / rect.height;
@@ -68,17 +110,29 @@ const mergeTwoCameraState = (cameraState1, cameraState2)=>{
         width: newWidth
     };
 };
-const capitalizeFirstLetter = (str)=>{
-    if ('string' != typeof str || 0 === str.length) return str;
-    return str.charAt(0).toUpperCase() + str.slice(1);
-};
 const allScriptsFromDump = (dump)=>{
+    var _normalizedDump_executions, _normalizedDump_executions1, _normalizedDump_modelBriefs;
+    if (!dump) {
+        console.warn('[allScriptsFromDump] dump is empty');
+        return {
+            scripts: [],
+            modelBriefs: []
+        };
+    }
+    const normalizedDump = Array.isArray(dump.executions) ? dump : {
+        sdkVersion: '',
+        groupName: 'Execution',
+        modelBriefs: [],
+        executions: [
+            dump
+        ]
+    };
     const dimensionsSet = new Set();
     let firstWidth;
     let firstHeight;
-    const sdkVersion = dump.sdkVersion;
+    const sdkVersion = normalizedDump.sdkVersion;
     const modelBriefsSet = new Set();
-    dump.executions.forEach((execution)=>{
+    null == (_normalizedDump_executions = normalizedDump.executions) || _normalizedDump_executions.filter(Boolean).forEach((execution)=>{
         execution.tasks.forEach((task)=>{
             var _task_uiContext_size, _task_uiContext;
             if (null == (_task_uiContext = task.uiContext) ? void 0 : null == (_task_uiContext_size = _task_uiContext.size) ? void 0 : _task_uiContext_size.width) {
@@ -101,13 +155,13 @@ const allScriptsFromDump = (dump)=>{
         };
     }
     const allScripts = [];
-    dump.executions.forEach((execution)=>{
+    null == (_normalizedDump_executions1 = normalizedDump.executions) || _normalizedDump_executions1.filter(Boolean).forEach((execution)=>{
         const scripts = generateAnimationScripts(execution, -1, firstWidth, firstHeight);
         if (scripts) allScripts.push(...scripts);
         execution.tasks.forEach((task)=>{
             if (task.usage) {
                 const { model_name, model_description, intent } = task.usage;
-                if (intent && model_name) modelBriefsSet.add(model_description ? `${capitalizeFirstLetter(intent)}/${model_name}(${model_description})` : model_name);
+                if (intent && model_name) modelBriefsSet.add(model_description ? `${intent}/${model_name}(${model_description})` : `${intent}/${model_name}`);
             }
         });
     });
@@ -115,9 +169,10 @@ const allScriptsFromDump = (dump)=>{
         if (index !== allScripts.length - 1 && 'Done' === script.title) return false;
         return true;
     });
+    const normalizedModelBriefs = (null == (_normalizedDump_modelBriefs = normalizedDump.modelBriefs) ? void 0 : _normalizedDump_modelBriefs.length) ? normalizedDump.modelBriefs : [];
     const modelBriefs = (()=>{
         var _list_;
-        const list = [
+        const list = normalizedModelBriefs.length ? normalizedModelBriefs : [
             ...modelBriefsSet
         ];
         if (!list.length) return list;
@@ -159,7 +214,7 @@ const generateAnimationScripts = (execution, task, imageWidth, imageHeight)=>{
         width: imageWidth,
         height: imageHeight
     }, imageWidth, imageHeight);
-    const pointerScript = (img, title, subTitle)=>({
+    const setPointerScript = (img, title, subTitle)=>({
             type: 'pointer',
             img,
             duration: 0,
@@ -168,22 +223,88 @@ const generateAnimationScripts = (execution, task, imageWidth, imageHeight)=>{
         });
     const scripts = [];
     let insightCameraState;
-    let currentCameraState = fullPageCameraState;
     let insightOnTop = false;
-    const taskCount = tasksIncluded.length;
+    tasksIncluded.length;
     let initSubTitle = '';
     let errorStateFlag = false;
     tasksIncluded.forEach((task, index)=>{
         if (errorStateFlag) return;
         if (0 === index) initSubTitle = (0, agent_namespaceObject.paramStr)(task);
         if ('Planning' === task.type) {
+            var _task_output;
+            let locateElements = [];
+            if ('Plan' === task.subType) {
+                var _planTask_output;
+                const planTask = task;
+                const actions = (null == (_planTask_output = planTask.output) ? void 0 : _planTask_output.actions) || [];
+                if (actions.length > 0) {
+                    const action = actions[0];
+                    const knownFields = [
+                        'locate',
+                        'start',
+                        'end'
+                    ];
+                    if (action.param) {
+                        knownFields.forEach((field)=>{
+                            if (action.param[field] && 'object' == typeof action.param[field] && 'center' in (action.param[field] || {})) locateElements.push(action.param[field]);
+                        });
+                        for(const key in action.param)if (!knownFields.includes(key)) {
+                            if ('object' == typeof action.param[key] && 'center' in (action.param[key] || {})) locateElements.push(action.param[key]);
+                        }
+                    }
+                }
+            } else if ('Locate' === task.subType && (null == (_task_output = task.output) ? void 0 : _task_output.element)) {
+                const locateTask = task;
+                locateElements = [
+                    locateTask.output.element
+                ];
+            }
+            const title = (0, agent_namespaceObject.typeStr)(task);
+            const subTitle = (0, agent_namespaceObject.paramStr)(task);
+            const context = task.uiContext;
+            if (null == context ? void 0 : context.screenshotBase64) {
+                var _context_size, _context_size1;
+                const width = (null == (_context_size = context.size) ? void 0 : _context_size.width) || imageWidth;
+                const height = (null == (_context_size1 = context.size) ? void 0 : _context_size1.height) || imageHeight;
+                scripts.push({
+                    type: 'img',
+                    img: context.screenshotBase64,
+                    duration: stillAfterInsightDuration,
+                    title,
+                    subTitle,
+                    imageWidth: width,
+                    imageHeight: height
+                });
+                locateElements.forEach((element)=>{
+                    var _task_log_taskInfo, _task_log, _context_size, _context_size1;
+                    insightCameraState = _object_spread_props(_object_spread({}, cameraStateForRect(element.rect, width, height)), {
+                        pointerLeft: element.center[0],
+                        pointerTop: element.center[1]
+                    });
+                    const newCameraState = insightCameraState;
+                    scripts.push({
+                        type: 'insight',
+                        img: context.screenshotBase64,
+                        context: context,
+                        camera: newCameraState,
+                        highlightElement: element,
+                        searchArea: null == (_task_log = task.log) ? void 0 : null == (_task_log_taskInfo = _task_log.taskInfo) ? void 0 : _task_log_taskInfo.searchArea,
+                        duration: 0.5 * locateDuration,
+                        insightCameraDuration: locateDuration,
+                        title,
+                        subTitle: element.description || subTitle,
+                        imageWidth: (null == (_context_size = context.size) ? void 0 : _context_size.width) || imageWidth,
+                        imageHeight: (null == (_context_size1 = context.size) ? void 0 : _context_size1.height) || imageHeight
+                    });
+                    insightOnTop = true;
+                });
+            }
             const planningTask = task;
             if (planningTask.recorder && planningTask.recorder.length > 0) {
                 var _planningTask_recorder_, _planningTask_recorder, _task_uiContext_size, _task_uiContext, _task_uiContext_size1, _task_uiContext1;
                 scripts.push({
                     type: 'img',
                     img: null == (_planningTask_recorder = planningTask.recorder) ? void 0 : null == (_planningTask_recorder_ = _planningTask_recorder[0]) ? void 0 : _planningTask_recorder_.screenshot,
-                    camera: 0 === index ? fullPageCameraState : void 0,
                     duration: stillDuration,
                     title: (0, agent_namespaceObject.typeStr)(task),
                     subTitle: (0, agent_namespaceObject.paramStr)(task),
@@ -191,73 +312,15 @@ const generateAnimationScripts = (execution, task, imageWidth, imageHeight)=>{
                     imageHeight: (null == (_task_uiContext1 = task.uiContext) ? void 0 : null == (_task_uiContext_size1 = _task_uiContext1.size) ? void 0 : _task_uiContext_size1.height) || imageHeight
                 });
             }
-        } else if ('Insight' === task.type && 'Locate' === task.subType) {
-            var _insightTask_output;
-            const insightTask = task;
-            const resultElement = null == (_insightTask_output = insightTask.output) ? void 0 : _insightTask_output.element;
+        } else if ('Action Space' === task.type) {
+            var _task_recorder_, _task_recorder, _task_uiContext_size2, _task_uiContext2, _task_uiContext_size3, _task_uiContext3;
             const title = (0, agent_namespaceObject.typeStr)(task);
             const subTitle = (0, agent_namespaceObject.paramStr)(task);
-            if (null == resultElement ? void 0 : resultElement.rect) insightCameraState = {
-                ...cameraStateForRect(resultElement.rect, imageWidth, imageHeight),
-                pointerLeft: resultElement.center[0],
-                pointerTop: resultElement.center[1]
-            };
-            const context = insightTask.uiContext;
-            if (null == context ? void 0 : context.screenshotBase64) {
-                var _insightTask_output1, _insightDump_taskInfo, _context_size, _context_size1;
-                const insightDump = insightTask.log;
-                const insightContentLength = context.tree ? (0, extractor_namespaceObject.treeToList)(context.tree).length : 0;
-                if (context.screenshotBase64) {
-                    var _context_size2, _context_size3;
-                    scripts.push({
-                        type: 'img',
-                        img: context.screenshotBase64,
-                        duration: stillAfterInsightDuration,
-                        title,
-                        subTitle,
-                        imageWidth: (null == (_context_size2 = context.size) ? void 0 : _context_size2.width) || imageWidth,
-                        imageHeight: (null == (_context_size3 = context.size) ? void 0 : _context_size3.height) || imageHeight
-                    });
-                }
-                let cameraState;
-                cameraState = currentCameraState === fullPageCameraState ? void 0 : insightCameraState ? mergeTwoCameraState(currentCameraState, insightCameraState) : void 0;
-                scripts.push({
-                    type: 'insight',
-                    img: context.screenshotBase64,
-                    context: context,
-                    camera: cameraState,
-                    highlightElement: (null == (_insightTask_output1 = insightTask.output) ? void 0 : _insightTask_output1.element) || void 0,
-                    searchArea: null == insightDump ? void 0 : null == (_insightDump_taskInfo = insightDump.taskInfo) ? void 0 : _insightDump_taskInfo.searchArea,
-                    duration: insightContentLength > 20 ? locateDuration : 0.5 * locateDuration,
-                    insightCameraDuration: locateDuration,
-                    title,
-                    subTitle,
-                    imageWidth: (null == (_context_size = context.size) ? void 0 : _context_size.width) || imageWidth,
-                    imageHeight: (null == (_context_size1 = context.size) ? void 0 : _context_size1.height) || imageHeight
-                });
-                scripts.push({
-                    type: 'sleep',
-                    duration: stillAfterInsightDuration,
-                    title,
-                    subTitle
-                });
-                insightOnTop = true;
-            }
-        } else if ('Action' === task.type && 'FalsyConditionStatement' !== task.subType) {
-            var _task_recorder_, _task_recorder, _task_uiContext_size2, _task_uiContext2, _task_uiContext_size3, _task_uiContext3, _task_recorder_1, _task_recorder1;
-            const title = (0, agent_namespaceObject.typeStr)(task);
-            const subTitle = (0, agent_namespaceObject.paramStr)(task);
-            scripts.push(pointerScript(external_index_js_namespaceObject.mousePointer, title, subTitle));
-            currentCameraState = null != insightCameraState ? insightCameraState : fullPageCameraState;
             scripts.push({
-                type: 'img',
-                img: null == (_task_recorder = task.recorder) ? void 0 : null == (_task_recorder_ = _task_recorder[0]) ? void 0 : _task_recorder_.screenshot,
-                duration: actionDuration,
-                camera: 'Sleep' === task.subType ? fullPageCameraState : insightCameraState,
+                type: 'spinning-pointer',
+                duration: actionSpinningPointerDuration,
                 title,
-                subTitle,
-                imageWidth: (null == (_task_uiContext2 = task.uiContext) ? void 0 : null == (_task_uiContext_size2 = _task_uiContext2.size) ? void 0 : _task_uiContext_size2.width) || imageWidth,
-                imageHeight: (null == (_task_uiContext3 = task.uiContext) ? void 0 : null == (_task_uiContext_size3 = _task_uiContext3.size) ? void 0 : _task_uiContext_size3.height) || imageHeight
+                subTitle
             });
             if (insightOnTop) {
                 scripts.push({
@@ -268,38 +331,24 @@ const generateAnimationScripts = (execution, task, imageWidth, imageHeight)=>{
                 });
                 insightOnTop = false;
             }
-            const imgStillDuration = index < taskCount - 1 ? stillDuration : 0;
-            if (null == (_task_recorder1 = task.recorder) ? void 0 : null == (_task_recorder_1 = _task_recorder1[1]) ? void 0 : _task_recorder_1.screenshot) {
-                var _task_recorder_2, _task_recorder2, _task_uiContext_size4, _task_uiContext4, _task_uiContext_size5, _task_uiContext5;
-                scripts.push({
-                    type: 'spinning-pointer',
-                    duration: actionSpinningPointerDuration,
-                    title,
-                    subTitle
-                });
-                scripts.push(pointerScript(external_index_js_namespaceObject.mousePointer, title, subTitle));
-                scripts.push({
-                    type: 'img',
-                    img: null == (_task_recorder2 = task.recorder) ? void 0 : null == (_task_recorder_2 = _task_recorder2[1]) ? void 0 : _task_recorder_2.screenshot,
-                    duration: imgStillDuration,
-                    title,
-                    subTitle,
-                    imageWidth: (null == (_task_uiContext4 = task.uiContext) ? void 0 : null == (_task_uiContext_size4 = _task_uiContext4.size) ? void 0 : _task_uiContext_size4.width) || imageWidth,
-                    imageHeight: (null == (_task_uiContext5 = task.uiContext) ? void 0 : null == (_task_uiContext_size5 = _task_uiContext5.size) ? void 0 : _task_uiContext_size5.height) || imageHeight
-                });
-            } else scripts.push({
-                type: 'sleep',
-                duration: imgStillDuration,
+            scripts.push(setPointerScript(external_index_js_namespaceObject.mousePointer, title, subTitle));
+            scripts.push({
+                type: 'img',
+                img: null == (_task_recorder = task.recorder) ? void 0 : null == (_task_recorder_ = _task_recorder[0]) ? void 0 : _task_recorder_.screenshot,
+                duration: actionDuration,
+                camera: 'Sleep' === task.subType ? fullPageCameraState : void 0,
                 title,
-                subTitle
+                subTitle,
+                imageWidth: (null == (_task_uiContext2 = task.uiContext) ? void 0 : null == (_task_uiContext_size2 = _task_uiContext2.size) ? void 0 : _task_uiContext_size2.width) || imageWidth,
+                imageHeight: (null == (_task_uiContext3 = task.uiContext) ? void 0 : null == (_task_uiContext_size3 = _task_uiContext3.size) ? void 0 : _task_uiContext_size3.height) || imageHeight
             });
         } else {
-            var _task_recorder_3, _task_recorder3;
+            var _task_recorder_1, _task_recorder1;
             const title = (0, agent_namespaceObject.typeStr)(task);
             const subTitle = (0, agent_namespaceObject.paramStr)(task);
-            const screenshot = null == (_task_recorder3 = task.recorder) ? void 0 : null == (_task_recorder_3 = _task_recorder3[task.recorder.length - 1]) ? void 0 : _task_recorder_3.screenshot;
+            const screenshot = null == (_task_recorder1 = task.recorder) ? void 0 : null == (_task_recorder_1 = _task_recorder1[task.recorder.length - 1]) ? void 0 : _task_recorder_1.screenshot;
             if (screenshot) {
-                var _task_uiContext_size6, _task_uiContext6, _task_uiContext_size7, _task_uiContext7;
+                var _task_uiContext_size4, _task_uiContext4, _task_uiContext_size5, _task_uiContext5;
                 scripts.push({
                     type: 'img',
                     img: screenshot,
@@ -307,13 +356,13 @@ const generateAnimationScripts = (execution, task, imageWidth, imageHeight)=>{
                     camera: fullPageCameraState,
                     title,
                     subTitle,
-                    imageWidth: (null == (_task_uiContext6 = task.uiContext) ? void 0 : null == (_task_uiContext_size6 = _task_uiContext6.size) ? void 0 : _task_uiContext_size6.width) || imageWidth,
-                    imageHeight: (null == (_task_uiContext7 = task.uiContext) ? void 0 : null == (_task_uiContext_size7 = _task_uiContext7.size) ? void 0 : _task_uiContext_size7.height) || imageHeight
+                    imageWidth: (null == (_task_uiContext4 = task.uiContext) ? void 0 : null == (_task_uiContext_size4 = _task_uiContext4.size) ? void 0 : _task_uiContext_size4.width) || imageWidth,
+                    imageHeight: (null == (_task_uiContext5 = task.uiContext) ? void 0 : null == (_task_uiContext_size5 = _task_uiContext5.size) ? void 0 : _task_uiContext_size5.height) || imageHeight
                 });
             }
         }
         if ('finished' !== task.status) {
-            var _task_uiContext_size8, _task_uiContext8, _task_uiContext_size9, _task_uiContext9;
+            var _task_uiContext_size6, _task_uiContext6, _task_uiContext_size7, _task_uiContext7;
             errorStateFlag = true;
             const errorTitle = (0, agent_namespaceObject.typeStr)(task);
             const errorMsg = task.errorMessage || 'unknown error';
@@ -325,8 +374,8 @@ const generateAnimationScripts = (execution, task, imageWidth, imageHeight)=>{
                 duration: stillDuration,
                 title: errorTitle,
                 subTitle: errorSubTitle,
-                imageWidth: (null == (_task_uiContext8 = task.uiContext) ? void 0 : null == (_task_uiContext_size8 = _task_uiContext8.size) ? void 0 : _task_uiContext_size8.width) || imageWidth,
-                imageHeight: (null == (_task_uiContext9 = task.uiContext) ? void 0 : null == (_task_uiContext_size9 = _task_uiContext9.size) ? void 0 : _task_uiContext_size9.height) || imageHeight
+                imageWidth: (null == (_task_uiContext6 = task.uiContext) ? void 0 : null == (_task_uiContext_size6 = _task_uiContext6.size) ? void 0 : _task_uiContext_size6.width) || imageWidth,
+                imageHeight: (null == (_task_uiContext7 = task.uiContext) ? void 0 : null == (_task_uiContext_size7 = _task_uiContext7.size) ? void 0 : _task_uiContext_size7.height) || imageHeight
             });
             return;
         }
@@ -335,7 +384,7 @@ const generateAnimationScripts = (execution, task, imageWidth, imageHeight)=>{
         title: 'Done',
         subTitle: initSubTitle,
         type: 'img',
-        duration: stillDuration,
+        duration: lastFrameDuration,
         camera: fullPageCameraState
     });
     return scripts;
@@ -344,12 +393,12 @@ exports.allScriptsFromDump = __webpack_exports__.allScriptsFromDump;
 exports.cameraStateForRect = __webpack_exports__.cameraStateForRect;
 exports.generateAnimationScripts = __webpack_exports__.generateAnimationScripts;
 exports.mergeTwoCameraState = __webpack_exports__.mergeTwoCameraState;
-for(var __webpack_i__ in __webpack_exports__)if (-1 === [
+for(var __rspack_i in __webpack_exports__)if (-1 === [
     "allScriptsFromDump",
     "cameraStateForRect",
     "generateAnimationScripts",
     "mergeTwoCameraState"
-].indexOf(__webpack_i__)) exports[__webpack_i__] = __webpack_exports__[__webpack_i__];
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
 Object.defineProperty(exports, '__esModule', {
     value: true
 });

package/dist/types/component/config-selector/index.d.ts

@@ -1,9 +1,11 @@
 import type React from 'react';
+import type { DeviceType } from '../../types';
 interface ConfigSelectorProps {
     showDeepThinkOption: boolean;
     enableTracking: boolean;
     showDataExtractionOptions: boolean;
     hideDomAndScreenshotOptions?: boolean;
+    deviceType?: DeviceType;
 }
 export declare const ConfigSelector: React.FC<ConfigSelectorProps>;
 export {};

package/dist/types/component/player/index.d.ts

@@ -9,5 +9,4 @@ export declare function Player(props?: {
     key?: string | number;
     fitMode?: 'width' | 'height';
     autoZoom?: boolean;
-    elementsVisible?: boolean;
 }): import("react").JSX.Element;

package/dist/types/component/playground-result/index.d.ts

@@ -15,6 +15,7 @@ interface PlaygroundResultProps {
     notReadyMessage?: React.ReactNode | string;
     fitMode?: 'width' | 'height';
     autoZoom?: boolean;
+    actionType?: string;
 }
 export declare const PlaygroundResultView: React.FC<PlaygroundResultProps>;
 export {};

package/dist/types/component/prompt-input/index.d.ts

@@ -1,6 +1,6 @@
 import './index.less';
 import React from 'react';
-import type { RunType } from '../../types';
+import type { DeviceType, RunType } from '../../types';
 import type { ServiceModeType } from '../../types';
 import './index.less';
 import type { DeviceAction } from '@midscene/core';
@@ -17,6 +17,7 @@ interface PromptInputProps {
     clearPromptAfterRun?: boolean;
     hideDomAndScreenshotOptions?: boolean;
     actionSpace: DeviceAction<any>[];
+    deviceType?: DeviceType;
 }
 export declare const PromptInput: React.FC<PromptInputProps>;
 export {};

package/dist/types/component/screenshot-viewer/index.d.ts

@@ -0,0 +1,15 @@
+import './index.less';
+interface ScreenshotViewerProps {
+    getScreenshot: () => Promise<{
+        screenshot: string;
+        timestamp: number;
+    } | null>;
+    getInterfaceInfo?: () => Promise<{
+        type: string;
+        description?: string;
+    } | null>;
+    serverOnline: boolean;
+    isUserOperating?: boolean;
+}
+export default function ScreenshotViewer({ getScreenshot, getInterfaceInfo, serverOnline, isUserOperating, }: ScreenshotViewerProps): import("react").JSX.Element;
+export {};

package/dist/types/hooks/usePlaygroundExecution.d.ts

@@ -3,7 +3,7 @@ import type { FormValue, InfoListItem, PlaygroundSDKLike, StorageProvider } from
 /**
  * Hook for handling playground execution logic
  */
-export declare function usePlaygroundExecution(playgroundSDK: PlaygroundSDKLike | null, storage: StorageProvider | undefined | null, actionSpace: DeviceAction<unknown>[], loading: boolean, setLoading: (loading: boolean) => void, infoList: InfoListItem[], setInfoList: React.Dispatch<React.SetStateAction<InfoListItem[]>>, replayCounter: number, setReplayCounter: React.Dispatch<React.SetStateAction<number>>, verticalMode: boolean, currentRunningIdRef: React.MutableRefObject<number | null>, interruptedFlagRef: React.MutableRefObject<Record<number, boolean>>): {
+export declare function usePlaygroundExecution(playgroundSDK: PlaygroundSDKLike | null, storage: StorageProvider | undefined | null, actionSpace: DeviceAction<unknown>[], loading: boolean, setLoading: (loading: boolean) => void, setInfoList: React.Dispatch<React.SetStateAction<InfoListItem[]>>, replayCounter: number, setReplayCounter: React.Dispatch<React.SetStateAction<number>>, verticalMode: boolean, currentRunningIdRef: React.MutableRefObject<number | null>, interruptedFlagRef: React.MutableRefObject<Record<number, boolean>>): {
     handleRun: (value: FormValue) => Promise<void>;
     handleStop: () => Promise<void>;
     canStop: boolean;

package/dist/types/hooks/usePlaygroundState.d.ts

@@ -8,10 +8,10 @@ export declare function usePlaygroundState(playgroundSDK: PlaygroundSDKLike | nu
     setLoading: import("react").Dispatch<import("react").SetStateAction<boolean>>;
     infoList: InfoListItem[];
     setInfoList: import("react").Dispatch<import("react").SetStateAction<InfoListItem[]>>;
-    actionSpace: DeviceAction<unknown>[];
+    actionSpace: DeviceAction<unknown, any>[];
     actionSpaceLoading: boolean;
-    uiContextPreview: UIContext<import("@midscene/core").BaseElement> | undefined;
-    setUiContextPreview: import("react").Dispatch<import("react").SetStateAction<UIContext<import("@midscene/core").BaseElement> | undefined>>;
+    uiContextPreview: UIContext | undefined;
+    setUiContextPreview: import("react").Dispatch<import("react").SetStateAction<UIContext | undefined>>;
     showScrollToBottomButton: boolean;
     verticalMode: boolean;
     replayCounter: number;

package/dist/types/hooks/useTheme.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Hook to detect and observe the current theme (light/dark mode)
+ * @returns Object containing isDarkMode boolean
+ */
+export declare function useTheme(): {
+    isDarkMode: boolean;
+};

package/dist/types/index.d.ts

@@ -1,7 +1,5 @@
-import './component/playground/index.less';
-import './component/universal-playground/index.less';
 export { type AnimationScript, type ReplayScriptsInfo, allScriptsFromDump, generateAnimationScripts, } from './utils/replay-scripts';
-export { useEnvConfig } from './store/store';
+export { useEnvConfig, useGlobalPreference } from './store/store';
 export { colorForName, highlightColorForType, globalThemeConfig, } from './utils/color';
 export { EnvConfig } from './component/env-config';
 export { EnvConfigReminder } from './component/env-config-reminder';
@@ -9,6 +7,7 @@ export { NavActions } from './component/nav-actions';
 export type { NavActionsProps } from './component/nav-actions';
 export { Logo } from './component/logo';
 export { iconForStatus, timeCostStrElement } from './component/misc';
+export { useTheme } from './hooks/useTheme';
 export { useServerValid } from './hooks/useServerValid';
 export { useSafeOverrideAIConfig, safeOverrideAIConfig, } from './hooks/useSafeOverrideAIConfig';
 export { PlaygroundResultView } from './component/playground-result';
@@ -18,6 +17,7 @@ export { ContextPreview } from './component/context-preview';
 export { PromptInput } from './component/prompt-input';
 export { Player } from './component/player';
 export { Blackboard } from './component/blackboard';
+export { default as ScreenshotViewer } from './component/screenshot-viewer';
 export { actionNameForType, staticAgentFromContext, getPlaceholderForType, } from './utils/playground-utils';
 export { timeStr, filterBase64Value } from './utils';
 export { default as ShinyText } from './component/shiny-text';

package/dist/types/store/store.d.ts

@@ -1,11 +1,16 @@
 import * as Z from 'zustand';
-export declare const useBlackboardPreference: Z.UseBoundStore<Z.StoreApi<{
+import type { PlanningStrategyType } from '../types';
+export declare const useGlobalPreference: Z.UseBoundStore<Z.StoreApi<{
     backgroundVisible: boolean;
     elementsVisible: boolean;
     autoZoom: boolean;
+    modelCallDetailsEnabled: boolean;
+    darkModeEnabled: boolean;
     setBackgroundVisible: (visible: boolean) => void;
     setElementsVisible: (visible: boolean) => void;
     setAutoZoom: (enabled: boolean) => void;
+    setModelCallDetailsEnabled: (enabled: boolean) => void;
+    setDarkModeEnabled: (enabled: boolean) => void;
 }>>;
 /**
  * Service Mode
@@ -15,6 +20,8 @@ export declare const useBlackboardPreference: Z.UseBoundStore<Z.StoreApi<{
  * - In-Browser-Extension: use browser's fetch API to run the code, but the page is running in the extension context
  */
 export type ServiceModeType = 'Server' | 'In-Browser' | 'In-Browser-Extension';
+export type ImeStrategyType = 'always-yadb' | 'yadb-for-non-ascii';
+export type KeyboardDismissStrategyType = 'esc-first' | 'back-first';
 export declare const useEnvConfig: Z.UseBoundStore<Z.StoreApi<{
     serviceMode: ServiceModeType;
     setServiceMode: (serviceMode: ServiceModeType) => void;
@@ -31,6 +38,16 @@ export declare const useEnvConfig: Z.UseBoundStore<Z.StoreApi<{
     setScreenshotIncluded: (screenshotIncluded: boolean) => void;
     domIncluded: boolean | "visible-only";
     setDomIncluded: (domIncluded: boolean | "visible-only") => void;
+    planningStrategy: PlanningStrategyType;
+    setPlanningStrategy: (planningStrategy: PlanningStrategyType) => void;
     popupTab: "playground" | "bridge" | "recorder";
     setPopupTab: (tab: "playground" | "bridge" | "recorder") => void;
+    imeStrategy: ImeStrategyType;
+    setImeStrategy: (imeStrategy: ImeStrategyType) => void;
+    autoDismissKeyboard: boolean;
+    setAutoDismissKeyboard: (autoDismissKeyboard: boolean) => void;
+    keyboardDismissStrategy: KeyboardDismissStrategyType;
+    setKeyboardDismissStrategy: (keyboardDismissStrategy: KeyboardDismissStrategyType) => void;
+    alwaysRefreshScreenInfo: boolean;
+    setAlwaysRefreshScreenInfo: (alwaysRefreshScreenInfo: boolean) => void;
 }>>;

package/dist/types/types.d.ts

@@ -58,7 +58,7 @@ export declare const VALIDATION_CONSTANTS: {
         readonly LOCATION: "midscene_location_field_flag";
     };
     readonly DEFAULT_VALUES: {
-        readonly ACTION_TYPE: "aiAction";
+        readonly ACTION_TYPE: "aiAct";
         readonly TIMEOUT_MS: 15000;
         readonly CHECK_INTERVAL_MS: 3000;
     };
@@ -71,11 +71,11 @@ export declare const unwrapZodType: (field: ZodType) => {
     hasDefault: boolean;
 };
 export declare const extractDefaultValue: (field: ZodType) => unknown;
-import type { GroupedActionDump, WebUIContext } from '@midscene/core';
+import type { ExecutionDump, WebUIContext } from '@midscene/core';
 import type { ExecutionOptions, PlaygroundAgent } from '@midscene/playground';
 export interface PlaygroundResult {
     result: any;
-    dump?: GroupedActionDump | null;
+    dump?: ExecutionDump | null;
     reportHTML?: string | null;
     error: string | null;
 }
@@ -89,7 +89,9 @@ export interface StaticPlaygroundProps {
     context: WebUIContext | null;
 }
 export type ServiceModeType = 'Server' | 'In-Browser' | 'In-Browser-Extension';
-export type RunType = 'aiAction' | 'aiQuery' | 'aiAssert' | 'aiTap' | 'aiDoubleClick' | 'aiHover' | 'aiInput' | 'aiRightClick' | 'aiKeyboardPress' | 'aiScroll' | 'aiLocate' | 'aiBoolean' | 'aiNumber' | 'aiString' | 'aiAsk' | 'aiWaitFor';
+export type DeviceType = 'web' | 'android' | 'ios';
+export type PlanningStrategyType = 'fast' | 'standard';
+export type RunType = 'aiAct' | 'aiQuery' | 'aiAssert' | 'aiTap' | 'aiDoubleClick' | 'aiHover' | 'aiInput' | 'aiRightClick' | 'aiKeyboardPress' | 'aiScroll' | 'aiLocate' | 'aiBoolean' | 'aiNumber' | 'aiString' | 'aiAsk' | 'aiWaitFor';
 export interface ReplayScriptsInfo {
     scripts: any[];
     width?: number;
@@ -108,9 +110,15 @@ export interface PlaygroundSDKLike {
     executeAction(actionType: string, value: FormValue, options: ExecutionOptions): Promise<unknown>;
     getActionSpace(context?: any): Promise<DeviceAction<unknown>[]>;
     onProgressUpdate?: (callback: ProgressCallback) => void;
+    onDumpUpdate?: (callback: (dump: string, executionDump?: ExecutionDump) => void) => void;
     cancelExecution?(requestId: string): Promise<void>;
+    getCurrentExecutionData?(): Promise<{
+        dump: ExecutionDump | null;
+        reportHTML: string | null;
+    }>;
     overrideConfig?(config: any): Promise<void>;
     checkStatus?(): Promise<boolean>;
+    getServiceMode?(): 'In-Browser-Extension' | 'Server';
     id?: string;
 }
 export interface StorageProvider {
@@ -134,6 +142,7 @@ export interface InfoListItem {
     replayCounter?: number;
     loadingProgressText?: string;
     verticalMode?: boolean;
+    actionType?: string;
 }
 export interface UniversalPlaygroundConfig {
     showContextPreview?: boolean;
@@ -143,6 +152,7 @@ export interface UniversalPlaygroundConfig {
     enableScrollToBottom?: boolean;
     serverMode?: boolean;
     showEnvConfigReminder?: boolean;
+    deviceType?: DeviceType;
 }
 export interface PlaygroundBranding {
     title?: string;