@midscene/visualizer 0.30.10 → 1.0.0

package/dist/es/utils/pixi-loader.mjs

@@ -1,16 +1,42 @@
 import "pixi.js/unsafe-eval";
 import { Assets } from "pixi.js";
+function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) {
+    try {
+        var info = gen[key](arg);
+        var value = info.value;
+    } catch (error) {
+        reject(error);
+        return;
+    }
+    if (info.done) resolve(value);
+    else Promise.resolve(value).then(_next, _throw);
+}
+function _async_to_generator(fn) {
+    return function() {
+        var self = this, args = arguments;
+        return new Promise(function(resolve, reject) {
+            var gen = fn.apply(self, args);
+            function _next(value) {
+                asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value);
+            }
+            function _throw(err) {
+                asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err);
+            }
+            _next(void 0);
+        });
+    };
+}
 const globalTextureMap = new Map();
-const loadTexture = async (img)=>{
-    if (globalTextureMap.has(img)) return;
-    return Assets.load(img).then((texture)=>{
-        globalTextureMap.set(img, texture);
-    });
-};
+const loadTexture = (img)=>_async_to_generator(function*() {
+        if (globalTextureMap.has(img)) return;
+        return Assets.load(img).then((texture)=>{
+            globalTextureMap.set(img, texture);
+        });
+    })();
 const getTextureFromCache = (name)=>globalTextureMap.get(name);
-const getTexture = async (name)=>{
-    if (globalTextureMap.has(name)) return globalTextureMap.get(name);
-    await loadTexture(name);
-    return globalTextureMap.get(name);
-};
+const getTexture = (name)=>_async_to_generator(function*() {
+        if (globalTextureMap.has(name)) return globalTextureMap.get(name);
+        yield loadTexture(name);
+        return globalTextureMap.get(name);
+    })();
 export { getTexture, getTextureFromCache, loadTexture };

package/dist/es/utils/replay-scripts.mjs

@@ -1,13 +1,55 @@
-'use client';
+"use client";
 import { mousePointer } from "./index.mjs";
 import { paramStr, typeStr } from "@midscene/core/agent";
-import { treeToList } from "@midscene/shared/extractor";
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+function _object_spread(target) {
+    for(var i = 1; i < arguments.length; i++){
+        var source = null != arguments[i] ? arguments[i] : {};
+        var ownKeys = Object.keys(source);
+        if ("function" == typeof Object.getOwnPropertySymbols) ownKeys = ownKeys.concat(Object.getOwnPropertySymbols(source).filter(function(sym) {
+            return Object.getOwnPropertyDescriptor(source, sym).enumerable;
+        }));
+        ownKeys.forEach(function(key) {
+            _define_property(target, key, source[key]);
+        });
+    }
+    return target;
+}
+function replay_scripts_ownKeys(object, enumerableOnly) {
+    var keys = Object.keys(object);
+    if (Object.getOwnPropertySymbols) {
+        var symbols = Object.getOwnPropertySymbols(object);
+        if (enumerableOnly) symbols = symbols.filter(function(sym) {
+            return Object.getOwnPropertyDescriptor(object, sym).enumerable;
+        });
+        keys.push.apply(keys, symbols);
+    }
+    return keys;
+}
+function _object_spread_props(target, source) {
+    source = null != source ? source : {};
+    if (Object.getOwnPropertyDescriptors) Object.defineProperties(target, Object.getOwnPropertyDescriptors(source));
+    else replay_scripts_ownKeys(Object(source)).forEach(function(key) {
+        Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key));
+    });
+    return target;
+}
 const stillDuration = 900;
 const actionSpinningPointerDuration = 300;
 const stillAfterInsightDuration = 300;
 const locateDuration = 800;
-const actionDuration = 1000;
+const actionDuration = 500;
 const clearInsightDuration = 200;
+const lastFrameDuration = 200;
 const cameraStateForRect = (rect, imageWidth, imageHeight)=>{
     const canvasRatio = imageWidth / imageHeight;
     const rectRatio = rect.width / rect.height;
@@ -37,17 +79,29 @@ const mergeTwoCameraState = (cameraState1, cameraState2)=>{
         width: newWidth
     };
 };
-const capitalizeFirstLetter = (str)=>{
-    if ('string' != typeof str || 0 === str.length) return str;
-    return str.charAt(0).toUpperCase() + str.slice(1);
-};
 const allScriptsFromDump = (dump)=>{
+    var _normalizedDump_executions, _normalizedDump_executions1, _normalizedDump_modelBriefs;
+    if (!dump) {
+        console.warn('[allScriptsFromDump] dump is empty');
+        return {
+            scripts: [],
+            modelBriefs: []
+        };
+    }
+    const normalizedDump = Array.isArray(dump.executions) ? dump : {
+        sdkVersion: '',
+        groupName: 'Execution',
+        modelBriefs: [],
+        executions: [
+            dump
+        ]
+    };
     const dimensionsSet = new Set();
     let firstWidth;
     let firstHeight;
-    const sdkVersion = dump.sdkVersion;
+    const sdkVersion = normalizedDump.sdkVersion;
     const modelBriefsSet = new Set();
-    dump.executions.forEach((execution)=>{
+    null == (_normalizedDump_executions = normalizedDump.executions) || _normalizedDump_executions.filter(Boolean).forEach((execution)=>{
         execution.tasks.forEach((task)=>{
             var _task_uiContext_size, _task_uiContext;
             if (null == (_task_uiContext = task.uiContext) ? void 0 : null == (_task_uiContext_size = _task_uiContext.size) ? void 0 : _task_uiContext_size.width) {
@@ -70,13 +124,13 @@ const allScriptsFromDump = (dump)=>{
         };
     }
     const allScripts = [];
-    dump.executions.forEach((execution)=>{
+    null == (_normalizedDump_executions1 = normalizedDump.executions) || _normalizedDump_executions1.filter(Boolean).forEach((execution)=>{
         const scripts = generateAnimationScripts(execution, -1, firstWidth, firstHeight);
         if (scripts) allScripts.push(...scripts);
         execution.tasks.forEach((task)=>{
             if (task.usage) {
                 const { model_name, model_description, intent } = task.usage;
-                if (intent && model_name) modelBriefsSet.add(model_description ? `${capitalizeFirstLetter(intent)}/${model_name}(${model_description})` : model_name);
+                if (intent && model_name) modelBriefsSet.add(model_description ? `${intent}/${model_name}(${model_description})` : `${intent}/${model_name}`);
             }
         });
     });
@@ -84,9 +138,10 @@ const allScriptsFromDump = (dump)=>{
         if (index !== allScripts.length - 1 && 'Done' === script.title) return false;
         return true;
     });
+    const normalizedModelBriefs = (null == (_normalizedDump_modelBriefs = normalizedDump.modelBriefs) ? void 0 : _normalizedDump_modelBriefs.length) ? normalizedDump.modelBriefs : [];
     const modelBriefs = (()=>{
         var _list_;
-        const list = [
+        const list = normalizedModelBriefs.length ? normalizedModelBriefs : [
             ...modelBriefsSet
         ];
         if (!list.length) return list;
@@ -128,7 +183,7 @@ const generateAnimationScripts = (execution, task, imageWidth, imageHeight)=>{
         width: imageWidth,
         height: imageHeight
     }, imageWidth, imageHeight);
-    const pointerScript = (img, title, subTitle)=>({
+    const setPointerScript = (img, title, subTitle)=>({
             type: 'pointer',
             img,
             duration: 0,
@@ -137,22 +192,88 @@ const generateAnimationScripts = (execution, task, imageWidth, imageHeight)=>{
         });
     const scripts = [];
     let insightCameraState;
-    let currentCameraState = fullPageCameraState;
     let insightOnTop = false;
-    const taskCount = tasksIncluded.length;
+    tasksIncluded.length;
     let initSubTitle = '';
     let errorStateFlag = false;
     tasksIncluded.forEach((task, index)=>{
         if (errorStateFlag) return;
         if (0 === index) initSubTitle = paramStr(task);
         if ('Planning' === task.type) {
+            var _task_output;
+            let locateElements = [];
+            if ('Plan' === task.subType) {
+                var _planTask_output;
+                const planTask = task;
+                const actions = (null == (_planTask_output = planTask.output) ? void 0 : _planTask_output.actions) || [];
+                if (actions.length > 0) {
+                    const action = actions[0];
+                    const knownFields = [
+                        'locate',
+                        'start',
+                        'end'
+                    ];
+                    if (action.param) {
+                        knownFields.forEach((field)=>{
+                            if (action.param[field] && 'object' == typeof action.param[field] && 'center' in (action.param[field] || {})) locateElements.push(action.param[field]);
+                        });
+                        for(const key in action.param)if (!knownFields.includes(key)) {
+                            if ('object' == typeof action.param[key] && 'center' in (action.param[key] || {})) locateElements.push(action.param[key]);
+                        }
+                    }
+                }
+            } else if ('Locate' === task.subType && (null == (_task_output = task.output) ? void 0 : _task_output.element)) {
+                const locateTask = task;
+                locateElements = [
+                    locateTask.output.element
+                ];
+            }
+            const title = typeStr(task);
+            const subTitle = paramStr(task);
+            const context = task.uiContext;
+            if (null == context ? void 0 : context.screenshotBase64) {
+                var _context_size, _context_size1;
+                const width = (null == (_context_size = context.size) ? void 0 : _context_size.width) || imageWidth;
+                const height = (null == (_context_size1 = context.size) ? void 0 : _context_size1.height) || imageHeight;
+                scripts.push({
+                    type: 'img',
+                    img: context.screenshotBase64,
+                    duration: stillAfterInsightDuration,
+                    title,
+                    subTitle,
+                    imageWidth: width,
+                    imageHeight: height
+                });
+                locateElements.forEach((element)=>{
+                    var _task_log_taskInfo, _task_log, _context_size, _context_size1;
+                    insightCameraState = _object_spread_props(_object_spread({}, cameraStateForRect(element.rect, width, height)), {
+                        pointerLeft: element.center[0],
+                        pointerTop: element.center[1]
+                    });
+                    const newCameraState = insightCameraState;
+                    scripts.push({
+                        type: 'insight',
+                        img: context.screenshotBase64,
+                        context: context,
+                        camera: newCameraState,
+                        highlightElement: element,
+                        searchArea: null == (_task_log = task.log) ? void 0 : null == (_task_log_taskInfo = _task_log.taskInfo) ? void 0 : _task_log_taskInfo.searchArea,
+                        duration: 0.5 * locateDuration,
+                        insightCameraDuration: locateDuration,
+                        title,
+                        subTitle: element.description || subTitle,
+                        imageWidth: (null == (_context_size = context.size) ? void 0 : _context_size.width) || imageWidth,
+                        imageHeight: (null == (_context_size1 = context.size) ? void 0 : _context_size1.height) || imageHeight
+                    });
+                    insightOnTop = true;
+                });
+            }
             const planningTask = task;
             if (planningTask.recorder && planningTask.recorder.length > 0) {
                 var _planningTask_recorder_, _planningTask_recorder, _task_uiContext_size, _task_uiContext, _task_uiContext_size1, _task_uiContext1;
                 scripts.push({
                     type: 'img',
                     img: null == (_planningTask_recorder = planningTask.recorder) ? void 0 : null == (_planningTask_recorder_ = _planningTask_recorder[0]) ? void 0 : _planningTask_recorder_.screenshot,
-                    camera: 0 === index ? fullPageCameraState : void 0,
                     duration: stillDuration,
                     title: typeStr(task),
                     subTitle: paramStr(task),
@@ -160,73 +281,15 @@ const generateAnimationScripts = (execution, task, imageWidth, imageHeight)=>{
                     imageHeight: (null == (_task_uiContext1 = task.uiContext) ? void 0 : null == (_task_uiContext_size1 = _task_uiContext1.size) ? void 0 : _task_uiContext_size1.height) || imageHeight
                 });
             }
-        } else if ('Insight' === task.type && 'Locate' === task.subType) {
-            var _insightTask_output;
-            const insightTask = task;
-            const resultElement = null == (_insightTask_output = insightTask.output) ? void 0 : _insightTask_output.element;
+        } else if ('Action Space' === task.type) {
+            var _task_recorder_, _task_recorder, _task_uiContext_size2, _task_uiContext2, _task_uiContext_size3, _task_uiContext3;
             const title = typeStr(task);
             const subTitle = paramStr(task);
-            if (null == resultElement ? void 0 : resultElement.rect) insightCameraState = {
-                ...cameraStateForRect(resultElement.rect, imageWidth, imageHeight),
-                pointerLeft: resultElement.center[0],
-                pointerTop: resultElement.center[1]
-            };
-            const context = insightTask.uiContext;
-            if (null == context ? void 0 : context.screenshotBase64) {
-                var _insightTask_output1, _insightDump_taskInfo, _context_size, _context_size1;
-                const insightDump = insightTask.log;
-                const insightContentLength = context.tree ? treeToList(context.tree).length : 0;
-                if (context.screenshotBase64) {
-                    var _context_size2, _context_size3;
-                    scripts.push({
-                        type: 'img',
-                        img: context.screenshotBase64,
-                        duration: stillAfterInsightDuration,
-                        title,
-                        subTitle,
-                        imageWidth: (null == (_context_size2 = context.size) ? void 0 : _context_size2.width) || imageWidth,
-                        imageHeight: (null == (_context_size3 = context.size) ? void 0 : _context_size3.height) || imageHeight
-                    });
-                }
-                let cameraState;
-                cameraState = currentCameraState === fullPageCameraState ? void 0 : insightCameraState ? mergeTwoCameraState(currentCameraState, insightCameraState) : void 0;
-                scripts.push({
-                    type: 'insight',
-                    img: context.screenshotBase64,
-                    context: context,
-                    camera: cameraState,
-                    highlightElement: (null == (_insightTask_output1 = insightTask.output) ? void 0 : _insightTask_output1.element) || void 0,
-                    searchArea: null == insightDump ? void 0 : null == (_insightDump_taskInfo = insightDump.taskInfo) ? void 0 : _insightDump_taskInfo.searchArea,
-                    duration: insightContentLength > 20 ? locateDuration : 0.5 * locateDuration,
-                    insightCameraDuration: locateDuration,
-                    title,
-                    subTitle,
-                    imageWidth: (null == (_context_size = context.size) ? void 0 : _context_size.width) || imageWidth,
-                    imageHeight: (null == (_context_size1 = context.size) ? void 0 : _context_size1.height) || imageHeight
-                });
-                scripts.push({
-                    type: 'sleep',
-                    duration: stillAfterInsightDuration,
-                    title,
-                    subTitle
-                });
-                insightOnTop = true;
-            }
-        } else if ('Action' === task.type && 'FalsyConditionStatement' !== task.subType) {
-            var _task_recorder_, _task_recorder, _task_uiContext_size2, _task_uiContext2, _task_uiContext_size3, _task_uiContext3, _task_recorder_1, _task_recorder1;
-            const title = typeStr(task);
-            const subTitle = paramStr(task);
-            scripts.push(pointerScript(mousePointer, title, subTitle));
-            currentCameraState = null != insightCameraState ? insightCameraState : fullPageCameraState;
             scripts.push({
-                type: 'img',
-                img: null == (_task_recorder = task.recorder) ? void 0 : null == (_task_recorder_ = _task_recorder[0]) ? void 0 : _task_recorder_.screenshot,
-                duration: actionDuration,
-                camera: 'Sleep' === task.subType ? fullPageCameraState : insightCameraState,
+                type: 'spinning-pointer',
+                duration: actionSpinningPointerDuration,
                 title,
-                subTitle,
-                imageWidth: (null == (_task_uiContext2 = task.uiContext) ? void 0 : null == (_task_uiContext_size2 = _task_uiContext2.size) ? void 0 : _task_uiContext_size2.width) || imageWidth,
-                imageHeight: (null == (_task_uiContext3 = task.uiContext) ? void 0 : null == (_task_uiContext_size3 = _task_uiContext3.size) ? void 0 : _task_uiContext_size3.height) || imageHeight
+                subTitle
             });
             if (insightOnTop) {
                 scripts.push({
@@ -237,38 +300,24 @@ const generateAnimationScripts = (execution, task, imageWidth, imageHeight)=>{
                 });
                 insightOnTop = false;
             }
-            const imgStillDuration = index < taskCount - 1 ? stillDuration : 0;
-            if (null == (_task_recorder1 = task.recorder) ? void 0 : null == (_task_recorder_1 = _task_recorder1[1]) ? void 0 : _task_recorder_1.screenshot) {
-                var _task_recorder_2, _task_recorder2, _task_uiContext_size4, _task_uiContext4, _task_uiContext_size5, _task_uiContext5;
-                scripts.push({
-                    type: 'spinning-pointer',
-                    duration: actionSpinningPointerDuration,
-                    title,
-                    subTitle
-                });
-                scripts.push(pointerScript(mousePointer, title, subTitle));
-                scripts.push({
-                    type: 'img',
-                    img: null == (_task_recorder2 = task.recorder) ? void 0 : null == (_task_recorder_2 = _task_recorder2[1]) ? void 0 : _task_recorder_2.screenshot,
-                    duration: imgStillDuration,
-                    title,
-                    subTitle,
-                    imageWidth: (null == (_task_uiContext4 = task.uiContext) ? void 0 : null == (_task_uiContext_size4 = _task_uiContext4.size) ? void 0 : _task_uiContext_size4.width) || imageWidth,
-                    imageHeight: (null == (_task_uiContext5 = task.uiContext) ? void 0 : null == (_task_uiContext_size5 = _task_uiContext5.size) ? void 0 : _task_uiContext_size5.height) || imageHeight
-                });
-            } else scripts.push({
-                type: 'sleep',
-                duration: imgStillDuration,
+            scripts.push(setPointerScript(mousePointer, title, subTitle));
+            scripts.push({
+                type: 'img',
+                img: null == (_task_recorder = task.recorder) ? void 0 : null == (_task_recorder_ = _task_recorder[0]) ? void 0 : _task_recorder_.screenshot,
+                duration: actionDuration,
+                camera: 'Sleep' === task.subType ? fullPageCameraState : void 0,
                 title,
-                subTitle
+                subTitle,
+                imageWidth: (null == (_task_uiContext2 = task.uiContext) ? void 0 : null == (_task_uiContext_size2 = _task_uiContext2.size) ? void 0 : _task_uiContext_size2.width) || imageWidth,
+                imageHeight: (null == (_task_uiContext3 = task.uiContext) ? void 0 : null == (_task_uiContext_size3 = _task_uiContext3.size) ? void 0 : _task_uiContext_size3.height) || imageHeight
             });
         } else {
-            var _task_recorder_3, _task_recorder3;
+            var _task_recorder_1, _task_recorder1;
             const title = typeStr(task);
             const subTitle = paramStr(task);
-            const screenshot = null == (_task_recorder3 = task.recorder) ? void 0 : null == (_task_recorder_3 = _task_recorder3[task.recorder.length - 1]) ? void 0 : _task_recorder_3.screenshot;
+            const screenshot = null == (_task_recorder1 = task.recorder) ? void 0 : null == (_task_recorder_1 = _task_recorder1[task.recorder.length - 1]) ? void 0 : _task_recorder_1.screenshot;
             if (screenshot) {
-                var _task_uiContext_size6, _task_uiContext6, _task_uiContext_size7, _task_uiContext7;
+                var _task_uiContext_size4, _task_uiContext4, _task_uiContext_size5, _task_uiContext5;
                 scripts.push({
                     type: 'img',
                     img: screenshot,
@@ -276,13 +325,13 @@ const generateAnimationScripts = (execution, task, imageWidth, imageHeight)=>{
                     camera: fullPageCameraState,
                     title,
                     subTitle,
-                    imageWidth: (null == (_task_uiContext6 = task.uiContext) ? void 0 : null == (_task_uiContext_size6 = _task_uiContext6.size) ? void 0 : _task_uiContext_size6.width) || imageWidth,
-                    imageHeight: (null == (_task_uiContext7 = task.uiContext) ? void 0 : null == (_task_uiContext_size7 = _task_uiContext7.size) ? void 0 : _task_uiContext_size7.height) || imageHeight
+                    imageWidth: (null == (_task_uiContext4 = task.uiContext) ? void 0 : null == (_task_uiContext_size4 = _task_uiContext4.size) ? void 0 : _task_uiContext_size4.width) || imageWidth,
+                    imageHeight: (null == (_task_uiContext5 = task.uiContext) ? void 0 : null == (_task_uiContext_size5 = _task_uiContext5.size) ? void 0 : _task_uiContext_size5.height) || imageHeight
                 });
             }
         }
         if ('finished' !== task.status) {
-            var _task_uiContext_size8, _task_uiContext8, _task_uiContext_size9, _task_uiContext9;
+            var _task_uiContext_size6, _task_uiContext6, _task_uiContext_size7, _task_uiContext7;
             errorStateFlag = true;
             const errorTitle = typeStr(task);
             const errorMsg = task.errorMessage || 'unknown error';
@@ -294,8 +343,8 @@ const generateAnimationScripts = (execution, task, imageWidth, imageHeight)=>{
                 duration: stillDuration,
                 title: errorTitle,
                 subTitle: errorSubTitle,
-                imageWidth: (null == (_task_uiContext8 = task.uiContext) ? void 0 : null == (_task_uiContext_size8 = _task_uiContext8.size) ? void 0 : _task_uiContext_size8.width) || imageWidth,
-                imageHeight: (null == (_task_uiContext9 = task.uiContext) ? void 0 : null == (_task_uiContext_size9 = _task_uiContext9.size) ? void 0 : _task_uiContext_size9.height) || imageHeight
+                imageWidth: (null == (_task_uiContext6 = task.uiContext) ? void 0 : null == (_task_uiContext_size6 = _task_uiContext6.size) ? void 0 : _task_uiContext_size6.width) || imageWidth,
+                imageHeight: (null == (_task_uiContext7 = task.uiContext) ? void 0 : null == (_task_uiContext_size7 = _task_uiContext7.size) ? void 0 : _task_uiContext_size7.height) || imageHeight
             });
             return;
         }
@@ -304,7 +353,7 @@ const generateAnimationScripts = (execution, task, imageWidth, imageHeight)=>{
         title: 'Done',
         subTitle: initSubTitle,
         type: 'img',
-        duration: stillDuration,
+        duration: lastFrameDuration,
         camera: fullPageCameraState
     });
     return scripts;

package/dist/lib/component/blackboard/index.css

@@ -31,3 +31,11 @@
   width: 100%;
 }
 
+[data-theme="dark"] .blackboard .footer, [data-theme="dark"] .blackboard .bottom-tip-item {
+  color: rgba(255, 255, 255, .45);
+}
+
+[data-theme="dark"] .blackboard-main-content canvas {
+  border-color: rgba(255, 255, 255, .12);
+}
+