@midscene/visualizer 0.28.1-beta-20250909063633.0 → 0.28.2-beta-20250909133629.0

package/dist/es/component/universal-playground/providers/storage-provider.mjs

@@ -0,0 +1,107 @@
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+class LocalStorageProvider {
+    async saveMessages(messages) {
+        try {
+            const lightMessages = messages.map((msg)=>({
+                    ...msg,
+                    result: void 0
+                }));
+            localStorage.setItem(this.messagesKey, JSON.stringify(lightMessages));
+        } catch (error) {
+            console.error('Failed to save messages to localStorage:', error);
+        }
+    }
+    async loadMessages() {
+        try {
+            const stored = localStorage.getItem(this.messagesKey);
+            if (!stored) return [];
+            const messages = JSON.parse(stored);
+            const restoredMessages = await Promise.all(messages.map(async (msg)=>{
+                if ('result' === msg.type && msg.id) {
+                    const resultKey = `${this.resultsKey}-${msg.id}`;
+                    const storedResult = localStorage.getItem(resultKey);
+                    if (storedResult) try {
+                        const resultItem = JSON.parse(storedResult);
+                        return {
+                            ...msg,
+                            ...resultItem
+                        };
+                    } catch (e) {
+                        console.warn('Failed to parse stored result:', e);
+                    }
+                }
+                return msg;
+            }));
+            return restoredMessages;
+        } catch (error) {
+            console.error('Failed to load messages from localStorage:', error);
+            return [];
+        }
+    }
+    async clearMessages() {
+        try {
+            localStorage.removeItem(this.messagesKey);
+            const keys = Object.keys(localStorage);
+            keys.forEach((key)=>{
+                if (key.startsWith(this.resultsKey)) localStorage.removeItem(key);
+            });
+        } catch (error) {
+            console.error('Failed to clear messages from localStorage:', error);
+        }
+    }
+    async saveResult(id, result) {
+        try {
+            const resultKey = `${this.resultsKey}-${id}`;
+            localStorage.setItem(resultKey, JSON.stringify(result));
+        } catch (error) {
+            console.error('Failed to save result to localStorage:', error);
+        }
+    }
+    constructor(namespace = 'playground'){
+        _define_property(this, "messagesKey", void 0);
+        _define_property(this, "resultsKey", void 0);
+        this.messagesKey = `${namespace}-messages`;
+        this.resultsKey = `${namespace}-results`;
+    }
+}
+class MemoryStorageProvider {
+    async saveMessages(messages) {
+        this.messages = [
+            ...messages
+        ];
+    }
+    async loadMessages() {
+        return [
+            ...this.messages
+        ];
+    }
+    async clearMessages() {
+        this.messages = [];
+        this.results.clear();
+    }
+    async saveResult(id, result) {
+        this.results.set(id, result);
+    }
+    constructor(){
+        _define_property(this, "messages", []);
+        _define_property(this, "results", new Map());
+    }
+}
+class NoOpStorageProvider {
+    async saveMessages(_messages) {}
+    async loadMessages() {
+        return [];
+    }
+    async clearMessages() {}
+    async saveResult(_id, _result) {}
+}
+export { LocalStorageProvider, MemoryStorageProvider, NoOpStorageProvider };

package/dist/es/hooks/usePlaygroundExecution.mjs

@@ -0,0 +1,170 @@
+import { useCallback } from "react";
+import { BLANK_RESULT } from "../types.mjs";
+import { allScriptsFromDump } from "../utils/replay-scripts.mjs";
+const noReplayAPIs = [
+    'aiQuery',
+    'aiAssert'
+];
+function usePlaygroundExecution(playgroundSDK, storage, actionSpace, loading, setLoading, infoList, setInfoList, replayCounter, setReplayCounter, verticalMode, currentRunningIdRef, interruptedFlagRef) {
+    const handleRun = useCallback(async (value)=>{
+        Date.now();
+        null == actionSpace || actionSpace.find((a)=>a.interfaceAlias === value.type || a.name === value.type);
+        const thisRunningId = Date.now();
+        const actionType = value.type;
+        const displayContent = `${value.type}: ${value.prompt || JSON.stringify(value.params)}`;
+        const userItem = {
+            id: `user-${Date.now()}`,
+            type: 'user',
+            content: displayContent,
+            timestamp: new Date()
+        };
+        setInfoList((prev)=>[
+                ...prev,
+                userItem
+            ]);
+        setLoading(true);
+        const result = {
+            ...BLANK_RESULT
+        };
+        const systemItem = {
+            id: `system-${thisRunningId}`,
+            type: 'system',
+            content: '',
+            timestamp: new Date(),
+            loading: true,
+            loadingProgressText: ''
+        };
+        setInfoList((prev)=>[
+                ...prev,
+                systemItem
+            ]);
+        try {
+            currentRunningIdRef.current = thisRunningId;
+            interruptedFlagRef.current[thisRunningId] = false;
+            if (playgroundSDK.onProgressUpdate) playgroundSDK.onProgressUpdate((tip)=>{
+                if (interruptedFlagRef.current[thisRunningId]) return;
+                const progressItem = {
+                    id: `progress-${thisRunningId}-${Date.now()}`,
+                    type: 'progress',
+                    content: tip,
+                    timestamp: new Date()
+                };
+                setInfoList((prev)=>[
+                        ...prev,
+                        progressItem
+                    ]);
+            });
+            result.result = await playgroundSDK.executeAction(actionType, value, {
+                requestId: thisRunningId.toString()
+            });
+            if ('object' == typeof result.result && null !== result.result) {
+                const resultObj = result.result;
+                if (resultObj.dump) result.dump = resultObj.dump;
+                if (resultObj.reportHTML) result.reportHTML = resultObj.reportHTML;
+                if (resultObj.error) result.error = resultObj.error;
+                if (void 0 !== resultObj.result) result.result = resultObj.result;
+            }
+        } catch (e) {
+            result.error = (null == e ? void 0 : e.message) || String(e);
+            console.error('Playground execution error:', e);
+        }
+        if (interruptedFlagRef.current[thisRunningId]) return;
+        setLoading(false);
+        currentRunningIdRef.current = null;
+        let replayInfo = null;
+        let counter = replayCounter;
+        if ((null == result ? void 0 : result.dump) && !noReplayAPIs.includes(actionType)) {
+            const info = allScriptsFromDump(result.dump);
+            setReplayCounter((c)=>c + 1);
+            replayInfo = info;
+            counter = replayCounter + 1;
+        }
+        setInfoList((prev)=>prev.map((item)=>item.id === `system-${thisRunningId}` ? {
+                    ...item,
+                    content: '',
+                    loading: false,
+                    loadingProgressText: ''
+                } : item));
+        const resultItem = {
+            id: `result-${thisRunningId}`,
+            type: 'result',
+            content: 'Execution result',
+            timestamp: new Date(),
+            result: result,
+            loading: false,
+            replayScriptsInfo: replayInfo,
+            replayCounter: counter,
+            loadingProgressText: '',
+            verticalMode: verticalMode
+        };
+        setInfoList((prev)=>[
+                ...prev,
+                resultItem
+            ]);
+        if (null == storage ? void 0 : storage.saveResult) try {
+            await storage.saveResult(resultItem.id, resultItem);
+        } catch (error) {
+            console.error('Failed to save result:', error);
+        }
+        const separatorItem = {
+            id: `separator-${thisRunningId}`,
+            type: 'separator',
+            content: 'New Session',
+            timestamp: new Date()
+        };
+        setInfoList((prev)=>[
+                ...prev,
+                separatorItem
+            ]);
+    }, [
+        playgroundSDK,
+        storage,
+        actionSpace,
+        setLoading,
+        setInfoList,
+        replayCounter,
+        setReplayCounter,
+        verticalMode,
+        currentRunningIdRef,
+        interruptedFlagRef
+    ]);
+    const handleStop = useCallback(async ()=>{
+        const thisRunningId = currentRunningIdRef.current;
+        if (thisRunningId && playgroundSDK.cancelExecution) try {
+            await playgroundSDK.cancelExecution(thisRunningId.toString());
+            interruptedFlagRef.current[thisRunningId] = true;
+            setLoading(false);
+            setInfoList((prev)=>prev.map((item)=>item.id === `system-${thisRunningId}` && item.loading ? {
+                        ...item,
+                        content: 'Operation stopped',
+                        loading: false,
+                        loadingProgressText: ''
+                    } : item));
+            const separatorItem = {
+                id: `separator-${thisRunningId}`,
+                type: 'separator',
+                content: 'New Session',
+                timestamp: new Date()
+            };
+            setInfoList((prev)=>[
+                    ...prev,
+                    separatorItem
+                ]);
+        } catch (error) {
+            console.error('Failed to stop execution:', error);
+        }
+    }, [
+        playgroundSDK,
+        currentRunningIdRef,
+        interruptedFlagRef,
+        setLoading,
+        setInfoList
+    ]);
+    const canStop = loading && !!currentRunningIdRef.current && !!playgroundSDK.cancelExecution;
+    return {
+        handleRun,
+        handleStop,
+        canStop
+    };
+}
+export { usePlaygroundExecution };

package/dist/es/hooks/usePlaygroundState.mjs

@@ -0,0 +1,173 @@
+import { useCallback, useEffect, useRef, useState } from "react";
+import { WELCOME_MESSAGE_TEMPLATE } from "../types.mjs";
+function usePlaygroundState(playgroundSDK, storage, contextProvider) {
+    const [loading, setLoading] = useState(false);
+    const [infoList, setInfoList] = useState([]);
+    const [actionSpace, setActionSpace] = useState([]);
+    const [actionSpaceLoading, setActionSpaceLoading] = useState(true);
+    const [uiContextPreview, setUiContextPreview] = useState();
+    const [showScrollToBottomButton, setShowScrollToBottomButton] = useState(false);
+    const [verticalMode, setVerticalMode] = useState(false);
+    const [replayCounter, setReplayCounter] = useState(0);
+    const infoListRef = useRef(null);
+    const currentRunningIdRef = useRef(null);
+    const interruptedFlagRef = useRef({});
+    const welcomeMessage = {
+        ...WELCOME_MESSAGE_TEMPLATE,
+        id: 'welcome',
+        timestamp: new Date()
+    };
+    useEffect(()=>{
+        const initializeMessages = async ()=>{
+            if (null == storage ? void 0 : storage.loadMessages) try {
+                const storedMessages = await storage.loadMessages();
+                setInfoList([
+                    welcomeMessage,
+                    ...storedMessages
+                ]);
+            } catch (error) {
+                console.error('Failed to load messages:', error);
+                setInfoList([
+                    welcomeMessage
+                ]);
+            }
+            else setInfoList([
+                welcomeMessage
+            ]);
+        };
+        initializeMessages();
+    }, [
+        storage
+    ]);
+    useEffect(()=>{
+        if ((null == storage ? void 0 : storage.saveMessages) && infoList.length > 1) storage.saveMessages(infoList).catch((error)=>{
+            console.error('Failed to save messages:', error);
+        });
+    }, [
+        infoList,
+        storage
+    ]);
+    useEffect(()=>{
+        if (!(null == contextProvider ? void 0 : contextProvider.getUIContext) || uiContextPreview) return;
+        contextProvider.getUIContext().then((context)=>setUiContextPreview(context)).catch((error)=>{
+            console.error('Failed to get UI context:', error);
+        });
+    }, [
+        contextProvider,
+        uiContextPreview
+    ]);
+    useEffect(()=>{
+        const loadActionSpace = async ()=>{
+            setActionSpaceLoading(true);
+            try {
+                var _contextProvider_getUIContext;
+                const context = uiContextPreview || await (null == contextProvider ? void 0 : null == (_contextProvider_getUIContext = contextProvider.getUIContext) ? void 0 : _contextProvider_getUIContext.call(contextProvider));
+                const space = await playgroundSDK.getActionSpace(context);
+                setActionSpace(space || []);
+            } catch (error) {
+                console.error('Failed to load action space:', error);
+                setActionSpace([]);
+            } finally{
+                setActionSpaceLoading(false);
+            }
+        };
+        loadActionSpace();
+    }, [
+        playgroundSDK,
+        uiContextPreview,
+        contextProvider
+    ]);
+    useEffect(()=>{
+        const sizeThreshold = 750;
+        setVerticalMode(window.innerWidth < sizeThreshold);
+        const handleResize = ()=>{
+            setVerticalMode(window.innerWidth < sizeThreshold);
+        };
+        window.addEventListener('resize', handleResize);
+        return ()=>window.removeEventListener('resize', handleResize);
+    }, []);
+    const scrollToBottom = useCallback(()=>{
+        setTimeout(()=>{
+            if (infoListRef.current) infoListRef.current.scrollTop = infoListRef.current.scrollHeight;
+        }, 100);
+    }, []);
+    const checkIfScrolledToBottom = useCallback(()=>{
+        if (infoListRef.current) {
+            const { scrollTop, scrollHeight, clientHeight } = infoListRef.current;
+            const isAtBottom = scrollTop + clientHeight >= scrollHeight - 10;
+            setShowScrollToBottomButton(!isAtBottom);
+        }
+    }, []);
+    const handleScrollToBottom = useCallback(()=>{
+        if (infoListRef.current) {
+            infoListRef.current.scrollTo({
+                top: infoListRef.current.scrollHeight,
+                behavior: 'smooth'
+            });
+            setShowScrollToBottomButton(false);
+        }
+    }, []);
+    useEffect(()=>{
+        if (infoList.length > 0) scrollToBottom();
+    }, [
+        infoList,
+        scrollToBottom
+    ]);
+    useEffect(()=>{
+        const container = infoListRef.current;
+        if (container) {
+            container.addEventListener('scroll', checkIfScrolledToBottom);
+            checkIfScrolledToBottom();
+            return ()=>{
+                container.removeEventListener('scroll', checkIfScrolledToBottom);
+            };
+        }
+    }, [
+        checkIfScrolledToBottom
+    ]);
+    const clearInfoList = useCallback(async ()=>{
+        setInfoList([
+            welcomeMessage
+        ]);
+        if (null == storage ? void 0 : storage.clearMessages) try {
+            await storage.clearMessages();
+        } catch (error) {
+            console.error('Failed to clear stored messages:', error);
+        }
+    }, [
+        storage,
+        welcomeMessage
+    ]);
+    const refreshContext = useCallback(async ()=>{
+        if (null == contextProvider ? void 0 : contextProvider.refreshContext) try {
+            const newContext = await contextProvider.refreshContext();
+            setUiContextPreview(newContext);
+        } catch (error) {
+            console.error('Failed to refresh context:', error);
+        }
+    }, [
+        contextProvider
+    ]);
+    return {
+        loading,
+        setLoading,
+        infoList,
+        setInfoList,
+        actionSpace,
+        actionSpaceLoading,
+        uiContextPreview,
+        setUiContextPreview,
+        showScrollToBottomButton,
+        verticalMode,
+        replayCounter,
+        setReplayCounter,
+        infoListRef,
+        currentRunningIdRef,
+        interruptedFlagRef,
+        clearInfoList,
+        refreshContext,
+        handleScrollToBottom,
+        scrollToBottom
+    };
+}
+export { usePlaygroundState };

package/dist/es/icons/avatar.mjs

@@ -0,0 +1,14 @@
+import { jsx } from "react/jsx-runtime";
+import "react";
+const SvgAvatar = (props)=>/*#__PURE__*/ jsx("svg", {
+        width: 20,
+        height: 20,
+        fill: "currentColor",
+        viewBox: "0 0 24 24",
+        ...props,
+        children: /*#__PURE__*/ jsx("path", {
+            d: "M12 2C6.48 2 2 6.48 2 12s4.48 10 10 10 10-4.48 10-10S17.52 2 12 2m-2 15-5-5 1.41-1.41L10 14.17l7.59-7.59L19 8z"
+        })
+    });
+const avatar = SvgAvatar;
+export { avatar as default };

package/dist/es/index.mjs

@@ -1,19 +1,23 @@
 import "./component/playground/index.css";
-import { allScriptsFromDump, generateAnimationScripts } from "./component/replay-scripts.mjs";
-import { useEnvConfig } from "./component/store/store.mjs";
-import { colorForName, globalThemeConfig, highlightColorForType } from "./component/color.mjs";
-import { EnvConfig } from "./component/env-config.mjs";
-import { Logo } from "./component/logo.mjs";
-import { iconForStatus, timeCostStrElement } from "./component/misc.mjs";
-import { useServerValid } from "./component/playground/useServerValid.mjs";
-import { PlaygroundResultView } from "./component/playground/PlaygroundResult.mjs";
-import { ServiceModeControl } from "./component/playground/ServiceModeControl.mjs";
-import { ContextPreview } from "./component/playground/ContextPreview.mjs";
-import { PromptInput } from "./component/playground/PromptInput.mjs";
-import { Player } from "./component/player.mjs";
-import { Blackboard } from "./component/blackboard.mjs";
-import { GithubStar } from "./component/github-star.mjs";
-import { actionNameForType, getPlaceholderForType, staticAgentFromContext } from "./component/playground/playground-utils.mjs";
-import { filterBase64Value, timeStr } from "./utils.mjs";
-import shiny_text from "./component/shiny-text.mjs";
-export { Blackboard, ContextPreview, EnvConfig, GithubStar, Logo, Player, PlaygroundResultView, PromptInput, ServiceModeControl, shiny_text as ShinyText, actionNameForType, allScriptsFromDump, colorForName, filterBase64Value, generateAnimationScripts, getPlaceholderForType, globalThemeConfig, highlightColorForType, iconForStatus, staticAgentFromContext, timeCostStrElement, timeStr, useEnvConfig, useServerValid };
+import "./component/universal-playground/index.css";
+import { allScriptsFromDump, generateAnimationScripts } from "./utils/replay-scripts.mjs";
+import { useEnvConfig } from "./store/store.mjs";
+import { colorForName, globalThemeConfig, highlightColorForType } from "./utils/color.mjs";
+import { EnvConfig } from "./component/env-config/index.mjs";
+import { Logo } from "./component/logo/index.mjs";
+import { iconForStatus, timeCostStrElement } from "./component/misc/index.mjs";
+import { useServerValid } from "./hooks/useServerValid.mjs";
+import { PlaygroundResultView } from "./component/playground-result/index.mjs";
+import { ServiceModeControl } from "./component/service-mode-control/index.mjs";
+import { ContextPreview } from "./component/context-preview/index.mjs";
+import { PromptInput } from "./component/prompt-input/index.mjs";
+import { Player } from "./component/player/index.mjs";
+import { Blackboard } from "./component/blackboard/index.mjs";
+import { GithubStar } from "./component/github-star/index.mjs";
+import { actionNameForType, getPlaceholderForType, staticAgentFromContext } from "./utils/playground-utils.mjs";
+import { filterBase64Value, timeStr } from "./utils/index.mjs";
+import shiny_text from "./component/shiny-text/index.mjs";
+import universal_playground, { UniversalPlayground } from "./component/universal-playground/index.mjs";
+import { LocalStorageProvider, MemoryStorageProvider, NoOpStorageProvider } from "./component/universal-playground/providers/storage-provider.mjs";
+import { AgentContextProvider, BaseContextProvider, NoOpContextProvider, StaticContextProvider } from "./component/universal-playground/providers/context-provider.mjs";
+export { AgentContextProvider, BaseContextProvider, Blackboard, ContextPreview, EnvConfig, GithubStar, LocalStorageProvider, Logo, MemoryStorageProvider, NoOpContextProvider, NoOpStorageProvider, Player, PlaygroundResultView, PromptInput, ServiceModeControl, shiny_text as ShinyText, StaticContextProvider, UniversalPlayground, universal_playground as UniversalPlaygroundDefault, actionNameForType, allScriptsFromDump, colorForName, filterBase64Value, generateAnimationScripts, getPlaceholderForType, globalThemeConfig, highlightColorForType, iconForStatus, staticAgentFromContext, timeCostStrElement, timeStr, useEnvConfig, useServerValid };

package/dist/es/{component/playground/types.mjs → types.mjs}

@@ -67,4 +67,26 @@ const extractDefaultValue = (field)=>{
         currentField = currentField._def.innerType;
     }
 };
-export { VALIDATION_CONSTANTS, extractDefaultValue, isLocateField, isZodObjectSchema, unwrapZodType };
+const WELCOME_MESSAGE_TEMPLATE = {
+    type: 'system',
+    content: `
+      Welcome to Midscene.js Playground!
+      
+      This is a panel for experimenting and testing Midscene.js features. You can use natural language instructions to operate the web page, such as clicking buttons, filling in forms, querying information, etc.
+      
+      Please enter your instructions in the input box below to start experiencing.
+    `,
+    loading: false,
+    result: void 0,
+    replayScriptsInfo: null,
+    replayCounter: 0,
+    loadingProgressText: '',
+    verticalMode: false
+};
+const BLANK_RESULT = {
+    result: void 0,
+    dump: null,
+    reportHTML: null,
+    error: null
+};
+export { BLANK_RESULT, VALIDATION_CONSTANTS, WELCOME_MESSAGE_TEMPLATE, extractDefaultValue, isLocateField, isZodObjectSchema, unwrapZodType };

package/dist/es/{component/playground/playground-constants.mjs → utils/constants.mjs}

@@ -1,7 +1,6 @@
 import { jsx, jsxs } from "react/jsx-runtime";
 import { Alert } from "antd";
-import shiny_text from "../shiny-text.mjs";
-import "./index.css";
+import shiny_text from "../component/shiny-text/index.mjs";
 const errorMessageServerNotReady = /*#__PURE__*/ jsxs("span", {
     children: [
         "Don't worry, just one more step to launch the playground server.",

package/dist/es/{component/playground → utils}/playground-utils.mjs

@@ -1,5 +1,5 @@
 import { StaticPage, StaticPageAgent } from "@midscene/web/static";
-import { isZodObjectSchema, unwrapZodType } from "./types.mjs";
+import { isZodObjectSchema, unwrapZodType } from "../types.mjs";
 const actionNameForType = (type)=>{
     const typeWithoutAi = type.startsWith('ai') ? type.slice(2) : type;
     const fullName = typeWithoutAi.replace(/([A-Z])/g, ' $1').trim();

package/dist/es/{component → utils}/replay-scripts.mjs

@@ -1,6 +1,5 @@
 'use client';
-import "./player.css";
-import { mousePointer } from "../utils.mjs";
+import { mousePointer } from "./index.mjs";
 import { paramStr, typeStr } from "@midscene/core/agent";
 import { treeToList } from "@midscene/shared/extractor";
 const stillDuration = 900;