@midscene/visualizer 0.26.2 → 0.26.3-beta-20250813021342.0

package/dist/es/component/playground/ContextPreview.mjs

@@ -0,0 +1,39 @@
+import { jsx, jsxs } from "react/jsx-runtime";
+import { Button } from "antd";
+import blackboard from "../blackboard.mjs";
+import { iconForStatus } from "../misc.mjs";
+import playground_demo_ui_context from "../playground-demo-ui-context.json";
+import "./index.css";
+const ContextPreview = (param)=>{
+    let { uiContextPreview, setUiContextPreview, showContextPreview } = param;
+    if (!showContextPreview) return null;
+    return /*#__PURE__*/ jsxs("div", {
+        className: "form-part context-panel",
+        children: [
+            /*#__PURE__*/ jsx("h3", {
+                children: "UI Context"
+            }),
+            uiContextPreview ? /*#__PURE__*/ jsx(blackboard, {
+                uiContext: uiContextPreview,
+                hideController: true
+            }) : /*#__PURE__*/ jsxs("div", {
+                children: [
+                    iconForStatus('failed'),
+                    " No UI context",
+                    /*#__PURE__*/ jsx(Button, {
+                        type: "link",
+                        onClick: (e)=>{
+                            e.preventDefault();
+                            setUiContextPreview(playground_demo_ui_context);
+                        },
+                        children: "Load Demo"
+                    }),
+                    /*#__PURE__*/ jsx("div", {
+                        children: "To load the UI context, you can either use the demo data above, or click the 'Send to Playground' in the report page."
+                    })
+                ]
+            })
+        ]
+    });
+};
+export { ContextPreview };

package/dist/es/component/playground/HistorySelector.mjs

@@ -0,0 +1,193 @@
+import { Fragment, jsx, jsxs } from "react/jsx-runtime";
+import { Button, Input, Modal, Typography } from "antd";
+import { useMemo, useState } from "react";
+import icons_close from "../../icons/close.mjs";
+import icons_history from "../../icons/history.mjs";
+import magnifying_glass from "../../icons/magnifying-glass.mjs";
+import { useHistoryStore } from "../store/history.mjs";
+import "./index.css";
+const { Text } = Typography;
+const HistorySelector = (param)=>{
+    let { onSelect } = param;
+    const [isModalOpen, setIsModalOpen] = useState(false);
+    const [searchText, setSearchText] = useState('');
+    const history = useHistoryStore((state)=>state.history);
+    const clearHistory = useHistoryStore((state)=>state.clearHistory);
+    const groupedHistory = useMemo(()=>{
+        const now = Date.now();
+        const sevenDaysAgo = now - 604800000;
+        const oneYearAgo = now - 31536000000;
+        const filteredHistory = history.filter((item)=>item.prompt.toLowerCase().includes(searchText.toLowerCase()));
+        const groups = {
+            recent7Days: filteredHistory.filter((item)=>item.timestamp >= sevenDaysAgo),
+            recent1Year: filteredHistory.filter((item)=>item.timestamp < sevenDaysAgo && item.timestamp >= oneYearAgo),
+            older: filteredHistory.filter((item)=>item.timestamp < oneYearAgo)
+        };
+        return groups;
+    }, [
+        history,
+        searchText
+    ]);
+    const handleHistoryClick = (item)=>{
+        onSelect(item);
+        setIsModalOpen(false);
+    };
+    const handleClearHistory = ()=>{
+        clearHistory();
+        setSearchText('');
+        setIsModalOpen(false);
+    };
+    const renderHistoryGroup = (title, items)=>{
+        if (0 === items.length) return null;
+        return /*#__PURE__*/ jsxs("div", {
+            className: "history-group",
+            children: [
+                /*#__PURE__*/ jsx("div", {
+                    className: "history-group-title",
+                    children: title
+                }),
+                items.map((item, index)=>/*#__PURE__*/ jsx("div", {
+                        className: "history-item",
+                        onClick: ()=>handleHistoryClick(item),
+                        children: item.prompt
+                    }, `${item.timestamp}-${index}`))
+            ]
+        }, title);
+    };
+    return /*#__PURE__*/ jsxs(Fragment, {
+        children: [
+            /*#__PURE__*/ jsx("div", {
+                className: "selector-trigger",
+                onClick: ()=>setIsModalOpen(true),
+                children: /*#__PURE__*/ jsx(icons_history, {
+                    width: 24,
+                    height: 24
+                })
+            }),
+            /*#__PURE__*/ jsx(Modal, {
+                open: isModalOpen,
+                onCancel: ()=>setIsModalOpen(false),
+                footer: null,
+                width: "100%",
+                closable: false,
+                centered: false,
+                transitionName: "",
+                maskTransitionName: "",
+                style: {
+                    margin: 0,
+                    padding: 0,
+                    maxWidth: 'none',
+                    top: 'auto',
+                    bottom: 0
+                },
+                styles: {
+                    wrapper: {
+                        alignItems: 'flex-end',
+                        justifyContent: 'center',
+                        paddingBottom: 0,
+                        display: 'flex'
+                    },
+                    body: {
+                        height: '70vh',
+                        padding: 0,
+                        margin: 0
+                    },
+                    content: {
+                        height: '70vh',
+                        borderRadius: '12px 12px 0 0',
+                        margin: 0,
+                        padding: 0,
+                        marginBottom: 0,
+                        position: 'fixed',
+                        bottom: 0,
+                        left: 0,
+                        right: 0
+                    }
+                },
+                maskClosable: true,
+                destroyOnClose: true,
+                children: /*#__PURE__*/ jsxs("div", {
+                    className: "history-modal-container",
+                    children: [
+                        /*#__PURE__*/ jsxs("div", {
+                            className: "history-modal-header",
+                            children: [
+                                /*#__PURE__*/ jsxs(Text, {
+                                    strong: true,
+                                    style: {
+                                        fontSize: '16px'
+                                    },
+                                    children: [
+                                        "History (",
+                                        history.length,
+                                        ")"
+                                    ]
+                                }),
+                                /*#__PURE__*/ jsx(Button, {
+                                    size: "small",
+                                    type: "text",
+                                    icon: /*#__PURE__*/ jsx(icons_close, {
+                                        width: 16,
+                                        height: 16
+                                    }),
+                                    onClick: ()=>setIsModalOpen(false),
+                                    className: "close-button"
+                                })
+                            ]
+                        }),
+                        /*#__PURE__*/ jsx("div", {
+                            className: "history-search-section",
+                            children: /*#__PURE__*/ jsxs("div", {
+                                className: "search-input-wrapper",
+                                children: [
+                                    /*#__PURE__*/ jsx(Input, {
+                                        placeholder: "Search",
+                                        value: searchText,
+                                        onChange: (e)=>setSearchText(e.target.value),
+                                        prefix: /*#__PURE__*/ jsx(magnifying_glass, {
+                                            width: 18,
+                                            height: 18
+                                        }),
+                                        className: "search-input",
+                                        allowClear: true
+                                    }),
+                                    /*#__PURE__*/ jsx(Button, {
+                                        type: "link",
+                                        onClick: handleClearHistory,
+                                        className: "clear-button",
+                                        disabled: 0 === history.length,
+                                        children: "Clear"
+                                    })
+                                ]
+                            })
+                        }),
+                        /*#__PURE__*/ jsx("div", {
+                            className: "history-content",
+                            children: 0 === history.length ? /*#__PURE__*/ jsx("div", {
+                                className: "no-results",
+                                children: /*#__PURE__*/ jsx(Text, {
+                                    type: "secondary",
+                                    children: "No history record"
+                                })
+                            }) : /*#__PURE__*/ jsxs(Fragment, {
+                                children: [
+                                    renderHistoryGroup('Last 7 days', groupedHistory.recent7Days),
+                                    renderHistoryGroup('Last 1 year', groupedHistory.recent1Year),
+                                    renderHistoryGroup('Earlier', groupedHistory.older),
+                                    searchText && 0 === groupedHistory.recent7Days.length && 0 === groupedHistory.recent1Year.length && 0 === groupedHistory.older.length && /*#__PURE__*/ jsx("div", {
+                                        className: "no-results",
+                                        children: /*#__PURE__*/ jsx(Text, {
+                                            type: "secondary",
+                                            children: "No matching history record"
+                                        })
+                                    })
+                                ]
+                            })
+                        })
+                    ]
+                })
+            })
+        ]
+    });
+};
+export { HistorySelector };

package/dist/es/component/playground/PlaygroundResult.mjs

@@ -0,0 +1,60 @@
+import { jsx, jsxs } from "react/jsx-runtime";
+import { LoadingOutlined } from "@ant-design/icons";
+import { Spin } from "antd";
+import { Player } from "../player.mjs";
+import shiny_text from "../shiny-text.mjs";
+import { emptyResultTip, serverLaunchTip } from "./playground-constants.mjs";
+import "./index.css";
+const PlaygroundResultView = (param)=>{
+    let { result, loading, serverValid, serviceMode, replayScriptsInfo, replayCounter, loadingProgressText, verticalMode = false, notReadyMessage, fitMode } = param;
+    let resultWrapperClassName = 'result-wrapper';
+    if (verticalMode) resultWrapperClassName += ' vertical-mode-result';
+    if (replayScriptsInfo && verticalMode) resultWrapperClassName += ' result-wrapper-compact';
+    let resultDataToShow = emptyResultTip;
+    if (serverValid || 'Server' !== serviceMode) {
+        if (loading) resultDataToShow = /*#__PURE__*/ jsxs("div", {
+            className: "loading-container",
+            children: [
+                /*#__PURE__*/ jsx(Spin, {
+                    spinning: loading,
+                    indicator: /*#__PURE__*/ jsx(LoadingOutlined, {
+                        spin: true
+                    })
+                }),
+                /*#__PURE__*/ jsx("div", {
+                    className: "loading-progress-text loading-progress-text-progress",
+                    children: /*#__PURE__*/ jsx(shiny_text, {
+                        text: loadingProgressText,
+                        speed: 3
+                    })
+                })
+            ]
+        });
+        else if (replayScriptsInfo) resultDataToShow = /*#__PURE__*/ jsx(Player, {
+            replayScripts: replayScriptsInfo.scripts,
+            imageWidth: replayScriptsInfo.width,
+            imageHeight: replayScriptsInfo.height,
+            reportFileContent: ('In-Browser-Extension' === serviceMode || 'Server' === serviceMode) && (null == result ? void 0 : result.reportHTML) ? null == result ? void 0 : result.reportHTML : null,
+            fitMode: fitMode
+        }, replayCounter);
+        else if (null == result ? void 0 : result.error) resultDataToShow = /*#__PURE__*/ jsx("pre", {
+            children: null == result ? void 0 : result.error
+        });
+        else if ((null == result ? void 0 : result.result) !== void 0) resultDataToShow = 'string' == typeof (null == result ? void 0 : result.result) ? /*#__PURE__*/ jsx("pre", {
+            children: null == result ? void 0 : result.result
+        }) : /*#__PURE__*/ jsx("pre", {
+            children: JSON.stringify(null == result ? void 0 : result.result, null, 2)
+        });
+    } else resultDataToShow = serverLaunchTip(notReadyMessage);
+    return /*#__PURE__*/ jsx("div", {
+        className: resultWrapperClassName,
+        style: {
+            height: '100%',
+            display: 'flex',
+            flexDirection: 'column',
+            flex: '1 1 auto'
+        },
+        children: resultDataToShow
+    });
+};
+export { PlaygroundResultView };

package/dist/es/component/playground/PromptInput.mjs

@@ -0,0 +1,225 @@
+import { jsx, jsxs } from "react/jsx-runtime";
+import { BorderOutlined, SendOutlined } from "@ant-design/icons";
+import { Button, Form, Input, Radio, Space, Tooltip } from "antd";
+import { useCallback, useEffect, useRef, useState } from "react";
+import { useHistoryStore } from "../store/history.mjs";
+import { ConfigSelector } from "./ConfigSelector.mjs";
+import { HistorySelector } from "./HistorySelector.mjs";
+import { actionNameForType, getPlaceholderForType } from "./playground-utils.mjs";
+import "./index.css";
+const { TextArea } = Input;
+const PromptInput = (param)=>{
+    let { runButtonEnabled, form, serviceMode, selectedType, dryMode, stoppable, loading, onRun, onStop, clearPromptAfterRun = true } = param;
+    const [hoveringSettings, setHoveringSettings] = useState(false);
+    const [promptValue, setPromptValue] = useState('');
+    const placeholder = getPlaceholderForType(selectedType);
+    const textAreaRef = useRef(null);
+    const history = useHistoryStore((state)=>state.history);
+    const addHistory = useHistoryStore((state)=>state.addHistory);
+    const lastHistory = history[0];
+    useEffect(()=>{
+        if (lastHistory) {
+            form.setFieldsValue({
+                type: lastHistory.type || 'aiAction',
+                prompt: lastHistory.prompt || ''
+            });
+            setPromptValue(lastHistory.prompt || '');
+        } else {
+            form.setFieldsValue({
+                type: 'aiAction',
+                prompt: ''
+            });
+            setPromptValue('');
+        }
+    }, []);
+    const handleSelectHistory = useCallback((historyItem)=>{
+        form.setFieldsValue({
+            prompt: historyItem.prompt,
+            type: historyItem.type
+        });
+        setPromptValue(historyItem.prompt);
+    }, [
+        form
+    ]);
+    const handlePromptChange = useCallback((e)=>{
+        const value = e.target.value;
+        setPromptValue(value);
+        form.setFieldValue('prompt', value);
+    }, [
+        form
+    ]);
+    const isRunButtonEnabled = runButtonEnabled && promptValue.trim().length > 0;
+    const handleRunWithHistory = useCallback(()=>{
+        const values = form.getFieldsValue();
+        if (values.prompt) addHistory({
+            type: values.type,
+            prompt: values.prompt,
+            timestamp: Date.now()
+        });
+        onRun();
+        if (clearPromptAfterRun) {
+            setPromptValue('');
+            form.setFieldValue('prompt', '');
+        }
+    }, [
+        form,
+        addHistory,
+        onRun
+    ]);
+    const handleKeyDown = useCallback((e)=>{
+        if ('Enter' === e.key && e.metaKey && isRunButtonEnabled) {
+            handleRunWithHistory();
+            e.preventDefault();
+            e.stopPropagation();
+        } else if ('Enter' === e.key) setTimeout(()=>{
+            if (textAreaRef.current) {
+                const textarea = textAreaRef.current.resizableTextArea.textArea;
+                const selectionStart = textarea.selectionStart;
+                const value = textarea.value;
+                const lastNewlineIndex = value.lastIndexOf('\n');
+                const isAtLastLine = -1 === lastNewlineIndex || selectionStart > lastNewlineIndex;
+                if (isAtLastLine) textarea.scrollTop = textarea.scrollHeight;
+            }
+        }, 0);
+    }, [
+        handleRunWithHistory,
+        isRunButtonEnabled
+    ]);
+    const handleMouseEnter = useCallback(()=>{
+        setHoveringSettings(true);
+    }, []);
+    const handleMouseLeave = useCallback(()=>{
+        setHoveringSettings(false);
+    }, []);
+    const renderActionButton = useCallback(()=>{
+        const runButton = (text)=>/*#__PURE__*/ jsx(Button, {
+                type: "primary",
+                icon: /*#__PURE__*/ jsx(SendOutlined, {}),
+                style: {
+                    borderRadius: 20,
+                    zIndex: 999
+                },
+                onClick: handleRunWithHistory,
+                disabled: !isRunButtonEnabled,
+                loading: loading,
+                children: text
+            });
+        if (dryMode) return 'aiAction' === selectedType ? /*#__PURE__*/ jsx(Tooltip, {
+            title: "Start executing until some interaction actions need to be performed. You can see the process of planning and locating.",
+            children: runButton('Dry Run')
+        }) : runButton('Run');
+        if (stoppable) return /*#__PURE__*/ jsx(Button, {
+            icon: /*#__PURE__*/ jsx(BorderOutlined, {}),
+            onClick: onStop,
+            style: {
+                borderRadius: 20,
+                zIndex: 999
+            },
+            children: "Stop"
+        });
+        return runButton('Run');
+    }, [
+        dryMode,
+        loading,
+        handleRunWithHistory,
+        onStop,
+        isRunButtonEnabled,
+        selectedType,
+        stoppable
+    ]);
+    return /*#__PURE__*/ jsxs("div", {
+        className: "prompt-input-wrapper",
+        children: [
+            /*#__PURE__*/ jsxs(Space, {
+                className: "mode-radio-group-wrapper",
+                children: [
+                    /*#__PURE__*/ jsx(Form.Item, {
+                        name: "type",
+                        style: {
+                            margin: 0
+                        },
+                        children: /*#__PURE__*/ jsxs(Radio.Group, {
+                            buttonStyle: "solid",
+                            disabled: !runButtonEnabled,
+                            className: "mode-radio-group",
+                            children: [
+                                /*#__PURE__*/ jsx(Tooltip, {
+                                    title: "Auto Planning: plan the steps and execute",
+                                    children: /*#__PURE__*/ jsx(Radio.Button, {
+                                        value: "aiAction",
+                                        children: actionNameForType('aiAction')
+                                    })
+                                }),
+                                /*#__PURE__*/ jsx(Tooltip, {
+                                    title: "Extract data directly from the UI",
+                                    children: /*#__PURE__*/ jsx(Radio.Button, {
+                                        value: "aiQuery",
+                                        children: actionNameForType('aiQuery')
+                                    })
+                                }),
+                                /*#__PURE__*/ jsx(Tooltip, {
+                                    title: "Understand the UI and determine if the assertion is true",
+                                    children: /*#__PURE__*/ jsx(Radio.Button, {
+                                        value: "aiAssert",
+                                        children: actionNameForType('aiAssert')
+                                    })
+                                }),
+                                /*#__PURE__*/ jsx(Tooltip, {
+                                    title: "Instant Action: click something",
+                                    children: /*#__PURE__*/ jsx(Radio.Button, {
+                                        value: "aiTap",
+                                        children: actionNameForType('aiTap')
+                                    })
+                                })
+                            ]
+                        })
+                    }),
+                    /*#__PURE__*/ jsxs("div", {
+                        className: "action-icons",
+                        children: [
+                            /*#__PURE__*/ jsx(HistorySelector, {
+                                onSelect: handleSelectHistory
+                            }),
+                            /*#__PURE__*/ jsx("div", {
+                                className: hoveringSettings ? 'settings-wrapper settings-wrapper-hover' : 'settings-wrapper',
+                                onMouseEnter: handleMouseEnter,
+                                onMouseLeave: handleMouseLeave,
+                                children: /*#__PURE__*/ jsx(ConfigSelector, {
+                                    enableTracking: 'In-Browser-Extension' === serviceMode,
+                                    showDeepThinkOption: 'aiTap' === selectedType
+                                })
+                            })
+                        ]
+                    })
+                ]
+            }),
+            /*#__PURE__*/ jsxs("div", {
+                className: `main-side-console-input ${!runButtonEnabled ? 'disabled' : ''} ${loading ? 'loading' : ''}`,
+                children: [
+                    /*#__PURE__*/ jsx(Form.Item, {
+                        name: "prompt",
+                        style: {
+                            margin: 0
+                        },
+                        children: /*#__PURE__*/ jsx(TextArea, {
+                            className: "main-side-console-input-textarea",
+                            disabled: !runButtonEnabled,
+                            rows: 4,
+                            placeholder: placeholder,
+                            autoFocus: true,
+                            onKeyDown: handleKeyDown,
+                            onChange: handlePromptChange,
+                            value: promptValue,
+                            ref: textAreaRef
+                        })
+                    }),
+                    /*#__PURE__*/ jsx("div", {
+                        className: "form-controller-wrapper",
+                        children: renderActionButton()
+                    })
+                ]
+            })
+        ]
+    });
+};
+export { PromptInput };

package/dist/es/component/playground/ServiceModeControl.mjs

@@ -0,0 +1,100 @@
+import { Fragment, jsx, jsxs } from "react/jsx-runtime";
+import { overrideAIConfig } from "@midscene/shared/env";
+import { Button, Tooltip } from "antd";
+import { useEffect } from "react";
+import { EnvConfig } from "../env-config.mjs";
+import { iconForStatus } from "../misc.mjs";
+import { useEnvConfig } from "../store/store.mjs";
+import { overrideServerConfig } from "./playground-utils.mjs";
+import { useServerValid } from "./useServerValid.mjs";
+import "./index.css";
+const TITLE_TEXT = {
+    Server: 'Server Status',
+    'In-Browser': 'In-Browser'
+};
+const SWITCH_BUTTON_TEXT = {
+    Server: 'Switch to In-Browser Mode',
+    'In-Browser': 'Switch to Server Mode'
+};
+const ServiceModeControl = (param)=>{
+    let { serviceMode } = param;
+    const { setServiceMode, config } = useEnvConfig();
+    const serverValid = useServerValid('Server' === serviceMode);
+    const renderServerTip = ()=>{
+        if (serverValid) return /*#__PURE__*/ jsx(Tooltip, {
+            title: "Connected",
+            children: /*#__PURE__*/ jsx("div", {
+                className: "server-tip",
+                children: iconForStatus('connected')
+            })
+        });
+        return /*#__PURE__*/ jsx(Tooltip, {
+            title: "Connection failed",
+            children: /*#__PURE__*/ jsx("div", {
+                className: "server-tip",
+                children: iconForStatus('failed')
+            })
+        });
+    };
+    const renderSwitchButton = ()=>{
+        const nextMode = 'Server' === serviceMode ? 'In-Browser' : 'Server';
+        const buttonText = SWITCH_BUTTON_TEXT[serviceMode];
+        return /*#__PURE__*/ jsx(Tooltip, {
+            title: /*#__PURE__*/ jsxs("span", {
+                children: [
+                    "Server Mode: send the request through the server ",
+                    /*#__PURE__*/ jsx("br", {}),
+                    "In-Browser Mode: send the request through the browser fetch API (The AI service should support CORS in this case)"
+                ]
+            }),
+            children: /*#__PURE__*/ jsx(Button, {
+                type: "link",
+                onClick: (e)=>{
+                    e.preventDefault();
+                    setServiceMode(nextMode);
+                },
+                children: buttonText
+            })
+        });
+    };
+    useEffect(()=>{
+        overrideAIConfig(config);
+        if ('Server' === serviceMode) overrideServerConfig(config);
+    }, [
+        config,
+        serviceMode
+    ]);
+    const statusContent = 'Server' === serviceMode && renderServerTip();
+    const title = TITLE_TEXT[serviceMode];
+    return /*#__PURE__*/ jsxs(Fragment, {
+        children: [
+            /*#__PURE__*/ jsxs("div", {
+                style: {
+                    display: 'flex',
+                    alignItems: 'center',
+                    justifyContent: 'space-between',
+                    gap: '10px'
+                },
+                children: [
+                    /*#__PURE__*/ jsx("h3", {
+                        style: {
+                            whiteSpace: 'nowrap',
+                            margin: 0,
+                            flexShrink: 0
+                        },
+                        children: title
+                    }),
+                    statusContent,
+                    /*#__PURE__*/ jsx(EnvConfig, {
+                        showTooltipWhenEmpty: 'Server' !== serviceMode
+                    })
+                ]
+            }),
+            /*#__PURE__*/ jsx("div", {
+                className: "switch-btn-wrapper",
+                children: renderSwitchButton()
+            })
+        ]
+    });
+};
+export { ServiceModeControl };