@midscene/visualizer 0.27.1-beta-20250822053848.0 → 0.27.1-beta-20250822094725.0

package/dist/es/component/playground/FormField.mjs

@@ -0,0 +1,122 @@
+import { jsx } from "react/jsx-runtime";
+import { Form, Input, InputNumber, Select } from "antd";
+const { TextArea } = Input;
+const renderLabel = (label, isOptional)=>`${label}${isOptional ? ' (Optional)' : ''}`;
+const TextField = (param)=>{
+    let { name, label, isRequired, marginBottom } = param;
+    const placeholder = `Enter ${name}`;
+    return /*#__PURE__*/ jsx(Form.Item, {
+        name: [
+            'params',
+            name
+        ],
+        label: renderLabel(label, !isRequired),
+        rules: isRequired ? [
+            {
+                required: true,
+                message: `Please input ${name}`
+            }
+        ] : [],
+        style: {
+            marginBottom
+        },
+        children: /*#__PURE__*/ jsx(Input, {
+            placeholder: placeholder
+        })
+    }, name);
+};
+const LocateField = (param)=>{
+    let { name, label, isRequired, marginBottom } = param;
+    return /*#__PURE__*/ jsx(Form.Item, {
+        name: [
+            'params',
+            name
+        ],
+        label: renderLabel(label, !isRequired),
+        rules: isRequired ? [
+            {
+                required: true,
+                message: `Please describe the ${name}`
+            }
+        ] : [],
+        style: {
+            marginBottom
+        },
+        children: /*#__PURE__*/ jsx(TextArea, {
+            rows: 2,
+            placeholder: `Describe the ${name}, use natural language`
+        })
+    }, name);
+};
+const EnumField = (param)=>{
+    let { name, label, fieldSchema, isRequired, marginBottom } = param;
+    const enumValues = fieldSchema._def.values || [];
+    const selectOptions = enumValues.map((value)=>({
+            value,
+            label: value.charAt(0).toUpperCase() + value.slice(1)
+        }));
+    return /*#__PURE__*/ jsx(Form.Item, {
+        name: [
+            'params',
+            name
+        ],
+        label: label,
+        rules: isRequired ? [
+            {
+                required: true,
+                message: `Please select ${name}`
+            }
+        ] : [],
+        style: {
+            marginBottom
+        },
+        children: /*#__PURE__*/ jsx(Select, {
+            placeholder: `Select ${name}`,
+            options: selectOptions
+        })
+    }, name);
+};
+const NumberField = (param)=>{
+    let { name, label, isRequired, marginBottom } = param;
+    const placeholder = 'distance' === name ? 500 : 0;
+    const min = 0;
+    const max = 'distance' === name ? 10000 : void 0;
+    return /*#__PURE__*/ jsx(Form.Item, {
+        name: [
+            'params',
+            name
+        ],
+        label: `${label}${'distance' === name ? ' (px)' : ''}`,
+        rules: isRequired ? [
+            {
+                required: true,
+                message: `Please input ${name}`
+            },
+            {
+                type: 'number',
+                min,
+                message: `${label} must be at least ${min}`
+            }
+        ] : [
+            {
+                type: 'number',
+                min,
+                message: `${label} must be at least ${min}`
+            }
+        ],
+        style: {
+            flex: 'distance' === name ? 1 : void 0,
+            marginBottom
+        },
+        children: /*#__PURE__*/ jsx(InputNumber, {
+            placeholder: placeholder.toString(),
+            min: min,
+            max: max,
+            step: 'distance' === name ? 10 : 1,
+            style: {
+                width: '100%'
+            }
+        })
+    }, name);
+};
+export { EnumField, LocateField, NumberField, TextField };

package/dist/es/component/playground/HistorySelector.mjs

@@ -8,10 +8,9 @@ import { useHistoryStore } from "../store/history.mjs";
 import "./index.css";
 const { Text } = Typography;
 const HistorySelector = (param)=>{
-    let { onSelect } = param;
+    let { onSelect, history, currentType } = param;
     const [isModalOpen, setIsModalOpen] = useState(false);
     const [searchText, setSearchText] = useState('');
-    const history = useHistoryStore((state)=>state.history);
     const clearHistory = useHistoryStore((state)=>state.clearHistory);
     const groupedHistory = useMemo(()=>{
         const now = Date.now();
@@ -33,7 +32,7 @@ const HistorySelector = (param)=>{
         setIsModalOpen(false);
     };
     const handleClearHistory = ()=>{
-        clearHistory();
+        clearHistory(currentType);
         setSearchText('');
         setIsModalOpen(false);
     };

package/dist/es/component/playground/PromptInput.mjs

@@ -1,41 +1,105 @@
 import { jsx, jsxs } from "react/jsx-runtime";
-import { BorderOutlined, SendOutlined } from "@ant-design/icons";
-import { Button, Form, Input, Radio, Space, Tooltip } from "antd";
-import { useCallback, useEffect, useRef, useState } from "react";
+import { BorderOutlined, DownOutlined, SendOutlined } from "@ant-design/icons";
+import { Button, Dropdown, Form, Input, Radio, Space, Tooltip } from "antd";
+import react, { useCallback, useEffect, useMemo, useRef, useState } from "react";
 import { useHistoryStore } from "../store/history.mjs";
 import { ConfigSelector } from "./ConfigSelector.mjs";
+import { EnumField, LocateField, NumberField, TextField } from "./FormField.mjs";
 import { HistorySelector } from "./HistorySelector.mjs";
-import { actionNameForType, getPlaceholderForType } from "./playground-utils.mjs";
+import { apiMetadata, defaultMainButtons } from "./playground-constants.mjs";
+import { actionNameForType, getPlaceholderForType, isRunButtonEnabled as external_playground_utils_mjs_isRunButtonEnabled } from "./playground-utils.mjs";
+import { extractDefaultValue, isLocateField, isZodObjectSchema, unwrapZodType } from "./types.mjs";
 import "./index.css";
 const { TextArea } = Input;
 const PromptInput = (param)=>{
-    let { runButtonEnabled, form, serviceMode, selectedType, dryMode, stoppable, loading, onRun, onStop, clearPromptAfterRun = true } = param;
+    let { runButtonEnabled, form, serviceMode, selectedType, dryMode, stoppable, loading, onRun, onStop, clearPromptAfterRun = true, actionSpace } = param;
     const [hoveringSettings, setHoveringSettings] = useState(false);
     const [promptValue, setPromptValue] = useState('');
     const placeholder = getPlaceholderForType(selectedType);
     const textAreaRef = useRef(null);
+    const params = Form.useWatch('params', form);
     const history = useHistoryStore((state)=>state.history);
+    const lastSelectedType = useHistoryStore((state)=>state.lastSelectedType);
     const addHistory = useHistoryStore((state)=>state.addHistory);
-    const lastHistory = history[0];
+    const setLastSelectedType = useHistoryStore((state)=>state.setLastSelectedType);
+    const historyForSelectedType = useMemo(()=>history[selectedType] || [], [
+        history,
+        selectedType
+    ]);
+    const needsStructuredParams = useMemo(()=>{
+        if (actionSpace) {
+            const action = actionSpace.find((a)=>a.interfaceAlias === selectedType || a.name === selectedType);
+            return null == action ? void 0 : action.paramSchema;
+        }
+        return [
+            'aiInput',
+            'aiKeyboardPress',
+            'aiScroll'
+        ].includes(selectedType);
+    }, [
+        selectedType,
+        actionSpace
+    ]);
+    const getDefaultParams = useCallback(()=>{
+        if (!needsStructuredParams || !actionSpace) return {};
+        const action = actionSpace.find((a)=>a.interfaceAlias === selectedType || a.name === selectedType);
+        if ((null == action ? void 0 : action.paramSchema) && isZodObjectSchema(action.paramSchema)) {
+            const defaultParams = {};
+            const schema = action.paramSchema;
+            Object.keys(schema.shape).forEach((key)=>{
+                const field = schema.shape[key];
+                const defaultValue = extractDefaultValue(field);
+                if (void 0 !== defaultValue) defaultParams[key] = defaultValue;
+            });
+            return defaultParams;
+        }
+        return {};
+    }, [
+        selectedType,
+        needsStructuredParams,
+        actionSpace
+    ]);
+    useEffect(()=>{
+        if (!form.getFieldValue('type') && lastSelectedType) form.setFieldValue('type', lastSelectedType);
+    }, [
+        form,
+        lastSelectedType
+    ]);
+    useEffect(()=>{
+        if (selectedType && selectedType !== lastSelectedType) setLastSelectedType(selectedType);
+    }, [
+        selectedType,
+        lastSelectedType,
+        setLastSelectedType
+    ]);
     useEffect(()=>{
+        const lastHistory = historyForSelectedType[0];
         if (lastHistory) {
             form.setFieldsValue({
-                type: lastHistory.type || 'aiAction',
-                prompt: lastHistory.prompt || ''
+                type: lastHistory.type,
+                prompt: lastHistory.prompt || '',
+                params: lastHistory.params
             });
             setPromptValue(lastHistory.prompt || '');
         } else {
+            const defaultParams = getDefaultParams();
             form.setFieldsValue({
-                type: 'aiAction',
-                prompt: ''
+                prompt: '',
+                params: defaultParams
             });
             setPromptValue('');
         }
-    }, []);
+    }, [
+        selectedType,
+        historyForSelectedType,
+        form,
+        getDefaultParams
+    ]);
     const handleSelectHistory = useCallback((historyItem)=>{
         form.setFieldsValue({
             prompt: historyItem.prompt,
-            type: historyItem.type
+            type: historyItem.type,
+            params: historyItem.params
         });
         setPromptValue(historyItem.prompt);
     }, [
@@ -48,23 +112,73 @@ const PromptInput = (param)=>{
     }, [
         form
     ]);
-    const isRunButtonEnabled = runButtonEnabled && promptValue.trim().length > 0;
+    const hasSingleStructuredParam = useMemo(()=>{
+        if (!needsStructuredParams || !actionSpace) return false;
+        const action = actionSpace.find((a)=>a.interfaceAlias === selectedType || a.name === selectedType);
+        if ((null == action ? void 0 : action.paramSchema) && isZodObjectSchema(action.paramSchema)) {
+            const schema = action.paramSchema;
+            return 1 === Object.keys(schema.shape).length;
+        }
+        return false;
+    }, [
+        selectedType,
+        needsStructuredParams,
+        actionSpace
+    ]);
+    const isRunButtonEnabled = useMemo(()=>external_playground_utils_mjs_isRunButtonEnabled(runButtonEnabled, !!needsStructuredParams, params, actionSpace, selectedType, promptValue), [
+        runButtonEnabled,
+        needsStructuredParams,
+        selectedType,
+        actionSpace,
+        promptValue,
+        params
+    ]);
     const handleRunWithHistory = useCallback(()=>{
         const values = form.getFieldsValue();
-        if (values.prompt) addHistory({
+        let historyPrompt = '';
+        if (needsStructuredParams && values.params && actionSpace) {
+            const action = actionSpace.find((a)=>a.interfaceAlias === selectedType || a.name === selectedType);
+            if ((null == action ? void 0 : action.paramSchema) && isZodObjectSchema(action.paramSchema)) {
+                let locateValue = '';
+                const otherValues = [];
+                const schema = action.paramSchema;
+                Object.keys(schema.shape).forEach((key)=>{
+                    const paramValue = values.params[key];
+                    if (null != paramValue && '' !== paramValue) {
+                        const field = schema.shape[key];
+                        const { actualField } = unwrapZodType(field);
+                        if (isLocateField(actualField)) locateValue = String(paramValue);
+                        else if ('distance' === key) otherValues.push(`${paramValue}`);
+                        else otherValues.push(String(paramValue));
+                    }
+                });
+                const mainPart = otherValues.join(' ');
+                historyPrompt = locateValue ? `${mainPart} | ${locateValue}` : mainPart;
+            } else historyPrompt = values.prompt || '';
+        } else historyPrompt = values.prompt || '';
+        addHistory({
             type: values.type,
-            prompt: values.prompt,
+            prompt: historyPrompt,
+            params: values.params,
             timestamp: Date.now()
         });
         onRun();
         if (clearPromptAfterRun) {
             setPromptValue('');
-            form.setFieldValue('prompt', '');
+            if (needsStructuredParams) {
+                const defaultParams = getDefaultParams();
+                form.setFieldValue('params', defaultParams);
+            } else form.setFieldValue('prompt', '');
         }
     }, [
         form,
         addHistory,
-        onRun
+        onRun,
+        needsStructuredParams,
+        selectedType,
+        clearPromptAfterRun,
+        actionSpace,
+        getDefaultParams
     ]);
     const handleKeyDown = useCallback((e)=>{
         if ('Enter' === e.key && e.metaKey && isRunButtonEnabled) {
@@ -85,6 +199,116 @@ const PromptInput = (param)=>{
         handleRunWithHistory,
         isRunButtonEnabled
     ]);
+    const handleStructuredKeyDown = useCallback((e)=>{
+        if ('Enter' === e.key && e.metaKey && isRunButtonEnabled) {
+            handleRunWithHistory();
+            e.preventDefault();
+            e.stopPropagation();
+        }
+    }, [
+        handleRunWithHistory,
+        isRunButtonEnabled
+    ]);
+    const renderStructuredParams = useCallback(()=>{
+        if (!needsStructuredParams) return null;
+        if (actionSpace) {
+            const action = actionSpace.find((a)=>a.interfaceAlias === selectedType || a.name === selectedType);
+            if ((null == action ? void 0 : action.paramSchema) && isZodObjectSchema(action.paramSchema)) {
+                const schema = action.paramSchema;
+                const schemaKeys = Object.keys(schema.shape);
+                if (1 === schemaKeys.length) {
+                    const key = schemaKeys[0];
+                    const field = schema.shape[key];
+                    const { actualField, isOptional } = unwrapZodType(field);
+                    const isLocateFieldFlag = isLocateField(actualField);
+                    let placeholderText = '';
+                    if (isLocateFieldFlag) placeholderText = 'Describe the element you want to interact with';
+                    else {
+                        placeholderText = `Enter ${key}`;
+                        if ('keyName' === key) placeholderText = 'Enter key name or text to type';
+                        if ('value' === key) placeholderText = 'Enter text to input';
+                    }
+                    return /*#__PURE__*/ jsx(Form.Item, {
+                        name: [
+                            'params',
+                            key
+                        ],
+                        style: {
+                            margin: 0
+                        },
+                        children: /*#__PURE__*/ jsx(Input.TextArea, {
+                            className: "main-side-console-input-textarea",
+                            rows: 4,
+                            placeholder: placeholderText,
+                            autoFocus: true,
+                            onKeyDown: handleStructuredKeyDown
+                        })
+                    });
+                }
+                const fields = [];
+                schemaKeys.forEach((key, index)=>{
+                    var _actualField__def, _actualField__def1;
+                    const fieldSchema = schema.shape[key];
+                    const { actualField, isOptional } = unwrapZodType(fieldSchema);
+                    const isLocateFieldFlag = isLocateField(actualField);
+                    const label = key.charAt(0).toUpperCase() + key.slice(1);
+                    const isRequired = !isOptional;
+                    const marginBottom = index === schemaKeys.length - 1 ? 0 : 12;
+                    const fieldProps = {
+                        name: key,
+                        label,
+                        fieldSchema: actualField,
+                        isRequired,
+                        marginBottom
+                    };
+                    if (isLocateFieldFlag) fields.push(/*#__PURE__*/ jsx(LocateField, {
+                        ...fieldProps
+                    }));
+                    else if ((null == (_actualField__def = actualField._def) ? void 0 : _actualField__def.typeName) === 'ZodEnum') fields.push(/*#__PURE__*/ jsx(EnumField, {
+                        ...fieldProps
+                    }));
+                    else if ((null == (_actualField__def1 = actualField._def) ? void 0 : _actualField__def1.typeName) === 'ZodNumber') fields.push(/*#__PURE__*/ jsx(NumberField, {
+                        ...fieldProps
+                    }));
+                    else fields.push(/*#__PURE__*/ jsx(TextField, {
+                        ...fieldProps
+                    }));
+                });
+                if ('aiScroll' === selectedType) {
+                    const directionField = fields.find((field)=>/*#__PURE__*/ react.isValidElement(field) && 'direction' === field.props.name);
+                    const distanceField = fields.find((field)=>/*#__PURE__*/ react.isValidElement(field) && 'distance' === field.props.name);
+                    const otherFields = fields.filter((field)=>/*#__PURE__*/ react.isValidElement(field) && 'direction' !== field.props.name && 'distance' !== field.props.name);
+                    if (directionField && distanceField) return /*#__PURE__*/ jsxs("div", {
+                        className: "structured-params",
+                        children: [
+                            /*#__PURE__*/ jsxs("div", {
+                                style: {
+                                    display: 'flex',
+                                    gap: 12,
+                                    marginBottom: 12
+                                },
+                                children: [
+                                    directionField,
+                                    distanceField
+                                ]
+                            }),
+                            otherFields
+                        ]
+                    });
+                }
+                return /*#__PURE__*/ jsx("div", {
+                    className: "structured-params",
+                    children: fields
+                });
+            }
+        }
+        return null;
+    }, [
+        selectedType,
+        needsStructuredParams,
+        actionSpace,
+        handleStructuredKeyDown
+    ]);
     const handleMouseEnter = useCallback(()=>{
         setHoveringSettings(true);
     }, []);
@@ -133,52 +357,106 @@ const PromptInput = (param)=>{
             /*#__PURE__*/ jsxs(Space, {
                 className: "mode-radio-group-wrapper",
                 children: [
-                    /*#__PURE__*/ jsx(Form.Item, {
-                        name: "type",
-                        style: {
-                            margin: 0
-                        },
-                        children: /*#__PURE__*/ jsxs(Radio.Group, {
-                            buttonStyle: "solid",
-                            disabled: !runButtonEnabled,
-                            className: "mode-radio-group",
-                            children: [
-                                /*#__PURE__*/ jsx(Tooltip, {
-                                    title: "Auto Planning: plan the steps and execute",
-                                    children: /*#__PURE__*/ jsx(Radio.Button, {
-                                        value: "aiAction",
-                                        children: actionNameForType('aiAction')
-                                    })
-                                }),
-                                /*#__PURE__*/ jsx(Tooltip, {
-                                    title: "Extract data directly from the UI",
-                                    children: /*#__PURE__*/ jsx(Radio.Button, {
-                                        value: "aiQuery",
-                                        children: actionNameForType('aiQuery')
-                                    })
-                                }),
-                                /*#__PURE__*/ jsx(Tooltip, {
-                                    title: "Understand the UI and determine if the assertion is true",
-                                    children: /*#__PURE__*/ jsx(Radio.Button, {
-                                        value: "aiAssert",
-                                        children: actionNameForType('aiAssert')
-                                    })
-                                }),
-                                /*#__PURE__*/ jsx(Tooltip, {
-                                    title: "Instant Action: click something",
-                                    children: /*#__PURE__*/ jsx(Radio.Button, {
-                                        value: "aiTap",
-                                        children: actionNameForType('aiTap')
+                    /*#__PURE__*/ jsxs("div", {
+                        className: "mode-radio-group",
+                        children: [
+                            /*#__PURE__*/ jsx(Form.Item, {
+                                name: "type",
+                                style: {
+                                    margin: 0
+                                },
+                                children: /*#__PURE__*/ jsx(Radio.Group, {
+                                    buttonStyle: "solid",
+                                    disabled: !runButtonEnabled,
+                                    children: defaultMainButtons.map((apiType)=>{
+                                        var _apiMetadata_apiType;
+                                        return /*#__PURE__*/ jsx(Tooltip, {
+                                            title: (null == (_apiMetadata_apiType = apiMetadata[apiType]) ? void 0 : _apiMetadata_apiType.title) || '',
+                                            children: /*#__PURE__*/ jsx(Radio.Button, {
+                                                value: apiType,
+                                                children: actionNameForType(apiType)
+                                            })
+                                        }, apiType);
                                     })
                                 })
-                            ]
-                        })
+                            }),
+                            /*#__PURE__*/ jsx(Dropdown, {
+                                menu: (()=>{
+                                    const hiddenAPIs = Object.keys(apiMetadata).filter((api)=>!defaultMainButtons.includes(api));
+                                    const groupedItems = [];
+                                    const interactionAPIs = hiddenAPIs.filter((api)=>'interaction' === apiMetadata[api].group);
+                                    if (interactionAPIs.length > 0) groupedItems.push({
+                                        key: 'interaction-group',
+                                        type: 'group',
+                                        label: 'Interaction APIs',
+                                        children: interactionAPIs.map((api)=>({
+                                                key: api,
+                                                label: actionNameForType(api),
+                                                title: apiMetadata[api].title,
+                                                onClick: ()=>{
+                                                    form.setFieldValue('type', api);
+                                                }
+                                            }))
+                                    });
+                                    const extractionAPIs = hiddenAPIs.filter((api)=>'extraction' === apiMetadata[api].group);
+                                    if (extractionAPIs.length > 0) groupedItems.push({
+                                        key: 'extraction-group',
+                                        type: 'group',
+                                        label: 'Data Extraction APIs',
+                                        children: extractionAPIs.map((api)=>({
+                                                key: api,
+                                                label: actionNameForType(api),
+                                                title: apiMetadata[api].title,
+                                                onClick: ()=>{
+                                                    form.setFieldValue('type', api);
+                                                }
+                                            }))
+                                    });
+                                    const validationAPIs = hiddenAPIs.filter((api)=>'validation' === apiMetadata[api].group);
+                                    if (validationAPIs.length > 0) groupedItems.push({
+                                        key: 'validation-group',
+                                        type: 'group',
+                                        label: 'Validation APIs',
+                                        children: validationAPIs.map((api)=>({
+                                                key: api,
+                                                label: actionNameForType(api),
+                                                title: apiMetadata[api].title,
+                                                onClick: ()=>{
+                                                    form.setFieldValue('type', api);
+                                                }
+                                            }))
+                                    });
+                                    return {
+                                        items: groupedItems
+                                    };
+                                })(),
+                                placement: "bottomLeft",
+                                trigger: [
+                                    'click'
+                                ],
+                                disabled: !runButtonEnabled,
+                                children: /*#__PURE__*/ jsxs(Button, {
+                                    className: `more-apis-button ${!defaultMainButtons.includes(selectedType) ? 'selected-from-dropdown' : ''}`,
+                                    children: [
+                                        selectedType && !defaultMainButtons.includes(selectedType) ? actionNameForType(selectedType) : 'more',
+                                        /*#__PURE__*/ jsx(DownOutlined, {
+                                            style: {
+                                                fontSize: '10px',
+                                                marginLeft: '2px'
+                                            }
+                                        })
+                                    ]
+                                })
+                            })
+                        ]
                     }),
                     /*#__PURE__*/ jsxs("div", {
                         className: "action-icons",
                         children: [
                             /*#__PURE__*/ jsx(HistorySelector, {
-                                onSelect: handleSelectHistory
+                                onSelect: handleSelectHistory,
+                                history: historyForSelectedType,
+                                currentType: selectedType
                             }),
                             /*#__PURE__*/ jsx("div", {
                                 className: hoveringSettings ? 'settings-wrapper settings-wrapper-hover' : 'settings-wrapper',
@@ -186,7 +464,7 @@ const PromptInput = (param)=>{
                                 onMouseLeave: handleMouseLeave,
                                 children: /*#__PURE__*/ jsx(ConfigSelector, {
                                     enableTracking: 'In-Browser-Extension' === serviceMode,
-                                    showDeepThinkOption: 'aiTap' === selectedType
+                                    showDeepThinkOption: 'aiTap' === selectedType || 'aiHover' === selectedType || 'aiInput' === selectedType || 'aiRightClick' === selectedType || 'aiLocate' === selectedType
                                 })
                             })
                         ]
@@ -196,7 +474,10 @@ const PromptInput = (param)=>{
             /*#__PURE__*/ jsxs("div", {
                 className: `main-side-console-input ${!runButtonEnabled ? 'disabled' : ''} ${loading ? 'loading' : ''}`,
                 children: [
-                    /*#__PURE__*/ jsx(Form.Item, {
+                    needsStructuredParams ? hasSingleStructuredParam ? renderStructuredParams() : /*#__PURE__*/ jsx("div", {
+                        className: "structured-params-container",
+                        children: renderStructuredParams()
+                    }) : /*#__PURE__*/ jsx(Form.Item, {
                         name: "prompt",
                         style: {
                             margin: 0