@midscene/visualizer 0.27.5-beta-20250827031507.0 → 0.27.5-beta-20250827160628.0

package/dist/es/component/playground/ConfigSelector.mjs

@@ -1,16 +1,20 @@
-import { jsx } from "react/jsx-runtime";
-import { Checkbox, Dropdown } from "antd";
+import { jsx, jsxs } from "react/jsx-runtime";
+import { Checkbox, Dropdown, Radio } from "antd";
 import setting from "../../icons/setting.mjs";
 import { useEnvConfig } from "../store/store.mjs";
-import { deepThinkTip, trackingTip } from "./playground-constants.mjs";
+import { deepThinkTip, domIncludedTip, screenshotIncludedTip, trackingTip } from "./playground-constants.mjs";
 import "./index.css";
 const ConfigSelector = (param)=>{
-    let { showDeepThinkOption = false, enableTracking = false } = param;
+    let { showDeepThinkOption = false, enableTracking = false, showDataExtractionOptions = false, hideDomAndScreenshotOptions = false } = param;
     const forceSameTabNavigation = useEnvConfig((state)=>state.forceSameTabNavigation);
     const setForceSameTabNavigation = useEnvConfig((state)=>state.setForceSameTabNavigation);
     const deepThink = useEnvConfig((state)=>state.deepThink);
     const setDeepThink = useEnvConfig((state)=>state.setDeepThink);
-    if (!enableTracking && !showDeepThinkOption) return null;
+    const screenshotIncluded = useEnvConfig((state)=>state.screenshotIncluded);
+    const setScreenshotIncluded = useEnvConfig((state)=>state.setScreenshotIncluded);
+    const domIncluded = useEnvConfig((state)=>state.domIncluded);
+    const setDomIncluded = useEnvConfig((state)=>state.setDomIncluded);
+    if (!enableTracking && !showDeepThinkOption && !showDataExtractionOptions) return null;
     const configItems = buildConfigItems();
     return /*#__PURE__*/ jsx("div", {
         className: "selector-trigger",
@@ -47,6 +51,54 @@ const ConfigSelector = (param)=>{
             }),
             key: 'deep-think-config'
         });
+        if (showDataExtractionOptions && !hideDomAndScreenshotOptions) {
+            items.push({
+                label: /*#__PURE__*/ jsx(Checkbox, {
+                    onChange: (e)=>{
+                        setScreenshotIncluded(e.target.checked);
+                    },
+                    checked: screenshotIncluded,
+                    children: screenshotIncludedTip
+                }),
+                key: 'screenshot-included-config'
+            });
+            items.push({
+                label: /*#__PURE__*/ jsxs("div", {
+                    style: {
+                        padding: '4px 0'
+                    },
+                    children: [
+                        /*#__PURE__*/ jsx("div", {
+                            style: {
+                                marginBottom: '4px',
+                                fontSize: '14px'
+                            },
+                            children: domIncludedTip
+                        }),
+                        /*#__PURE__*/ jsxs(Radio.Group, {
+                            size: "small",
+                            value: domIncluded,
+                            onChange: (e)=>setDomIncluded(e.target.value),
+                            children: [
+                                /*#__PURE__*/ jsx(Radio, {
+                                    value: false,
+                                    children: "Off"
+                                }),
+                                /*#__PURE__*/ jsx(Radio, {
+                                    value: true,
+                                    children: "All"
+                                }),
+                                /*#__PURE__*/ jsx(Radio, {
+                                    value: 'visible-only',
+                                    children: "Visible only"
+                                })
+                            ]
+                        })
+                    ]
+                }),
+                key: 'dom-included-config'
+            });
+        }
         return items;
     }
 };

package/dist/es/component/playground/PromptInput.mjs

@@ -12,12 +12,13 @@ import { extractDefaultValue, isLocateField, isZodObjectSchema, unwrapZodType }
 import "./index.css";
 const { TextArea } = Input;
 const PromptInput = (param)=>{
-    let { runButtonEnabled, form, serviceMode, selectedType, dryMode, stoppable, loading, onRun, onStop, clearPromptAfterRun = true, actionSpace } = param;
+    let { runButtonEnabled, form, serviceMode, selectedType, dryMode, stoppable, loading, onRun, onStop, clearPromptAfterRun = true, actionSpace, hideDomAndScreenshotOptions = false } = param;
     const [hoveringSettings, setHoveringSettings] = useState(false);
     const [promptValue, setPromptValue] = useState('');
     const placeholder = getPlaceholderForType(selectedType);
     const textAreaRef = useRef(null);
     const params = Form.useWatch('params', form);
+    const lastHistoryRef = useRef(null);
     const history = useHistoryStore((state)=>state.history);
     const lastSelectedType = useHistoryStore((state)=>state.lastSelectedType);
     const addHistory = useHistoryStore((state)=>state.addHistory);
@@ -29,13 +30,28 @@ const PromptInput = (param)=>{
     const needsStructuredParams = useMemo(()=>{
         if (actionSpace) {
             const action = actionSpace.find((a)=>a.interfaceAlias === selectedType || a.name === selectedType);
-            return null == action ? void 0 : action.paramSchema;
+            return !!(null == action ? void 0 : action.paramSchema);
         }
+        return false;
     }, [
         selectedType,
         actionSpace
     ]);
+    const showDataExtractionOptions = useMemo(()=>{
+        const dataExtractionMethods = [
+            'aiQuery',
+            'aiBoolean',
+            'aiNumber',
+            'aiString',
+            'aiAsk',
+            'aiAssert'
+        ];
+        return dataExtractionMethods.includes(selectedType);
+    }, [
+        selectedType
+    ]);
     const showDeepThinkOption = useMemo(()=>{
+        if ('aiLocate' === selectedType) return true;
         if (actionSpace) {
             const action = actionSpace.find((a)=>a.interfaceAlias === selectedType || a.name === selectedType);
             if ((null == action ? void 0 : action.paramSchema) && isZodObjectSchema(action.paramSchema)) {
@@ -46,15 +62,8 @@ const PromptInput = (param)=>{
                     return isLocateField(actualField);
                 });
             }
-            return false;
         }
-        return [
-            'aiTap',
-            'aiHover',
-            'aiInput',
-            'aiRightClick',
-            'aiLocate'
-        ].includes(selectedType);
+        return false;
     }, [
         selectedType,
         actionSpace
@@ -93,6 +102,7 @@ const PromptInput = (param)=>{
     ]);
     useEffect(()=>{
         const lastHistory = historyForSelectedType[0];
+        if (lastHistory && lastHistoryRef.current && lastHistory.timestamp === lastHistoryRef.current.timestamp) return;
         if (lastHistory) {
             form.setFieldsValue({
                 type: lastHistory.type,
@@ -100,6 +110,7 @@ const PromptInput = (param)=>{
                 params: lastHistory.params
             });
             setPromptValue(lastHistory.prompt || '');
+            lastHistoryRef.current = lastHistory;
         } else {
             const defaultParams = getDefaultParams();
             form.setFieldsValue({
@@ -107,6 +118,7 @@ const PromptInput = (param)=>{
                 params: defaultParams
             });
             setPromptValue('');
+            lastHistoryRef.current = null;
         }
     }, [
         selectedType,
@@ -114,6 +126,13 @@ const PromptInput = (param)=>{
         form,
         getDefaultParams
     ]);
+    const formPromptValue = Form.useWatch('prompt', form);
+    useEffect(()=>{
+        if (formPromptValue !== promptValue) setPromptValue(formPromptValue || '');
+    }, [
+        formPromptValue,
+        promptValue
+    ]);
     const handleSelectHistory = useCallback((historyItem)=>{
         form.setFieldsValue({
             prompt: historyItem.prompt,
@@ -127,10 +146,7 @@ const PromptInput = (param)=>{
     const handlePromptChange = useCallback((e)=>{
         const value = e.target.value;
         setPromptValue(value);
-        form.setFieldValue('prompt', value);
-    }, [
-        form
-    ]);
+    }, []);
     const hasSingleStructuredParam = useMemo(()=>{
         if (!needsStructuredParams || !actionSpace) return false;
         const action = actionSpace.find((a)=>a.interfaceAlias === selectedType || a.name === selectedType);
@@ -175,14 +191,16 @@ const PromptInput = (param)=>{
                 historyPrompt = locateValue ? `${mainPart} | ${locateValue}` : mainPart;
             } else historyPrompt = values.prompt || '';
         } else historyPrompt = values.prompt || '';
-        addHistory({
+        const newHistoryItem = {
             type: values.type,
             prompt: historyPrompt,
             params: values.params,
             timestamp: Date.now()
-        });
+        };
+        addHistory(newHistoryItem);
         onRun();
         if (clearPromptAfterRun) {
+            lastHistoryRef.current = newHistoryItem;
             setPromptValue('');
             if (needsStructuredParams) {
                 const defaultParams = getDefaultParams();
@@ -483,7 +501,9 @@ const PromptInput = (param)=>{
                                 onMouseLeave: handleMouseLeave,
                                 children: /*#__PURE__*/ jsx(ConfigSelector, {
                                     enableTracking: 'In-Browser-Extension' === serviceMode,
-                                    showDeepThinkOption: showDeepThinkOption
+                                    showDeepThinkOption: showDeepThinkOption,
+                                    showDataExtractionOptions: showDataExtractionOptions,
+                                    hideDomAndScreenshotOptions: hideDomAndScreenshotOptions
                                 })
                             })
                         ]
@@ -509,7 +529,6 @@ const PromptInput = (param)=>{
                             autoFocus: true,
                             onKeyDown: handleKeyDown,
                             onChange: handlePromptChange,
-                            value: promptValue,
                             ref: textAreaRef
                         })
                     }),

package/dist/es/component/playground/playground-constants.mjs

@@ -42,6 +42,8 @@ const emptyResultTip = /*#__PURE__*/ jsx("div", {
 });
 const trackingTip = 'limit popup to current tab';
 const deepThinkTip = 'deep think';
+const screenshotIncludedTip = 'include screenshot in request';
+const domIncludedTip = 'include DOM info in request';
 const apiMetadata = {
     aiAction: {
         group: 'interaction',
@@ -110,4 +112,4 @@ const defaultMainButtons = [
     'aiQuery',
     'aiAssert'
 ];
-export { apiMetadata, deepThinkTip, defaultMainButtons, emptyResultTip, errorMessageServerNotReady, serverLaunchTip, trackingTip };
+export { apiMetadata, deepThinkTip, defaultMainButtons, domIncludedTip, emptyResultTip, errorMessageServerNotReady, screenshotIncludedTip, serverLaunchTip, trackingTip };

package/dist/es/component/playground/playground-utils.mjs

@@ -11,7 +11,7 @@ const checkServerStatus = async ()=>{
     }
 };
 const requestPlaygroundServer = async function(context, type, prompt) {
-    let { requestId, deepThink } = arguments.length > 3 && void 0 !== arguments[3] ? arguments[3] : {};
+    let { requestId, deepThink, screenshotIncluded, domIncluded } = arguments.length > 3 && void 0 !== arguments[3] ? arguments[3] : {};
     const payload = {
         context,
         type,
@@ -19,6 +19,8 @@ const requestPlaygroundServer = async function(context, type, prompt) {
     };
     if (requestId) payload.requestId = requestId;
     if (deepThink) payload.deepThink = deepThink;
+    if (void 0 !== screenshotIncluded) payload.screenshotIncluded = screenshotIncluded;
+    if (void 0 !== domIncluded) payload.domIncluded = domIncluded;
     const res = await fetch(`${serverBase}/execute`, {
         method: 'POST',
         headers: {

package/dist/es/component/store/store.mjs

@@ -62,6 +62,8 @@ const CONFIG_KEY = 'midscene-env-config';
 const SERVICE_MODE_KEY = 'midscene-service-mode';
 const TRACKING_ACTIVE_TAB_KEY = 'midscene-tracking-active-tab';
 const DEEP_THINK_KEY = 'midscene-deep-think';
+const SCREENSHOT_INCLUDED_KEY = 'midscene-screenshot-included';
+const DOM_INCLUDED_KEY = 'midscene-dom-included';
 const getConfigStringFromLocalStorage = ()=>{
     const configString = localStorage.getItem(CONFIG_KEY);
     return configString || '';
@@ -90,6 +92,8 @@ const useEnvConfig = store_create((set, get)=>{
     const savedServiceMode = localStorage.getItem(SERVICE_MODE_KEY);
     const savedForceSameTabNavigation = 'false' !== localStorage.getItem(TRACKING_ACTIVE_TAB_KEY);
     const savedDeepThink = 'true' === localStorage.getItem(DEEP_THINK_KEY);
+    const savedScreenshotIncluded = 'false' !== localStorage.getItem(SCREENSHOT_INCLUDED_KEY);
+    const savedDomIncluded = localStorage.getItem(DOM_INCLUDED_KEY) || 'false';
     return {
         serviceMode: ifInExtension ? 'In-Browser-Extension' : savedServiceMode || 'Server',
         setServiceMode: (serviceMode)=>{
@@ -134,6 +138,20 @@ const useEnvConfig = store_create((set, get)=>{
             });
             localStorage.setItem(DEEP_THINK_KEY, deepThink.toString());
         },
+        screenshotIncluded: savedScreenshotIncluded,
+        setScreenshotIncluded: (screenshotIncluded)=>{
+            set({
+                screenshotIncluded
+            });
+            localStorage.setItem(SCREENSHOT_INCLUDED_KEY, screenshotIncluded.toString());
+        },
+        domIncluded: 'visible-only' === savedDomIncluded ? 'visible-only' : 'true' === savedDomIncluded,
+        setDomIncluded: (domIncluded)=>{
+            set({
+                domIncluded
+            });
+            localStorage.setItem(DOM_INCLUDED_KEY, domIncluded.toString());
+        },
         popupTab: 'playground',
         setPopupTab: (tab)=>{
             set({

package/dist/lib/component/playground/ConfigSelector.js

@@ -43,12 +43,16 @@ const store_js_namespaceObject = require("../store/store.js");
 const external_playground_constants_js_namespaceObject = require("./playground-constants.js");
 require("./index.css");
 const ConfigSelector = (param)=>{
-    let { showDeepThinkOption = false, enableTracking = false } = param;
+    let { showDeepThinkOption = false, enableTracking = false, showDataExtractionOptions = false, hideDomAndScreenshotOptions = false } = param;
     const forceSameTabNavigation = (0, store_js_namespaceObject.useEnvConfig)((state)=>state.forceSameTabNavigation);
     const setForceSameTabNavigation = (0, store_js_namespaceObject.useEnvConfig)((state)=>state.setForceSameTabNavigation);
     const deepThink = (0, store_js_namespaceObject.useEnvConfig)((state)=>state.deepThink);
     const setDeepThink = (0, store_js_namespaceObject.useEnvConfig)((state)=>state.setDeepThink);
-    if (!enableTracking && !showDeepThinkOption) return null;
+    const screenshotIncluded = (0, store_js_namespaceObject.useEnvConfig)((state)=>state.screenshotIncluded);
+    const setScreenshotIncluded = (0, store_js_namespaceObject.useEnvConfig)((state)=>state.setScreenshotIncluded);
+    const domIncluded = (0, store_js_namespaceObject.useEnvConfig)((state)=>state.domIncluded);
+    const setDomIncluded = (0, store_js_namespaceObject.useEnvConfig)((state)=>state.setDomIncluded);
+    if (!enableTracking && !showDeepThinkOption && !showDataExtractionOptions) return null;
     const configItems = buildConfigItems();
     return /*#__PURE__*/ (0, jsx_runtime_namespaceObject.jsx)("div", {
         className: "selector-trigger",
@@ -85,6 +89,54 @@ const ConfigSelector = (param)=>{
             }),
             key: 'deep-think-config'
         });
+        if (showDataExtractionOptions && !hideDomAndScreenshotOptions) {
+            items.push({
+                label: /*#__PURE__*/ (0, jsx_runtime_namespaceObject.jsx)(external_antd_namespaceObject.Checkbox, {
+                    onChange: (e)=>{
+                        setScreenshotIncluded(e.target.checked);
+                    },
+                    checked: screenshotIncluded,
+                    children: external_playground_constants_js_namespaceObject.screenshotIncludedTip
+                }),
+                key: 'screenshot-included-config'
+            });
+            items.push({
+                label: /*#__PURE__*/ (0, jsx_runtime_namespaceObject.jsxs)("div", {
+                    style: {
+                        padding: '4px 0'
+                    },
+                    children: [
+                        /*#__PURE__*/ (0, jsx_runtime_namespaceObject.jsx)("div", {
+                            style: {
+                                marginBottom: '4px',
+                                fontSize: '14px'
+                            },
+                            children: external_playground_constants_js_namespaceObject.domIncludedTip
+                        }),
+                        /*#__PURE__*/ (0, jsx_runtime_namespaceObject.jsxs)(external_antd_namespaceObject.Radio.Group, {
+                            size: "small",
+                            value: domIncluded,
+                            onChange: (e)=>setDomIncluded(e.target.value),
+                            children: [
+                                /*#__PURE__*/ (0, jsx_runtime_namespaceObject.jsx)(external_antd_namespaceObject.Radio, {
+                                    value: false,
+                                    children: "Off"
+                                }),
+                                /*#__PURE__*/ (0, jsx_runtime_namespaceObject.jsx)(external_antd_namespaceObject.Radio, {
+                                    value: true,
+                                    children: "All"
+                                }),
+                                /*#__PURE__*/ (0, jsx_runtime_namespaceObject.jsx)(external_antd_namespaceObject.Radio, {
+                                    value: 'visible-only',
+                                    children: "Visible only"
+                                })
+                            ]
+                        })
+                    ]
+                }),
+                key: 'dom-included-config'
+            });
+        }
         return items;
     }
 };

package/dist/lib/component/playground/PromptInput.js

@@ -50,12 +50,13 @@ const external_types_js_namespaceObject = require("./types.js");
 require("./index.css");
 const { TextArea } = external_antd_namespaceObject.Input;
 const PromptInput = (param)=>{
-    let { runButtonEnabled, form, serviceMode, selectedType, dryMode, stoppable, loading, onRun, onStop, clearPromptAfterRun = true, actionSpace } = param;
+    let { runButtonEnabled, form, serviceMode, selectedType, dryMode, stoppable, loading, onRun, onStop, clearPromptAfterRun = true, actionSpace, hideDomAndScreenshotOptions = false } = param;
     const [hoveringSettings, setHoveringSettings] = (0, external_react_namespaceObject.useState)(false);
     const [promptValue, setPromptValue] = (0, external_react_namespaceObject.useState)('');
     const placeholder = (0, external_playground_utils_js_namespaceObject.getPlaceholderForType)(selectedType);
     const textAreaRef = (0, external_react_namespaceObject.useRef)(null);
     const params = external_antd_namespaceObject.Form.useWatch('params', form);
+    const lastHistoryRef = (0, external_react_namespaceObject.useRef)(null);
     const history = (0, history_js_namespaceObject.useHistoryStore)((state)=>state.history);
     const lastSelectedType = (0, history_js_namespaceObject.useHistoryStore)((state)=>state.lastSelectedType);
     const addHistory = (0, history_js_namespaceObject.useHistoryStore)((state)=>state.addHistory);
@@ -67,13 +68,28 @@ const PromptInput = (param)=>{
     const needsStructuredParams = (0, external_react_namespaceObject.useMemo)(()=>{
         if (actionSpace) {
             const action = actionSpace.find((a)=>a.interfaceAlias === selectedType || a.name === selectedType);
-            return null == action ? void 0 : action.paramSchema;
+            return !!(null == action ? void 0 : action.paramSchema);
         }
+        return false;
     }, [
         selectedType,
         actionSpace
     ]);
+    const showDataExtractionOptions = (0, external_react_namespaceObject.useMemo)(()=>{
+        const dataExtractionMethods = [
+            'aiQuery',
+            'aiBoolean',
+            'aiNumber',
+            'aiString',
+            'aiAsk',
+            'aiAssert'
+        ];
+        return dataExtractionMethods.includes(selectedType);
+    }, [
+        selectedType
+    ]);
     const showDeepThinkOption = (0, external_react_namespaceObject.useMemo)(()=>{
+        if ('aiLocate' === selectedType) return true;
         if (actionSpace) {
             const action = actionSpace.find((a)=>a.interfaceAlias === selectedType || a.name === selectedType);
             if ((null == action ? void 0 : action.paramSchema) && (0, external_types_js_namespaceObject.isZodObjectSchema)(action.paramSchema)) {
@@ -84,15 +100,8 @@ const PromptInput = (param)=>{
                     return (0, external_types_js_namespaceObject.isLocateField)(actualField);
                 });
             }
-            return false;
         }
-        return [
-            'aiTap',
-            'aiHover',
-            'aiInput',
-            'aiRightClick',
-            'aiLocate'
-        ].includes(selectedType);
+        return false;
     }, [
         selectedType,
         actionSpace
@@ -131,6 +140,7 @@ const PromptInput = (param)=>{
     ]);
     (0, external_react_namespaceObject.useEffect)(()=>{
         const lastHistory = historyForSelectedType[0];
+        if (lastHistory && lastHistoryRef.current && lastHistory.timestamp === lastHistoryRef.current.timestamp) return;
         if (lastHistory) {
             form.setFieldsValue({
                 type: lastHistory.type,
@@ -138,6 +148,7 @@ const PromptInput = (param)=>{
                 params: lastHistory.params
             });
             setPromptValue(lastHistory.prompt || '');
+            lastHistoryRef.current = lastHistory;
         } else {
             const defaultParams = getDefaultParams();
             form.setFieldsValue({
@@ -145,6 +156,7 @@ const PromptInput = (param)=>{
                 params: defaultParams
             });
             setPromptValue('');
+            lastHistoryRef.current = null;
         }
     }, [
         selectedType,
@@ -152,6 +164,13 @@ const PromptInput = (param)=>{
         form,
         getDefaultParams
     ]);
+    const formPromptValue = external_antd_namespaceObject.Form.useWatch('prompt', form);
+    (0, external_react_namespaceObject.useEffect)(()=>{
+        if (formPromptValue !== promptValue) setPromptValue(formPromptValue || '');
+    }, [
+        formPromptValue,
+        promptValue
+    ]);
     const handleSelectHistory = (0, external_react_namespaceObject.useCallback)((historyItem)=>{
         form.setFieldsValue({
             prompt: historyItem.prompt,
@@ -165,10 +184,7 @@ const PromptInput = (param)=>{
     const handlePromptChange = (0, external_react_namespaceObject.useCallback)((e)=>{
         const value = e.target.value;
         setPromptValue(value);
-        form.setFieldValue('prompt', value);
-    }, [
-        form
-    ]);
+    }, []);
     const hasSingleStructuredParam = (0, external_react_namespaceObject.useMemo)(()=>{
         if (!needsStructuredParams || !actionSpace) return false;
         const action = actionSpace.find((a)=>a.interfaceAlias === selectedType || a.name === selectedType);
@@ -213,14 +229,16 @@ const PromptInput = (param)=>{
                 historyPrompt = locateValue ? `${mainPart} | ${locateValue}` : mainPart;
             } else historyPrompt = values.prompt || '';
         } else historyPrompt = values.prompt || '';
-        addHistory({
+        const newHistoryItem = {
             type: values.type,
             prompt: historyPrompt,
             params: values.params,
             timestamp: Date.now()
-        });
+        };
+        addHistory(newHistoryItem);
         onRun();
         if (clearPromptAfterRun) {
+            lastHistoryRef.current = newHistoryItem;
             setPromptValue('');
             if (needsStructuredParams) {
                 const defaultParams = getDefaultParams();
@@ -521,7 +539,9 @@ const PromptInput = (param)=>{
                                 onMouseLeave: handleMouseLeave,
                                 children: /*#__PURE__*/ (0, jsx_runtime_namespaceObject.jsx)(external_ConfigSelector_js_namespaceObject.ConfigSelector, {
                                     enableTracking: 'In-Browser-Extension' === serviceMode,
-                                    showDeepThinkOption: showDeepThinkOption
+                                    showDeepThinkOption: showDeepThinkOption,
+                                    showDataExtractionOptions: showDataExtractionOptions,
+                                    hideDomAndScreenshotOptions: hideDomAndScreenshotOptions
                                 })
                             })
                         ]
@@ -547,7 +567,6 @@ const PromptInput = (param)=>{
                             autoFocus: true,
                             onKeyDown: handleKeyDown,
                             onChange: handlePromptChange,
-                            value: promptValue,
                             ref: textAreaRef
                         })
                     }),

package/dist/lib/component/playground/playground-constants.js

@@ -33,12 +33,14 @@ var __webpack_require__ = {};
 var __webpack_exports__ = {};
 __webpack_require__.r(__webpack_exports__);
 __webpack_require__.d(__webpack_exports__, {
-    apiMetadata: ()=>apiMetadata,
+    screenshotIncludedTip: ()=>screenshotIncludedTip,
     errorMessageServerNotReady: ()=>errorMessageServerNotReady,
     serverLaunchTip: ()=>serverLaunchTip,
     defaultMainButtons: ()=>defaultMainButtons,
-    deepThinkTip: ()=>deepThinkTip,
+    apiMetadata: ()=>apiMetadata,
     trackingTip: ()=>trackingTip,
+    deepThinkTip: ()=>deepThinkTip,
+    domIncludedTip: ()=>domIncludedTip,
     emptyResultTip: ()=>emptyResultTip
 });
 const jsx_runtime_namespaceObject = require("react/jsx-runtime");
@@ -86,6 +88,8 @@ const emptyResultTip = /*#__PURE__*/ (0, jsx_runtime_namespaceObject.jsx)("div",
 });
 const trackingTip = 'limit popup to current tab';
 const deepThinkTip = 'deep think';
+const screenshotIncludedTip = 'include screenshot in request';
+const domIncludedTip = 'include DOM info in request';
 const apiMetadata = {
     aiAction: {
         group: 'interaction',
@@ -157,16 +161,20 @@ const defaultMainButtons = [
 exports.apiMetadata = __webpack_exports__.apiMetadata;
 exports.deepThinkTip = __webpack_exports__.deepThinkTip;
 exports.defaultMainButtons = __webpack_exports__.defaultMainButtons;
+exports.domIncludedTip = __webpack_exports__.domIncludedTip;
 exports.emptyResultTip = __webpack_exports__.emptyResultTip;
 exports.errorMessageServerNotReady = __webpack_exports__.errorMessageServerNotReady;
+exports.screenshotIncludedTip = __webpack_exports__.screenshotIncludedTip;
 exports.serverLaunchTip = __webpack_exports__.serverLaunchTip;
 exports.trackingTip = __webpack_exports__.trackingTip;
 for(var __webpack_i__ in __webpack_exports__)if (-1 === [
     "apiMetadata",
     "deepThinkTip",
     "defaultMainButtons",
+    "domIncludedTip",
     "emptyResultTip",
     "errorMessageServerNotReady",
+    "screenshotIncludedTip",
     "serverLaunchTip",
     "trackingTip"
 ].indexOf(__webpack_i__)) exports[__webpack_i__] = __webpack_exports__[__webpack_i__];

package/dist/lib/component/playground/playground-utils.js

@@ -50,7 +50,7 @@ const checkServerStatus = async ()=>{
     }
 };
 const requestPlaygroundServer = async function(context, type, prompt) {
-    let { requestId, deepThink } = arguments.length > 3 && void 0 !== arguments[3] ? arguments[3] : {};
+    let { requestId, deepThink, screenshotIncluded, domIncluded } = arguments.length > 3 && void 0 !== arguments[3] ? arguments[3] : {};
     const payload = {
         context,
         type,
@@ -58,6 +58,8 @@ const requestPlaygroundServer = async function(context, type, prompt) {
     };
     if (requestId) payload.requestId = requestId;
     if (deepThink) payload.deepThink = deepThink;
+    if (void 0 !== screenshotIncluded) payload.screenshotIncluded = screenshotIncluded;
+    if (void 0 !== domIncluded) payload.domIncluded = domIncluded;
     const res = await fetch(`${serverBase}/execute`, {
         method: 'POST',
         headers: {

package/dist/lib/component/store/store.js

@@ -64,6 +64,8 @@ const CONFIG_KEY = 'midscene-env-config';
 const SERVICE_MODE_KEY = 'midscene-service-mode';
 const TRACKING_ACTIVE_TAB_KEY = 'midscene-tracking-active-tab';
 const DEEP_THINK_KEY = 'midscene-deep-think';
+const SCREENSHOT_INCLUDED_KEY = 'midscene-screenshot-included';
+const DOM_INCLUDED_KEY = 'midscene-dom-included';
 const getConfigStringFromLocalStorage = ()=>{
     const configString = localStorage.getItem(CONFIG_KEY);
     return configString || '';
@@ -92,6 +94,8 @@ const useEnvConfig = create((set, get)=>{
     const savedServiceMode = localStorage.getItem(SERVICE_MODE_KEY);
     const savedForceSameTabNavigation = 'false' !== localStorage.getItem(TRACKING_ACTIVE_TAB_KEY);
     const savedDeepThink = 'true' === localStorage.getItem(DEEP_THINK_KEY);
+    const savedScreenshotIncluded = 'false' !== localStorage.getItem(SCREENSHOT_INCLUDED_KEY);
+    const savedDomIncluded = localStorage.getItem(DOM_INCLUDED_KEY) || 'false';
     return {
         serviceMode: ifInExtension ? 'In-Browser-Extension' : savedServiceMode || 'Server',
         setServiceMode: (serviceMode)=>{
@@ -136,6 +140,20 @@ const useEnvConfig = create((set, get)=>{
             });
             localStorage.setItem(DEEP_THINK_KEY, deepThink.toString());
         },
+        screenshotIncluded: savedScreenshotIncluded,
+        setScreenshotIncluded: (screenshotIncluded)=>{
+            set({
+                screenshotIncluded
+            });
+            localStorage.setItem(SCREENSHOT_INCLUDED_KEY, screenshotIncluded.toString());
+        },
+        domIncluded: 'visible-only' === savedDomIncluded ? 'visible-only' : 'true' === savedDomIncluded,
+        setDomIncluded: (domIncluded)=>{
+            set({
+                domIncluded
+            });
+            localStorage.setItem(DOM_INCLUDED_KEY, domIncluded.toString());
+        },
         popupTab: 'playground',
         setPopupTab: (tab)=>{
             set({

package/dist/types/component/playground/ConfigSelector.d.ts

@@ -3,6 +3,8 @@ import './index.less';
 interface ConfigSelectorProps {
     showDeepThinkOption: boolean;
     enableTracking: boolean;
+    showDataExtractionOptions: boolean;
+    hideDomAndScreenshotOptions?: boolean;
 }
 export declare const ConfigSelector: React.FC<ConfigSelectorProps>;
 export {};

package/dist/types/component/playground/PromptInput.d.ts

@@ -15,6 +15,7 @@ interface PromptInputProps {
     onStop: () => void;
     clearPromptAfterRun?: boolean;
     actionSpace?: DeviceAction<any>[];
+    hideDomAndScreenshotOptions?: boolean;
 }
 export declare const PromptInput: React.FC<PromptInputProps>;
 export {};

package/dist/types/component/playground/playground-constants.d.ts

@@ -5,6 +5,8 @@ export declare const serverLaunchTip: (notReadyMessage?: React.ReactNode | strin
 export declare const emptyResultTip: React.JSX.Element;
 export declare const trackingTip = "limit popup to current tab";
 export declare const deepThinkTip = "deep think";
+export declare const screenshotIncludedTip = "include screenshot in request";
+export declare const domIncludedTip = "include DOM info in request";
 export declare const apiMetadata: {
     aiAction: {
         group: string;

package/dist/types/component/playground/playground-utils.d.ts

@@ -3,9 +3,11 @@ import type { WebUIContext } from '@midscene/web';
 import { StaticPageAgent } from '@midscene/web/playground';
 export declare const serverBase = "http://localhost:5800";
 export declare const checkServerStatus: () => Promise<boolean>;
-export declare const requestPlaygroundServer: (context: UIContext | string, type: string, prompt: string, { requestId, deepThink, }?: {
+export declare const requestPlaygroundServer: (context: UIContext | string, type: string, prompt: string, { requestId, deepThink, screenshotIncluded, domIncluded, }?: {
     requestId?: string;
     deepThink?: boolean;
+    screenshotIncluded?: boolean;
+    domIncluded?: boolean | "visible-only";
 }) => Promise<any>;
 export declare const overrideServerConfig: (aiConfig: any) => Promise<Response>;
 export declare const cancelTask: (requestId: string) => Promise<any>;

package/dist/types/component/store/store.d.ts

@@ -27,6 +27,10 @@ export declare const useEnvConfig: Z.UseBoundStore<Z.StoreApi<{
     setForceSameTabNavigation: (forceSameTabNavigation: boolean) => void;
     deepThink: boolean;
     setDeepThink: (deepThink: boolean) => void;
+    screenshotIncluded: boolean;
+    setScreenshotIncluded: (screenshotIncluded: boolean) => void;
+    domIncluded: boolean | "visible-only";
+    setDomIncluded: (domIncluded: boolean | "visible-only") => void;
     popupTab: "playground" | "bridge" | "recorder";
     setPopupTab: (tab: "playground" | "bridge" | "recorder") => void;
 }>>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@midscene/visualizer",
-  "version": "0.27.5-beta-20250827031507.0",
+  "version": "0.27.5-beta-20250827160628.0",
   "repository": "https://github.com/web-infra-dev/midscene",
   "homepage": "https://midscenejs.com/",
   "types": "./dist/types/index.d.ts",
@@ -70,9 +70,9 @@
     "antd": "^5.21.6",
     "buffer": "6.0.3",
     "dayjs": "^1.11.11",
-    "@midscene/core": "0.27.5-beta-20250827031507.0",
-    "@midscene/shared": "0.27.5-beta-20250827031507.0",
-    "@midscene/web": "0.27.5-beta-20250827031507.0"
+    "@midscene/shared": "0.27.5-beta-20250827160628.0",
+    "@midscene/core": "0.27.5-beta-20250827160628.0",
+    "@midscene/web": "0.27.5-beta-20250827160628.0"
   },
   "license": "MIT",
   "scripts": {