@midscene/visualizer 0.27.5-beta-20250828012323.0 → 0.27.5-beta-20250828025824.0

package/dist/lib/component/playground/playground-utils.js

@@ -34,6 +34,7 @@ __webpack_require__.d(__webpack_exports__, {
     requestPlaygroundServer: ()=>requestPlaygroundServer,
     staticAgentFromContext: ()=>staticAgentFromContext,
     checkServerStatus: ()=>checkServerStatus,
+    getActionSpace: ()=>getActionSpace,
     overrideServerConfig: ()=>overrideServerConfig,
     getTaskProgress: ()=>getTaskProgress
 });
@@ -50,7 +51,7 @@ const checkServerStatus = async ()=>{
     }
 };
 const requestPlaygroundServer = async function(context, type, prompt) {
-    let { requestId, deepThink, screenshotIncluded, domIncluded } = arguments.length > 3 && void 0 !== arguments[3] ? arguments[3] : {};
+    let { requestId, deepThink, params, screenshotIncluded, domIncluded } = arguments.length > 3 && void 0 !== arguments[3] ? arguments[3] : {};
     const payload = {
         context,
         type,
@@ -60,6 +61,7 @@ const requestPlaygroundServer = async function(context, type, prompt) {
     if (deepThink) payload.deepThink = deepThink;
     if (void 0 !== screenshotIncluded) payload.screenshotIncluded = screenshotIncluded;
     if (void 0 !== domIncluded) payload.domIncluded = domIncluded;
+    if (params) payload.params = params;
     const res = await fetch(`${serverBase}/execute`, {
         method: 'POST',
         headers: {
@@ -100,9 +102,31 @@ const getTaskProgress = async (requestId)=>{
         };
     }
 };
+const getActionSpace = async (context)=>{
+    try {
+        if (!context) return [];
+        const response = await fetch(`${serverBase}/action-space`, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json'
+            },
+            body: JSON.stringify({
+                context
+            })
+        });
+        if (response.ok) return await response.json();
+        return [];
+    } catch (error) {
+        console.error('Failed to get action space:', error);
+        return [];
+    }
+};
 const actionNameForType = (type)=>{
     const typeWithoutAi = type.startsWith('ai') ? type.slice(2) : type;
-    return typeWithoutAi.replace(/([A-Z])/g, ' $1').trim();
+    const fullName = typeWithoutAi.replace(/([A-Z])/g, ' $1').trim();
+    const words = fullName.split(' ');
+    if (words.length > 3) return words.slice(-3).join(' ');
+    return fullName;
 };
 const staticAgentFromContext = (context)=>{
     const page = new playground_namespaceObject.StaticPage(context);
@@ -111,8 +135,7 @@ const staticAgentFromContext = (context)=>{
 const formatErrorMessage = (e)=>{
     const errorMessage = (null == e ? void 0 : e.message) || '';
     if (errorMessage.includes('of different extension')) return 'Conflicting extension detected. Please disable the suspicious plugins and refresh the page. Guide: https://midscenejs.com/quick-experience.html#faq';
-    if (!(null == errorMessage ? void 0 : errorMessage.includes(playground_namespaceObject.ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED))) return errorMessage;
-    return 'Unknown error';
+    return errorMessage || 'Unknown error';
 };
 const getPlaceholderForType = (type)=>{
     if ('aiQuery' === type) return 'What do you want to query?';
@@ -139,13 +162,29 @@ const blankResult = {
 };
 const isRunButtonEnabled = (runButtonEnabled, needsStructuredParams, params, actionSpace, selectedType, promptValue)=>{
     if (!runButtonEnabled) return false;
+    const needsAnyInput = (()=>{
+        if (actionSpace) {
+            const action = actionSpace.find((a)=>a.interfaceAlias === selectedType || a.name === selectedType);
+            if (action) {
+                if (!action.paramSchema) return false;
+                if ('object' == typeof action.paramSchema && 'shape' in action.paramSchema) {
+                    const shape = action.paramSchema.shape || {};
+                    const shapeKeys = Object.keys(shape);
+                    return shapeKeys.length > 0;
+                }
+            }
+        }
+        return true;
+    })();
+    if (!needsAnyInput) return true;
     if (needsStructuredParams) {
         const currentParams = params || {};
         const action = null == actionSpace ? void 0 : actionSpace.find((a)=>a.interfaceAlias === selectedType || a.name === selectedType);
         if ((null == action ? void 0 : action.paramSchema) && (0, external_types_js_namespaceObject.isZodObjectSchema)(action.paramSchema)) {
             const schema = action.paramSchema;
-            return Object.keys(schema.shape).every((key)=>{
-                const field = schema.shape[key];
+            const shape = schema.shape || {};
+            return Object.keys(shape).every((key)=>{
+                const field = shape[key];
                 const { isOptional } = (0, external_types_js_namespaceObject.unwrapZodType)(field);
                 const value = currentParams[key];
                 return isOptional || void 0 !== value && '' !== value && null !== value;
@@ -160,6 +199,7 @@ exports.blankResult = __webpack_exports__.blankResult;
 exports.cancelTask = __webpack_exports__.cancelTask;
 exports.checkServerStatus = __webpack_exports__.checkServerStatus;
 exports.formatErrorMessage = __webpack_exports__.formatErrorMessage;
+exports.getActionSpace = __webpack_exports__.getActionSpace;
 exports.getPlaceholderForType = __webpack_exports__.getPlaceholderForType;
 exports.getTaskProgress = __webpack_exports__.getTaskProgress;
 exports.isRunButtonEnabled = __webpack_exports__.isRunButtonEnabled;
@@ -173,6 +213,7 @@ for(var __webpack_i__ in __webpack_exports__)if (-1 === [
     "cancelTask",
     "checkServerStatus",
     "formatErrorMessage",
+    "getActionSpace",
     "getPlaceholderForType",
     "getTaskProgress",
     "isRunButtonEnabled",

package/dist/lib/component/playground/types.js

@@ -49,10 +49,29 @@ const VALIDATION_CONSTANTS = {
         CHECK_INTERVAL_MS: 3000
     }
 };
-const isZodObjectSchema = (schema)=>'object' == typeof schema && null !== schema && 'shape' in schema && 'parse' in schema && 'function' == typeof schema.parse;
+const isZodObjectSchema = (schema)=>'object' == typeof schema && null !== schema && ('shape' in schema || 'ZodObject' === schema.type);
 const isLocateField = (field)=>{
-    var _field__def, _field__def_shape, _field__def1, _field__def_shape1, _field__def2;
-    return !!((null == (_field__def = field._def) ? void 0 : _field__def.typeName) === VALIDATION_CONSTANTS.ZOD_TYPES.OBJECT && (null == (_field__def_shape = (_field__def1 = field._def).shape) ? void 0 : _field__def_shape.call(_field__def1)) && VALIDATION_CONSTANTS.FIELD_FLAGS.LOCATION in ((null == (_field__def_shape1 = (_field__def2 = field._def).shape) ? void 0 : _field__def_shape1.call(_field__def2)) || {}));
+    var _field__def;
+    const fieldAsAny = field;
+    if ((null == (_field__def = field._def) ? void 0 : _field__def.typeName) === VALIDATION_CONSTANTS.ZOD_TYPES.OBJECT) {
+        var _field__def1;
+        let shape;
+        if (field._def.shape) shape = 'function' == typeof field._def.shape ? field._def.shape() : field._def.shape;
+        if (!shape && fieldAsAny.shape) shape = fieldAsAny.shape;
+        if (shape && VALIDATION_CONSTANTS.FIELD_FLAGS.LOCATION in shape) return true;
+        const description = (null == (_field__def1 = field._def) ? void 0 : _field__def1.description) || fieldAsAny.description || '';
+        if ('string' == typeof description && description.toLowerCase().includes('input field')) return true;
+    }
+    if ('object' == typeof field && null !== field) {
+        var _fieldAsAny__def;
+        const description = fieldAsAny.description || (null == (_fieldAsAny__def = fieldAsAny._def) ? void 0 : _fieldAsAny__def.description) || '';
+        if ('string' == typeof description) {
+            const desc = description.toLowerCase();
+            if (desc.includes('input field') || desc.includes('element') || desc.includes('locate')) return true;
+        }
+        if ('ZodObject' === fieldAsAny.typeName || 'ZodObject' === fieldAsAny.type) return 'string' == typeof description && description.toLowerCase().includes('input field');
+    }
+    return false;
 };
 const unwrapZodType = (field)=>{
     var _actualField__def, _actualField__def1, _actualField__def2;

package/dist/lib/index.js

@@ -43,22 +43,24 @@ __webpack_require__.d(__webpack_exports__, {
     generateAnimationScripts: ()=>replay_scripts_js_namespaceObject.generateAnimationScripts,
     getPlaceholderForType: ()=>playground_utils_js_namespaceObject.getPlaceholderForType,
     ShinyText: ()=>shiny_text_js_default(),
+    getActionSpace: ()=>playground_utils_js_namespaceObject.getActionSpace,
     staticAgentFromContext: ()=>playground_utils_js_namespaceObject.staticAgentFromContext,
-    useStaticPageAgent: ()=>useStaticPageAgent_js_namespaceObject.useStaticPageAgent,
     useServerValid: ()=>useServerValid_js_namespaceObject.useServerValid,
     ServiceModeControl: ()=>ServiceModeControl_js_namespaceObject.ServiceModeControl,
     colorForName: ()=>color_js_namespaceObject.colorForName,
+    validateStructuredParams: ()=>playground_execution_js_namespaceObject.validateStructuredParams,
     GithubStar: ()=>github_star_js_namespaceObject.GithubStar,
     Logo: ()=>logo_js_namespaceObject.Logo,
     checkServerStatus: ()=>playground_utils_js_namespaceObject.checkServerStatus,
     blankResult: ()=>playground_utils_js_namespaceObject.blankResult,
     useEnvConfig: ()=>store_js_namespaceObject.useEnvConfig,
+    executeAction: ()=>playground_execution_js_namespaceObject.executeAction,
     PromptInput: ()=>PromptInput_js_namespaceObject.PromptInput,
     ContextPreview: ()=>ContextPreview_js_namespaceObject.ContextPreview,
     requestPlaygroundServer: ()=>playground_utils_js_namespaceObject.requestPlaygroundServer,
     Blackboard: ()=>blackboard_js_namespaceObject.Blackboard,
     overrideServerConfig: ()=>playground_utils_js_namespaceObject.overrideServerConfig,
-    formatErrorMessage: ()=>playground_utils_js_namespaceObject.formatErrorMessage,
+    formatErrorMessage: ()=>playground_execution_js_namespaceObject.formatErrorMessage,
     Player: ()=>player_js_namespaceObject.Player,
     filterBase64Value: ()=>external_utils_js_namespaceObject.filterBase64Value,
     highlightColorForType: ()=>color_js_namespaceObject.highlightColorForType,
@@ -66,7 +68,6 @@ __webpack_require__.d(__webpack_exports__, {
     globalThemeConfig: ()=>color_js_namespaceObject.globalThemeConfig,
     getTaskProgress: ()=>playground_utils_js_namespaceObject.getTaskProgress
 });
-const useStaticPageAgent_js_namespaceObject = require("./component/playground/useStaticPageAgent.js");
 require("./component/playground/index.css");
 const replay_scripts_js_namespaceObject = require("./component/replay-scripts.js");
 const store_js_namespaceObject = require("./component/store/store.js");
@@ -83,6 +84,7 @@ const player_js_namespaceObject = require("./component/player.js");
 const blackboard_js_namespaceObject = require("./component/blackboard.js");
 const github_star_js_namespaceObject = require("./component/github-star.js");
 const playground_utils_js_namespaceObject = require("./component/playground/playground-utils.js");
+const playground_execution_js_namespaceObject = require("./component/playground/playground-execution.js");
 const external_utils_js_namespaceObject = require("./utils.js");
 const shiny_text_js_namespaceObject = require("./component/shiny-text.js");
 var shiny_text_js_default = /*#__PURE__*/ __webpack_require__.n(shiny_text_js_namespaceObject);
@@ -102,9 +104,11 @@ exports.blankResult = __webpack_exports__.blankResult;
 exports.cancelTask = __webpack_exports__.cancelTask;
 exports.checkServerStatus = __webpack_exports__.checkServerStatus;
 exports.colorForName = __webpack_exports__.colorForName;
+exports.executeAction = __webpack_exports__.executeAction;
 exports.filterBase64Value = __webpack_exports__.filterBase64Value;
 exports.formatErrorMessage = __webpack_exports__.formatErrorMessage;
 exports.generateAnimationScripts = __webpack_exports__.generateAnimationScripts;
+exports.getActionSpace = __webpack_exports__.getActionSpace;
 exports.getPlaceholderForType = __webpack_exports__.getPlaceholderForType;
 exports.getTaskProgress = __webpack_exports__.getTaskProgress;
 exports.globalThemeConfig = __webpack_exports__.globalThemeConfig;
@@ -117,7 +121,7 @@ exports.timeCostStrElement = __webpack_exports__.timeCostStrElement;
 exports.timeStr = __webpack_exports__.timeStr;
 exports.useEnvConfig = __webpack_exports__.useEnvConfig;
 exports.useServerValid = __webpack_exports__.useServerValid;
-exports.useStaticPageAgent = __webpack_exports__.useStaticPageAgent;
+exports.validateStructuredParams = __webpack_exports__.validateStructuredParams;
 for(var __webpack_i__ in __webpack_exports__)if (-1 === [
     "Blackboard",
     "ContextPreview",
@@ -135,9 +139,11 @@ for(var __webpack_i__ in __webpack_exports__)if (-1 === [
     "cancelTask",
     "checkServerStatus",
     "colorForName",
+    "executeAction",
     "filterBase64Value",
     "formatErrorMessage",
     "generateAnimationScripts",
+    "getActionSpace",
     "getPlaceholderForType",
     "getTaskProgress",
     "globalThemeConfig",
@@ -150,7 +156,7 @@ for(var __webpack_i__ in __webpack_exports__)if (-1 === [
     "timeStr",
     "useEnvConfig",
     "useServerValid",
-    "useStaticPageAgent"
+    "validateStructuredParams"
 ].indexOf(__webpack_i__)) exports[__webpack_i__] = __webpack_exports__[__webpack_i__];
 Object.defineProperty(exports, '__esModule', {
     value: true

package/dist/types/component/playground/FormField.d.ts

@@ -7,6 +7,7 @@ interface FormFieldProps {
     isRequired: boolean;
     isLocateField: boolean;
     marginBottom: number;
+    placeholder?: string;
 }
 export declare const TextField: React.FC<Omit<FormFieldProps, 'isLocateField'>>;
 export declare const LocateField: React.FC<Omit<FormFieldProps, 'isLocateField'>>;

package/dist/types/component/playground/PromptInput.d.ts

@@ -14,8 +14,8 @@ interface PromptInputProps {
     onRun: () => void;
     onStop: () => void;
     clearPromptAfterRun?: boolean;
-    actionSpace?: DeviceAction<any>[];
     hideDomAndScreenshotOptions?: boolean;
+    actionSpace: DeviceAction<any>[];
 }
 export declare const PromptInput: React.FC<PromptInputProps>;
 export {};

package/dist/types/component/playground/playground-execution.d.ts

@@ -0,0 +1,7 @@
+import type { DeviceAction } from '@midscene/core';
+export declare const formatErrorMessage: (e: any) => string;
+export declare function validateStructuredParams(value: any, action: DeviceAction<any> | undefined): {
+    valid: boolean;
+    errorMessage?: string;
+};
+export declare function executeAction(activeAgent: any, actionType: string, actionSpace: DeviceAction<any>[], value: any, deepThink: boolean): Promise<any>;

package/dist/types/component/playground/playground-utils.d.ts

@@ -3,15 +3,17 @@ import type { WebUIContext } from '@midscene/web';
 import { StaticPageAgent } from '@midscene/web/playground';
 export declare const serverBase = "http://localhost:5800";
 export declare const checkServerStatus: () => Promise<boolean>;
-export declare const requestPlaygroundServer: (context: UIContext | string, type: string, prompt: string, { requestId, deepThink, screenshotIncluded, domIncluded, }?: {
+export declare const requestPlaygroundServer: (context: UIContext | string, type: string, prompt: string, { requestId, deepThink, params, screenshotIncluded, domIncluded, }?: {
     requestId?: string;
     deepThink?: boolean;
+    params?: any;
     screenshotIncluded?: boolean;
     domIncluded?: boolean | "visible-only";
 }) => Promise<any>;
 export declare const overrideServerConfig: (aiConfig: any) => Promise<Response>;
 export declare const cancelTask: (requestId: string) => Promise<any>;
 export declare const getTaskProgress: (requestId: string) => Promise<any>;
+export declare const getActionSpace: (context?: string) => Promise<any>;
 export declare const actionNameForType: (type: string) => string;
 export declare const staticAgentFromContext: (context: WebUIContext) => StaticPageAgent;
 export declare const formatErrorMessage: (e: any) => string;

package/dist/types/index.d.ts

@@ -1,4 +1,3 @@
-export { useStaticPageAgent } from './component/playground/useStaticPageAgent';
 import './component/playground/index.less';
 export { type AnimationScript, type ReplayScriptsInfo, allScriptsFromDump, generateAnimationScripts, } from './component/replay-scripts';
 export { useEnvConfig } from './component/store/store';
@@ -15,6 +14,7 @@ export { PromptInput } from './component/playground/PromptInput';
 export { Player } from './component/player';
 export { Blackboard } from './component/blackboard';
 export { GithubStar } from './component/github-star';
-export { requestPlaygroundServer, cancelTask, overrideServerConfig, getTaskProgress, checkServerStatus, actionNameForType, staticAgentFromContext, formatErrorMessage, getPlaceholderForType, blankResult, } from './component/playground/playground-utils';
+export { requestPlaygroundServer, cancelTask, overrideServerConfig, getTaskProgress, getActionSpace, checkServerStatus, actionNameForType, staticAgentFromContext, getPlaceholderForType, blankResult, } from './component/playground/playground-utils';
+export { executeAction, validateStructuredParams, formatErrorMessage, } from './component/playground/playground-execution';
 export { timeStr, filterBase64Value } from './utils';
 export { default as ShinyText } from './component/shiny-text';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@midscene/visualizer",
-  "version": "0.27.5-beta-20250828012323.0",
+  "version": "0.27.5-beta-20250828025824.0",
   "repository": "https://github.com/web-infra-dev/midscene",
   "homepage": "https://midscenejs.com/",
   "types": "./dist/types/index.d.ts",
@@ -70,9 +70,9 @@
     "antd": "^5.21.6",
     "buffer": "6.0.3",
     "dayjs": "^1.11.11",
-    "@midscene/core": "0.27.5-beta-20250828012323.0",
-    "@midscene/shared": "0.27.5-beta-20250828012323.0",
-    "@midscene/web": "0.27.5-beta-20250828012323.0"
+    "@midscene/shared": "0.27.5-beta-20250828025824.0",
+    "@midscene/web": "0.27.5-beta-20250828025824.0",
+    "@midscene/core": "0.27.5-beta-20250828025824.0"
   },
   "license": "MIT",
   "scripts": {

package/dist/es/component/playground/useStaticPageAgent.mjs

@@ -1,12 +0,0 @@
-import { useMemo } from "react";
-import { staticAgentFromContext } from "./playground-utils.mjs";
-const useStaticPageAgent = (context)=>{
-    const agent = useMemo(()=>{
-        if (!context) return null;
-        return staticAgentFromContext(context);
-    }, [
-        context
-    ]);
-    return agent;
-};
-export { staticAgentFromContext, useStaticPageAgent };

package/dist/lib/component/playground/useStaticPageAgent.js

@@ -1,49 +0,0 @@
-"use strict";
-var __webpack_require__ = {};
-(()=>{
-    __webpack_require__.d = (exports1, definition)=>{
-        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
-            enumerable: true,
-            get: definition[key]
-        });
-    };
-})();
-(()=>{
-    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
-})();
-(()=>{
-    __webpack_require__.r = (exports1)=>{
-        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
-            value: 'Module'
-        });
-        Object.defineProperty(exports1, '__esModule', {
-            value: true
-        });
-    };
-})();
-var __webpack_exports__ = {};
-__webpack_require__.r(__webpack_exports__);
-__webpack_require__.d(__webpack_exports__, {
-    useStaticPageAgent: ()=>useStaticPageAgent,
-    staticAgentFromContext: ()=>external_playground_utils_js_namespaceObject.staticAgentFromContext
-});
-const external_react_namespaceObject = require("react");
-const external_playground_utils_js_namespaceObject = require("./playground-utils.js");
-const useStaticPageAgent = (context)=>{
-    const agent = (0, external_react_namespaceObject.useMemo)(()=>{
-        if (!context) return null;
-        return (0, external_playground_utils_js_namespaceObject.staticAgentFromContext)(context);
-    }, [
-        context
-    ]);
-    return agent;
-};
-exports.staticAgentFromContext = __webpack_exports__.staticAgentFromContext;
-exports.useStaticPageAgent = __webpack_exports__.useStaticPageAgent;
-for(var __webpack_i__ in __webpack_exports__)if (-1 === [
-    "staticAgentFromContext",
-    "useStaticPageAgent"
-].indexOf(__webpack_i__)) exports[__webpack_i__] = __webpack_exports__[__webpack_i__];
-Object.defineProperty(exports, '__esModule', {
-    value: true
-});

package/dist/types/component/playground/useStaticPageAgent.d.ts

@@ -1,5 +0,0 @@
-import type { WebUIContext } from '@midscene/web';
-import type { StaticPageAgent } from '@midscene/web/playground';
-import { staticAgentFromContext } from './playground-utils';
-export { staticAgentFromContext };
-export declare const useStaticPageAgent: (context: WebUIContext | undefined | null) => StaticPageAgent | null;