@midscene/shared 1.10.8-beta-20260724035136.0 → 1.10.8

package/dist/es/agent-tools/tool-generator.mjs

@@ -417,6 +417,8 @@ function generateCommonTools(getAgent, initArgSchema = {}, initArgCliMetadata, t
                 prompt: z.string().describe('Natural language description of the action to perform, e.g. "press Command+Space, type Safari, press Enter"'),
                 deepLocate: z.boolean().optional().describe('Use deep locate for every element this action targets. Improves precision for small or ambiguous targets at the cost of speed. Defaults to the server --deep-locate setting.'),
                 deepThink: z.boolean().optional().describe('Plan this action with deep thinking (richer context and sub-goal decomposition). Helps with complex multi-step instructions at the cost of speed. Defaults to the server --deep-think setting.'),
+                fileChooserAllowedDir: z.string().optional().describe('Directory that model-planned file uploads may access. Required when the prompt asks the model to upload files.'),
+                ...promptInputExtraSchema,
                 ...initArgSchema
             },
             cli: mergeToolCliMetadata(void 0, initArgCliMetadata),
@@ -439,7 +441,14 @@ function generateCommonTools(getAgent, initArgSchema = {}, initArgCliMetadata, t
                         };
                         if (void 0 !== args.deepLocate) actOptions.deepLocate = args.deepLocate;
                         if (void 0 !== args.deepThink) actOptions.deepThink = args.deepThink;
-                        const result = await agent.aiAction(prompt, actOptions);
+                        if (void 0 !== args.fileChooserAllowedDir) actOptions.fileChooserAllowedDir = args.fileChooserAllowedDir;
+                        const userPrompt = composeUserPrompt({
+                            prompt,
+                            image: args.image,
+                            imageName: args.imageName,
+                            convertHttpImage2Base64: args.convertHttpImage2Base64
+                        });
+                        const result = await agent.aiAction(userPrompt, actOptions);
                         return await captureScreenshotResult(agent, 'act', result);
                     } finally{
                         unsubscribeVerbose();

package/dist/es/env/parse-model-config.mjs

@@ -5,7 +5,7 @@ import { assert } from "../utils.mjs";
 import { maskConfig, parseJson } from "./helper.mjs";
 import { initDebugConfig } from "./init-debug.mjs";
 const MODEL_CONFIG_DOC_URL = 'https://midscenejs.com/model-common-config.html';
-const getCurrentVersion = ()=>"1.10.8-beta-20260724035136.0";
+const getCurrentVersion = ()=>"1.10.8";
 const getInvalidModelFamilyMessage = (modelFamily)=>`Invalid MIDSCENE_MODEL_FAMILY value: ${modelFamily}. Current version v${getCurrentVersion()} accepts the following model families: ${MODEL_FAMILY_VALUES.join(', ')}. You can also visit ${MODEL_CONFIG_DOC_URL} for the latest configuration information.`;
 const KEYS_MAP = {
     insight: INSIGHT_MODEL_CONFIG_KEYS,

package/dist/es/env/types.mjs

@@ -203,6 +203,7 @@ const MODEL_FAMILY_VALUES = [
     'auto-glm-multilingual',
     'gpt-5',
     'kimi',
+    'kimi3',
     'xiaomi-mimo'
 ];
 var types_UITarsModelVersion = /*#__PURE__*/ function(UITarsModelVersion) {

package/dist/lib/agent-tools/tool-generator.js

@@ -447,6 +447,8 @@ function generateCommonTools(getAgent, initArgSchema = {}, initArgCliMetadata, t
                 prompt: external_zod_namespaceObject.z.string().describe('Natural language description of the action to perform, e.g. "press Command+Space, type Safari, press Enter"'),
                 deepLocate: external_zod_namespaceObject.z.boolean().optional().describe('Use deep locate for every element this action targets. Improves precision for small or ambiguous targets at the cost of speed. Defaults to the server --deep-locate setting.'),
                 deepThink: external_zod_namespaceObject.z.boolean().optional().describe('Plan this action with deep thinking (richer context and sub-goal decomposition). Helps with complex multi-step instructions at the cost of speed. Defaults to the server --deep-think setting.'),
+                fileChooserAllowedDir: external_zod_namespaceObject.z.string().optional().describe('Directory that model-planned file uploads may access. Required when the prompt asks the model to upload files.'),
+                ...external_user_prompt_js_namespaceObject.promptInputExtraSchema,
                 ...initArgSchema
             },
             cli: mergeToolCliMetadata(void 0, initArgCliMetadata),
@@ -469,7 +471,14 @@ function generateCommonTools(getAgent, initArgSchema = {}, initArgCliMetadata, t
                         };
                         if (void 0 !== args.deepLocate) actOptions.deepLocate = args.deepLocate;
                         if (void 0 !== args.deepThink) actOptions.deepThink = args.deepThink;
-                        const result = await agent.aiAction(prompt, actOptions);
+                        if (void 0 !== args.fileChooserAllowedDir) actOptions.fileChooserAllowedDir = args.fileChooserAllowedDir;
+                        const userPrompt = (0, external_user_prompt_js_namespaceObject.composeUserPrompt)({
+                            prompt,
+                            image: args.image,
+                            imageName: args.imageName,
+                            convertHttpImage2Base64: args.convertHttpImage2Base64
+                        });
+                        const result = await agent.aiAction(userPrompt, actOptions);
                         return await captureScreenshotResult(agent, 'act', result);
                     } finally{
                         unsubscribeVerbose();

package/dist/lib/env/parse-model-config.js

@@ -37,7 +37,7 @@ const external_utils_js_namespaceObject = require("../utils.js");
 const external_helper_js_namespaceObject = require("./helper.js");
 const external_init_debug_js_namespaceObject = require("./init-debug.js");
 const MODEL_CONFIG_DOC_URL = 'https://midscenejs.com/model-common-config.html';
-const getCurrentVersion = ()=>"1.10.8-beta-20260724035136.0";
+const getCurrentVersion = ()=>"1.10.8";
 const getInvalidModelFamilyMessage = (modelFamily)=>`Invalid MIDSCENE_MODEL_FAMILY value: ${modelFamily}. Current version v${getCurrentVersion()} accepts the following model families: ${external_types_js_namespaceObject.MODEL_FAMILY_VALUES.join(', ')}. You can also visit ${MODEL_CONFIG_DOC_URL} for the latest configuration information.`;
 const KEYS_MAP = {
     insight: external_constants_js_namespaceObject.INSIGHT_MODEL_CONFIG_KEYS,

package/dist/lib/env/types.js

@@ -323,6 +323,7 @@ const MODEL_FAMILY_VALUES = [
     'auto-glm-multilingual',
     'gpt-5',
     'kimi',
+    'kimi3',
     'xiaomi-mimo'
 ];
 var UITarsModelVersion = /*#__PURE__*/ function(UITarsModelVersion) {

package/dist/types/agent-tools/types.d.ts

@@ -81,9 +81,10 @@ export interface ActionSpaceItem {
  * Structural shape compatible with @midscene/core `TUserPrompt`.
  * Declared locally to avoid a circular dep on `@midscene/core` from `@midscene/shared`.
  *
- * Currently consumed only by the `assert` tool in `generateCommonTools`.
- * `aiAction` and `aiWaitFor` stay string-only at the CLI surface because the
- * tools generator does not yet expose multimodal entry points for them.
+ * Consumed by the `assert` and `act` tools in `generateCommonTools`, both of
+ * which forward reference images to core (`aiAssert` / `aiAct`). `aiWaitFor`
+ * stays string-only at the CLI surface because the tools generator does not
+ * yet expose a multimodal entry point for it.
  */
 export type UserPromptLike = string | {
     prompt: string;
@@ -131,7 +132,7 @@ export interface BaseAgent {
     addProgressListener?: (listener: (event: BaseAgentProgressEvent) => void) => () => void;
     recordToReport?: (title?: string, opt?: RecordToReportOptions) => Promise<void>;
     callActionInActionSpace?: (actionName: string, params?: unknown) => Promise<unknown>;
-    aiAction?: (description: string, params?: Record<string, unknown>) => Promise<unknown>;
+    aiAction?: (description: UserPromptLike, params?: Record<string, unknown>) => Promise<unknown>;
     aiWaitFor?: (assertion: string, options: Record<string, unknown>) => Promise<unknown>;
     aiAssert?: (assertion: UserPromptLike, msg?: string, options?: Record<string, unknown>) => Promise<unknown>;
 }

package/dist/types/env/types.d.ts

@@ -131,7 +131,7 @@ export type TGlobalConfig = Record<TEnvKeys, string | undefined>;
 /**
  * valid Model family types
  */
-export type TModelFamily = 'qwen2.5-vl' | 'qwen3-vl' | 'qwen3' | 'qwen3.5' | 'qwen3.6' | 'doubao-vision' | 'doubao-seed' | 'gemini' | 'vlm-ui-tars' | 'vlm-ui-tars-doubao' | 'vlm-ui-tars-doubao-1.5' | 'glm-v' | 'auto-glm' | 'auto-glm-multilingual' | 'gpt-5' | 'kimi' | 'xiaomi-mimo';
+export type TModelFamily = 'qwen2.5-vl' | 'qwen3-vl' | 'qwen3' | 'qwen3.5' | 'qwen3.6' | 'doubao-vision' | 'doubao-seed' | 'gemini' | 'vlm-ui-tars' | 'vlm-ui-tars-doubao' | 'vlm-ui-tars-doubao-1.5' | 'glm-v' | 'auto-glm' | 'auto-glm-multilingual' | 'gpt-5' | 'kimi' | 'kimi3' | 'xiaomi-mimo';
 export declare const MODEL_FAMILY_VALUES: TModelFamily[];
 export interface IModelConfigForInsight {
     [MIDSCENE_INSIGHT_MODEL_NAME]: string;

package/dist/types/types/index.d.ts

@@ -31,4 +31,9 @@ export type LocateResultElement = {
     description: string;
     center: [number, number];
     rect: Rect;
+    /**
+     * Web-only compatibility field returned by `Agent.aiLocate()`.
+     * It is the ratio between physical screenshot pixels and logical CSS pixels.
+     */
+    dpr?: number;
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@midscene/shared",
-  "version": "1.10.8-beta-20260724035136.0",
+  "version": "1.10.8",
   "repository": "https://github.com/web-infra-dev/midscene",
   "homepage": "https://midscenejs.com/",
   "types": "./dist/types/index.d.ts",

package/src/agent-tools/tool-generator.ts

@@ -712,6 +712,13 @@ export function generateCommonTools(
           .describe(
             'Plan this action with deep thinking (richer context and sub-goal decomposition). Helps with complex multi-step instructions at the cost of speed. Defaults to the server --deep-think setting.',
           ),
+        fileChooserAllowedDir: z
+          .string()
+          .optional()
+          .describe(
+            'Directory that model-planned file uploads may access. Required when the prompt asks the model to upload files.',
+          ),
+        ...promptInputExtraSchema,
         ...initArgSchema,
       },
       cli: mergeToolCliMetadata(undefined, initArgCliMetadata),
@@ -744,7 +751,16 @@ export function generateCommonTools(
             if (args.deepThink !== undefined) {
               actOptions.deepThink = args.deepThink;
             }
-            const result = await agent.aiAction(prompt, actOptions);
+            if (args.fileChooserAllowedDir !== undefined) {
+              actOptions.fileChooserAllowedDir = args.fileChooserAllowedDir;
+            }
+            const userPrompt = composeUserPrompt({
+              prompt,
+              image: args.image,
+              imageName: args.imageName,
+              convertHttpImage2Base64: args.convertHttpImage2Base64,
+            });
+            const result = await agent.aiAction(userPrompt, actOptions);
             return await captureScreenshotResult(agent, 'act', result);
           } finally {
             unsubscribeVerbose();

package/src/agent-tools/types.ts

@@ -84,9 +84,10 @@ export interface ActionSpaceItem {
  * Structural shape compatible with @midscene/core `TUserPrompt`.
  * Declared locally to avoid a circular dep on `@midscene/core` from `@midscene/shared`.
  *
- * Currently consumed only by the `assert` tool in `generateCommonTools`.
- * `aiAction` and `aiWaitFor` stay string-only at the CLI surface because the
- * tools generator does not yet expose multimodal entry points for them.
+ * Consumed by the `assert` and `act` tools in `generateCommonTools`, both of
+ * which forward reference images to core (`aiAssert` / `aiAct`). `aiWaitFor`
+ * stays string-only at the CLI surface because the tools generator does not
+ * yet expose a multimodal entry point for it.
  */
 export type UserPromptLike =
   | string
@@ -148,7 +149,7 @@ export interface BaseAgent {
     params?: unknown,
   ) => Promise<unknown>;
   aiAction?: (
-    description: string,
+    description: UserPromptLike,
     params?: Record<string, unknown>,
   ) => Promise<unknown>;
   aiWaitFor?: (

package/src/env/types.ts

@@ -310,6 +310,7 @@ export type TModelFamily =
   | 'auto-glm-multilingual'
   | 'gpt-5'
   | 'kimi'
+  | 'kimi3'
   | 'xiaomi-mimo';
 
 export const MODEL_FAMILY_VALUES: TModelFamily[] = [
@@ -329,6 +330,7 @@ export const MODEL_FAMILY_VALUES: TModelFamily[] = [
   'auto-glm-multilingual',
   'gpt-5',
   'kimi',
+  'kimi3',
   'xiaomi-mimo',
 ];
 

package/src/types/index.ts

@@ -49,4 +49,9 @@ export type LocateResultElement = {
   description: string; // the description of the element
   center: [number, number];
   rect: Rect;
+  /**
+   * Web-only compatibility field returned by `Agent.aiLocate()`.
+   * It is the ratio between physical screenshot pixels and logical CSS pixels.
+   */
+  dpr?: number;
 };