@midscene/core 1.2.2-beta-20260116114131.0 → 1.2.2-beta-20260119111553.0

package/dist/types/agent/agent.d.ts

@@ -86,7 +86,7 @@ export declare class Agent<InterfaceType extends AbstractInterface = AbstractInt
     } & {
         autoDismissKeyboard?: boolean;
     } & {
-        mode?: 'replace' | 'clear' | 'append';
+        mode?: 'replace' | 'clear' | 'typeOnly' | 'append';
     }): Promise<any>;
     /**
      * @deprecated Use aiInput(locatePrompt, opt) instead where opt contains the value
@@ -94,7 +94,7 @@ export declare class Agent<InterfaceType extends AbstractInterface = AbstractInt
     aiInput(value: string | number, locatePrompt: TUserPrompt, opt?: LocateOption & {
         autoDismissKeyboard?: boolean;
     } & {
-        mode?: 'replace' | 'clear' | 'append';
+        mode?: 'replace' | 'clear' | 'typeOnly' | 'append';
     }): Promise<any>;
     aiKeyboardPress(locatePrompt: TUserPrompt, opt: LocateOption & {
         keyName: string;

package/dist/types/ai-model/auto-glm/index.d.ts

@@ -2,5 +2,5 @@ export { getAutoGLMLocatePrompt, getAutoGLMPlanPrompt } from './prompt';
 export { parseAutoGLMLocateResponse, parseAutoGLMResponse, parseAction, } from './parser';
 export { autoGLMPlanning } from './planning';
 export { transformAutoGLMAction } from './actions';
-export { isAutoGLM } from './util';
+export { isAutoGLM, isUITars } from './util';
 export type { ParsedAction } from './actions';

package/dist/types/ai-model/auto-glm/prompt.d.ts

@@ -22,6 +22,6 @@
  * Modifications:
  * - Adapted prompts for Midscene.js integration
  */
-import type { TVlModeTypes } from '@midscene/shared/env';
-export declare const getAutoGLMPlanPrompt: (vlMode: TVlModeTypes | undefined) => string;
-export declare const getAutoGLMLocatePrompt: (vlMode: TVlModeTypes | undefined) => string;
+import type { TModelFamily } from '@midscene/shared/env';
+export declare const getAutoGLMPlanPrompt: (modelFamily: TModelFamily | undefined) => string;
+export declare const getAutoGLMLocatePrompt: (modelFamily: TModelFamily | undefined) => string;

package/dist/types/ai-model/auto-glm/util.d.ts

@@ -1,7 +1,13 @@
-import type { TVlModeTypes } from '@midscene/shared/env';
+import type { TModelFamily } from '@midscene/shared/env';
 /**
- * Check if the vlMode is auto-glm or auto-glm-multilingual
- * @param vlMode The VL mode to check
- * @returns true if vlMode is auto-glm or auto-glm-multilingual
+ * Check if the modelFamily is auto-glm or auto-glm-multilingual
+ * @param modelFamily The model family to check
+ * @returns true if modelFamily is auto-glm or auto-glm-multilingual
  */
-export declare function isAutoGLM(vlMode: TVlModeTypes | undefined): boolean;
+export declare function isAutoGLM(modelFamily: TModelFamily | undefined): boolean;
+/**
+ * Check if the modelFamily is a UI-TARS variant
+ * @param modelFamily The model family to check
+ * @returns true if modelFamily is any UI-TARS variant
+ */
+export declare function isUITars(modelFamily: TModelFamily | undefined): boolean;

package/dist/types/ai-model/index.d.ts

@@ -8,7 +8,7 @@ export { AiLocateElement, AiExtractElementInfo, AiLocateSection, AiJudgeOrderSen
 export { plan } from './llm-planning';
 export { autoGLMPlanning } from './auto-glm/planning';
 export { adaptBboxToRect } from '../common';
-export { uiTarsPlanning, resizeImageForUiTars } from './ui-tars-planning';
+export { uiTarsPlanning } from './ui-tars-planning';
 export { ConversationHistory, type ConversationHistoryOptions, } from './conversation-history';
 export type { AIArgs } from '../common';
 export { getMidsceneLocationSchema, type MidsceneLocationResultType, PointSchema, SizeSchema, RectSchema, TMultimodalPromptSchema, TUserPromptSchema, type TMultimodalPrompt, type TUserPrompt, findAllMidsceneLocatorField, dumpActionParam, parseActionParam, } from '../common';

package/dist/types/ai-model/prompt/common.d.ts

@@ -1,2 +1,2 @@
-import type { TVlModeTypes } from '@midscene/shared/env';
-export declare function bboxDescription(vlMode: TVlModeTypes | undefined): "box_2d bounding box for the target element, should be [ymin, xmin, ymax, xmax] normalized to 0-1000." | "2d bounding box as [xmin, ymin, xmax, ymax]";
+import type { TModelFamily } from '@midscene/shared/env';
+export declare function bboxDescription(modelFamily: TModelFamily | undefined): "box_2d bounding box for the target element, should be [ymin, xmin, ymax, xmax] normalized to 0-1000." | "2d bounding box as [xmin, ymin, xmax, ymax]";

package/dist/types/ai-model/prompt/llm-locator.d.ts

@@ -1,3 +1,3 @@
-import type { TVlModeTypes } from '@midscene/shared/env';
-export declare function systemPromptToLocateElement(vlMode: TVlModeTypes | undefined): string;
+import type { TModelFamily } from '@midscene/shared/env';
+export declare function systemPromptToLocateElement(modelFamily: TModelFamily | undefined): string;
 export declare const findElementPrompt: (targetElementDescription: string) => string;

package/dist/types/ai-model/prompt/llm-planning.d.ts

@@ -1,9 +1,9 @@
 import type { DeviceAction } from '../../types';
-import type { TVlModeTypes } from '@midscene/shared/env';
+import type { TModelFamily } from '@midscene/shared/env';
 export declare const descriptionForAction: (action: DeviceAction<any>, locatorSchemaTypeDescription: string) => string;
-export declare function systemPromptToTaskPlanning({ actionSpace, vlMode, includeBbox, includeThought, }: {
+export declare function systemPromptToTaskPlanning({ actionSpace, modelFamily, includeBbox, includeThought, }: {
     actionSpace: DeviceAction<any>[];
-    vlMode: TVlModeTypes | undefined;
+    modelFamily: TModelFamily | undefined;
     includeBbox: boolean;
     includeThought?: boolean;
 }): Promise<string>;

package/dist/types/ai-model/prompt/llm-section-locator.d.ts

@@ -1,3 +1,3 @@
-import type { TVlModeTypes } from '@midscene/shared/env';
-export declare function systemPromptToLocateSection(vlMode: TVlModeTypes | undefined): string;
+import type { TModelFamily } from '@midscene/shared/env';
+export declare function systemPromptToLocateSection(modelFamily: TModelFamily | undefined): string;
 export declare const sectionLocatorInstruction: (sectionDescription: string) => string;

package/dist/types/ai-model/service-caller/index.d.ts

@@ -1,6 +1,6 @@
 import type { AIUsageInfo, DeepThinkOption } from '../../types';
 import type { StreamingCallback } from '../../types';
-import { type IModelConfig, type TModelFamily, type TVlModeTypes } from '@midscene/shared/env';
+import { type IModelConfig, type TModelFamily } from '@midscene/shared/env';
 import type { ChatCompletionMessageParam } from 'openai/resources/index';
 import type { AIArgs } from '../../common';
 export declare function callAI(messages: ChatCompletionMessageParam[], modelConfig: IModelConfig, options?: {
@@ -35,4 +35,4 @@ export declare function resolveDeepThinkConfig({ deepThink, modelFamily, }: {
     debugMessage?: string;
     warningMessage?: string;
 };
-export declare function safeParseJson(input: string, vlMode: TVlModeTypes | undefined): any;
+export declare function safeParseJson(input: string, modelFamily: TModelFamily | undefined): any;

package/dist/types/ai-model/ui-tars-planning.d.ts

@@ -1,5 +1,5 @@
-import type { PlanningAIResponse, Size, UIContext } from '../types';
-import { type IModelConfig, UITarsModelVersion } from '@midscene/shared/env';
+import type { PlanningAIResponse, UIContext } from '../types';
+import { type IModelConfig } from '@midscene/shared/env';
 import type { ConversationHistory } from './conversation-history';
 type ActionType = 'click' | 'left_double' | 'right_single' | 'drag' | 'type' | 'hotkey' | 'finished' | 'scroll' | 'wait';
 export declare function uiTarsPlanning(userInstruction: string, options: {
@@ -68,5 +68,4 @@ interface FinishedAction extends BaseAction {
     action_inputs: Record<string, never>;
 }
 export type Action = ClickAction | LeftDoubleAction | RightSingleAction | DragAction | TypeAction | HotkeyAction | ScrollAction | FinishedAction | WaitAction;
-export declare function resizeImageForUiTars(imageBase64: string, size: Size, uiTarsVersion: UITarsModelVersion | undefined): Promise<string>;
 export {};

package/dist/types/common.d.ts

@@ -1,7 +1,7 @@
 import type { BaseElement, DeviceAction, ElementTreeNode, MidsceneYamlFlowItem, PlanningAction, Rect, Size } from './types';
 import type { ChatCompletionMessageParam } from 'openai/resources/index';
 import type { PlanningLocateParam } from './types';
-import type { TVlModeTypes } from '@midscene/shared/env';
+import type { TModelFamily } from '@midscene/shared/env';
 import { z } from 'zod';
 export type AIArgs = ChatCompletionMessageParam[];
 type AdaptBboxInput = number[] | string[] | string | (number[] | string[])[];
@@ -15,20 +15,20 @@ type AdaptBboxInput = number[] | string[] | string | (number[] | string[])[];
  * @returns [x1, y1, x2, y2] bbox in [0, 1000] coordinate space
  */
 export declare function pointToBbox(x: number, y: number, bboxSize?: number): [number, number, number, number];
-export declare function fillBboxParam(locate: PlanningLocateParam, width: number, height: number, rightLimit: number, bottomLimit: number, vlMode: TVlModeTypes | undefined): PlanningLocateParam;
+export declare function fillBboxParam(locate: PlanningLocateParam, width: number, height: number, rightLimit: number, bottomLimit: number, modelFamily: TModelFamily | undefined): PlanningLocateParam;
 export declare function adaptQwen2_5Bbox(bbox: number[]): [number, number, number, number];
 export declare function adaptDoubaoBbox(bbox: string[] | number[] | string, width: number, height: number): [number, number, number, number];
-export declare function adaptBbox(bbox: AdaptBboxInput, width: number, height: number, rightLimit: number, bottomLimit: number, vlMode: TVlModeTypes | undefined): [number, number, number, number];
+export declare function adaptBbox(bbox: AdaptBboxInput, width: number, height: number, rightLimit: number, bottomLimit: number, modelFamily: TModelFamily | undefined): [number, number, number, number];
 export declare function normalized01000(bbox: number[], width: number, height: number): [number, number, number, number];
 export declare function adaptGeminiBbox(bbox: number[], width: number, height: number): [number, number, number, number];
-export declare function adaptBboxToRect(bbox: number[], width: number, height: number, offsetX?: number, offsetY?: number, rightLimit?: number, bottomLimit?: number, vlMode?: TVlModeTypes | undefined): Rect;
+export declare function adaptBboxToRect(bbox: number[], width: number, height: number, offsetX?: number, offsetY?: number, rightLimit?: number, bottomLimit?: number, modelFamily?: TModelFamily | undefined): Rect;
 export declare function mergeRects(rects: Rect[]): {
     left: number;
     top: number;
     width: number;
     height: number;
 };
-export declare function expandSearchArea(rect: Rect, screenSize: Size, vlMode: TVlModeTypes | undefined): Rect;
+export declare function expandSearchArea(rect: Rect, screenSize: Size, modelFamily: TModelFamily | undefined): Rect;
 export declare function markupImageForLLM(screenshotBase64: string, tree: ElementTreeNode<BaseElement>, size: Size): Promise<string>;
 export declare function buildYamlFlowFromPlans(plans: PlanningAction[], actionSpace: DeviceAction<any>[]): MidsceneYamlFlowItem[];
 export declare const PointSchema: z.ZodObject<{

package/dist/types/device/index.d.ts

@@ -728,7 +728,7 @@ export declare const actionInputParamSchema: z.ZodObject<{
         cacheable: z.ZodOptional<z.ZodBoolean>;
         xpath: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodBoolean]>>;
     }, z.ZodTypeAny, "passthrough">>>;
-    mode: z.ZodOptional<z.ZodDefault<z.ZodEnum<["replace", "clear", "append"]>>>;
+    mode: z.ZodEffects<z.ZodOptional<z.ZodDefault<z.ZodEnum<["replace", "clear", "typeOnly"]>>>, "replace" | "clear" | "typeOnly" | undefined, unknown>;
 }, "strip", z.ZodTypeAny, {
     value: string;
     locate?: z.objectOutputType<{
@@ -767,7 +767,7 @@ export declare const actionInputParamSchema: z.ZodObject<{
         cacheable: z.ZodOptional<z.ZodBoolean>;
         xpath: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodBoolean]>>;
     }, z.ZodTypeAny, "passthrough"> | undefined;
-    mode?: "replace" | "clear" | "append" | undefined;
+    mode?: "replace" | "clear" | "typeOnly" | undefined;
 }, {
     value: string | number;
     locate?: z.objectInputType<{
@@ -806,12 +806,12 @@ export declare const actionInputParamSchema: z.ZodObject<{
         cacheable: z.ZodOptional<z.ZodBoolean>;
         xpath: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodBoolean]>>;
     }, z.ZodTypeAny, "passthrough"> | undefined;
-    mode?: "replace" | "clear" | "append" | undefined;
+    mode?: unknown;
 }>;
 export type ActionInputParam = {
     value: string;
     locate?: LocateResultElement;
-    mode?: 'replace' | 'clear' | 'append';
+    mode?: 'replace' | 'clear' | 'typeOnly' | 'append';
 };
 export declare const defineActionInput: (call: (param: ActionInputParam) => Promise<void>) => DeviceAction<ActionInputParam>;
 export declare const actionKeyboardPressParamSchema: z.ZodObject<{

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@midscene/core",
   "description": "Automate browser actions, extract data, and perform assertions using AI. It offers JavaScript SDK, Chrome extension, and support for scripting in YAML. See https://midscenejs.com/ for details.",
-  "version": "1.2.2-beta-20260116114131.0",
+  "version": "1.2.2-beta-20260119111553.0",
   "repository": "https://github.com/web-infra-dev/midscene",
   "homepage": "https://midscenejs.com/",
   "main": "./dist/lib/index.js",
@@ -89,7 +89,7 @@
     "semver": "7.5.2",
     "undici": "^6.0.0",
     "zod": "3.24.3",
-    "@midscene/shared": "1.2.2-beta-20260116114131.0"
+    "@midscene/shared": "1.2.2-beta-20260119111553.0"
   },
   "devDependencies": {
     "@rslib/core": "^0.18.3",

package/dist/es/ai-model/prompt/assertion.mjs

@@ -1,31 +0,0 @@
-const assertSchema = {
-    type: 'json_schema',
-    json_schema: {
-        name: 'assert',
-        strict: true,
-        schema: {
-            type: 'object',
-            properties: {
-                pass: {
-                    type: 'boolean',
-                    description: 'Whether the assertion passed or failed'
-                },
-                thought: {
-                    type: [
-                        'string',
-                        'null'
-                    ],
-                    description: 'The thought process behind the assertion'
-                }
-            },
-            required: [
-                'pass',
-                'thought'
-            ],
-            additionalProperties: false
-        }
-    }
-};
-export { assertSchema };
-
-//# sourceMappingURL=assertion.mjs.map

package/dist/es/ai-model/prompt/assertion.mjs.map

@@ -1 +0,0 @@
-{"version":3,"file":"ai-model/prompt/assertion.mjs","sources":["../../../../src/ai-model/prompt/assertion.ts"],"sourcesContent":["import type { ResponseFormatJSONSchema } from 'openai/resources/index';\n\nexport const assertSchema: ResponseFormatJSONSchema = {\n  type: 'json_schema',\n  json_schema: {\n    name: 'assert',\n    strict: true,\n    schema: {\n      type: 'object',\n      properties: {\n        pass: {\n          type: 'boolean',\n          description: 'Whether the assertion passed or failed',\n        },\n        thought: {\n          type: ['string', 'null'],\n          description: 'The thought process behind the assertion',\n        },\n      },\n      required: ['pass', 'thought'],\n      additionalProperties: false,\n    },\n  },\n};\n"],"names":["assertSchema"],"mappings":"AAEO,MAAMA,eAAyC;IACpD,MAAM;IACN,aAAa;QACX,MAAM;QACN,QAAQ;QACR,QAAQ;YACN,MAAM;YACN,YAAY;gBACV,MAAM;oBACJ,MAAM;oBACN,aAAa;gBACf;gBACA,SAAS;oBACP,MAAM;wBAAC;wBAAU;qBAAO;oBACxB,aAAa;gBACf;YACF;YACA,UAAU;gBAAC;gBAAQ;aAAU;YAC7B,sBAAsB;QACxB;IACF;AACF"}

package/dist/lib/ai-model/prompt/assertion.js

@@ -1,65 +0,0 @@
-"use strict";
-var __webpack_require__ = {};
-(()=>{
-    __webpack_require__.d = (exports1, definition)=>{
-        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
-            enumerable: true,
-            get: definition[key]
-        });
-    };
-})();
-(()=>{
-    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
-})();
-(()=>{
-    __webpack_require__.r = (exports1)=>{
-        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
-            value: 'Module'
-        });
-        Object.defineProperty(exports1, '__esModule', {
-            value: true
-        });
-    };
-})();
-var __webpack_exports__ = {};
-__webpack_require__.r(__webpack_exports__);
-__webpack_require__.d(__webpack_exports__, {
-    assertSchema: ()=>assertSchema
-});
-const assertSchema = {
-    type: 'json_schema',
-    json_schema: {
-        name: 'assert',
-        strict: true,
-        schema: {
-            type: 'object',
-            properties: {
-                pass: {
-                    type: 'boolean',
-                    description: 'Whether the assertion passed or failed'
-                },
-                thought: {
-                    type: [
-                        'string',
-                        'null'
-                    ],
-                    description: 'The thought process behind the assertion'
-                }
-            },
-            required: [
-                'pass',
-                'thought'
-            ],
-            additionalProperties: false
-        }
-    }
-};
-exports.assertSchema = __webpack_exports__.assertSchema;
-for(var __rspack_i in __webpack_exports__)if (-1 === [
-    "assertSchema"
-].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
-Object.defineProperty(exports, '__esModule', {
-    value: true
-});
-
-//# sourceMappingURL=assertion.js.map

package/dist/lib/ai-model/prompt/assertion.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"ai-model/prompt/assertion.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/prompt/assertion.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { ResponseFormatJSONSchema } from 'openai/resources/index';\n\nexport const assertSchema: ResponseFormatJSONSchema = {\n  type: 'json_schema',\n  json_schema: {\n    name: 'assert',\n    strict: true,\n    schema: {\n      type: 'object',\n      properties: {\n        pass: {\n          type: 'boolean',\n          description: 'Whether the assertion passed or failed',\n        },\n        thought: {\n          type: ['string', 'null'],\n          description: 'The thought process behind the assertion',\n        },\n      },\n      required: ['pass', 'thought'],\n      additionalProperties: false,\n    },\n  },\n};\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","assertSchema"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;ACJO,MAAMI,eAAyC;IACpD,MAAM;IACN,aAAa;QACX,MAAM;QACN,QAAQ;QACR,QAAQ;YACN,MAAM;YACN,YAAY;gBACV,MAAM;oBACJ,MAAM;oBACN,aAAa;gBACf;gBACA,SAAS;oBACP,MAAM;wBAAC;wBAAU;qBAAO;oBACxB,aAAa;gBACf;YACF;YACA,UAAU;gBAAC;gBAAQ;aAAU;YAC7B,sBAAsB;QACxB;IACF;AACF"}

package/dist/types/ai-model/prompt/assertion.d.ts

@@ -1,2 +0,0 @@
-import type { ResponseFormatJSONSchema } from 'openai/resources/index';
-export declare const assertSchema: ResponseFormatJSONSchema;