@midscene/core 1.0.1-beta-20251208075922.0 → 1.0.1-beta-20251208112226.0

package/dist/lib/yaml/builder.js

@@ -47,9 +47,9 @@ function buildYaml(env, tasks) {
     });
 }
 exports.buildYaml = __webpack_exports__.buildYaml;
-for(var __rspack_i in __webpack_exports__)if (-1 === [
+for(var __webpack_i__ in __webpack_exports__)if (-1 === [
     "buildYaml"
-].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+].indexOf(__webpack_i__)) exports[__webpack_i__] = __webpack_exports__[__webpack_i__];
 Object.defineProperty(exports, '__esModule', {
     value: true
 });

package/dist/lib/yaml/index.js

@@ -1,12 +1,12 @@
 "use strict";
 var __webpack_modules__ = {
-    "./builder" (module) {
+    "./builder": function(module) {
         module.exports = require("./builder.js");
     },
-    "./player" (module) {
+    "./player": function(module) {
         module.exports = require("./player.js");
     },
-    "./utils" (module) {
+    "./utils": function(module) {
         module.exports = require("./utils.js");
     }
 };
@@ -66,7 +66,7 @@ var __webpack_exports__ = {};
     for(const __rspack_import_key in _utils__rspack_import_2)if ("default" !== __rspack_import_key) __rspack_reexport[__rspack_import_key] = ()=>_utils__rspack_import_2[__rspack_import_key];
     __webpack_require__.d(__webpack_exports__, __rspack_reexport);
 })();
-for(var __rspack_i in __webpack_exports__)exports[__rspack_i] = __webpack_exports__[__rspack_i];
+for(var __webpack_i__ in __webpack_exports__)exports[__webpack_i__] = __webpack_exports__[__webpack_i__];
 Object.defineProperty(exports, '__esModule', {
     value: true
 });

package/dist/lib/yaml/player.js

@@ -414,9 +414,9 @@ class ScriptPlayer {
     }
 }
 exports.ScriptPlayer = __webpack_exports__.ScriptPlayer;
-for(var __rspack_i in __webpack_exports__)if (-1 === [
+for(var __webpack_i__ in __webpack_exports__)if (-1 === [
     "ScriptPlayer"
-].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+].indexOf(__webpack_i__)) exports[__webpack_i__] = __webpack_exports__[__webpack_i__];
 Object.defineProperty(exports, '__esModule', {
     value: true
 });

package/dist/lib/yaml/utils.js

@@ -113,12 +113,12 @@ exports.buildDetailedLocateParam = __webpack_exports__.buildDetailedLocateParam;
 exports.buildDetailedLocateParamAndRestParams = __webpack_exports__.buildDetailedLocateParamAndRestParams;
 exports.interpolateEnvVars = __webpack_exports__.interpolateEnvVars;
 exports.parseYamlScript = __webpack_exports__.parseYamlScript;
-for(var __rspack_i in __webpack_exports__)if (-1 === [
+for(var __webpack_i__ in __webpack_exports__)if (-1 === [
     "buildDetailedLocateParam",
     "buildDetailedLocateParamAndRestParams",
     "interpolateEnvVars",
     "parseYamlScript"
-].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+].indexOf(__webpack_i__)) exports[__webpack_i__] = __webpack_exports__[__webpack_i__];
 Object.defineProperty(exports, '__esModule', {
     value: true
 });

package/dist/lib/yaml.js

@@ -12,7 +12,7 @@ var __webpack_require__ = {};
 })();
 var __webpack_exports__ = {};
 __webpack_require__.r(__webpack_exports__);
-for(var __rspack_i in __webpack_exports__)exports[__rspack_i] = __webpack_exports__[__rspack_i];
+for(var __webpack_i__ in __webpack_exports__)exports[__webpack_i__] = __webpack_exports__[__webpack_i__];
 Object.defineProperty(exports, '__esModule', {
     value: true
 });

package/dist/types/agent/agent.d.ts

@@ -95,7 +95,7 @@ export declare class Agent<InterfaceType extends AbstractInterface = AbstractInt
     aiScroll(scrollParam: ScrollParam, locatePrompt?: TUserPrompt, opt?: LocateOption): Promise<any>;
     aiAct(taskPrompt: string, opt?: {
         cacheable?: boolean;
-        _deepThink?: boolean;
+        planningStrategy?: 'fast' | 'standard' | 'max';
     }): Promise<{
         result: Record<string, any>;
     } | {

package/dist/types/agent/task-builder.d.ts

@@ -1,6 +1,6 @@
 import type { AbstractInterface } from '../device';
 import type Service from '../service';
-import type { DetailedLocateParam, ExecutionTaskActionApply, ExecutionTaskApply, PlanningAction, PlanningActionParamSleep, PlanningLocateParam } from '../types';
+import type { DetailedLocateParam, DeviceAction, ExecutionTaskActionApply, ExecutionTaskApply, PlanningAction, PlanningActionParamSleep, PlanningLocateParam } from '../types';
 import type { IModelConfig } from '@midscene/shared/env';
 import type { TaskCache } from './task-cache';
 export declare function locatePlanForLocate(param: string | DetailedLocateParam): PlanningAction<PlanningLocateParam>;
@@ -8,6 +8,7 @@ interface TaskBuilderDeps {
     interfaceInstance: AbstractInterface;
     service: Service;
     taskCache?: TaskCache;
+    actionSpace: DeviceAction[];
 }
 interface BuildOptions {
     cacheable?: boolean;
@@ -17,7 +18,8 @@ export declare class TaskBuilder {
     private readonly interface;
     private readonly service;
     private readonly taskCache?;
-    constructor({ interfaceInstance, service, taskCache }: TaskBuilderDeps);
+    private readonly actionSpace;
+    constructor({ interfaceInstance, service, taskCache, actionSpace, }: TaskBuilderDeps);
     build(plans: PlanningAction[], modelConfigForPlanning: IModelConfig, modelConfigForDefaultIntent: IModelConfig, options?: BuildOptions): Promise<{
         tasks: ExecutionTaskApply[];
     }>;

package/dist/types/agent/tasks.d.ts

@@ -3,7 +3,7 @@ import type { AbstractInterface } from '../device';
 import type Service from '../service';
 import type { TaskRunner } from '../task-runner';
 import { TaskExecutionError } from '../task-runner';
-import type { ExecutionTaskApply, ExecutionTaskProgressOptions, MidsceneYamlFlowItem, PlanningAction, PlanningActionParamSleep, PlanningActionParamWaitFor, ServiceExtractOption, ServiceExtractParam } from '../types';
+import type { ExecutionTaskApply, ExecutionTaskProgressOptions, MidsceneYamlFlowItem, PlanningAction, PlanningActionParamSleep, PlanningActionParamWaitFor, ServiceExtractOption, ServiceExtractParam, DeviceAction } from '../types';
 import type { IModelConfig } from '@midscene/shared/env';
 import type { TaskCache } from './task-cache';
 export { locatePlanForLocate } from './task-builder';
@@ -20,6 +20,7 @@ export declare class TaskExecutor {
     interface: AbstractInterface;
     service: Service;
     taskCache?: TaskCache;
+    private readonly providedActionSpace;
     private readonly taskBuilder;
     private conversationHistory;
     onTaskStartCallback?: ExecutionTaskProgressOptions['onTaskStart'];
@@ -31,8 +32,10 @@ export declare class TaskExecutor {
         onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];
         replanningCycleLimit?: number;
         hooks?: TaskExecutorHooks;
+        actionSpace: DeviceAction[];
     });
     private createExecutionSession;
+    private getActionSpace;
     convertPlanToExecutable(plans: PlanningAction[], modelConfigForPlanning: IModelConfig, modelConfigForDefaultIntent: IModelConfig, options?: {
         cacheable?: boolean;
         subTask?: boolean;
@@ -43,7 +46,7 @@ export declare class TaskExecutor {
         runner: TaskRunner;
     }>;
     runPlans(title: string, plans: PlanningAction[], modelConfigForPlanning: IModelConfig, modelConfigForDefaultIntent: IModelConfig): Promise<ExecutionResult>;
-    action(userPrompt: string, modelConfigForPlanning: IModelConfig, modelConfigForDefaultIntent: IModelConfig, includeBboxInPlanning: boolean, backgroundKnowledge?: string, cacheable?: boolean, replanningCycleLimitOverride?: number): Promise<ExecutionResult<{
+    action(userPrompt: string, modelConfigForPlanning: IModelConfig, modelConfigForDefaultIntent: IModelConfig, includeBboxInPlanning: boolean, backgroundKnowledge?: string, cacheable?: boolean, replanningCycleLimitOverride?: number, imagesIncludeCount?: number): Promise<ExecutionResult<{
         yamlFlow?: MidsceneYamlFlowItem[];
     } | undefined>>;
     private createTypeQueryTask;

package/dist/types/ai-model/conversation-history.d.ts

@@ -10,7 +10,12 @@ export declare class ConversationHistory {
     append(message: ChatCompletionMessageParam): void;
     seed(messages: ChatCompletionMessageParam[]): void;
     reset(): void;
-    snapshot(): ChatCompletionMessageParam[];
+    /**
+     * Snapshot the conversation history, and replace the images with text if the number of images exceeds the limit.
+     * @param maxImages - The maximum number of images to include in the snapshot. Undefined means no limit.
+     * @returns The snapshot of the conversation history.
+     */
+    snapshot(maxImages?: number): ChatCompletionMessageParam[];
     get length(): number;
     [Symbol.iterator](): IterableIterator<ChatCompletionMessageParam>;
     toJSON(): ChatCompletionMessageParam[];

package/dist/types/ai-model/llm-planning.d.ts

@@ -9,4 +9,5 @@ export declare function plan(userInstruction: string, opts: {
     modelConfig: IModelConfig;
     conversationHistory: ConversationHistory;
     includeBbox: boolean;
+    imagesIncludeCount?: number;
 }): Promise<PlanningAIResponse>;

package/dist/types/ai-model/prompt/llm-planning.d.ts

@@ -1,10 +1,8 @@
 import type { DeviceAction } from '../../types';
 import type { TVlModeTypes } from '@midscene/shared/env';
-import type { ResponseFormatJSONSchema } from 'openai/resources/index';
 export declare const descriptionForAction: (action: DeviceAction<any>, locatorSchemaTypeDescription: string) => string;
 export declare function systemPromptToTaskPlanning({ actionSpace, vlMode, includeBbox, }: {
     actionSpace: DeviceAction<any>[];
     vlMode: TVlModeTypes | undefined;
     includeBbox: boolean;
 }): Promise<string>;
-export declare const planSchema: ResponseFormatJSONSchema;

package/dist/types/ai-model/service-caller/index.d.ts

@@ -1,9 +1,8 @@
 import { type AIUsageInfo } from '../../types';
 import type { StreamingCallback } from '../../types';
 import { type IModelConfig, type TVlModeTypes } from '@midscene/shared/env';
-import OpenAI from 'openai';
 import type { ChatCompletionMessageParam } from 'openai/resources/index';
-import { AIActionType, type AIArgs } from '../../common';
+import type { AIActionType, AIArgs } from '../../common';
 export declare function callAI(messages: ChatCompletionMessageParam[], AIActionTypeValue: AIActionType, modelConfig: IModelConfig, options?: {
     stream?: boolean;
     onChunk?: StreamingCallback;
@@ -12,7 +11,6 @@ export declare function callAI(messages: ChatCompletionMessageParam[], AIActionT
     usage?: AIUsageInfo;
     isStreamed: boolean;
 }>;
-export declare const getResponseFormat: (modelName: string, AIActionTypeValue: AIActionType) => OpenAI.ChatCompletionCreateParams["response_format"] | OpenAI.ResponseFormatJSONObject;
 export declare function callAIWithObjectResponse<T>(messages: ChatCompletionMessageParam[], AIActionTypeValue: AIActionType, modelConfig: IModelConfig): Promise<{
     content: T;
     contentString: string;

package/dist/types/device/index.d.ts

@@ -7,7 +7,7 @@ export declare abstract class AbstractInterface {
     abstract interfaceType: string;
     abstract screenshotBase64(): Promise<string>;
     abstract size(): Promise<Size>;
-    abstract actionSpace(): DeviceAction[] | Promise<DeviceAction[]>;
+    abstract actionSpace(): DeviceAction[];
     abstract cacheFeatureForRect?(rect: Rect, options?: {
         targetDescription?: string;
         modelConfig?: IModelConfig;
@@ -2153,14 +2153,22 @@ export type ActionClearInputParam = {
 export declare const defineActionClearInput: (call: (param: ActionClearInputParam) => Promise<void>) => DeviceAction<ActionClearInputParam>;
 export declare const actionAssertParamSchema: z.ZodObject<{
     condition: z.ZodString;
+    thought: z.ZodString;
+    result: z.ZodBoolean;
 }, "strip", z.ZodTypeAny, {
     condition: string;
+    thought: string;
+    result: boolean;
 }, {
     condition: string;
+    thought: string;
+    result: boolean;
 }>;
 export type ActionAssertParam = {
     condition: string;
+    thought: string;
+    result: boolean;
 };
-export declare const defineActionAssert: (call: (param: ActionAssertParam) => Promise<void>) => DeviceAction<ActionAssertParam>;
+export declare const defineActionAssert: () => DeviceAction<ActionAssertParam>;
 export type { DeviceAction } from '../types';
 export type { AndroidDeviceOpt, AndroidDeviceInputOpt, IOSDeviceOpt, IOSDeviceInputOpt, } from './device-options';

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@midscene/core",
   "description": "Automate browser actions, extract data, and perform assertions using AI. It offers JavaScript SDK, Chrome extension, and support for scripting in YAML. See https://midscenejs.com/ for details.",
-  "version": "1.0.1-beta-20251208075922.0",
+  "version": "1.0.1-beta-20251208112226.0",
   "repository": "https://github.com/web-infra-dev/midscene",
   "homepage": "https://midscenejs.com/",
   "main": "./dist/lib/index.js",
@@ -82,17 +82,17 @@
     "@ui-tars/action-parser": "1.2.3",
     "dayjs": "^1.11.11",
     "dotenv": "^16.4.5",
-    "https-proxy-agent": "7.0.2",
+    "fetch-socks": "^1.3.0",
     "openai": "6.3.0",
+    "undici": "^6.0.0",
     "jsonrepair": "3.12.0",
     "semver": "7.5.2",
     "js-yaml": "4.1.0",
     "zod": "3.24.3",
-    "socks-proxy-agent": "8.0.4",
-    "@midscene/shared": "1.0.1-beta-20251208075922.0"
+    "@midscene/shared": "1.0.1-beta-20251208112226.0"
   },
   "devDependencies": {
-    "@rslib/core": "^0.18.3",
+    "@rslib/core": "^0.18.2",
     "@types/node": "^18.0.0",
     "@types/node-fetch": "2.6.11",
     "@types/js-yaml": "4.0.9",