@midscene/core 1.11.0 → 1.11.1-beta-20260819031908.0

package/dist/types/agent/agent.d.ts

@@ -1,6 +1,6 @@
 import type { TUserPrompt } from '../ai-model/index';
 import Service from '../service/index';
-import { type ActionParam, type ActionReturn, type AgentAssertResult, type AgentOpt, type AgentProgressListener, type AgentWaitForOpt, type AssertOptions, type DeepThinkOption, type DeviceAction, ExecutionDump, type InsightAPI, type LocateOption, type LocateResultElement, type OnTaskStartTip, type QueryOptions, type RecordToReportOptions, ReportActionDump, type ScrollParam, type ServiceAction, type ServiceExtractParam, type UIContext } from '../types';
+import { type ActionParam, type ActionReturn, type AgentAssertResult, type AgentOpt, type AgentProgressListener, type AgentWaitForOpt, type AiActEffort, type AssertOptions, type DeepThinkOption, type DeviceAction, ExecutionDump, type InsightAPI, type LocateOption, type LocateResultElement, type OnTaskStartTip, type QueryOptions, type RecordToReportOptions, ReportActionDump, type ScrollParam, type ServiceAction, type ServiceExtractParam, type UIContext } from '../types';
 import type { AbstractInterface } from '../device';
 import type { TaskRunner } from '../task-runner';
 import { ModelConfigManager } from '@midscene/shared/env';
@@ -13,6 +13,7 @@ export type AiActOptions = {
     cacheable?: boolean;
     fileChooserAccept?: string | string[];
     fileChooserAllowedDir?: string;
+    effort?: AiActEffort;
     deepThink?: DeepThinkOption;
     deepLocate?: boolean;
     abortSignal?: AbortSignal;

package/dist/types/agent/tasks.d.ts

@@ -4,7 +4,7 @@ import type { AbstractInterface } from '../device';
 import type Service from '../service';
 import type { TaskRunner } from '../task-runner';
 import { TaskExecutionError } from '../task-runner';
-import type { DeviceAction, ExecutionTask, ExecutionTaskApply, ExecutionTaskProgressOptions, MidsceneYamlFlowItem, PlanningAction, PlanningActionParamWaitFor, ServiceExtractOption, ServiceExtractParam, UIContext } from '../types';
+import type { AiActEffort, DeviceAction, ExecutionTask, ExecutionTaskApply, ExecutionTaskProgressOptions, MidsceneYamlFlowItem, PlanningAction, PlanningActionParamWaitFor, ServiceExtractOption, ServiceExtractParam, UIContext } from '../types';
 import { type AgentProgressPublisher } from './progress';
 import type { TaskCache } from './task-cache';
 export { locatePlanForLocate } from './task-builder';
@@ -94,7 +94,7 @@ export declare class TaskExecutor {
     runPlans(title: string, plans: PlanningAction[], planningModel: ModelRuntime, defaultModel: ModelRuntime, options?: {
         uiContext?: UIContext;
     }): Promise<ExecutionResult>;
-    action(userPrompt: TUserPrompt, planningModel: ModelRuntime, defaultModel: ModelRuntime, includeLocateInPlanning: boolean, aiActContext?: string, cacheable?: boolean, replanningCycleLimitOverride?: number, imagesIncludeCount?: number, deepThink?: boolean, fileChooserAccept?: string[], deepLocate?: boolean, abortSignal?: AbortSignal, reportOptions?: ActionReportOptions): Promise<ExecutionResult<{
+    action(userPrompt: TUserPrompt, planningModel: ModelRuntime, defaultModel: ModelRuntime, aiActContext?: string, cacheable?: boolean, replanningCycleLimitOverride?: number, effort?: AiActEffort, fileChooserAccept?: string[], deepLocate?: boolean, abortSignal?: AbortSignal, reportOptions?: ActionReportOptions): Promise<ExecutionResult<{
         yamlFlow?: MidsceneYamlFlowItem[];
         output?: string;
     } | undefined>>;

package/dist/types/ai-model/prompt/planning/multi-turn-example.d.ts

@@ -1,7 +1,9 @@
 import type { LocateResultPromptSpec } from '../../shared/model-locate-result';
 import type { PlanningActionOutputProtocol } from './action-output-protocol';
-export declare const buildPlanningMultiTurnExample: ({ includeSubGoals, locatePromptSpec, actionOutputProtocol, }: {
+export declare const buildPlanningMultiTurnExample: ({ includeSubGoals, includeThought, includeLog, locatePromptSpec, actionOutputProtocol, }: {
     includeSubGoals: boolean;
+    includeThought: boolean;
+    includeLog: boolean;
     locatePromptSpec?: LocateResultPromptSpec;
     actionOutputProtocol: PlanningActionOutputProtocol;
 }) => string;

package/dist/types/ai-model/prompt/planning/planning-response-example.d.ts

@@ -5,7 +5,7 @@ type PlanningSubGoalExample = {
     description: string;
 };
 type PlanningResponseActionExample = {
-    log: string;
+    log?: string;
     actionExample: string | undefined;
     complete?: never;
 };
@@ -18,7 +18,7 @@ type PlanningResponseCompleteExample = {
     actionExample?: never;
 };
 type BuildPlanningResponseExampleInput = {
-    planning: string;
+    planning?: string;
     updateSubGoals?: PlanningSubGoalExample[];
     markSubGoalsDone?: number[];
     memory?: string;

package/dist/types/ai-model/prompt/planning/system-prompt.d.ts

@@ -4,6 +4,8 @@ import { type PlanningActionOutputProtocol } from './action-output-protocol';
 type BuildStandardPlanningSystemPromptInput = {
     actionSpace: DeviceAction<any>[];
     includeSubGoals?: boolean;
+    includeThought?: boolean;
+    includeLog?: boolean;
     actionOutputProtocol?: PlanningActionOutputProtocol;
 } & ({
     includeLocateInPlanning: true;

package/dist/types/ai-model/workflows/planning/index.d.ts

@@ -1,3 +1,3 @@
 export { ConversationHistory, type ConversationHistoryOptions, } from './conversation-history';
 export { standardPlan } from './standard-planning';
-export { parseXMLPlanningResponse } from './standard-planning-parser';
+export { parseStandardPlanningResponse, parseXMLPlanningResponse, } from './standard-planning-parser';

package/dist/types/ai-model/workflows/planning/planning-action-log.d.ts

@@ -0,0 +1,2 @@
+import type { DeviceAction, PlanningAction } from '../../../types';
+export declare function buildPlanningActionLog(action: PlanningAction, actionSpace: DeviceAction<any>[]): string;

package/dist/types/ai-model/workflows/planning/standard-planning-parser.d.ts

@@ -1,4 +1,4 @@
-import type { RawResponsePlanningAIResponse, SubGoal } from '../../../types';
+import type { DeviceAction, RawResponsePlanningAIResponse, SubGoal } from '../../../types';
 import type { JsonParser } from '../../shared/json';
 /**
  * Parse sub-goals from XML content.
@@ -13,4 +13,16 @@ export declare function parseSubGoalsFromXML(xmlContent: string): SubGoal[];
  */
 export declare function parseMarkFinishedIndexes(xmlContent: string): number[];
 /** Parse an XML model response into a planning response. */
-export declare function parseXMLPlanningResponse(xmlString: string, jsonParser: JsonParser): RawResponsePlanningAIResponse;
+export declare function parseXMLPlanningResponse(xmlString: string, jsonParser: JsonParser, options: {
+    includeThought: boolean;
+}): RawResponsePlanningAIResponse;
+type ParseStandardPlanningResponseOptions = {
+    includeThought: boolean;
+} & ({
+    logSource?: 'model';
+} | {
+    logSource: 'action';
+    actionSpace: DeviceAction<any>[];
+});
+export declare function parseStandardPlanningResponse(xmlString: string, jsonParser: JsonParser, options: ParseStandardPlanningResponseOptions): RawResponsePlanningAIResponse;
+export {};

package/dist/types/ai-model/workflows/planning/types.d.ts

@@ -1,5 +1,5 @@
 import type { TUserPrompt } from '../../../common';
-import type { DeviceAction, PlanningAIResponse, UIContext } from '../../../types';
+import type { AiActEffort, DeviceAction, PlanningAIResponse, UIContext } from '../../../types';
 import type { ChatCompletionUserMessageParam } from 'openai/resources/index';
 import type { ModelRuntime } from '../../models';
 import type { ConversationHistory } from './conversation-history';
@@ -11,7 +11,7 @@ export interface PlanOptions {
     conversationHistory: ConversationHistory;
     includeLocateInPlanning: boolean;
     imagesIncludeCount?: number;
-    deepThink?: boolean;
+    effort: AiActEffort;
     referenceImageMessages?: ChatCompletionUserMessageParam[];
     abortSignal?: AbortSignal;
 }

package/dist/types/device/index.d.ts

@@ -347,8 +347,8 @@ export declare const actionTapParamSchema: z.ZodObject<{
             }[] | undefined;
             convertHttpImage2Base64?: boolean | undefined;
         });
-        deepLocate?: boolean | undefined;
         deepThink?: boolean | undefined;
+        deepLocate?: boolean | undefined;
         cacheable?: boolean | undefined;
         xpath?: string | boolean | undefined;
     } & {
@@ -365,8 +365,8 @@ export declare const actionTapParamSchema: z.ZodObject<{
             }[] | undefined;
             convertHttpImage2Base64?: boolean | undefined;
         });
-        deepLocate?: boolean | undefined;
         deepThink?: boolean | undefined;
+        deepLocate?: boolean | undefined;
         cacheable?: boolean | undefined;
         xpath?: string | boolean | undefined;
     } & {
@@ -509,8 +509,8 @@ export declare const actionRightClickParamSchema: z.ZodObject<{
             }[] | undefined;
             convertHttpImage2Base64?: boolean | undefined;
         });
-        deepLocate?: boolean | undefined;
         deepThink?: boolean | undefined;
+        deepLocate?: boolean | undefined;
         cacheable?: boolean | undefined;
         xpath?: string | boolean | undefined;
     } & {
@@ -527,8 +527,8 @@ export declare const actionRightClickParamSchema: z.ZodObject<{
             }[] | undefined;
             convertHttpImage2Base64?: boolean | undefined;
         });
-        deepLocate?: boolean | undefined;
         deepThink?: boolean | undefined;
+        deepLocate?: boolean | undefined;
         cacheable?: boolean | undefined;
         xpath?: string | boolean | undefined;
     } & {
@@ -660,8 +660,8 @@ export declare const actionDoubleClickParamSchema: z.ZodObject<{
             }[] | undefined;
             convertHttpImage2Base64?: boolean | undefined;
         });
-        deepLocate?: boolean | undefined;
         deepThink?: boolean | undefined;
+        deepLocate?: boolean | undefined;
         cacheable?: boolean | undefined;
         xpath?: string | boolean | undefined;
     } & {
@@ -678,8 +678,8 @@ export declare const actionDoubleClickParamSchema: z.ZodObject<{
             }[] | undefined;
             convertHttpImage2Base64?: boolean | undefined;
         });
-        deepLocate?: boolean | undefined;
         deepThink?: boolean | undefined;
+        deepLocate?: boolean | undefined;
         cacheable?: boolean | undefined;
         xpath?: string | boolean | undefined;
     } & {
@@ -811,8 +811,8 @@ export declare const actionHoverParamSchema: z.ZodObject<{
             }[] | undefined;
             convertHttpImage2Base64?: boolean | undefined;
         });
-        deepLocate?: boolean | undefined;
         deepThink?: boolean | undefined;
+        deepLocate?: boolean | undefined;
         cacheable?: boolean | undefined;
         xpath?: string | boolean | undefined;
     } & {
@@ -829,8 +829,8 @@ export declare const actionHoverParamSchema: z.ZodObject<{
             }[] | undefined;
             convertHttpImage2Base64?: boolean | undefined;
         });
-        deepLocate?: boolean | undefined;
         deepThink?: boolean | undefined;
+        deepLocate?: boolean | undefined;
         cacheable?: boolean | undefined;
         xpath?: string | boolean | undefined;
     } & {
@@ -1670,8 +1670,8 @@ export declare const actionDragAndDropParamSchema: z.ZodObject<{
             }[] | undefined;
             convertHttpImage2Base64?: boolean | undefined;
         });
-        deepLocate?: boolean | undefined;
         deepThink?: boolean | undefined;
+        deepLocate?: boolean | undefined;
         cacheable?: boolean | undefined;
         xpath?: string | boolean | undefined;
     } & {
@@ -1687,8 +1687,8 @@ export declare const actionDragAndDropParamSchema: z.ZodObject<{
             }[] | undefined;
             convertHttpImage2Base64?: boolean | undefined;
         });
-        deepLocate?: boolean | undefined;
         deepThink?: boolean | undefined;
+        deepLocate?: boolean | undefined;
         cacheable?: boolean | undefined;
         xpath?: string | boolean | undefined;
     } & {
@@ -1705,8 +1705,8 @@ export declare const actionDragAndDropParamSchema: z.ZodObject<{
             }[] | undefined;
             convertHttpImage2Base64?: boolean | undefined;
         });
-        deepLocate?: boolean | undefined;
         deepThink?: boolean | undefined;
+        deepLocate?: boolean | undefined;
         cacheable?: boolean | undefined;
         xpath?: string | boolean | undefined;
     } & {
@@ -1722,8 +1722,8 @@ export declare const actionDragAndDropParamSchema: z.ZodObject<{
             }[] | undefined;
             convertHttpImage2Base64?: boolean | undefined;
         });
-        deepLocate?: boolean | undefined;
         deepThink?: boolean | undefined;
+        deepLocate?: boolean | undefined;
         cacheable?: boolean | undefined;
         xpath?: string | boolean | undefined;
     } & {
@@ -1857,8 +1857,8 @@ export declare const ActionLongPressParamSchema: z.ZodObject<{
             }[] | undefined;
             convertHttpImage2Base64?: boolean | undefined;
         });
-        deepLocate?: boolean | undefined;
         deepThink?: boolean | undefined;
+        deepLocate?: boolean | undefined;
         cacheable?: boolean | undefined;
         xpath?: string | boolean | undefined;
     } & {
@@ -1876,8 +1876,8 @@ export declare const ActionLongPressParamSchema: z.ZodObject<{
             }[] | undefined;
             convertHttpImage2Base64?: boolean | undefined;
         });
-        deepLocate?: boolean | undefined;
         deepThink?: boolean | undefined;
+        deepLocate?: boolean | undefined;
         cacheable?: boolean | undefined;
         xpath?: string | boolean | undefined;
     } & {

package/dist/types/types.d.ts

@@ -147,6 +147,7 @@ export interface LocateResult {
     rect?: Rect;
 }
 export type ThinkingLevel = 'off' | 'medium' | 'high';
+export type AiActEffort = 'fast' | 'balance' | 'deepThink';
 export type DeepThinkOption = 'unset' | true | false;
 export interface ServiceTaskInfo {
     durationMs: number;
@@ -524,7 +525,7 @@ export interface ExecutionTaskPlanningParam {
     replanningCycleLimit?: number;
     aiActContext?: string;
     imagesIncludeCount?: number;
-    deepThink?: DeepThinkOption;
+    effort?: AiActEffort;
     subGoalStatus?: string;
     memoriesStatus?: string;
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@midscene/core",
   "description": "Automate browser actions, extract data, and perform assertions using AI. It offers JavaScript SDK, Chrome extension, and support for scripting in YAML. See https://midscenejs.com/ for details.",
-  "version": "1.11.0",
+  "version": "1.11.1-beta-20260819031908.0",
   "repository": "https://github.com/web-infra-dev/midscene",
   "homepage": "https://midscenejs.com/",
   "main": "./dist/lib/index.js",
@@ -115,7 +115,7 @@
     "semver": "7.5.2",
     "undici": "^6.0.0",
     "zod": "^3.25.1",
-    "@midscene/shared": "1.11.0"
+    "@midscene/shared": "1.11.1-beta-20260819031908.0"
   },
   "devDependencies": {
     "@rslib/core": "^0.18.3",