@midscene/core 0.28.12-beta-20250923124135.0 → 0.28.12-beta-20250924031347.0

package/dist/types/agent/tasks.d.ts

@@ -1,4 +1,3 @@
-import { type ChatCompletionMessageParam } from '../ai-model';
 import type { AbstractInterface } from '../device';
 import { type DetailedLocateParam, type ExecutionTaskApply, type ExecutionTaskProgressOptions, Executor, type Insight, type InsightExtractOption, type InsightExtractParam, type MidsceneYamlFlowItem, type PlanningAction, type PlanningActionParamWaitFor, type PlanningLocateParam, type TMultimodalPrompt, type TUserPrompt } from '../index';
 import { type IModelConfig } from '@midscene/shared/env';
@@ -13,7 +12,7 @@ export declare class TaskExecutor {
     interface: AbstractInterface;
     insight: Insight;
     taskCache?: TaskCache;
-    conversationHistory: ChatCompletionMessageParam[];
+    private conversationHistory;
     onTaskStartCallback?: ExecutionTaskProgressOptions['onTaskStart'];
     replanningCycleLimit?: number;
     get page(): AbstractInterface;
@@ -32,29 +31,17 @@ export declare class TaskExecutor {
     loadYamlFlowAsPlanning(userInstruction: string, yamlString: string): Promise<{
         executor: Executor;
     }>;
-    private planningTaskFromPrompt;
-    private planningTaskToGoal;
+    private createPlanningTask;
     runPlans(title: string, plans: PlanningAction[], modelConfig: IModelConfig): Promise<ExecutionResult>;
+    private getReplanningCycleLimit;
     action(userPrompt: string, modelConfig: IModelConfig, actionContext?: string): Promise<ExecutionResult<{
         yamlFlow?: MidsceneYamlFlowItem[];
     } | undefined>>;
-    actionToGoal(userPrompt: string, modelConfig: IModelConfig): Promise<ExecutionResult<{
-        yamlFlow?: MidsceneYamlFlowItem[];
-    } | undefined>>;
     private createTypeQueryTask;
     createTypeQueryExecution<T>(type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert', demand: InsightExtractParam, modelConfig: IModelConfig, opt?: InsightExtractOption, multimodalPrompt?: TMultimodalPrompt): Promise<ExecutionResult<T>>;
     assert(assertion: TUserPrompt, modelConfig: IModelConfig, opt?: InsightExtractOption): Promise<ExecutionResult<boolean>>;
-    /**
-     * Append a message to the conversation history
-     * For user messages with images:
-     * - Keep max 4 user image messages in history
-     * - Remove oldest user image message when limit reached
-     * For assistant messages:
-     * - Simply append to history
-     * @param conversationHistory Message to append
-     */
-    private appendConversationHistory;
     private appendErrorPlan;
+    taskForSleep(timeMs: number, modelConfig: IModelConfig): Promise<ExecutionTaskApply<any, any, any, any>>;
     waitFor(assertion: TUserPrompt, opt: PlanningActionParamWaitFor, modelConfig: IModelConfig): Promise<ExecutionResult<void>>;
 }
 export {};

package/dist/types/ai-model/conversation-history.d.ts

@@ -0,0 +1,18 @@
+import type { ChatCompletionMessageParam } from 'openai/resources/index';
+export interface ConversationHistoryOptions {
+    maxUserImageMessages?: number;
+    initialMessages?: ChatCompletionMessageParam[];
+}
+export declare class ConversationHistory {
+    private readonly maxUserImageMessages;
+    private readonly messages;
+    constructor(options?: ConversationHistoryOptions);
+    append(message: ChatCompletionMessageParam): void;
+    seed(messages: ChatCompletionMessageParam[]): void;
+    reset(): void;
+    snapshot(): ChatCompletionMessageParam[];
+    get length(): number;
+    [Symbol.iterator](): IterableIterator<ChatCompletionMessageParam>;
+    toJSON(): ChatCompletionMessageParam[];
+    private pruneOldestUserMessageIfNecessary;
+}

package/dist/types/ai-model/index.d.ts

@@ -7,6 +7,7 @@ export type { ChatCompletionMessageParam } from 'openai/resources/index';
 export { AiLocateElement, AiExtractElementInfo, AiLocateSection, } from './inspect';
 export { plan } from './llm-planning';
 export { adaptBboxToRect } from './common';
-export { vlmPlanning, resizeImageForUiTars } from './ui-tars-planning';
+export { uiTarsPlanning, resizeImageForUiTars } from './ui-tars-planning';
+export { ConversationHistory, type ConversationHistoryOptions, } from './conversation-history';
 export { AIActionType, type AIArgs } from './common';
 export { getMidsceneLocationSchema, type MidsceneLocationResultType, PointSchema, SizeSchema, RectSchema, TMultimodalPromptSchema, TUserPromptSchema, type TMultimodalPrompt, type TUserPrompt, findAllMidsceneLocatorField, dumpActionParam, loadActionParam, } from './common';

package/dist/types/ai-model/llm-planning.d.ts

@@ -1,10 +1,11 @@
 import type { DeviceAction, InterfaceType, PlanningAIResponse, UIContext } from '../types';
 import type { IModelConfig } from '@midscene/shared/env';
+import type { ConversationHistory } from './conversation-history';
 export declare function plan(userInstruction: string, opts: {
     context: UIContext;
     interfaceType: InterfaceType;
     actionSpace: DeviceAction<any>[];
-    log?: string;
     actionContext?: string;
     modelConfig: IModelConfig;
+    conversationHistory?: ConversationHistory;
 }): Promise<PlanningAIResponse>;

package/dist/types/ai-model/prompt/llm-planning.d.ts

@@ -1,5 +1,4 @@
 import type { DeviceAction } from '../../types';
-import { PromptTemplate } from '@langchain/core/prompts';
 import type { TVlModeTypes } from '@midscene/shared/env';
 import type { ResponseFormatJSONSchema } from 'openai/resources/index';
 export declare const descriptionForAction: (action: DeviceAction<any>, locatorSchemaTypeDescription: string) => string;
@@ -8,8 +7,3 @@ export declare function systemPromptToTaskPlanning({ actionSpace, vlMode, }: {
     vlMode: TVlModeTypes | undefined;
 }): Promise<string>;
 export declare const planSchema: ResponseFormatJSONSchema;
-export declare const generateTaskBackgroundContext: (userInstruction: string, log?: string, userActionContext?: string) => string;
-export declare const automationUserPrompt: (vlMode: TVlModeTypes | undefined) => PromptTemplate<{
-    pageDescription: any;
-    taskBackgroundContext: any;
-}, any>;

package/dist/types/ai-model/service-caller/index.d.ts

@@ -13,7 +13,7 @@ export declare function callAI(messages: ChatCompletionMessageParam[], AIActionT
     isStreamed: boolean;
 }>;
 export declare const getResponseFormat: (modelName: string, AIActionTypeValue: AIActionType) => OpenAI.ChatCompletionCreateParams["response_format"] | OpenAI.ResponseFormatJSONObject;
-export declare function callAIWithObjectResponse<T>(messages: AIArgs, AIActionTypeValue: AIActionType, modelConfig: IModelConfig): Promise<{
+export declare function callAIWithObjectResponse<T>(messages: ChatCompletionMessageParam[], AIActionTypeValue: AIActionType, modelConfig: IModelConfig): Promise<{
     content: T;
     usage?: AIUsageInfo;
 }>;

package/dist/types/ai-model/ui-tars-planning.d.ts

@@ -1,24 +1,12 @@
-import type { AIUsageInfo, MidsceneYamlFlowItem, PlanningAction, Size } from '../types';
+import type { PlanningAIResponse, Size, UIContext } from '../types';
 import { type IModelConfig, UITarsModelVersion } from '@midscene/shared/env';
-import { actionParser } from '@ui-tars/action-parser';
-import type { ChatCompletionMessageParam } from 'openai/resources/index';
+import type { ConversationHistory } from './conversation-history';
 type ActionType = 'click' | 'drag' | 'type' | 'hotkey' | 'finished' | 'scroll' | 'wait';
-export declare function vlmPlanning(options: {
-    userInstruction: string;
-    conversationHistory: ChatCompletionMessageParam[];
-    size: {
-        width: number;
-        height: number;
-    };
+export declare function uiTarsPlanning(userInstruction: string, options: {
+    conversationHistory: ConversationHistory;
+    context: UIContext;
     modelConfig: IModelConfig;
-}): Promise<{
-    actions: PlanningAction<any>[];
-    actionsFromModel: ReturnType<typeof actionParser>['parsed'];
-    action_summary: string;
-    yamlFlow?: MidsceneYamlFlowItem[];
-    usage?: AIUsageInfo;
-    rawResponse?: string;
-}>;
+}): Promise<PlanningAIResponse>;
 interface BaseAction {
     action_type: ActionType;
     action_inputs: Record<string, any>;

package/dist/types/index.d.ts

@@ -8,5 +8,5 @@ export type * from './types';
 export { z };
 export default Insight;
 export { Executor, Insight, getVersion };
-export type { MidsceneYamlScript, MidsceneYamlTask, MidsceneYamlFlowItem, MidsceneYamlConfigResult, MidsceneYamlConfig, MidsceneYamlScriptWebEnv, MidsceneYamlScriptAndroidEnv, MidsceneYamlScriptIOSEnv, MidsceneYamlScriptEnv, LocateOption, DetailedLocateParam, } from './yaml';
+export type { MidsceneYamlScript, MidsceneYamlTask, MidsceneYamlFlowItem, MidsceneYamlConfigResult, LocateOption, DetailedLocateParam, } from './yaml';
 export { Agent, type AgentOpt, createAgent } from './agent';

package/dist/types/types.d.ts

@@ -318,7 +318,6 @@ export type ExecutionTaskLogApply<LogParam = {
 export type ExecutionTaskLog = ExecutionTask<ExecutionTaskLogApply>;
 export type ExecutionTaskPlanningApply = ExecutionTaskApply<'Planning', {
     userInstruction: string;
-    log?: string;
 }, PlanningAIResponse>;
 export type ExecutionTaskPlanning = ExecutionTask<ExecutionTaskPlanningApply>;
 export interface GroupedActionDump {

package/dist/types/yaml.d.ts

@@ -32,7 +32,6 @@ export interface MidsceneYamlScript {
     target?: MidsceneYamlScriptWebEnv;
     web?: MidsceneYamlScriptWebEnv;
     android?: MidsceneYamlScriptAndroidEnv;
-    ios?: MidsceneYamlScriptIOSEnv;
     interface?: MidsceneYamlScriptEnvGeneralInterface;
     config?: MidsceneYamlScriptConfig;
     agent?: MidsceneYamlScriptAgentOpt;
@@ -74,15 +73,7 @@ export interface MidsceneYamlScriptAndroidEnv extends MidsceneYamlScriptConfig {
     deviceId?: string;
     launch?: string;
 }
-export interface MidsceneYamlScriptIOSEnv extends MidsceneYamlScriptConfig {
-    deviceId?: string;
-    wdaPort?: number;
-    wdaHost?: string;
-    autoDismissKeyboard?: boolean;
-    keyboardDismissStrategy?: 'done-first' | 'escape-first';
-    launch?: string;
-}
-export type MidsceneYamlScriptEnv = MidsceneYamlScriptWebEnv | MidsceneYamlScriptAndroidEnv | MidsceneYamlScriptIOSEnv;
+export type MidsceneYamlScriptEnv = MidsceneYamlScriptWebEnv | MidsceneYamlScriptAndroidEnv;
 export interface MidsceneYamlFlowItemAIAction {
     ai?: string;
     aiAction?: string;
@@ -152,7 +143,6 @@ export interface MidsceneYamlConfig {
     shareBrowserContext?: boolean;
     web?: MidsceneYamlScriptWebEnv;
     android?: MidsceneYamlScriptAndroidEnv;
-    ios?: MidsceneYamlScriptIOSEnv;
     files: string[];
     headed?: boolean;
     keepWindow?: boolean;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@midscene/core",
   "description": "Automate browser actions, extract data, and perform assertions using AI. It offers JavaScript SDK, Chrome extension, and support for scripting in YAML. See https://midscenejs.com/ for details.",
-  "version": "0.28.12-beta-20250923124135.0",
+  "version": "0.28.12-beta-20250924031347.0",
   "repository": "https://github.com/web-infra-dev/midscene",
   "homepage": "https://midscenejs.com/",
   "main": "./dist/lib/index.js",
@@ -87,8 +87,8 @@
     "zod": "3.24.3",
     "semver": "7.5.2",
     "js-yaml": "4.1.0",
-    "@midscene/recorder": "0.28.12-beta-20250923124135.0",
-    "@midscene/shared": "0.28.12-beta-20250923124135.0"
+    "@midscene/recorder": "0.28.12-beta-20250924031347.0",
+    "@midscene/shared": "0.28.12-beta-20250924031347.0"
   },
   "devDependencies": {
     "@microsoft/api-extractor": "^7.52.10",