@midscene/core 1.2.1-beta-20260113073450.0 → 1.2.1-beta-20260114072539.0

package/dist/types/agent/agent.d.ts

@@ -47,7 +47,6 @@ export declare class Agent<InterfaceType extends AbstractInterface = AbstractInt
      */
     private screenshotScalePromise?;
     private executionDumpIndexByRunner;
-    private fullActionSpace;
     get page(): InterfaceType;
     /**
      * Ensures VL model warning is shown once when needed
@@ -108,11 +107,19 @@ export declare class Agent<InterfaceType extends AbstractInterface = AbstractInt
      * @deprecated Use aiScroll(locatePrompt, opt) instead where opt contains the scroll parameters
      */
     aiScroll(scrollParam: ScrollParam, locatePrompt?: TUserPrompt, opt?: LocateOption): Promise<any>;
-    aiAct(taskPrompt: string, opt?: AiActOptions): Promise<string | undefined>;
+    aiAct(taskPrompt: string, opt?: AiActOptions): Promise<{
+        result: Record<string, any>;
+    } | {
+        yamlFlow?: import("../yaml").MidsceneYamlFlowItem[];
+    } | undefined>;
     /**
      * @deprecated Use {@link Agent.aiAct} instead.
      */
-    aiAction(taskPrompt: string, opt?: AiActOptions): Promise<string | undefined>;
+    aiAction(taskPrompt: string, opt?: AiActOptions): Promise<{
+        result: Record<string, any>;
+    } | {
+        yamlFlow?: import("../yaml").MidsceneYamlFlowItem[];
+    } | undefined>;
     aiQuery<ReturnType = any>(demand: ServiceExtractParam, opt?: ServiceExtractOption): Promise<ReturnType>;
     aiBoolean(prompt: TUserPrompt, opt?: ServiceExtractOption): Promise<boolean>;
     aiNumber(prompt: TUserPrompt, opt?: ServiceExtractOption): Promise<number>;
@@ -133,7 +140,11 @@ export declare class Agent<InterfaceType extends AbstractInterface = AbstractInt
         message: string | undefined;
     } | undefined>;
     aiWaitFor(assertion: TUserPrompt, opt?: AgentWaitForOpt): Promise<void>;
-    ai(...args: Parameters<typeof this.aiAct>): Promise<string | undefined>;
+    ai(...args: Parameters<typeof this.aiAct>): Promise<{
+        result: Record<string, any>;
+    } | {
+        yamlFlow?: import("../yaml").MidsceneYamlFlowItem[];
+    } | undefined>;
     runYaml(yamlScriptContent: string): Promise<{
         result: Record<string, any>;
     }>;

package/dist/types/agent/tasks.d.ts

@@ -1,9 +1,9 @@
-import { type TMultimodalPrompt, type TUserPrompt } from '../common';
+import type { TMultimodalPrompt, TUserPrompt } from '../common';
 import type { AbstractInterface } from '../device';
 import type Service from '../service';
 import type { TaskRunner } from '../task-runner';
 import { TaskExecutionError } from '../task-runner';
-import type { DeepThinkOption, DeviceAction, ExecutionTaskApply, ExecutionTaskProgressOptions, MidsceneYamlFlowItem, PlanningAction, PlanningActionParamWaitFor, ServiceExtractOption, ServiceExtractParam } from '../types';
+import type { DeepThinkOption, DeviceAction, ExecutionTaskApply, ExecutionTaskProgressOptions, MidsceneYamlFlowItem, PlanningAction, PlanningActionParamSleep, PlanningActionParamWaitFor, ServiceExtractOption, ServiceExtractParam } from '../types';
 import type { IModelConfig } from '@midscene/shared/env';
 import type { TaskCache } from './task-cache';
 export { locatePlanForLocate } from './task-builder';
@@ -48,12 +48,12 @@ export declare class TaskExecutor {
     runPlans(title: string, plans: PlanningAction[], modelConfigForPlanning: IModelConfig, modelConfigForDefaultIntent: IModelConfig): Promise<ExecutionResult>;
     action(userPrompt: string, modelConfigForPlanning: IModelConfig, modelConfigForDefaultIntent: IModelConfig, includeBboxInPlanning: boolean, aiActContext?: string, cacheable?: boolean, replanningCycleLimitOverride?: number, imagesIncludeCount?: number, deepThink?: DeepThinkOption, fileChooserAccept?: string[]): Promise<ExecutionResult<{
         yamlFlow?: MidsceneYamlFlowItem[];
-        output?: string;
     } | undefined>>;
     private runAction;
     private createTypeQueryTask;
     createTypeQueryExecution<T>(type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert', demand: ServiceExtractParam, modelConfig: IModelConfig, opt?: ServiceExtractOption, multimodalPrompt?: TMultimodalPrompt): Promise<ExecutionResult<T>>;
-    taskForSleep(timeMs: number, _modelConfig: IModelConfig): Promise<import("../types").ExecutionTaskActionApply<import("../types").PlanningActionParamSleep>>;
+    private sleepPlan;
+    taskForSleep(timeMs: number, _modelConfig: IModelConfig): Promise<import("../types").ExecutionTaskActionApply<PlanningActionParamSleep>>;
     waitFor(assertion: TUserPrompt, opt: PlanningActionParamWaitFor, modelConfig: IModelConfig): Promise<ExecutionResult<void>>;
 }
 export declare function withFileChooser<T>(interfaceInstance: AbstractInterface, fileChooserAccept: string[] | undefined, action: () => Promise<T>): Promise<T>;

package/dist/types/ai-model/auto-glm/actions.d.ts

@@ -0,0 +1,77 @@
+import type { PlanningAction } from '../../types';
+export interface BaseAction {
+    _metadata: string;
+    think?: string;
+}
+export interface TapAction extends BaseAction {
+    _metadata: 'do';
+    action: 'Tap';
+    element: [number, number];
+}
+export interface DoubleTapAction extends BaseAction {
+    _metadata: 'do';
+    action: 'Double Tap';
+    element: [number, number];
+}
+export interface TypeAction extends BaseAction {
+    _metadata: 'do';
+    action: 'Type';
+    text: string;
+}
+export interface SwipeAction extends BaseAction {
+    _metadata: 'do';
+    action: 'Swipe';
+    start: [number, number];
+    end: [number, number];
+}
+export interface LongPressAction extends BaseAction {
+    _metadata: 'do';
+    action: 'Long Press';
+    element: [number, number];
+}
+export interface LaunchAction extends BaseAction {
+    _metadata: 'do';
+    action: 'Launch';
+    app: string;
+}
+export interface BackAction extends BaseAction {
+    _metadata: 'do';
+    action: 'Back';
+}
+export interface HomeAction extends BaseAction {
+    _metadata: 'do';
+    action: 'Home';
+}
+export interface WaitAction extends BaseAction {
+    _metadata: 'do';
+    action: 'Wait';
+    durationMs: number;
+}
+export interface InteractAction extends BaseAction {
+    _metadata: 'do';
+    action: 'Interact';
+}
+export interface CallAPIAction extends BaseAction {
+    _metadata: 'do';
+    action: 'Call_API';
+    instruction: string;
+}
+export interface TakeoverAction extends BaseAction {
+    _metadata: 'do';
+    action: 'Take_over';
+    message: string;
+}
+export interface NoteAction extends BaseAction {
+    _metadata: 'do';
+    action: 'Note';
+    message: string;
+}
+export interface FinishAction extends BaseAction {
+    _metadata: 'finish';
+    message: string;
+}
+export type ParsedAction = TapAction | DoubleTapAction | TypeAction | SwipeAction | LongPressAction | LaunchAction | BackAction | HomeAction | WaitAction | InteractAction | CallAPIAction | TakeoverAction | NoteAction | FinishAction;
+export declare function transformAutoGLMAction(action: ParsedAction, size: {
+    width: number;
+    height: number;
+}): PlanningAction[];

package/dist/types/ai-model/auto-glm/index.d.ts

@@ -0,0 +1,5 @@
+export { getAutoGLMLocatePrompt, getAutoGLMPlanPrompt } from './prompt';
+export { parseAutoGLMLocateResponse, parseAutoGLMResponse, parseAction, } from './parser';
+export { autoGLMPlanning } from './planning';
+export { autoGLMCoordinateToBbox } from './util';
+export type { ParsedAction } from './actions';

package/dist/types/ai-model/auto-glm/parser.d.ts

@@ -0,0 +1,18 @@
+import type { ParsedAction } from './actions';
+export declare const extractValueAfter: (src: string, key: string) => string;
+export declare function parseAction(response: {
+    think: string;
+    content: string;
+}): ParsedAction;
+export declare function parseAutoGLMResponse(content: string): {
+    think: string;
+    content: string;
+};
+export declare function parseAutoGLMLocateResponse(rawResponse: string): {
+    think: string;
+    coordinates: {
+        x: number;
+        y: number;
+    } | null;
+    error?: string;
+};

package/dist/types/ai-model/auto-glm/planning.d.ts

@@ -0,0 +1,9 @@
+import type { PlanningAIResponse, UIContext } from '../../types';
+import type { IModelConfig } from '@midscene/shared/env';
+import type { ConversationHistory } from '../conversation-history';
+export declare function autoGLMPlanning(userInstruction: string, options: {
+    conversationHistory: ConversationHistory;
+    context: UIContext;
+    modelConfig: IModelConfig;
+    actionContext?: string;
+}): Promise<PlanningAIResponse>;

package/dist/types/ai-model/auto-glm/prompt.d.ts

@@ -0,0 +1,27 @@
+/**
+ * Auto-GLM Prompt Templates
+ *
+ * Portions of this file are derived from Open-AutoGLM
+ * Copyright (c) 2024 zai-org
+ * Licensed under the Apache License, Version 2.0
+ *
+ * Source: https://github.com/zai-org/Open-AutoGLM
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Modifications:
+ * - Adapted prompts for Midscene.js integration
+ */
+import type { TVlModeTypes } from '@midscene/shared/env';
+export declare const getAutoGLMPlanPrompt: (vlMode: TVlModeTypes | undefined) => string;
+export declare const getAutoGLMLocatePrompt: (vlMode: TVlModeTypes | undefined) => string;

package/dist/types/ai-model/auto-glm/util.d.ts

@@ -0,0 +1,16 @@
+import type { TVlModeTypes } from '@midscene/shared/env';
+/**
+ * Auto-GLM coordinate system range: [0, AUTO_GLM_COORDINATE_MAX]
+ */
+export declare const AUTO_GLM_COORDINATE_MAX = 1000;
+/**
+ * Check if the vlMode is auto-glm or auto-glm-multilingual
+ * @param vlMode The VL mode to check
+ * @returns true if vlMode is auto-glm or auto-glm-multilingual
+ */
+export declare function isAutoGLM(vlMode: TVlModeTypes | undefined): boolean;
+/**
+ * Convert auto-glm coordinate [0,999] to bbox
+ * Auto-glm uses [0,999] coordinate system, maps to image size, and creates a 10x10 bbox around the point
+ */
+export declare function autoGLMCoordinateToBbox(x: number, y: number, width: number, height: number): [number, number, number, number];

package/dist/types/ai-model/index.d.ts

@@ -6,6 +6,7 @@ export { generateYamlTest, generateYamlTestStream, } from './prompt/yaml-generat
 export type { ChatCompletionMessageParam } from 'openai/resources/index';
 export { AiLocateElement, AiExtractElementInfo, AiLocateSection, AiJudgeOrderSensitive, } from './inspect';
 export { plan } from './llm-planning';
+export { autoGLMPlanning } from './auto-glm/planning';
 export { adaptBboxToRect } from '../common';
 export { uiTarsPlanning, resizeImageForUiTars } from './ui-tars-planning';
 export { ConversationHistory, type ConversationHistoryOptions, } from './conversation-history';

package/dist/types/ai-model/latest-locate-recorder.d.ts

@@ -0,0 +1,14 @@
+interface ILocate {
+    prompt: string;
+    bbox: [number, number, number, number];
+}
+export declare class LatestLocateRecorder {
+    latestLocate: ILocate | undefined;
+    source: string;
+    recordLocate(locate: ILocate, source: string): void;
+    getLatestLocate(): {
+        locate: ILocate | undefined;
+        source: string;
+    };
+}
+export {};

package/dist/types/common.d.ts

@@ -20,7 +20,7 @@ export declare function mergeRects(rects: Rect[]): {
 };
 export declare function expandSearchArea(rect: Rect, screenSize: Size, vlMode: TVlModeTypes | undefined): Rect;
 export declare function markupImageForLLM(screenshotBase64: string, tree: ElementTreeNode<BaseElement>, size: Size): Promise<string>;
-export declare function buildYamlFlowFromPlans(plans: PlanningAction[], actionSpace: DeviceAction<any>[]): MidsceneYamlFlowItem[];
+export declare function buildYamlFlowFromPlans(plans: PlanningAction[], actionSpace: DeviceAction<any>[], sleep?: number): MidsceneYamlFlowItem[];
 export declare const PointSchema: z.ZodObject<{
     left: z.ZodNumber;
     top: z.ZodNumber;
@@ -558,11 +558,4 @@ export declare const loadActionParam: (jsonObject: Record<string, any>, zodSchem
  * so they are intentionally excluded from Zod parsing and use existing validation logic.
  */
 export declare const parseActionParam: (rawParam: Record<string, any> | undefined, zodSchema?: z.ZodType<any>) => Record<string, any> | undefined;
-export declare const finalizeActionName = "Finalize";
-/**
- * Get a readable time string for the current time
- * @param format - Optional format string. Supports: YYYY, MM, DD, HH, mm, ss. Default: 'YYYY-MM-DD HH:mm:ss'
- * @returns A formatted time string with format label
- */
-export declare const getReadableTimeString: (format?: string) => string;
 export {};

package/dist/types/device/index.d.ts

@@ -2177,27 +2177,5 @@ export type ActionAssertParam = {
     result: boolean;
 };
 export declare const defineActionAssert: () => DeviceAction<ActionAssertParam>;
-export declare const ActionSleepParamSchema: z.ZodObject<{
-    millisecond: z.ZodOptional<z.ZodDefault<z.ZodNumber>>;
-}, "strip", z.ZodTypeAny, {
-    millisecond?: number | undefined;
-}, {
-    millisecond?: number | undefined;
-}>;
-export type ActionSleepParam = {
-    millisecond?: number;
-};
-export declare const defineActionSleep: () => DeviceAction<ActionSleepParam>;
-export declare const actionFinalizeParamSchema: z.ZodObject<{
-    message: z.ZodOptional<z.ZodString>;
-}, "strip", z.ZodTypeAny, {
-    message?: string | undefined;
-}, {
-    message?: string | undefined;
-}>;
-export type ActionFinalizeParam = {
-    message?: string;
-};
-export declare const defineActionFinalize: () => DeviceAction<ActionFinalizeParam>;
 export type { DeviceAction } from '../types';
 export type { AndroidDeviceOpt, AndroidDeviceInputOpt, IOSDeviceOpt, IOSDeviceInputOpt, } from './device-options';

package/dist/types/types.d.ts

@@ -176,8 +176,9 @@ export interface PlanningAction<ParamType = any> {
 }
 export interface RawResponsePlanningAIResponse {
     action: PlanningAction;
+    more_actions_needed_by_instruction: boolean;
     log: string;
-    note?: string;
+    sleep?: number;
     error?: string;
 }
 export interface PlanningAIResponse extends Omit<RawResponsePlanningAIResponse, 'action'> {
@@ -188,7 +189,6 @@ export interface PlanningAIResponse extends Omit<RawResponsePlanningAIResponse,
     yamlString?: string;
     error?: string;
     reasoning_content?: string;
-    shouldContinuePlanning: boolean;
 }
 export interface PlanningActionParamSleep {
     timeMs: number;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@midscene/core",
   "description": "Automate browser actions, extract data, and perform assertions using AI. It offers JavaScript SDK, Chrome extension, and support for scripting in YAML. See https://midscenejs.com/ for details.",
-  "version": "1.2.1-beta-20260113073450.0",
+  "version": "1.2.1-beta-20260114072539.0",
   "repository": "https://github.com/web-infra-dev/midscene",
   "homepage": "https://midscenejs.com/",
   "main": "./dist/lib/index.js",
@@ -89,7 +89,7 @@
     "semver": "7.5.2",
     "undici": "^6.0.0",
     "zod": "3.24.3",
-    "@midscene/shared": "1.2.1-beta-20260113073450.0"
+    "@midscene/shared": "1.2.1-beta-20260114072539.0"
   },
   "devDependencies": {
     "@rslib/core": "^0.18.3",