@midscene/core 0.24.2-beta-20250731102409.0 → 0.24.2-beta-20250731132300.0

package/dist/es/index.d.ts

@@ -1,7 +1,7 @@
-import { E as ExecutionTask, a as ExecutionTaskProgressOptions, b as ExecutionTaskApply, c as ExecutionDump, U as UIContext, I as InsightAction, D as DumpSubscriber, d as InsightTaskInfo, e as InsightOptions, f as DetailedLocateParam, L as LocateResult, g as InsightExtractOption, h as InsightAssertionResponse, A as AIDescribeElementResponse } from './types-93fa8419.js';
-export { w as AIAssertionResponse, u as AIDataExtractionResponse, s as AIElementCoordinatesResponse, r as AIElementLocatorResponse, t as AIElementResponse, n as AIResponseFormat, v as AISectionLocatorResponse, q as AISingleElementResponse, o as AISingleElementResponseById, p as AISingleElementResponseByPosition, m as AIUsageInfo, S as AgentAssertOpt, z as AgentDescribeElementAtPointResult, Q as AgentWaitForOpt, a5 as BaseAgentParserOpt, C as CallAIFn, az as CodeGenerationChunk, a4 as Color, H as DumpMeta, N as ElementById, B as EnsureObject, a8 as ExecutionRecorderItem, aq as ExecutionTaskAction, ap as ExecutionTaskActionApply, ab as ExecutionTaskHitBy, ao as ExecutionTaskInsightAssertion, an as ExecutionTaskInsightAssertionApply, am as ExecutionTaskInsightAssertionParam, af as ExecutionTaskInsightDumpLog, ah as ExecutionTaskInsightLocate, ag as ExecutionTaskInsightLocateApply, ae as ExecutionTaskInsightLocateOutput, ad as ExecutionTaskInsightLocateParam, al as ExecutionTaskInsightQuery, ak as ExecutionTaskInsightQueryApply, aj as ExecutionTaskInsightQueryOutput, ai as ExecutionTaskInsightQueryParam, as as ExecutionTaskLog, ar as ExecutionTaskLogApply, au as ExecutionTaskPlanning, at as ExecutionTaskPlanningApply, ac as ExecutionTaskReturn, a9 as ExecutionTaskType, aa as ExecutorContext, aZ as FreeFn, av as GroupedActionDump, J as InsightDump, F as InsightExtractParam, K as LiteUISection, aB as LocateOption, G as LocateResultElement, y as LocateValidatorResult, x as LocatorValidatorOption, b0 as MidsceneYamlConfig, b1 as MidsceneYamlConfigOutput, l as MidsceneYamlConfigResult, j as MidsceneYamlFlowItem, aI as MidsceneYamlFlowItemAIAction, aN as MidsceneYamlFlowItemAIAsk, aJ as MidsceneYamlFlowItemAIAssert, aO as MidsceneYamlFlowItemAIBoolean, aS as MidsceneYamlFlowItemAIHover, aT as MidsceneYamlFlowItemAIInput, aU as MidsceneYamlFlowItemAIKeyboardPress, aP as MidsceneYamlFlowItemAILocate, aL as MidsceneYamlFlowItemAINumber, aK as MidsceneYamlFlowItemAIQuery, k as MidsceneYamlFlowItemAIRightClick, aV as MidsceneYamlFlowItemAIScroll, aM as MidsceneYamlFlowItemAIString, aR as MidsceneYamlFlowItemAITap, aQ as MidsceneYamlFlowItemAIWaitFor, aW as MidsceneYamlFlowItemEvaluateJavaScript, aY as MidsceneYamlFlowItemLogScreenshot, aX as MidsceneYamlFlowItemSleep, M as MidsceneYamlScript, aG as MidsceneYamlScriptAndroidEnv, aH as MidsceneYamlScriptEnv, aE as MidsceneYamlScriptEnvBase, aF as MidsceneYamlScriptWebEnv, i as MidsceneYamlTask, O as OnTaskStartTip, aw as PageType, P as PartialInsightDumpFromSDK, W as PlanningAIResponse, V as PlanningAction, a0 as PlanningActionParamAssert, a2 as PlanningActionParamError, Y as PlanningActionParamHover, _ as PlanningActionParamInputOrKeyPress, Z as PlanningActionParamRightClick, $ as PlanningActionParamScroll, a1 as PlanningActionParamSleep, X as PlanningActionParamTap, a3 as PlanningActionParamWaitFor, T as PlanningLocateParam, a7 as PlaywrightParserOpt, a6 as PuppeteerParserOpt, aC as ReferenceImage, R as ReportDumpWithAttributes, a$ as ScriptPlayerStatusValue, a_ as ScriptPlayerTaskStatus, aA as StreamingAIResponse, ay as StreamingCallback, ax as StreamingCodeGenerationOptions, aD as scrollParam } from './types-93fa8419.js';
-import { c as callAiFn } from './llm-planning-4bc82162.js';
-export { a as AiAssert, A as AiLocateElement, d as describeUserPage, p as plan } from './llm-planning-4bc82162.js';
+import { E as ExecutionTask, a as ExecutionTaskProgressOptions, b as ExecutionTaskApply, c as ExecutionDump, U as UIContext, I as InsightAction, D as DumpSubscriber, d as InsightTaskInfo, e as InsightOptions, f as DetailedLocateParam, L as LocateResult, g as InsightExtractOption, h as InsightAssertionResponse, A as AIDescribeElementResponse } from './types-d836fa73.js';
+export { w as AIAssertionResponse, u as AIDataExtractionResponse, s as AIElementCoordinatesResponse, r as AIElementLocatorResponse, t as AIElementResponse, n as AIResponseFormat, v as AISectionLocatorResponse, q as AISingleElementResponse, o as AISingleElementResponseById, p as AISingleElementResponseByPosition, m as AIUsageInfo, S as AgentAssertOpt, z as AgentDescribeElementAtPointResult, Q as AgentWaitForOpt, a7 as BaseAgentParserOpt, C as CallAIFn, aB as CodeGenerationChunk, a6 as Color, H as DumpMeta, N as ElementById, B as EnsureObject, aa as ExecutionRecorderItem, as as ExecutionTaskAction, ar as ExecutionTaskActionApply, ad as ExecutionTaskHitBy, aq as ExecutionTaskInsightAssertion, ap as ExecutionTaskInsightAssertionApply, ao as ExecutionTaskInsightAssertionParam, ah as ExecutionTaskInsightDumpLog, aj as ExecutionTaskInsightLocate, ai as ExecutionTaskInsightLocateApply, ag as ExecutionTaskInsightLocateOutput, af as ExecutionTaskInsightLocateParam, an as ExecutionTaskInsightQuery, am as ExecutionTaskInsightQueryApply, al as ExecutionTaskInsightQueryOutput, ak as ExecutionTaskInsightQueryParam, au as ExecutionTaskLog, at as ExecutionTaskLogApply, aw as ExecutionTaskPlanning, av as ExecutionTaskPlanningApply, ae as ExecutionTaskReturn, ab as ExecutionTaskType, ac as ExecutorContext, a$ as FreeFn, ax as GroupedActionDump, J as InsightDump, F as InsightExtractParam, K as LiteUISection, aD as LocateOption, G as LocateResultElement, y as LocateValidatorResult, x as LocatorValidatorOption, b2 as MidsceneYamlConfig, b3 as MidsceneYamlConfigOutput, l as MidsceneYamlConfigResult, j as MidsceneYamlFlowItem, aK as MidsceneYamlFlowItemAIAction, aP as MidsceneYamlFlowItemAIAsk, aL as MidsceneYamlFlowItemAIAssert, aQ as MidsceneYamlFlowItemAIBoolean, aU as MidsceneYamlFlowItemAIHover, aV as MidsceneYamlFlowItemAIInput, aW as MidsceneYamlFlowItemAIKeyboardPress, aR as MidsceneYamlFlowItemAILocate, aN as MidsceneYamlFlowItemAINumber, aM as MidsceneYamlFlowItemAIQuery, k as MidsceneYamlFlowItemAIRightClick, aX as MidsceneYamlFlowItemAIScroll, aO as MidsceneYamlFlowItemAIString, aT as MidsceneYamlFlowItemAITap, aS as MidsceneYamlFlowItemAIWaitFor, aY as MidsceneYamlFlowItemEvaluateJavaScript, a_ as MidsceneYamlFlowItemLogScreenshot, aZ as MidsceneYamlFlowItemSleep, M as MidsceneYamlScript, aI as MidsceneYamlScriptAndroidEnv, aJ as MidsceneYamlScriptEnv, aG as MidsceneYamlScriptEnvBase, aH as MidsceneYamlScriptWebEnv, i as MidsceneYamlTask, O as OnTaskStartTip, ay as PageType, P as PartialInsightDumpFromSDK, W as PlanningAIResponse, V as PlanningAction, a4 as PlanningActionParamAndroidLongPress, a5 as PlanningActionParamAndroidPull, a0 as PlanningActionParamAssert, a2 as PlanningActionParamError, Y as PlanningActionParamHover, _ as PlanningActionParamInputOrKeyPress, Z as PlanningActionParamRightClick, $ as PlanningActionParamScroll, a1 as PlanningActionParamSleep, X as PlanningActionParamTap, a3 as PlanningActionParamWaitFor, T as PlanningLocateParam, a9 as PlaywrightParserOpt, a8 as PuppeteerParserOpt, aE as ReferenceImage, R as ReportDumpWithAttributes, b1 as ScriptPlayerStatusValue, b0 as ScriptPlayerTaskStatus, aC as StreamingAIResponse, aA as StreamingCallback, az as StreamingCodeGenerationOptions, aF as scrollParam } from './types-d836fa73.js';
+import { c as callAiFn } from './llm-planning-d7096b0d.js';
+export { a as AiAssert, A as AiLocateElement, d as describeUserPage, p as plan } from './llm-planning-d7096b0d.js';
 import { BaseElement, Rect } from '@midscene/shared/types';
 export { BaseElement, ElementTreeNode, Point, Rect, Size } from '@midscene/shared/types';
 export { getVersion } from './utils.js';

package/dist/es/index.js

@@ -1,6 +1,6 @@
 import {
   getVersion
-} from "./chunk-TYKJHDNK.js";
+} from "./chunk-IMZJSEAX.js";
 import {
   AiAssert,
   AiExtractElementInfo,
@@ -11,7 +11,7 @@ import {
   describeUserPage,
   expandSearchArea,
   plan
-} from "./chunk-YCHAOUOW.js";
+} from "./chunk-FKQMUAXP.js";
 
 // src/ai-model/action-executor.ts
 import {

package/dist/es/{llm-planning-4bc82162.d.ts → llm-planning-d7096b0d.d.ts}

@@ -1,4 +1,4 @@
-import { m as AIUsageInfo, U as UIContext, aC as ReferenceImage, r as AIElementLocatorResponse, N as ElementById, g as InsightExtractOption, u as AIDataExtractionResponse, w as AIAssertionResponse, aw as PageType, W as PlanningAIResponse } from './types-93fa8419.js';
+import { m as AIUsageInfo, U as UIContext, aE as ReferenceImage, r as AIElementLocatorResponse, N as ElementById, g as InsightExtractOption, u as AIDataExtractionResponse, w as AIAssertionResponse, ay as PageType, W as PlanningAIResponse } from './types-d836fa73.js';
 import { Rect, ElementTreeNode, BaseElement } from '@midscene/shared/types';
 import { ChatCompletionSystemMessageParam, ChatCompletionUserMessageParam } from 'openai/resources';
 

package/dist/es/{types-93fa8419.d.ts → types-d836fa73.d.ts}

@@ -357,7 +357,7 @@ interface PlanningLocateParam extends DetailedLocateParam {
 }
 interface PlanningAction<ParamType = any> {
     thought?: string;
-    type: 'Locate' | 'Tap' | 'RightClick' | 'Hover' | 'Drag' | 'Input' | 'KeyboardPress' | 'Scroll' | 'Error' | 'ExpectedFalsyCondition' | 'Assert' | 'AssertWithoutThrow' | 'Sleep' | 'Finished' | 'AndroidBackButton' | 'AndroidHomeButton' | 'AndroidRecentAppsButton';
+    type: 'Locate' | 'Tap' | 'RightClick' | 'Hover' | 'Drag' | 'Input' | 'KeyboardPress' | 'Scroll' | 'Error' | 'ExpectedFalsyCondition' | 'Assert' | 'AssertWithoutThrow' | 'Sleep' | 'Finished' | 'AndroidBackButton' | 'AndroidHomeButton' | 'AndroidRecentAppsButton' | 'AndroidLongPress' | 'AndroidPull';
     param: ParamType;
     locate?: PlanningLocateParam | null;
 }
@@ -393,6 +393,20 @@ interface PlanningActionParamError {
 type PlanningActionParamWaitFor = AgentWaitForOpt & {
     assertion: string;
 };
+interface PlanningActionParamAndroidLongPress {
+    x: number;
+    y: number;
+    duration?: number;
+}
+interface PlanningActionParamAndroidPull {
+    direction: 'up' | 'down';
+    startPoint?: {
+        x: number;
+        y: number;
+    };
+    distance?: number;
+    duration?: number;
+}
 /**
  * misc
  */
@@ -529,4 +543,4 @@ interface StreamingAIResponse {
     isStreamed: boolean;
 }
 
-export { type PlanningActionParamScroll as $, type AIDescribeElementResponse as A, type EnsureObject as B, type CallAIFn as C, type DumpSubscriber as D, type ExecutionTask as E, type InsightExtractParam as F, type LocateResultElement as G, type DumpMeta as H, type InsightAction as I, type InsightDump as J, type LiteUISection as K, type LocateResult as L, type MidsceneYamlScript as M, type ElementById as N, type OnTaskStartTip as O, type PartialInsightDumpFromSDK as P, type AgentWaitForOpt as Q, type ReportDumpWithAttributes as R, type AgentAssertOpt as S, type PlanningLocateParam as T, UIContext as U, type PlanningAction as V, type PlanningAIResponse as W, type PlanningActionParamTap as X, type PlanningActionParamHover as Y, type PlanningActionParamRightClick as Z, type PlanningActionParamInputOrKeyPress as _, type ExecutionTaskProgressOptions as a, type ScriptPlayerStatusValue as a$, type PlanningActionParamAssert as a0, type PlanningActionParamSleep as a1, type PlanningActionParamError as a2, type PlanningActionParamWaitFor as a3, type Color as a4, type BaseAgentParserOpt as a5, type PuppeteerParserOpt as a6, type PlaywrightParserOpt as a7, type ExecutionRecorderItem as a8, type ExecutionTaskType as a9, type StreamingAIResponse as aA, type LocateOption as aB, type ReferenceImage as aC, type scrollParam as aD, type MidsceneYamlScriptEnvBase as aE, type MidsceneYamlScriptWebEnv as aF, type MidsceneYamlScriptAndroidEnv as aG, type MidsceneYamlScriptEnv as aH, type MidsceneYamlFlowItemAIAction as aI, type MidsceneYamlFlowItemAIAssert as aJ, type MidsceneYamlFlowItemAIQuery as aK, type MidsceneYamlFlowItemAINumber as aL, type MidsceneYamlFlowItemAIString as aM, type MidsceneYamlFlowItemAIAsk as aN, type MidsceneYamlFlowItemAIBoolean as aO, type MidsceneYamlFlowItemAILocate as aP, type MidsceneYamlFlowItemAIWaitFor as aQ, type MidsceneYamlFlowItemAITap as aR, type MidsceneYamlFlowItemAIHover as aS, type MidsceneYamlFlowItemAIInput as aT, type MidsceneYamlFlowItemAIKeyboardPress as aU, type MidsceneYamlFlowItemAIScroll as aV, type MidsceneYamlFlowItemEvaluateJavaScript as aW, type MidsceneYamlFlowItemSleep as aX, type MidsceneYamlFlowItemLogScreenshot as aY, type FreeFn as aZ, type ScriptPlayerTaskStatus as a_, type ExecutorContext as aa, type ExecutionTaskHitBy as ab, type ExecutionTaskReturn as ac, type ExecutionTaskInsightLocateParam as ad, type ExecutionTaskInsightLocateOutput as ae, type ExecutionTaskInsightDumpLog as af, type ExecutionTaskInsightLocateApply as ag, type ExecutionTaskInsightLocate as ah, type ExecutionTaskInsightQueryParam as ai, type ExecutionTaskInsightQueryOutput as aj, type ExecutionTaskInsightQueryApply as ak, type ExecutionTaskInsightQuery as al, type ExecutionTaskInsightAssertionParam as am, type ExecutionTaskInsightAssertionApply as an, type ExecutionTaskInsightAssertion as ao, type ExecutionTaskActionApply as ap, type ExecutionTaskAction as aq, type ExecutionTaskLogApply as ar, type ExecutionTaskLog as as, type ExecutionTaskPlanningApply as at, type ExecutionTaskPlanning as au, type GroupedActionDump as av, type PageType as aw, type StreamingCodeGenerationOptions as ax, type StreamingCallback as ay, type CodeGenerationChunk as az, type ExecutionTaskApply as b, type MidsceneYamlConfig as b0, type MidsceneYamlConfigOutput as b1, type ExecutionDump as c, type InsightTaskInfo as d, type InsightOptions as e, type DetailedLocateParam as f, type InsightExtractOption as g, type InsightAssertionResponse as h, type MidsceneYamlTask as i, type MidsceneYamlFlowItem as j, type MidsceneYamlFlowItemAIRightClick as k, type MidsceneYamlConfigResult as l, type AIUsageInfo as m, AIResponseFormat as n, type AISingleElementResponseById as o, type AISingleElementResponseByPosition as p, type AISingleElementResponse as q, type AIElementLocatorResponse as r, type AIElementCoordinatesResponse as s, type AIElementResponse as t, type AIDataExtractionResponse as u, type AISectionLocatorResponse as v, type AIAssertionResponse as w, type LocatorValidatorOption as x, type LocateValidatorResult as y, type AgentDescribeElementAtPointResult as z };
+export { type PlanningActionParamScroll as $, type AIDescribeElementResponse as A, type EnsureObject as B, type CallAIFn as C, type DumpSubscriber as D, type ExecutionTask as E, type InsightExtractParam as F, type LocateResultElement as G, type DumpMeta as H, type InsightAction as I, type InsightDump as J, type LiteUISection as K, type LocateResult as L, type MidsceneYamlScript as M, type ElementById as N, type OnTaskStartTip as O, type PartialInsightDumpFromSDK as P, type AgentWaitForOpt as Q, type ReportDumpWithAttributes as R, type AgentAssertOpt as S, type PlanningLocateParam as T, UIContext as U, type PlanningAction as V, type PlanningAIResponse as W, type PlanningActionParamTap as X, type PlanningActionParamHover as Y, type PlanningActionParamRightClick as Z, type PlanningActionParamInputOrKeyPress as _, type ExecutionTaskProgressOptions as a, type FreeFn as a$, type PlanningActionParamAssert as a0, type PlanningActionParamSleep as a1, type PlanningActionParamError as a2, type PlanningActionParamWaitFor as a3, type PlanningActionParamAndroidLongPress as a4, type PlanningActionParamAndroidPull as a5, type Color as a6, type BaseAgentParserOpt as a7, type PuppeteerParserOpt as a8, type PlaywrightParserOpt as a9, type StreamingCallback as aA, type CodeGenerationChunk as aB, type StreamingAIResponse as aC, type LocateOption as aD, type ReferenceImage as aE, type scrollParam as aF, type MidsceneYamlScriptEnvBase as aG, type MidsceneYamlScriptWebEnv as aH, type MidsceneYamlScriptAndroidEnv as aI, type MidsceneYamlScriptEnv as aJ, type MidsceneYamlFlowItemAIAction as aK, type MidsceneYamlFlowItemAIAssert as aL, type MidsceneYamlFlowItemAIQuery as aM, type MidsceneYamlFlowItemAINumber as aN, type MidsceneYamlFlowItemAIString as aO, type MidsceneYamlFlowItemAIAsk as aP, type MidsceneYamlFlowItemAIBoolean as aQ, type MidsceneYamlFlowItemAILocate as aR, type MidsceneYamlFlowItemAIWaitFor as aS, type MidsceneYamlFlowItemAITap as aT, type MidsceneYamlFlowItemAIHover as aU, type MidsceneYamlFlowItemAIInput as aV, type MidsceneYamlFlowItemAIKeyboardPress as aW, type MidsceneYamlFlowItemAIScroll as aX, type MidsceneYamlFlowItemEvaluateJavaScript as aY, type MidsceneYamlFlowItemSleep as aZ, type MidsceneYamlFlowItemLogScreenshot as a_, type ExecutionRecorderItem as aa, type ExecutionTaskType as ab, type ExecutorContext as ac, type ExecutionTaskHitBy as ad, type ExecutionTaskReturn as ae, type ExecutionTaskInsightLocateParam as af, type ExecutionTaskInsightLocateOutput as ag, type ExecutionTaskInsightDumpLog as ah, type ExecutionTaskInsightLocateApply as ai, type ExecutionTaskInsightLocate as aj, type ExecutionTaskInsightQueryParam as ak, type ExecutionTaskInsightQueryOutput as al, type ExecutionTaskInsightQueryApply as am, type ExecutionTaskInsightQuery as an, type ExecutionTaskInsightAssertionParam as ao, type ExecutionTaskInsightAssertionApply as ap, type ExecutionTaskInsightAssertion as aq, type ExecutionTaskActionApply as ar, type ExecutionTaskAction as as, type ExecutionTaskLogApply as at, type ExecutionTaskLog as au, type ExecutionTaskPlanningApply as av, type ExecutionTaskPlanning as aw, type GroupedActionDump as ax, type PageType as ay, type StreamingCodeGenerationOptions as az, type ExecutionTaskApply as b, type ScriptPlayerTaskStatus as b0, type ScriptPlayerStatusValue as b1, type MidsceneYamlConfig as b2, type MidsceneYamlConfigOutput as b3, type ExecutionDump as c, type InsightTaskInfo as d, type InsightOptions as e, type DetailedLocateParam as f, type InsightExtractOption as g, type InsightAssertionResponse as h, type MidsceneYamlTask as i, type MidsceneYamlFlowItem as j, type MidsceneYamlFlowItemAIRightClick as k, type MidsceneYamlConfigResult as l, type AIUsageInfo as m, AIResponseFormat as n, type AISingleElementResponseById as o, type AISingleElementResponseByPosition as p, type AISingleElementResponse as q, type AIElementLocatorResponse as r, type AIElementCoordinatesResponse as s, type AIElementResponse as t, type AIDataExtractionResponse as u, type AISectionLocatorResponse as v, type AIAssertionResponse as w, type LocatorValidatorOption as x, type LocateValidatorResult as y, type AgentDescribeElementAtPointResult as z };

package/dist/es/utils.d.ts

@@ -1,4 +1,4 @@
-import { R as ReportDumpWithAttributes } from './types-93fa8419.js';
+import { R as ReportDumpWithAttributes } from './types-d836fa73.js';
 import { Rect } from '@midscene/shared/types';
 import '@midscene/shared/constants';
 import 'openai/resources';

package/dist/es/utils.js

@@ -12,7 +12,7 @@ import {
   uploadTestInfoToServer,
   writeDumpReport,
   writeLogFile
-} from "./chunk-TYKJHDNK.js";
+} from "./chunk-IMZJSEAX.js";
 export {
   getTmpDir,
   getTmpFile,

package/dist/lib/ai-model.d.ts

@@ -1,9 +1,9 @@
-import { ay as StreamingCallback, m as AIUsageInfo, ax as StreamingCodeGenerationOptions, aA as StreamingAIResponse, V as PlanningAction, j as MidsceneYamlFlowItem } from './types-93fa8419.js';
+import { aA as StreamingCallback, m as AIUsageInfo, az as StreamingCodeGenerationOptions, aC as StreamingAIResponse, V as PlanningAction, j as MidsceneYamlFlowItem } from './types-d836fa73.js';
 import OpenAI from 'openai';
 import { ChatCompletionMessageParam } from 'openai/resources';
 export { ChatCompletionMessageParam } from 'openai/resources';
-import { b as AIActionType } from './llm-planning-4bc82162.js';
-export { a as AiAssert, f as AiExtractElementInfo, A as AiLocateElement, g as AiLocateSection, h as adaptBboxToRect, c as callAiFn, d as describeUserPage, e as elementByPositionWithElementInfo, p as plan } from './llm-planning-4bc82162.js';
+import { b as AIActionType } from './llm-planning-d7096b0d.js';
+export { a as AiAssert, f as AiExtractElementInfo, A as AiLocateElement, g as AiLocateSection, h as adaptBboxToRect, c as callAiFn, d as describeUserPage, e as elementByPositionWithElementInfo, p as plan } from './llm-planning-d7096b0d.js';
 import { vlLocateMode } from '@midscene/shared/env';
 import { actionParser } from '@ui-tars/action-parser';
 import { Size } from '@midscene/shared/types';

package/dist/lib/ai-model.js

@@ -18,7 +18,7 @@
 
 
 
-var _chunkYCHAOUOWjs = require('./chunk-YCHAOUOW.js');
+var _chunkFKQMUAXPjs = require('./chunk-FKQMUAXP.js');
 
 
 
@@ -39,4 +39,4 @@ var _chunkYCHAOUOWjs = require('./chunk-YCHAOUOW.js');
 
 
 
-exports.AIActionType = _chunkYCHAOUOWjs.AIActionType; exports.AiAssert = _chunkYCHAOUOWjs.AiAssert; exports.AiExtractElementInfo = _chunkYCHAOUOWjs.AiExtractElementInfo; exports.AiLocateElement = _chunkYCHAOUOWjs.AiLocateElement; exports.AiLocateSection = _chunkYCHAOUOWjs.AiLocateSection; exports.adaptBboxToRect = _chunkYCHAOUOWjs.adaptBboxToRect; exports.callAi = _chunkYCHAOUOWjs.call; exports.callAiFn = _chunkYCHAOUOWjs.callAiFn; exports.callToGetJSONObject = _chunkYCHAOUOWjs.callToGetJSONObject; exports.describeUserPage = _chunkYCHAOUOWjs.describeUserPage; exports.elementByPositionWithElementInfo = _chunkYCHAOUOWjs.elementByPositionWithElementInfo; exports.generatePlaywrightTest = _chunkYCHAOUOWjs.generatePlaywrightTest; exports.generatePlaywrightTestStream = _chunkYCHAOUOWjs.generatePlaywrightTestStream; exports.generateYamlTest = _chunkYCHAOUOWjs.generateYamlTest; exports.generateYamlTestStream = _chunkYCHAOUOWjs.generateYamlTestStream; exports.plan = _chunkYCHAOUOWjs.plan; exports.resizeImageForUiTars = _chunkYCHAOUOWjs.resizeImageForUiTars; exports.systemPromptToLocateElement = _chunkYCHAOUOWjs.systemPromptToLocateElement; exports.vlmPlanning = _chunkYCHAOUOWjs.vlmPlanning;
+exports.AIActionType = _chunkFKQMUAXPjs.AIActionType; exports.AiAssert = _chunkFKQMUAXPjs.AiAssert; exports.AiExtractElementInfo = _chunkFKQMUAXPjs.AiExtractElementInfo; exports.AiLocateElement = _chunkFKQMUAXPjs.AiLocateElement; exports.AiLocateSection = _chunkFKQMUAXPjs.AiLocateSection; exports.adaptBboxToRect = _chunkFKQMUAXPjs.adaptBboxToRect; exports.callAi = _chunkFKQMUAXPjs.call; exports.callAiFn = _chunkFKQMUAXPjs.callAiFn; exports.callToGetJSONObject = _chunkFKQMUAXPjs.callToGetJSONObject; exports.describeUserPage = _chunkFKQMUAXPjs.describeUserPage; exports.elementByPositionWithElementInfo = _chunkFKQMUAXPjs.elementByPositionWithElementInfo; exports.generatePlaywrightTest = _chunkFKQMUAXPjs.generatePlaywrightTest; exports.generatePlaywrightTestStream = _chunkFKQMUAXPjs.generatePlaywrightTestStream; exports.generateYamlTest = _chunkFKQMUAXPjs.generateYamlTest; exports.generateYamlTestStream = _chunkFKQMUAXPjs.generateYamlTestStream; exports.plan = _chunkFKQMUAXPjs.plan; exports.resizeImageForUiTars = _chunkFKQMUAXPjs.resizeImageForUiTars; exports.systemPromptToLocateElement = _chunkFKQMUAXPjs.systemPromptToLocateElement; exports.vlmPlanning = _chunkFKQMUAXPjs.vlmPlanning;

package/dist/lib/{chunk-YCHAOUOW.js → chunk-FKQMUAXP.js}

@@ -300,7 +300,7 @@ function buildYamlFlowFromPlans(plans, sleep) {
       flow.push({
         sleep: param.timeMs
       });
-    } else if (type === "AndroidBackButton" || type === "AndroidHomeButton" || type === "AndroidRecentAppsButton") {
+    } else if (type === "AndroidBackButton" || type === "AndroidHomeButton" || type === "AndroidRecentAppsButton" || type === "AndroidLongPress" || type === "AndroidPull") {
     } else if (type === "Error" || type === "ExpectedFalsyCondition" || type === "Assert" || type === "AssertWithoutThrow" || type === "Finished") {
     } else {
       console.warn(
@@ -794,7 +794,7 @@ Target: User will give you a screenshot, an instruction and some previous logs i
 
 Restriction:
 - Don't give extra actions or plans beyond the instruction. ONLY plan for what the instruction requires. For example, don't try to submit the form if the instruction is only to fill something.
-- Always give ONLY ONE action in \`log\` field (or null if no action should be done), instead of multiple actions. Supported actions are Tap, Hover, Input, KeyboardPress, Scroll${pageType === "android" ? ", AndroidBackButton, AndroidHomeButton, AndroidRecentAppsButton." : "."}
+- Always give ONLY ONE action in \`log\` field (or null if no action should be done), instead of multiple actions. Supported actions are Tap, Hover, Input, KeyboardPress, Scroll${pageType === "android" ? ", AndroidBackButton, AndroidHomeButton, AndroidRecentAppsButton, AndroidLongPress, AndroidPull." : "."}
 - Don't repeat actions in the previous logs.
 - Bbox is the bounding box of the element to be located. It's an array of 4 numbers, representing ${bboxDescription(vlMode)}.
 
@@ -807,7 +807,9 @@ Supporting actions:
 - Scroll: { type: "Scroll", ${vlLocateParam} | null, param: { direction: 'down'(default) | 'up' | 'right' | 'left', scrollType: 'once' (default) | 'untilBottom' | 'untilTop' | 'untilRight' | 'untilLeft', distance: null | number }} // locate is the element to scroll. If it's a page scroll, put \`null\` in the \`locate\` field.
 ${pageType === "android" ? `- AndroidBackButton: { type: "AndroidBackButton", param: {} }
 - AndroidHomeButton: { type: "AndroidHomeButton", param: {} }
-- AndroidRecentAppsButton: { type: "AndroidRecentAppsButton", param: {} }` : ""}
+- AndroidRecentAppsButton: { type: "AndroidRecentAppsButton", param: {} }
+- AndroidLongPress: { type: "AndroidLongPress", param: { x: number, y: number, duration?: number } }
+- AndroidPull: { type: "AndroidPull", param: { direction: 'up' | 'down', startPoint?: { x: number, y: number }, distance?: number, duration?: number } } // Pull down to refresh (direction: 'down') or pull up to load more (direction: 'up')` : ""}
 
 Field description:
 * The \`prompt\` field inside the \`locate\` field is a short description that could be used to locate the element.
@@ -857,7 +859,7 @@ You are a versatile professional in software UI automation. Your outstanding con
 ## Workflow
 
 1. Receive the screenshot, element description of screenshot(if any), user's instruction and previous logs.
-2. Decompose the user's task into a sequence of actions, and place it in the \`actions\` field. There are different types of actions (Tap / Hover / Input / KeyboardPress / Scroll / FalsyConditionStatement / Sleep ${pageType === "android" ? "/ AndroidBackButton / AndroidHomeButton / AndroidRecentAppsButton" : ""}). The "About the action" section below will give you more details.
+2. Decompose the user's task into a sequence of actions, and place it in the \`actions\` field. There are different types of actions (Tap / Hover / Input / KeyboardPress / Scroll / FalsyConditionStatement / Sleep ${pageType === "android" ? "/ AndroidBackButton / AndroidHomeButton / AndroidRecentAppsButton / AndroidLongPress / AndroidPull" : ""}). The "About the action" section below will give you more details.
 3. Precisely locate the target element if it's already shown in the screenshot, put the location info in the \`locate\` field of the action.
 4. If some target elements is not shown in the screenshot, consider the user's instruction is not feasible on this page. Follow the next steps.
 5. Consider whether the user's instruction will be accomplished after all the actions
@@ -916,7 +918,11 @@ ${pageType === "android" ? `- type: 'AndroidBackButton', trigger the system "bac
 - type: 'AndroidHomeButton', trigger the system "home" operation on Android devices
   * {{ param: {{}} }}
 - type: 'AndroidRecentAppsButton', trigger the system "recent apps" operation on Android devices
-  * {{ param: {{}} }}` : ""}
+  * {{ param: {{}} }}
+- type: 'AndroidLongPress', trigger a long press on the screen at specified coordinates on Android devices
+  * {{ param: {{ x: number, y: number, duration?: number }} }}
+- type: 'AndroidPull', trigger pull down to refresh or pull up actions on Android devices
+  * {{ param: {{ direction: 'up' | 'down', startPoint?: {{ x: number, y: number }}, distance?: number, duration?: number }} }}` : ""}
 `;
 var outputTemplate = `
 ## Output JSON Format:
@@ -1030,7 +1036,7 @@ var planSchema = {
               },
               type: {
                 type: "string",
-                description: 'Type of action, one of "Tap", "RightClick", "Hover" , "Input", "KeyboardPress", "Scroll", "ExpectedFalsyCondition", "Sleep", "AndroidBackButton", "AndroidHomeButton", "AndroidRecentAppsButton"'
+                description: 'Type of action, one of "Tap", "RightClick", "Hover" , "Input", "KeyboardPress", "Scroll", "ExpectedFalsyCondition", "Sleep", "AndroidBackButton", "AndroidHomeButton", "AndroidRecentAppsButton", "AndroidLongPress"'
               },
               param: {
                 anyOf: [
@@ -2185,14 +2191,6 @@ async function AiLocateElement(options) {
       context.size
     );
   }
-  let referenceImagePayload;
-  if (_optionalChain([options, 'access', _55 => _55.referenceImage, 'optionalAccess', _56 => _56.rect]) && options.referenceImage.base64) {
-    referenceImagePayload = await _img.cropByRect.call(void 0, 
-      options.referenceImage.base64,
-      options.referenceImage.rect,
-      _env.getAIConfigInBoolean.call(void 0, _env.MIDSCENE_USE_QWEN_VL)
-    );
-  }
   const msgs = [
     { role: "system", content: systemPrompt },
     {
@@ -2222,10 +2220,10 @@ async function AiLocateElement(options) {
     if ("bbox" in res.content && Array.isArray(res.content.bbox)) {
       resRect = adaptBboxToRect(
         res.content.bbox,
-        _optionalChain([options, 'access', _57 => _57.searchConfig, 'optionalAccess', _58 => _58.rect, 'optionalAccess', _59 => _59.width]) || context.size.width,
-        _optionalChain([options, 'access', _60 => _60.searchConfig, 'optionalAccess', _61 => _61.rect, 'optionalAccess', _62 => _62.height]) || context.size.height,
-        _optionalChain([options, 'access', _63 => _63.searchConfig, 'optionalAccess', _64 => _64.rect, 'optionalAccess', _65 => _65.left]),
-        _optionalChain([options, 'access', _66 => _66.searchConfig, 'optionalAccess', _67 => _67.rect, 'optionalAccess', _68 => _68.top])
+        _optionalChain([options, 'access', _55 => _55.searchConfig, 'optionalAccess', _56 => _56.rect, 'optionalAccess', _57 => _57.width]) || context.size.width,
+        _optionalChain([options, 'access', _58 => _58.searchConfig, 'optionalAccess', _59 => _59.rect, 'optionalAccess', _60 => _60.height]) || context.size.height,
+        _optionalChain([options, 'access', _61 => _61.searchConfig, 'optionalAccess', _62 => _62.rect, 'optionalAccess', _63 => _63.left]),
+        _optionalChain([options, 'access', _64 => _64.searchConfig, 'optionalAccess', _65 => _65.rect, 'optionalAccess', _66 => _66.top])
       );
       debugInspect("resRect", resRect);
       const rectCenter = {
@@ -2244,7 +2242,7 @@ async function AiLocateElement(options) {
     }
   } catch (e) {
     const msg = e instanceof Error ? `Failed to parse bbox: ${e.message}` : "unknown error in locate";
-    if (!errors || _optionalChain([errors, 'optionalAccess', _69 => _69.length]) === 0) {
+    if (!errors || _optionalChain([errors, 'optionalAccess', _67 => _67.length]) === 0) {
       errors = [msg];
     } else {
       errors.push(`(${msg})`);
@@ -2336,14 +2334,14 @@ async function AiExtractElementInfo(options) {
     truncateTextLength: 200,
     filterNonTextContent: false,
     visibleOnly: false,
-    domIncluded: _optionalChain([extractOption, 'optionalAccess', _70 => _70.domIncluded])
+    domIncluded: _optionalChain([extractOption, 'optionalAccess', _68 => _68.domIncluded])
   });
   const extractDataPromptText = await extractDataQueryPrompt(
     description,
     dataQuery
   );
   const userContent = [];
-  if (_optionalChain([extractOption, 'optionalAccess', _71 => _71.screenshotIncluded]) !== false) {
+  if (_optionalChain([extractOption, 'optionalAccess', _69 => _69.screenshotIncluded]) !== false) {
     userContent.push({
       type: "image_url",
       image_url: {
@@ -2471,7 +2469,7 @@ async function plan(userInstruction, opts) {
   const { content, usage } = await call2(msgs, 3 /* PLAN */);
   const rawResponse = JSON.stringify(content, void 0, 2);
   const planFromAI = content;
-  const actions = (_optionalChain([planFromAI, 'access', _72 => _72.action, 'optionalAccess', _73 => _73.type]) ? [planFromAI.action] : planFromAI.actions) || [];
+  const actions = (_optionalChain([planFromAI, 'access', _70 => _70.action, 'optionalAccess', _71 => _71.type]) ? [planFromAI.action] : planFromAI.actions) || [];
   const returnValue = {
     ...planFromAI,
     actions,
@@ -2498,7 +2496,7 @@ async function plan(userInstruction, opts) {
     _utils.assert.call(void 0, !planFromAI.error, `Failed to plan actions: ${planFromAI.error}`);
   } else {
     actions.forEach((action) => {
-      if (_optionalChain([action, 'access', _74 => _74.locate, 'optionalAccess', _75 => _75.id])) {
+      if (_optionalChain([action, 'access', _72 => _72.locate, 'optionalAccess', _73 => _73.id])) {
         const element = elementById(action.locate.id);
         if (element) {
           action.locate.id = element.id;
@@ -2710,6 +2708,39 @@ async function vlmPlanning(options) {
         type: "AndroidRecentAppsButton",
         param: {}
       });
+    } else if (action.action_type === "androidLongPress") {
+      _utils.assert.call(void 0, 
+        action.action_inputs.start_coords,
+        "start_coords is required for androidLongPress"
+      );
+      const point = action.action_inputs.start_coords;
+      transformActions.push({
+        type: "AndroidLongPress",
+        param: {
+          x: point[0],
+          y: point[1],
+          duration: 1e3
+        },
+        locate: null,
+        thought: action.thought || ""
+      });
+    } else if (action.action_type === "androidPull") {
+      const pullDirection = action.action_inputs.direction || "down";
+      const startPoint = action.action_inputs.start_coords ? {
+        x: action.action_inputs.start_coords[0],
+        y: action.action_inputs.start_coords[1]
+      } : void 0;
+      transformActions.push({
+        type: "AndroidPull",
+        param: {
+          direction: pullDirection,
+          startPoint,
+          distance: action.action_inputs.distance,
+          duration: action.action_inputs.duration || 500
+        },
+        locate: null,
+        thought: action.thought || ""
+      });
     }
   });
   if (transformActions.length === 0) {
@@ -2793,4 +2824,4 @@ async function resizeImageForUiTars(imageBase64, size) {
 
 exports.systemPromptToLocateElement = systemPromptToLocateElement; exports.elementByPositionWithElementInfo = elementByPositionWithElementInfo; exports.describeUserPage = describeUserPage; exports.call = call; exports.callToGetJSONObject = callToGetJSONObject; exports.AIActionType = AIActionType; exports.callAiFn = callAiFn; exports.adaptBboxToRect = adaptBboxToRect; exports.expandSearchArea = expandSearchArea; exports.generateYamlTest = generateYamlTest; exports.generateYamlTestStream = generateYamlTestStream; exports.generatePlaywrightTest = generatePlaywrightTest; exports.generatePlaywrightTestStream = generatePlaywrightTestStream; exports.AiLocateElement = AiLocateElement; exports.AiLocateSection = AiLocateSection; exports.AiExtractElementInfo = AiExtractElementInfo; exports.AiAssert = AiAssert; exports.plan = plan; exports.vlmPlanning = vlmPlanning; exports.resizeImageForUiTars = resizeImageForUiTars;
 
-//# sourceMappingURL=chunk-YCHAOUOW.js.map
+//# sourceMappingURL=chunk-FKQMUAXP.js.map