@midscene/core 0.26.2-beta-20250812035614.0 → 0.26.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/ai-model.d.ts +3 -3
- package/dist/es/ai-model.js +1 -1
- package/dist/es/{chunk-I5LBWOQA.js → chunk-DDYIQHOA.js} +268 -310
- package/dist/es/chunk-DDYIQHOA.js.map +1 -0
- package/dist/es/{chunk-KAYSYGXR.js → chunk-O3KUKF2A.js} +3 -3
- package/dist/es/index.d.ts +4 -4
- package/dist/es/index.js +2 -2
- package/dist/es/{llm-planning-92cec090.d.ts → llm-planning-4e0c16fe.d.ts} +2 -1
- package/dist/es/{types-b4a208c6.d.ts → types-8a6be57c.d.ts} +11 -2
- package/dist/es/utils.d.ts +1 -1
- package/dist/es/utils.js +1 -1
- package/dist/lib/ai-model.d.ts +3 -3
- package/dist/lib/ai-model.js +2 -2
- package/dist/lib/{chunk-I5LBWOQA.js → chunk-DDYIQHOA.js} +257 -299
- package/dist/lib/chunk-DDYIQHOA.js.map +1 -0
- package/dist/lib/{chunk-KAYSYGXR.js → chunk-O3KUKF2A.js} +3 -3
- package/dist/lib/index.d.ts +4 -4
- package/dist/lib/index.js +12 -12
- package/dist/lib/{llm-planning-92cec090.d.ts → llm-planning-4e0c16fe.d.ts} +2 -1
- package/dist/{types/types-b4a208c6.d.ts → lib/types-8a6be57c.d.ts} +11 -2
- package/dist/lib/utils.d.ts +1 -1
- package/dist/lib/utils.js +2 -2
- package/dist/types/ai-model.d.ts +3 -3
- package/dist/types/index.d.ts +4 -4
- package/dist/types/{llm-planning-92cec090.d.ts → llm-planning-4e0c16fe.d.ts} +2 -1
- package/dist/{lib/types-b4a208c6.d.ts → types/types-8a6be57c.d.ts} +11 -2
- package/dist/types/utils.d.ts +1 -1
- package/package.json +3 -3
- package/dist/es/chunk-I5LBWOQA.js.map +0 -1
- package/dist/lib/chunk-I5LBWOQA.js.map +0 -1
- /package/dist/es/{chunk-KAYSYGXR.js.map → chunk-O3KUKF2A.js.map} +0 -0
- /package/dist/lib/{chunk-KAYSYGXR.js.map → chunk-O3KUKF2A.js.map} +0 -0
package/dist/es/index.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { E as ExecutionTask, a as ExecutionTaskProgressOptions, b as ExecutionTaskApply, c as ExecutionDump, U as UIContext, I as InsightAction, D as DumpSubscriber, d as InsightTaskInfo, e as InsightOptions, f as DetailedLocateParam, L as LocateResult, g as InsightExtractParam, h as InsightExtractOption, T as TMultimodalPrompt, A as AIUsageInfo, i as TUserPrompt, j as InsightAssertionResponse, k as AIDescribeElementResponse } from './types-
|
|
2
|
-
export { z as AIAssertionResponse, x as AIDataExtractionResponse, v as AIElementCoordinatesResponse, u as AIElementLocatorResponse, w as AIElementResponse, q as AIResponseFormat, y as AISectionLocatorResponse, t as AISingleElementResponse, r as AISingleElementResponseById, s as AISingleElementResponseByPosition, W as AgentAssertOpt, F as AgentDescribeElementAtPointResult, V as AgentWaitForOpt, aa as BaseAgentParserOpt, G as CallAIFn, aE as CodeGenerationChunk, a9 as Color, K as DumpMeta, Q as ElementById, H as EnsureObject, ad as ExecutionRecorderItem, av as ExecutionTaskAction, au as ExecutionTaskActionApply, ag as ExecutionTaskHitBy, at as ExecutionTaskInsightAssertion, as as ExecutionTaskInsightAssertionApply, ar as ExecutionTaskInsightAssertionParam, ak as ExecutionTaskInsightDumpLog, am as ExecutionTaskInsightLocate, al as ExecutionTaskInsightLocateApply, aj as ExecutionTaskInsightLocateOutput, ai as ExecutionTaskInsightLocateParam, aq as ExecutionTaskInsightQuery, ap as ExecutionTaskInsightQueryApply, ao as ExecutionTaskInsightQueryOutput, an as ExecutionTaskInsightQueryParam, ax as ExecutionTaskLog, aw as ExecutionTaskLogApply, az as ExecutionTaskPlanning, ay as ExecutionTaskPlanningApply, ah as ExecutionTaskReturn, ae as ExecutionTaskType, af as ExecutorContext,
|
|
3
|
-
import { c as callAiFn } from './llm-planning-
|
|
4
|
-
export { a as AiAssert, A as AiLocateElement, d as describeUserPage, p as plan } from './llm-planning-
|
|
1
|
+
import { E as ExecutionTask, a as ExecutionTaskProgressOptions, b as ExecutionTaskApply, c as ExecutionDump, U as UIContext, I as InsightAction, D as DumpSubscriber, d as InsightTaskInfo, e as InsightOptions, f as DetailedLocateParam, L as LocateResult, g as InsightExtractParam, h as InsightExtractOption, T as TMultimodalPrompt, A as AIUsageInfo, i as TUserPrompt, j as InsightAssertionResponse, k as AIDescribeElementResponse } from './types-8a6be57c.js';
|
|
2
|
+
export { z as AIAssertionResponse, x as AIDataExtractionResponse, v as AIElementCoordinatesResponse, u as AIElementLocatorResponse, w as AIElementResponse, q as AIResponseFormat, y as AISectionLocatorResponse, t as AISingleElementResponse, r as AISingleElementResponseById, s as AISingleElementResponseByPosition, W as AgentAssertOpt, F as AgentDescribeElementAtPointResult, V as AgentWaitForOpt, aa as BaseAgentParserOpt, G as CallAIFn, aE as CodeGenerationChunk, a9 as Color, aG as DeviceAction, K as DumpMeta, Q as ElementById, H as EnsureObject, ad as ExecutionRecorderItem, av as ExecutionTaskAction, au as ExecutionTaskActionApply, ag as ExecutionTaskHitBy, at as ExecutionTaskInsightAssertion, as as ExecutionTaskInsightAssertionApply, ar as ExecutionTaskInsightAssertionParam, ak as ExecutionTaskInsightDumpLog, am as ExecutionTaskInsightLocate, al as ExecutionTaskInsightLocateApply, aj as ExecutionTaskInsightLocateOutput, ai as ExecutionTaskInsightLocateParam, aq as ExecutionTaskInsightQuery, ap as ExecutionTaskInsightQueryApply, ao as ExecutionTaskInsightQueryOutput, an as ExecutionTaskInsightQueryParam, ax as ExecutionTaskLog, aw as ExecutionTaskLogApply, az as ExecutionTaskPlanning, ay as ExecutionTaskPlanningApply, ah as ExecutionTaskReturn, ae as ExecutionTaskType, af as ExecutorContext, b2 as FreeFn, aA as GroupedActionDump, N as InsightDump, O as LiteUISection, p as LocateOption, J as LocateResultElement, C as LocateValidatorResult, B as LocatorValidatorOption, b5 as MidsceneYamlConfig, b6 as MidsceneYamlConfigOutput, o as MidsceneYamlConfigResult, m as MidsceneYamlFlowItem, aN as MidsceneYamlFlowItemAIAction, aS as MidsceneYamlFlowItemAIAsk, aO as MidsceneYamlFlowItemAIAssert, aT as MidsceneYamlFlowItemAIBoolean, aX as MidsceneYamlFlowItemAIHover, aY as MidsceneYamlFlowItemAIInput, aZ as MidsceneYamlFlowItemAIKeyboardPress, aU as MidsceneYamlFlowItemAILocate, aQ as MidsceneYamlFlowItemAINumber, aP as MidsceneYamlFlowItemAIQuery, n as MidsceneYamlFlowItemAIRightClick, a_ as MidsceneYamlFlowItemAIScroll, aR as MidsceneYamlFlowItemAIString, aW as MidsceneYamlFlowItemAITap, aV as MidsceneYamlFlowItemAIWaitFor, a$ as MidsceneYamlFlowItemEvaluateJavaScript, b1 as MidsceneYamlFlowItemLogScreenshot, b0 as MidsceneYamlFlowItemSleep, M as MidsceneYamlScript, aL as MidsceneYamlScriptAndroidEnv, aM as MidsceneYamlScriptEnv, aJ as MidsceneYamlScriptEnvBase, aK as MidsceneYamlScriptWebEnv, l as MidsceneYamlTask, S as OnTaskStartTip, aB as PageType, P as PartialInsightDumpFromSDK, Z as PlanningAIResponse, Y as PlanningAction, a7 as PlanningActionParamAndroidLongPress, a8 as PlanningActionParamAndroidPull, a3 as PlanningActionParamAssert, a5 as PlanningActionParamError, $ as PlanningActionParamHover, a1 as PlanningActionParamInputOrKeyPress, a0 as PlanningActionParamRightClick, a2 as PlanningActionParamScroll, a4 as PlanningActionParamSleep, _ as PlanningActionParamTap, a6 as PlanningActionParamWaitFor, X as PlanningLocateParam, ac as PlaywrightParserOpt, ab as PuppeteerParserOpt, aH as ReferenceImage, R as ReportDumpWithAttributes, b4 as ScriptPlayerStatusValue, b3 as ScriptPlayerTaskStatus, aF as StreamingAIResponse, aD as StreamingCallback, aC as StreamingCodeGenerationOptions, aI as scrollParam } from './types-8a6be57c.js';
|
|
3
|
+
import { c as callAiFn } from './llm-planning-4e0c16fe.js';
|
|
4
|
+
export { a as AiAssert, A as AiLocateElement, d as describeUserPage, p as plan } from './llm-planning-4e0c16fe.js';
|
|
5
5
|
import { BaseElement, Rect } from '@midscene/shared/types';
|
|
6
6
|
export { BaseElement, ElementTreeNode, Point, Rect, Size } from '@midscene/shared/types';
|
|
7
7
|
export { getVersion } from './utils.js';
|
package/dist/es/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
getVersion
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-O3KUKF2A.js";
|
|
4
4
|
import {
|
|
5
5
|
AiAssert,
|
|
6
6
|
AiExtractElementInfo,
|
|
@@ -11,7 +11,7 @@ import {
|
|
|
11
11
|
describeUserPage,
|
|
12
12
|
expandSearchArea,
|
|
13
13
|
plan
|
|
14
|
-
} from "./chunk-
|
|
14
|
+
} from "./chunk-DDYIQHOA.js";
|
|
15
15
|
|
|
16
16
|
// src/ai-model/action-executor.ts
|
|
17
17
|
import {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { A as AIUsageInfo, U as UIContext, i as TUserPrompt,
|
|
1
|
+
import { A as AIUsageInfo, U as UIContext, i as TUserPrompt, aH as ReferenceImage, u as AIElementLocatorResponse, Q as ElementById, T as TMultimodalPrompt, h as InsightExtractOption, x as AIDataExtractionResponse, z as AIAssertionResponse, aB as PageType, aG as DeviceAction, Z as PlanningAIResponse } from './types-8a6be57c.js';
|
|
2
2
|
import { Rect, ElementTreeNode, BaseElement } from '@midscene/shared/types';
|
|
3
3
|
import { ChatCompletionSystemMessageParam, ChatCompletionUserMessageParam } from 'openai/resources';
|
|
4
4
|
|
|
@@ -97,6 +97,7 @@ declare function AiAssert<ElementType extends BaseElement = BaseElement>(options
|
|
|
97
97
|
declare function plan(userInstruction: string, opts: {
|
|
98
98
|
context: UIContext;
|
|
99
99
|
pageType: PageType;
|
|
100
|
+
actionSpace: DeviceAction[];
|
|
100
101
|
callAI?: typeof callAiFn<PlanningAIResponse>;
|
|
101
102
|
log?: string;
|
|
102
103
|
actionContext?: string;
|
|
@@ -359,7 +359,7 @@ interface PlanningLocateParam extends DetailedLocateParam {
|
|
|
359
359
|
}
|
|
360
360
|
interface PlanningAction<ParamType = any> {
|
|
361
361
|
thought?: string;
|
|
362
|
-
type: 'Locate' | 'Tap' | 'RightClick' | 'Hover' | 'Drag' | 'Input' | 'KeyboardPress' | 'Scroll' | 'Error' | '
|
|
362
|
+
type: 'Locate' | 'Tap' | 'RightClick' | 'Hover' | 'Drag' | 'Input' | 'KeyboardPress' | 'Scroll' | 'Error' | 'Assert' | 'AssertWithoutThrow' | 'Sleep' | 'Finished' | 'AndroidBackButton' | 'AndroidHomeButton' | 'AndroidRecentAppsButton' | 'AndroidLongPress' | 'AndroidPull';
|
|
363
363
|
param: ParamType;
|
|
364
364
|
locate?: PlanningLocateParam | null;
|
|
365
365
|
}
|
|
@@ -564,5 +564,14 @@ type TMultimodalPrompt = {
|
|
|
564
564
|
type TUserPrompt = string | ({
|
|
565
565
|
prompt: string;
|
|
566
566
|
} & Partial<TMultimodalPrompt>);
|
|
567
|
+
interface DeviceAction<ParamType = any> {
|
|
568
|
+
name: string;
|
|
569
|
+
description?: string;
|
|
570
|
+
paramSchema?: string;
|
|
571
|
+
paramDescription?: string;
|
|
572
|
+
location?: 'required' | 'optional' | false;
|
|
573
|
+
whatToLocate?: string;
|
|
574
|
+
call: (param: ParamType) => Promise<void> | void;
|
|
575
|
+
}
|
|
567
576
|
|
|
568
|
-
export { type PlanningActionParamHover as $, type AIUsageInfo as A, type LocatorValidatorOption as B, type LocateValidatorResult as C, type DumpSubscriber as D, type ExecutionTask as E, type AgentDescribeElementAtPointResult as F, type CallAIFn as G, type EnsureObject as H, type InsightAction as I, type LocateResultElement as J, type DumpMeta as K, type LocateResult as L, type MidsceneYamlScript as M, type InsightDump as N, type LiteUISection as O, type PartialInsightDumpFromSDK as P, type ElementById as Q, type ReportDumpWithAttributes as R, type OnTaskStartTip as S, type TMultimodalPrompt as T, UIContext as U, type AgentWaitForOpt as V, type AgentAssertOpt as W, type PlanningLocateParam as X, type PlanningAction as Y, type PlanningAIResponse as Z, type PlanningActionParamTap as _, type ExecutionTaskProgressOptions as a, type
|
|
577
|
+
export { type PlanningActionParamHover as $, type AIUsageInfo as A, type LocatorValidatorOption as B, type LocateValidatorResult as C, type DumpSubscriber as D, type ExecutionTask as E, type AgentDescribeElementAtPointResult as F, type CallAIFn as G, type EnsureObject as H, type InsightAction as I, type LocateResultElement as J, type DumpMeta as K, type LocateResult as L, type MidsceneYamlScript as M, type InsightDump as N, type LiteUISection as O, type PartialInsightDumpFromSDK as P, type ElementById as Q, type ReportDumpWithAttributes as R, type OnTaskStartTip as S, type TMultimodalPrompt as T, UIContext as U, type AgentWaitForOpt as V, type AgentAssertOpt as W, type PlanningLocateParam as X, type PlanningAction as Y, type PlanningAIResponse as Z, type PlanningActionParamTap as _, type ExecutionTaskProgressOptions as a, type MidsceneYamlFlowItemEvaluateJavaScript as a$, type PlanningActionParamRightClick as a0, type PlanningActionParamInputOrKeyPress as a1, type PlanningActionParamScroll as a2, type PlanningActionParamAssert as a3, type PlanningActionParamSleep as a4, type PlanningActionParamError as a5, type PlanningActionParamWaitFor as a6, type PlanningActionParamAndroidLongPress as a7, type PlanningActionParamAndroidPull as a8, type Color as a9, type GroupedActionDump as aA, type PageType as aB, type StreamingCodeGenerationOptions as aC, type StreamingCallback as aD, type CodeGenerationChunk as aE, type StreamingAIResponse as aF, type DeviceAction as aG, type ReferenceImage as aH, type scrollParam as aI, type MidsceneYamlScriptEnvBase as aJ, type MidsceneYamlScriptWebEnv as aK, type MidsceneYamlScriptAndroidEnv as aL, type MidsceneYamlScriptEnv as aM, type MidsceneYamlFlowItemAIAction as aN, type MidsceneYamlFlowItemAIAssert as aO, type MidsceneYamlFlowItemAIQuery as aP, type MidsceneYamlFlowItemAINumber as aQ, type MidsceneYamlFlowItemAIString as aR, type MidsceneYamlFlowItemAIAsk as aS, type MidsceneYamlFlowItemAIBoolean as aT, type MidsceneYamlFlowItemAILocate as aU, type MidsceneYamlFlowItemAIWaitFor as aV, type MidsceneYamlFlowItemAITap as aW, type MidsceneYamlFlowItemAIHover as aX, type MidsceneYamlFlowItemAIInput as aY, type MidsceneYamlFlowItemAIKeyboardPress as aZ, type MidsceneYamlFlowItemAIScroll as a_, type BaseAgentParserOpt as aa, type PuppeteerParserOpt as ab, type PlaywrightParserOpt as ac, type ExecutionRecorderItem as ad, type ExecutionTaskType as ae, type ExecutorContext as af, type ExecutionTaskHitBy as ag, type ExecutionTaskReturn as ah, type ExecutionTaskInsightLocateParam as ai, type ExecutionTaskInsightLocateOutput as aj, type ExecutionTaskInsightDumpLog as ak, type ExecutionTaskInsightLocateApply as al, type ExecutionTaskInsightLocate as am, type ExecutionTaskInsightQueryParam as an, type ExecutionTaskInsightQueryOutput as ao, type ExecutionTaskInsightQueryApply as ap, type ExecutionTaskInsightQuery as aq, type ExecutionTaskInsightAssertionParam as ar, type ExecutionTaskInsightAssertionApply as as, type ExecutionTaskInsightAssertion as at, type ExecutionTaskActionApply as au, type ExecutionTaskAction as av, type ExecutionTaskLogApply as aw, type ExecutionTaskLog as ax, type ExecutionTaskPlanningApply as ay, type ExecutionTaskPlanning as az, type ExecutionTaskApply as b, type MidsceneYamlFlowItemSleep as b0, type MidsceneYamlFlowItemLogScreenshot as b1, type FreeFn as b2, type ScriptPlayerTaskStatus as b3, type ScriptPlayerStatusValue as b4, type MidsceneYamlConfig as b5, type MidsceneYamlConfigOutput as b6, type ExecutionDump as c, type InsightTaskInfo as d, type InsightOptions as e, type DetailedLocateParam as f, type InsightExtractParam as g, type InsightExtractOption as h, type TUserPrompt as i, type InsightAssertionResponse as j, type AIDescribeElementResponse as k, type MidsceneYamlTask as l, type MidsceneYamlFlowItem as m, type MidsceneYamlFlowItemAIRightClick as n, type MidsceneYamlConfigResult as o, type LocateOption as p, AIResponseFormat as q, type AISingleElementResponseById as r, type AISingleElementResponseByPosition as s, type AISingleElementResponse as t, type AIElementLocatorResponse as u, type AIElementCoordinatesResponse as v, type AIElementResponse as w, type AIDataExtractionResponse as x, type AISectionLocatorResponse as y, type AIAssertionResponse as z };
|
package/dist/es/utils.d.ts
CHANGED
package/dist/es/utils.js
CHANGED
package/dist/lib/ai-model.d.ts
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { aD as StreamingCallback, A as AIUsageInfo, aC as StreamingCodeGenerationOptions, aF as StreamingAIResponse, Y as PlanningAction, m as MidsceneYamlFlowItem } from './types-
|
|
1
|
+
import { aD as StreamingCallback, A as AIUsageInfo, aC as StreamingCodeGenerationOptions, aF as StreamingAIResponse, Y as PlanningAction, m as MidsceneYamlFlowItem } from './types-8a6be57c.js';
|
|
2
2
|
import OpenAI from 'openai';
|
|
3
3
|
import { ChatCompletionMessageParam } from 'openai/resources';
|
|
4
4
|
export { ChatCompletionMessageParam } from 'openai/resources';
|
|
5
|
-
import { b as AIActionType, e as AIArgs } from './llm-planning-
|
|
6
|
-
export { a as AiAssert, g as AiExtractElementInfo, A as AiLocateElement, h as AiLocateSection, i as adaptBboxToRect, c as callAiFn, d as describeUserPage, f as elementByPositionWithElementInfo, p as plan } from './llm-planning-
|
|
5
|
+
import { b as AIActionType, e as AIArgs } from './llm-planning-4e0c16fe.js';
|
|
6
|
+
export { a as AiAssert, g as AiExtractElementInfo, A as AiLocateElement, h as AiLocateSection, i as adaptBboxToRect, c as callAiFn, d as describeUserPage, f as elementByPositionWithElementInfo, p as plan } from './llm-planning-4e0c16fe.js';
|
|
7
7
|
import { vlLocateMode } from '@midscene/shared/env';
|
|
8
8
|
import { actionParser } from '@ui-tars/action-parser';
|
|
9
9
|
import { Size } from '@midscene/shared/types';
|
package/dist/lib/ai-model.js
CHANGED
|
@@ -19,7 +19,7 @@
|
|
|
19
19
|
|
|
20
20
|
|
|
21
21
|
|
|
22
|
-
var
|
|
22
|
+
var _chunkDDYIQHOAjs = require('./chunk-DDYIQHOA.js');
|
|
23
23
|
|
|
24
24
|
|
|
25
25
|
|
|
@@ -41,4 +41,4 @@ var _chunkI5LBWOQAjs = require('./chunk-I5LBWOQA.js');
|
|
|
41
41
|
|
|
42
42
|
|
|
43
43
|
|
|
44
|
-
exports.AIActionType =
|
|
44
|
+
exports.AIActionType = _chunkDDYIQHOAjs.AIActionType; exports.AiAssert = _chunkDDYIQHOAjs.AiAssert; exports.AiExtractElementInfo = _chunkDDYIQHOAjs.AiExtractElementInfo; exports.AiLocateElement = _chunkDDYIQHOAjs.AiLocateElement; exports.AiLocateSection = _chunkDDYIQHOAjs.AiLocateSection; exports.adaptBboxToRect = _chunkDDYIQHOAjs.adaptBboxToRect; exports.callAi = _chunkDDYIQHOAjs.call; exports.callAiFn = _chunkDDYIQHOAjs.callAiFn; exports.callAiFnWithStringResponse = _chunkDDYIQHOAjs.callAiFnWithStringResponse; exports.callToGetJSONObject = _chunkDDYIQHOAjs.callToGetJSONObject; exports.describeUserPage = _chunkDDYIQHOAjs.describeUserPage; exports.elementByPositionWithElementInfo = _chunkDDYIQHOAjs.elementByPositionWithElementInfo; exports.generatePlaywrightTest = _chunkDDYIQHOAjs.generatePlaywrightTest; exports.generatePlaywrightTestStream = _chunkDDYIQHOAjs.generatePlaywrightTestStream; exports.generateYamlTest = _chunkDDYIQHOAjs.generateYamlTest; exports.generateYamlTestStream = _chunkDDYIQHOAjs.generateYamlTestStream; exports.plan = _chunkDDYIQHOAjs.plan; exports.resizeImageForUiTars = _chunkDDYIQHOAjs.resizeImageForUiTars; exports.systemPromptToLocateElement = _chunkDDYIQHOAjs.systemPromptToLocateElement; exports.vlmPlanning = _chunkDDYIQHOAjs.vlmPlanning;
|