@midscene/core 0.1.4 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.md +7 -0
- package/dist/es/ai-model.js +52 -30
- package/dist/es/image.js +16 -10
- package/dist/es/index.js +594 -476
- package/dist/es/utils.js +14 -9
- package/dist/lib/ai-model.js +58 -42
- package/dist/lib/image.js +24 -24
- package/dist/lib/index.js +611 -499
- package/dist/lib/utils.js +32 -33
- package/dist/types/ai-model.d.ts +3 -5
- package/dist/types/image.d.ts +1 -1
- package/dist/types/index.d.ts +19 -19
- package/dist/types/{types-1f7912d5.d.ts → types-2c92867c.d.ts} +33 -15
- package/dist/types/{util-3a13ce3d.d.ts → util-3931e76e.d.ts} +1 -1
- package/dist/types/utils.d.ts +1 -1
- package/package.json +5 -4
package/dist/lib/utils.js
CHANGED
|
@@ -44,34 +44,33 @@ __export(utils_exports, {
|
|
|
44
44
|
writeDumpFile: () => writeDumpFile
|
|
45
45
|
});
|
|
46
46
|
module.exports = __toCommonJS(utils_exports);
|
|
47
|
-
var
|
|
48
|
-
var
|
|
49
|
-
var
|
|
50
|
-
var
|
|
51
|
-
var
|
|
47
|
+
var import_node_assert = __toESM(require("assert"));
|
|
48
|
+
var import_node_crypto = require("crypto");
|
|
49
|
+
var import_node_fs = require("fs");
|
|
50
|
+
var import_node_os = require("os");
|
|
51
|
+
var import_node_path = require("path");
|
|
52
52
|
var pkg;
|
|
53
53
|
function getPkgInfo() {
|
|
54
54
|
if (pkg) {
|
|
55
55
|
return pkg;
|
|
56
56
|
}
|
|
57
57
|
let pkgJsonFile = "";
|
|
58
|
-
if ((0,
|
|
59
|
-
pkgJsonFile = (0,
|
|
60
|
-
} else if ((0,
|
|
61
|
-
pkgJsonFile = (0,
|
|
58
|
+
if ((0, import_node_fs.existsSync)((0, import_node_path.join)(__dirname, "../package.json"))) {
|
|
59
|
+
pkgJsonFile = (0, import_node_path.join)(__dirname, "../package.json");
|
|
60
|
+
} else if ((0, import_node_fs.existsSync)((0, import_node_path.join)(__dirname, "../../../package.json"))) {
|
|
61
|
+
pkgJsonFile = (0, import_node_path.join)(__dirname, "../../../package.json");
|
|
62
62
|
}
|
|
63
63
|
if (pkgJsonFile) {
|
|
64
|
-
const { name, version } = JSON.parse((0,
|
|
64
|
+
const { name, version } = JSON.parse((0, import_node_fs.readFileSync)(pkgJsonFile, "utf-8"));
|
|
65
65
|
pkg = { name, version };
|
|
66
66
|
return pkg;
|
|
67
|
-
} else {
|
|
68
|
-
return {
|
|
69
|
-
name: "midscene-unknown-page-name",
|
|
70
|
-
version: "0.0.0"
|
|
71
|
-
};
|
|
72
67
|
}
|
|
68
|
+
return {
|
|
69
|
+
name: "midscene-unknown-page-name",
|
|
70
|
+
version: "0.0.0"
|
|
71
|
+
};
|
|
73
72
|
}
|
|
74
|
-
var logDir = (0,
|
|
73
|
+
var logDir = (0, import_node_path.join)(process.cwd(), "./midscene_run/");
|
|
75
74
|
var logEnvReady = false;
|
|
76
75
|
var insightDumpFileExt = "insight-dump.json";
|
|
77
76
|
var groupedActionDumpFileExt = "web-dump.json";
|
|
@@ -82,27 +81,27 @@ function setDumpDir(dir) {
|
|
|
82
81
|
logDir = dir;
|
|
83
82
|
}
|
|
84
83
|
function getDumpDirPath(type) {
|
|
85
|
-
return (0,
|
|
84
|
+
return (0, import_node_path.join)(getDumpDir(), type);
|
|
86
85
|
}
|
|
87
86
|
function writeDumpFile(opts) {
|
|
88
87
|
const { fileName, fileExt, fileContent, type = "dump" } = opts;
|
|
89
88
|
const targetDir = getDumpDirPath(type);
|
|
90
|
-
if (!(0,
|
|
91
|
-
(0,
|
|
89
|
+
if (!(0, import_node_fs.existsSync)(targetDir)) {
|
|
90
|
+
(0, import_node_fs.mkdirSync)(targetDir, { recursive: true });
|
|
92
91
|
}
|
|
93
92
|
if (!logEnvReady) {
|
|
94
|
-
(0,
|
|
95
|
-
const gitIgnorePath = (0,
|
|
93
|
+
(0, import_node_assert.default)(targetDir, "logDir should be set before writing dump file");
|
|
94
|
+
const gitIgnorePath = (0, import_node_path.join)(targetDir, "../../.gitignore");
|
|
96
95
|
let gitIgnoreContent = "";
|
|
97
|
-
if ((0,
|
|
98
|
-
gitIgnoreContent = (0,
|
|
96
|
+
if ((0, import_node_fs.existsSync)(gitIgnorePath)) {
|
|
97
|
+
gitIgnoreContent = (0, import_node_fs.readFileSync)(gitIgnorePath, "utf-8");
|
|
99
98
|
}
|
|
100
|
-
const logDirName = (0,
|
|
99
|
+
const logDirName = (0, import_node_path.basename)(logDir);
|
|
101
100
|
if (!gitIgnoreContent.includes(`${logDirName}/`)) {
|
|
102
|
-
(0,
|
|
101
|
+
(0, import_node_fs.writeFileSync)(
|
|
103
102
|
gitIgnorePath,
|
|
104
103
|
`${gitIgnoreContent}
|
|
105
|
-
#
|
|
104
|
+
# Midscene.js dump files
|
|
106
105
|
${logDirName}/report
|
|
107
106
|
${logDirName}/dump-logger
|
|
108
107
|
`,
|
|
@@ -111,21 +110,21 @@ ${logDirName}/dump-logger
|
|
|
111
110
|
}
|
|
112
111
|
logEnvReady = true;
|
|
113
112
|
}
|
|
114
|
-
const filePath = (0,
|
|
115
|
-
(0,
|
|
113
|
+
const filePath = (0, import_node_path.join)(targetDir, `${fileName}.${fileExt}`);
|
|
114
|
+
(0, import_node_fs.writeFileSync)(filePath, fileContent);
|
|
116
115
|
if (type === "dump") {
|
|
117
|
-
(0,
|
|
116
|
+
(0, import_node_fs.copyFileSync)(filePath, (0, import_node_path.join)(targetDir, `latest.${fileExt}`));
|
|
118
117
|
}
|
|
119
118
|
return filePath;
|
|
120
119
|
}
|
|
121
120
|
function getTmpDir() {
|
|
122
|
-
const path = (0,
|
|
123
|
-
(0,
|
|
121
|
+
const path = (0, import_node_path.join)((0, import_node_os.tmpdir)(), getPkgInfo().name);
|
|
122
|
+
(0, import_node_fs.mkdirSync)(path, { recursive: true });
|
|
124
123
|
return path;
|
|
125
124
|
}
|
|
126
125
|
function getTmpFile(fileExtWithoutDot) {
|
|
127
|
-
const filename = `${(0,
|
|
128
|
-
return (0,
|
|
126
|
+
const filename = `${(0, import_node_crypto.randomUUID)()}.${fileExtWithoutDot}`;
|
|
127
|
+
return (0, import_node_path.join)(getTmpDir(), filename);
|
|
129
128
|
}
|
|
130
129
|
function overlapped(container, target) {
|
|
131
130
|
return container.left < target.left + target.width && container.left + container.width > target.left && container.top < target.top + target.height && container.top + container.height > target.top;
|
package/dist/types/ai-model.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { c as callToGetJSONObject } from './util-
|
|
2
|
-
export { d as describeUserPage } from './util-
|
|
1
|
+
import { c as callToGetJSONObject } from './util-3931e76e.js';
|
|
2
|
+
export { d as describeUserPage } from './util-3931e76e.js';
|
|
3
3
|
export { ChatCompletionMessageParam } from 'openai/resources';
|
|
4
|
-
import { B as BaseElement, U as UIContext,
|
|
4
|
+
import { B as BaseElement, U as UIContext, f as AIElementParseResponse, g as AISectionParseResponse } from './types-2c92867c.js';
|
|
5
5
|
|
|
6
6
|
declare function systemPromptToFindElement(description: string, multi?: boolean): string;
|
|
7
7
|
|
|
@@ -13,7 +13,6 @@ declare function AiInspectElement<ElementType extends BaseElement = BaseElement>
|
|
|
13
13
|
}): Promise<{
|
|
14
14
|
parseResult: AIElementParseResponse;
|
|
15
15
|
elementById: (id: string) => ElementType;
|
|
16
|
-
systemPrompt: string;
|
|
17
16
|
}>;
|
|
18
17
|
declare function AiExtractElementInfo<T, ElementType extends BaseElement = BaseElement>(options: {
|
|
19
18
|
dataQuery: string | Record<string, string>;
|
|
@@ -26,7 +25,6 @@ declare function AiExtractElementInfo<T, ElementType extends BaseElement = BaseE
|
|
|
26
25
|
}): Promise<{
|
|
27
26
|
parseResult: AISectionParseResponse<T>;
|
|
28
27
|
elementById: (id: string) => ElementType;
|
|
29
|
-
systemPrompt: string;
|
|
30
28
|
}>;
|
|
31
29
|
|
|
32
30
|
export { AiExtractElementInfo, AiInspectElement, callToGetJSONObject, systemPromptToFindElement };
|
package/dist/types/image.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { Buffer } from 'node:buffer';
|
|
2
|
-
import { S as Size, R as Rect,
|
|
2
|
+
import { S as Size, R as Rect, i as UISection, U as UIContext, w as Color } from './types-2c92867c.js';
|
|
3
3
|
import 'openai/resources';
|
|
4
4
|
|
|
5
5
|
/**
|
package/dist/types/index.d.ts
CHANGED
|
@@ -1,9 +1,23 @@
|
|
|
1
|
-
import { c as
|
|
2
|
-
|
|
3
|
-
|
|
1
|
+
import { E as ExecutionTask, a as ExecutionTaskApply, b as ExecutionDump, B as BaseElement, U as UIContext, D as DumpSubscriber, I as InsightTaskInfo, c as InsightOptions, d as InsightAssertionResponse, P as PlanningAction } from './types-2c92867c.js';
|
|
2
|
+
export { h as AIAssertionResponse, f as AIElementParseResponse, A as AIResponseFormat, g as AISectionParseResponse, x as BaseAgentParserOpt, k as BasicSectionQuery, C as CallAIFn, w as Color, m as DumpMeta, p as ElementById, j as EnsureObject, F as ExecutionRecorderItem, a1 as ExecutionTaskAction, a0 as ExecutionTaskActionApply, $ as ExecutionTaskInsightAssertion, _ as ExecutionTaskInsightAssertionApply, Z as ExecutionTaskInsightAssertionParam, N as ExecutionTaskInsightDumpLog, Q as ExecutionTaskInsightLocate, O as ExecutionTaskInsightLocateApply, M as ExecutionTaskInsightLocateOutput, K as ExecutionTaskInsightLocateParam, Y as ExecutionTaskInsightQuery, X as ExecutionTaskInsightQueryApply, W as ExecutionTaskInsightQueryOutput, V as ExecutionTaskInsightQueryParam, a3 as ExecutionTaskPlanning, a2 as ExecutionTaskPlanningApply, J as ExecutionTaskReturn, G as ExecutionTaskType, H as ExecutorContext, a4 as GroupedActionDump, n as InsightDump, l as InsightExtractParam, L as LiteUISection, o as PartialInsightDumpFromSDK, q as PlanningAIResponse, v as PlanningActionParamAssert, s as PlanningActionParamHover, t as PlanningActionParamInputOrKeyPress, u as PlanningActionParamScroll, r as PlanningActionParamTap, z as PlaywrightParserOpt, e as Point, y as PuppeteerParserOpt, R as Rect, S as Size, T as TaskCacheInfo, i as UISection } from './types-2c92867c.js';
|
|
3
|
+
import { c as callToGetJSONObject, r as retrieveElement, a as retrieveSection } from './util-3931e76e.js';
|
|
4
4
|
export { setDumpDir } from './utils.js';
|
|
5
5
|
import 'openai/resources';
|
|
6
6
|
|
|
7
|
+
declare class Executor {
|
|
8
|
+
name: string;
|
|
9
|
+
description?: string;
|
|
10
|
+
tasks: ExecutionTask[];
|
|
11
|
+
status: 'init' | 'pending' | 'running' | 'completed' | 'error';
|
|
12
|
+
errorMsg?: string;
|
|
13
|
+
dumpFileName?: string;
|
|
14
|
+
constructor(name: string, description?: string, tasks?: ExecutionTaskApply[]);
|
|
15
|
+
private markTaskAsPending;
|
|
16
|
+
append(task: ExecutionTaskApply[] | ExecutionTaskApply): Promise<void>;
|
|
17
|
+
flush(): Promise<any>;
|
|
18
|
+
dump(): ExecutionDump;
|
|
19
|
+
}
|
|
20
|
+
|
|
7
21
|
interface LocateOpts {
|
|
8
22
|
multi?: boolean;
|
|
9
23
|
callAI?: typeof callToGetJSONObject<AIElementParseResponse>;
|
|
@@ -23,21 +37,7 @@ declare class Insight<ElementType extends BaseElement = BaseElement, ContextType
|
|
|
23
37
|
extract<T = any>(input: string): Promise<T>;
|
|
24
38
|
extract<T extends Record<string, string>>(input: T): Promise<Record<keyof T, any>>;
|
|
25
39
|
extract<T extends object>(input: Record<keyof T, string>): Promise<T>;
|
|
26
|
-
|
|
27
|
-
}
|
|
28
|
-
|
|
29
|
-
declare class Executor {
|
|
30
|
-
name: string;
|
|
31
|
-
description?: string;
|
|
32
|
-
tasks: ExecutionTask[];
|
|
33
|
-
status: 'init' | 'pending' | 'running' | 'completed' | 'error';
|
|
34
|
-
errorMsg?: string;
|
|
35
|
-
dumpFileName?: string;
|
|
36
|
-
constructor(name: string, description?: string, tasks?: ExecutionTaskApply[]);
|
|
37
|
-
private markTaskAsPending;
|
|
38
|
-
append(task: ExecutionTaskApply[] | ExecutionTaskApply): Promise<void>;
|
|
39
|
-
flush(): Promise<void>;
|
|
40
|
-
dump(): ExecutionDump;
|
|
40
|
+
assert(assertion: string): Promise<InsightAssertionResponse>;
|
|
41
41
|
}
|
|
42
42
|
|
|
43
43
|
declare const getElement: typeof retrieveElement;
|
|
@@ -50,4 +50,4 @@ declare function plan(userPrompt: string, opts: {
|
|
|
50
50
|
plans: PlanningAction[];
|
|
51
51
|
}>;
|
|
52
52
|
|
|
53
|
-
export { BaseElement, DumpSubscriber, ExecutionDump, ExecutionTask, ExecutionTaskApply, Executor, InsightOptions, InsightTaskInfo, PlanningAction, UIContext, Insight as default, getElement, getSection, plan };
|
|
53
|
+
export { BaseElement, DumpSubscriber, ExecutionDump, ExecutionTask, ExecutionTaskApply, Executor, InsightAssertionResponse, InsightOptions, InsightTaskInfo, PlanningAction, UIContext, Insight as default, getElement, getSection, plan };
|
|
@@ -10,15 +10,15 @@ interface Size {
|
|
|
10
10
|
}
|
|
11
11
|
type Rect = Point & Size;
|
|
12
12
|
declare enum NodeType {
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
13
|
+
INPUT = "INPUT Node",
|
|
14
|
+
BUTTON = "BUTTON Node",
|
|
15
|
+
IMG = "IMG Node",
|
|
16
|
+
TEXT = "TEXT Node"
|
|
17
17
|
}
|
|
18
18
|
declare abstract class BaseElement {
|
|
19
19
|
abstract id: string;
|
|
20
20
|
abstract attributes: {
|
|
21
|
-
|
|
21
|
+
nodeType: NodeType;
|
|
22
22
|
[key: string]: string;
|
|
23
23
|
};
|
|
24
24
|
abstract content: string;
|
|
@@ -47,6 +47,10 @@ interface AISectionParseResponse<DataShape> {
|
|
|
47
47
|
sections?: LiteUISection[];
|
|
48
48
|
errors?: string[];
|
|
49
49
|
}
|
|
50
|
+
interface AIAssertionResponse {
|
|
51
|
+
pass: boolean;
|
|
52
|
+
thought: string;
|
|
53
|
+
}
|
|
50
54
|
/**
|
|
51
55
|
* context
|
|
52
56
|
*/
|
|
@@ -80,7 +84,6 @@ interface BasicSectionQuery {
|
|
|
80
84
|
type InsightExtractParam = string | Record<string, string>;
|
|
81
85
|
interface InsightTaskInfo {
|
|
82
86
|
durationMs: number;
|
|
83
|
-
systemPrompt?: string;
|
|
84
87
|
rawResponse?: string;
|
|
85
88
|
}
|
|
86
89
|
interface DumpMeta {
|
|
@@ -88,17 +91,20 @@ interface DumpMeta {
|
|
|
88
91
|
logTime: number;
|
|
89
92
|
}
|
|
90
93
|
interface InsightDump extends DumpMeta {
|
|
91
|
-
type: 'locate' | 'extract';
|
|
94
|
+
type: 'locate' | 'extract' | 'assert';
|
|
92
95
|
logId: string;
|
|
93
96
|
context: UIContext;
|
|
94
97
|
userQuery: {
|
|
95
98
|
element?: string;
|
|
96
99
|
dataDemand?: InsightExtractParam;
|
|
97
100
|
sections?: Record<string, string>;
|
|
101
|
+
assertion?: string;
|
|
98
102
|
};
|
|
99
103
|
matchedSection: UISection[];
|
|
100
104
|
matchedElement: BaseElement[];
|
|
101
105
|
data: any;
|
|
106
|
+
assertionPass?: boolean;
|
|
107
|
+
assertionThought?: string;
|
|
102
108
|
taskInfo: InsightTaskInfo;
|
|
103
109
|
error?: string;
|
|
104
110
|
}
|
|
@@ -111,13 +117,14 @@ interface LiteUISection {
|
|
|
111
117
|
textIds: string[];
|
|
112
118
|
}
|
|
113
119
|
type ElementById = (id: string) => BaseElement | null;
|
|
120
|
+
type InsightAssertionResponse = AIAssertionResponse;
|
|
114
121
|
/**
|
|
115
122
|
* planning
|
|
116
123
|
*
|
|
117
124
|
*/
|
|
118
125
|
interface PlanningAction<ParamType = any> {
|
|
119
|
-
thought
|
|
120
|
-
type: 'Locate' | 'Tap' | 'Hover' | 'Input' | 'KeyboardPress' | 'Scroll' | 'Error';
|
|
126
|
+
thought?: string;
|
|
127
|
+
type: 'Locate' | 'Tap' | 'Hover' | 'Input' | 'KeyboardPress' | 'Scroll' | 'Error' | 'Assert';
|
|
121
128
|
param: ParamType;
|
|
122
129
|
}
|
|
123
130
|
interface PlanningAIResponse {
|
|
@@ -133,6 +140,9 @@ interface PlanningActionParamInputOrKeyPress {
|
|
|
133
140
|
interface PlanningActionParamScroll {
|
|
134
141
|
scrollType: 'ScrollUntilBottom' | 'ScrollUntilTop' | 'ScrollDown' | 'ScrollUp';
|
|
135
142
|
}
|
|
143
|
+
interface PlanningActionParamAssert {
|
|
144
|
+
assertion: string;
|
|
145
|
+
}
|
|
136
146
|
/**
|
|
137
147
|
* misc
|
|
138
148
|
*/
|
|
@@ -158,16 +168,20 @@ interface ExecutorContext {
|
|
|
158
168
|
task: ExecutionTask;
|
|
159
169
|
element?: BaseElement | null;
|
|
160
170
|
}
|
|
171
|
+
interface TaskCacheInfo {
|
|
172
|
+
hit: boolean;
|
|
173
|
+
}
|
|
161
174
|
interface ExecutionTaskApply<Type extends ExecutionTaskType = any, TaskParam = any, TaskOutput = any, TaskLog = any> {
|
|
162
175
|
type: Type;
|
|
163
176
|
subType?: string;
|
|
164
177
|
param?: TaskParam;
|
|
165
|
-
executor: (param: TaskParam, context: ExecutorContext) => Promise<ExecutionTaskReturn<TaskOutput, TaskLog> | void> | void;
|
|
178
|
+
executor: (param: TaskParam, context: ExecutorContext) => Promise<ExecutionTaskReturn<TaskOutput, TaskLog> | undefined | void> | undefined | void;
|
|
166
179
|
}
|
|
167
180
|
interface ExecutionTaskReturn<TaskOutput = unknown, TaskLog = unknown> {
|
|
168
181
|
output?: TaskOutput;
|
|
169
182
|
log?: TaskLog;
|
|
170
183
|
recorder?: ExecutionRecorderItem[];
|
|
184
|
+
cache?: TaskCacheInfo;
|
|
171
185
|
}
|
|
172
186
|
type ExecutionTask<E extends ExecutionTaskApply<any, any, any> = ExecutionTaskApply<any, any, any>> = E & ExecutionTaskReturn<E extends ExecutionTaskApply<any, any, infer TaskOutput, any> ? TaskOutput : unknown, E extends ExecutionTaskApply<any, any, any, infer TaskLog> ? TaskLog : unknown> & {
|
|
173
187
|
status: 'pending' | 'running' | 'success' | 'fail' | 'cancelled';
|
|
@@ -189,10 +203,10 @@ interface ExecutionTaskInsightLocateParam {
|
|
|
189
203
|
interface ExecutionTaskInsightLocateOutput {
|
|
190
204
|
element: BaseElement | null;
|
|
191
205
|
}
|
|
192
|
-
interface
|
|
206
|
+
interface ExecutionTaskInsightDumpLog {
|
|
193
207
|
dump?: InsightDump;
|
|
194
208
|
}
|
|
195
|
-
type ExecutionTaskInsightLocateApply = ExecutionTaskApply<'Insight', ExecutionTaskInsightLocateParam, ExecutionTaskInsightLocateOutput,
|
|
209
|
+
type ExecutionTaskInsightLocateApply = ExecutionTaskApply<'Insight', ExecutionTaskInsightLocateParam, ExecutionTaskInsightLocateOutput, ExecutionTaskInsightDumpLog>;
|
|
196
210
|
type ExecutionTaskInsightLocate = ExecutionTask<ExecutionTaskInsightLocateApply>;
|
|
197
211
|
interface ExecutionTaskInsightQueryParam {
|
|
198
212
|
dataDemand: InsightExtractParam;
|
|
@@ -200,11 +214,15 @@ interface ExecutionTaskInsightQueryParam {
|
|
|
200
214
|
interface ExecutionTaskInsightQueryOutput {
|
|
201
215
|
data: any;
|
|
202
216
|
}
|
|
203
|
-
type ExecutionTaskInsightQueryApply = ExecutionTaskApply<'Insight', ExecutionTaskInsightQueryParam>;
|
|
217
|
+
type ExecutionTaskInsightQueryApply = ExecutionTaskApply<'Insight', ExecutionTaskInsightQueryParam, any, ExecutionTaskInsightDumpLog>;
|
|
204
218
|
type ExecutionTaskInsightQuery = ExecutionTask<ExecutionTaskInsightQueryApply>;
|
|
219
|
+
interface ExecutionTaskInsightAssertionParam {
|
|
220
|
+
assertion: string;
|
|
221
|
+
}
|
|
222
|
+
type ExecutionTaskInsightAssertionApply = ExecutionTaskApply<'Insight', ExecutionTaskInsightAssertionParam, InsightAssertionResponse, ExecutionTaskInsightDumpLog>;
|
|
223
|
+
type ExecutionTaskInsightAssertion = ExecutionTask<ExecutionTaskInsightAssertionApply>;
|
|
205
224
|
type ExecutionTaskActionApply<ActionParam = any> = ExecutionTaskApply<'Action', ActionParam, void, void>;
|
|
206
225
|
type ExecutionTaskAction = ExecutionTask<ExecutionTaskActionApply>;
|
|
207
|
-
type ExectuionTaskPlanningParam = PlanningAIResponse;
|
|
208
226
|
type ExecutionTaskPlanningApply = ExecutionTaskApply<'Planning', {
|
|
209
227
|
userPrompt: string;
|
|
210
228
|
}, {
|
|
@@ -216,4 +234,4 @@ interface GroupedActionDump {
|
|
|
216
234
|
executions: ExecutionDump[];
|
|
217
235
|
}
|
|
218
236
|
|
|
219
|
-
export { AIResponseFormat as A, BaseElement as B, type CallAIFn as C, type DumpSubscriber as D, type ExecutionTask as E, type
|
|
237
|
+
export { type ExecutionTaskInsightAssertion as $, AIResponseFormat as A, BaseElement as B, type CallAIFn as C, type DumpSubscriber as D, type ExecutionTask as E, type ExecutionRecorderItem as F, type ExecutionTaskType as G, type ExecutorContext as H, type InsightTaskInfo as I, type ExecutionTaskReturn as J, type ExecutionTaskInsightLocateParam as K, type LiteUISection as L, type ExecutionTaskInsightLocateOutput as M, type ExecutionTaskInsightDumpLog as N, type ExecutionTaskInsightLocateApply as O, type PlanningAction as P, type ExecutionTaskInsightLocate as Q, type Rect as R, type Size as S, type TaskCacheInfo as T, UIContext as U, type ExecutionTaskInsightQueryParam as V, type ExecutionTaskInsightQueryOutput as W, type ExecutionTaskInsightQueryApply as X, type ExecutionTaskInsightQuery as Y, type ExecutionTaskInsightAssertionParam as Z, type ExecutionTaskInsightAssertionApply as _, type ExecutionTaskApply as a, type ExecutionTaskActionApply as a0, type ExecutionTaskAction as a1, type ExecutionTaskPlanningApply as a2, type ExecutionTaskPlanning as a3, type GroupedActionDump as a4, type ExecutionDump as b, type InsightOptions as c, type InsightAssertionResponse as d, type Point as e, type AIElementParseResponse as f, type AISectionParseResponse as g, type AIAssertionResponse as h, type UISection as i, type EnsureObject as j, type BasicSectionQuery as k, type InsightExtractParam as l, type DumpMeta as m, type InsightDump as n, type PartialInsightDumpFromSDK as o, type ElementById as p, type PlanningAIResponse as q, type PlanningActionParamTap as r, type PlanningActionParamHover as s, type PlanningActionParamInputOrKeyPress as t, type PlanningActionParamScroll as u, type PlanningActionParamAssert as v, type Color as w, type BaseAgentParserOpt as x, type PuppeteerParserOpt as y, type PlaywrightParserOpt as z };
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { ChatCompletionMessageParam } from 'openai/resources';
|
|
2
|
-
import { B as BaseElement, U as UIContext } from './types-
|
|
2
|
+
import { B as BaseElement, U as UIContext } from './types-2c92867c.js';
|
|
3
3
|
|
|
4
4
|
declare function callToGetJSONObject<T>(messages: ChatCompletionMessageParam[]): Promise<T>;
|
|
5
5
|
|
package/dist/types/utils.d.ts
CHANGED
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@midscene/core",
|
|
3
|
-
"description": "Hello, It's
|
|
4
|
-
"version": "0.
|
|
3
|
+
"description": "Hello, It's Midscene",
|
|
4
|
+
"version": "0.2.0",
|
|
5
5
|
"jsnext:source": "./src/index.ts",
|
|
6
6
|
"main": "./dist/lib/index.js",
|
|
7
7
|
"module": "./dist/es/index.js",
|
|
@@ -63,11 +63,12 @@
|
|
|
63
63
|
"sharp": "0.33.3"
|
|
64
64
|
},
|
|
65
65
|
"devDependencies": {
|
|
66
|
-
"@modern-js/module-tools": "^2.
|
|
66
|
+
"@modern-js/module-tools": "^2.56.1",
|
|
67
67
|
"@types/node": "^18.0.0",
|
|
68
68
|
"langsmith": "0.1.36",
|
|
69
69
|
"typescript": "~5.0.4",
|
|
70
|
-
"vitest": "^1.6.0"
|
|
70
|
+
"vitest": "^1.6.0",
|
|
71
|
+
"dotenv": "16.4.5"
|
|
71
72
|
},
|
|
72
73
|
"engines": {
|
|
73
74
|
"node": ">=16.0.0"
|