@midscene/core 0.8.16 → 0.8.17-beta-20250103010342.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/lib/ai-model.js +387 -342
- package/dist/lib/env.js +3 -0
- package/dist/lib/index.js +388 -343
- package/dist/lib/types/ai-model.d.ts +3 -3
- package/dist/lib/types/{index-43fd19f4.d.ts → automation-1fa20bde.d.ts} +21 -44
- package/dist/lib/types/env.d.ts +2 -1
- package/dist/lib/types/index.d.ts +4 -4
- package/dist/lib/types/{types-55182ae1.d.ts → types-7702c41a.d.ts} +5 -16
- package/dist/lib/types/utils.d.ts +1 -1
- package/dist/lib/utils.js +1 -1
- package/package.json +6 -3
- package/report/index.html +2 -2
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { g as AIUsageInfo } from './types-
|
|
1
|
+
import { g as AIUsageInfo } from './types-7702c41a.js';
|
|
2
2
|
import { ChatCompletionMessageParam } from 'openai/resources';
|
|
3
3
|
export { ChatCompletionMessageParam } from 'openai/resources';
|
|
4
|
-
import { A as AIActionType } from './
|
|
5
|
-
export { f as AiAssert, e as AiExtractElementInfo, b as AiInspectElement, c as callAiFn, d as describeUserPage, p as plan, t as transformElementPositionToId } from './
|
|
4
|
+
import { A as AIActionType } from './automation-1fa20bde.js';
|
|
5
|
+
export { f as AiAssert, e as AiExtractElementInfo, b as AiInspectElement, c as callAiFn, d as describeUserPage, p as plan, t as transformElementPositionToId } from './automation-1fa20bde.js';
|
|
6
6
|
|
|
7
7
|
declare function callToGetJSONObject<T>(messages: ChatCompletionMessageParam[], AIActionTypeValue: AIActionType): Promise<{
|
|
8
8
|
content: T;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { g as AIUsageInfo, B as BaseElement, U as UIContext,
|
|
1
|
+
import { g as AIUsageInfo, B as BaseElement, U as UIContext, l as AIElementResponse, A as AISingleElementResponse, u as ElementById, m as AISectionParseResponse, n as AIAssertionResponse, z as PlanningAIResponse } from './types-7702c41a.js';
|
|
2
2
|
import { ChatCompletionSystemMessageParam, ChatCompletionUserMessageParam } from 'openai/resources';
|
|
3
3
|
|
|
4
4
|
type AIArgs = [
|
|
@@ -11,10 +11,7 @@ declare enum AIActionType {
|
|
|
11
11
|
EXTRACT_DATA = 2,
|
|
12
12
|
PLAN = 3
|
|
13
13
|
}
|
|
14
|
-
declare function callAiFn<T>(
|
|
15
|
-
msgs: AIArgs;
|
|
16
|
-
AIActionType: AIActionType;
|
|
17
|
-
}): Promise<{
|
|
14
|
+
declare function callAiFn<T>(msgs: AIArgs, AIActionTypeValue: AIActionType): Promise<{
|
|
18
15
|
content: T;
|
|
19
16
|
usage?: AIUsageInfo;
|
|
20
17
|
}>;
|
|
@@ -28,7 +25,14 @@ declare function describeUserPage<ElementType extends BaseElement = BaseElement>
|
|
|
28
25
|
elementByPosition(position: {
|
|
29
26
|
x: number;
|
|
30
27
|
y: number;
|
|
28
|
+
}, size: {
|
|
29
|
+
width: number;
|
|
30
|
+
height: number;
|
|
31
31
|
}): BaseElement | undefined;
|
|
32
|
+
size: {
|
|
33
|
+
width: number;
|
|
34
|
+
height: number;
|
|
35
|
+
};
|
|
32
36
|
}>;
|
|
33
37
|
/**
|
|
34
38
|
* elements
|
|
@@ -41,56 +45,29 @@ declare function retrieveElement(prompt: string, opt?: {
|
|
|
41
45
|
*/
|
|
42
46
|
declare function retrieveSection(prompt: string): string;
|
|
43
47
|
|
|
44
|
-
declare function transformElementPositionToId(aiResult: AIElementResponse, elementsInfo: BaseElement[]
|
|
48
|
+
declare function transformElementPositionToId(aiResult: AIElementResponse | [number, number], elementsInfo: BaseElement[], size: {
|
|
49
|
+
width: number;
|
|
50
|
+
height: number;
|
|
51
|
+
}, screenshotBase64: string): Promise<{
|
|
45
52
|
errors: string[] | undefined;
|
|
46
53
|
elements: {
|
|
47
54
|
id: string;
|
|
48
|
-
reason
|
|
49
|
-
text
|
|
55
|
+
reason?: string | undefined;
|
|
56
|
+
text?: string | undefined;
|
|
50
57
|
}[];
|
|
51
|
-
}
|
|
58
|
+
}>;
|
|
52
59
|
declare function AiInspectElement<ElementType extends BaseElement = BaseElement>(options: {
|
|
53
60
|
context: UIContext<ElementType>;
|
|
54
61
|
multi: boolean;
|
|
55
62
|
targetElementDescription: string;
|
|
56
|
-
callAI?: typeof callAiFn<AIElementResponse>;
|
|
63
|
+
callAI?: typeof callAiFn<AIElementResponse | [number, number]>;
|
|
57
64
|
useModel?: 'coze' | 'openAI';
|
|
58
65
|
quickAnswer?: AISingleElementResponse;
|
|
59
66
|
}): Promise<{
|
|
60
|
-
parseResult:
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
elementById: (id: string) => ElementType;
|
|
65
|
-
rawResponse?: undefined;
|
|
66
|
-
usage?: undefined;
|
|
67
|
-
} | {
|
|
68
|
-
parseResult: {
|
|
69
|
-
errors: string[] | undefined;
|
|
70
|
-
elements: {
|
|
71
|
-
id: string;
|
|
72
|
-
reason: string;
|
|
73
|
-
text: string;
|
|
74
|
-
}[];
|
|
75
|
-
};
|
|
76
|
-
elementById: (id: string) => ElementType;
|
|
77
|
-
rawResponse?: undefined;
|
|
78
|
-
usage?: undefined;
|
|
79
|
-
} | {
|
|
80
|
-
parseResult: {
|
|
81
|
-
errors: string[] | undefined;
|
|
82
|
-
elements: {
|
|
83
|
-
id: string;
|
|
84
|
-
reason: string;
|
|
85
|
-
text: string;
|
|
86
|
-
}[];
|
|
87
|
-
};
|
|
88
|
-
rawResponse: {
|
|
89
|
-
content: AIElementResponse;
|
|
90
|
-
usage?: AIUsageInfo | undefined;
|
|
91
|
-
};
|
|
92
|
-
elementById: (id: string) => ElementType;
|
|
93
|
-
usage: AIUsageInfo | undefined;
|
|
67
|
+
parseResult: AIElementResponse;
|
|
68
|
+
rawResponse: any;
|
|
69
|
+
elementById: ElementById;
|
|
70
|
+
usage?: AIUsageInfo;
|
|
94
71
|
}>;
|
|
95
72
|
declare function AiExtractElementInfo<T, ElementType extends BaseElement = BaseElement>(options: {
|
|
96
73
|
dataQuery: string | Record<string, string>;
|
package/dist/lib/types/env.d.ts
CHANGED
|
@@ -11,6 +11,7 @@ declare const OPENAI_MAX_TOKENS = "OPENAI_MAX_TOKENS";
|
|
|
11
11
|
declare const MIDSCENE_MODEL_TEXT_ONLY = "MIDSCENE_MODEL_TEXT_ONLY";
|
|
12
12
|
declare const MIDSCENE_CACHE = "MIDSCENE_CACHE";
|
|
13
13
|
declare const MATCH_BY_POSITION = "MATCH_BY_POSITION";
|
|
14
|
+
declare const MIDSCENE_API_TYPE = "MIDSCENE_API_TYPE";
|
|
14
15
|
declare const MIDSCENE_REPORT_TAG_NAME = "MIDSCENE_REPORT_TAG_NAME";
|
|
15
16
|
declare const MIDSCENE_USE_AZURE_OPENAI = "MIDSCENE_USE_AZURE_OPENAI";
|
|
16
17
|
declare const MIDSCENE_AZURE_OPENAI_SCOPE = "MIDSCENE_AZURE_OPENAI_SCOPE";
|
|
@@ -79,4 +80,4 @@ declare const allAIConfig: () => {
|
|
|
79
80
|
};
|
|
80
81
|
declare const overrideAIConfig: (newConfig: ReturnType<typeof allConfigFromEnv>, extendMode?: boolean) => void;
|
|
81
82
|
|
|
82
|
-
export { ANTHROPIC_API_KEY, AZURE_OPENAI_API_VERSION, AZURE_OPENAI_DEPLOYMENT, AZURE_OPENAI_ENDPOINT, AZURE_OPENAI_KEY, MATCH_BY_POSITION, MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_AZURE_OPENAI_SCOPE, MIDSCENE_CACHE, MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG, MIDSCENE_DEBUG_AI_PROFILE, MIDSCENE_DEBUG_MODE, MIDSCENE_LANGSMITH_DEBUG, MIDSCENE_MODEL_NAME, MIDSCENE_MODEL_TEXT_ONLY, MIDSCENE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_OPENAI_SOCKS_PROXY, MIDSCENE_REPORT_TAG_NAME, MIDSCENE_USE_ANTHROPIC_SDK, MIDSCENE_USE_AZURE_OPENAI, OPENAI_API_KEY, OPENAI_BASE_URL, OPENAI_MAX_TOKENS, OPENAI_USE_AZURE, allAIConfig, getAIConfig, getAIConfigInJson, overrideAIConfig };
|
|
83
|
+
export { ANTHROPIC_API_KEY, AZURE_OPENAI_API_VERSION, AZURE_OPENAI_DEPLOYMENT, AZURE_OPENAI_ENDPOINT, AZURE_OPENAI_KEY, MATCH_BY_POSITION, MIDSCENE_API_TYPE, MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_AZURE_OPENAI_SCOPE, MIDSCENE_CACHE, MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG, MIDSCENE_DEBUG_AI_PROFILE, MIDSCENE_DEBUG_MODE, MIDSCENE_LANGSMITH_DEBUG, MIDSCENE_MODEL_NAME, MIDSCENE_MODEL_TEXT_ONLY, MIDSCENE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_OPENAI_SOCKS_PROXY, MIDSCENE_REPORT_TAG_NAME, MIDSCENE_USE_ANTHROPIC_SDK, MIDSCENE_USE_AZURE_OPENAI, OPENAI_API_KEY, OPENAI_BASE_URL, OPENAI_MAX_TOKENS, OPENAI_USE_AZURE, allAIConfig, getAIConfig, getAIConfigInJson, overrideAIConfig };
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { E as ExecutionTask, a as ExecutionTaskProgressOptions, b as ExecutionTaskApply, c as ExecutionDump, B as BaseElement, U as UIContext, I as InsightAction, D as DumpSubscriber, d as InsightOptions, e as InsightTaskInfo, A as AISingleElementResponse, f as InsightAssertionResponse } from './types-
|
|
2
|
-
export {
|
|
1
|
+
import { E as ExecutionTask, a as ExecutionTaskProgressOptions, b as ExecutionTaskApply, c as ExecutionDump, B as BaseElement, U as UIContext, I as InsightAction, D as DumpSubscriber, d as InsightOptions, e as InsightTaskInfo, A as AISingleElementResponse, f as InsightAssertionResponse } from './types-7702c41a.js';
|
|
2
|
+
export { n as AIAssertionResponse, k as AIElementIdResponse, l as AIElementResponse, h as AIResponseFormat, m as AISectionParseResponse, i as AISingleElementResponseById, j as AISingleElementResponseByPosition, g as AIUsageInfo, w as AgentAssertOpt, v as AgentWaitForOpt, W as BaseAgentParserOpt, C as CallAIFn, V as Color, q as DumpMeta, u as ElementById, o as EnsureObject, Z as ExecutionRecorderItem, af as ExecutionTaskAction, ae as ExecutionTaskActionApply, ad as ExecutionTaskInsightAssertion, ac as ExecutionTaskInsightAssertionApply, ab as ExecutionTaskInsightAssertionParam, a4 as ExecutionTaskInsightDumpLog, a6 as ExecutionTaskInsightLocate, a5 as ExecutionTaskInsightLocateApply, a3 as ExecutionTaskInsightLocateOutput, a2 as ExecutionTaskInsightLocateParam, aa as ExecutionTaskInsightQuery, a9 as ExecutionTaskInsightQueryApply, a8 as ExecutionTaskInsightQueryOutput, a7 as ExecutionTaskInsightQueryParam, ah as ExecutionTaskPlanning, ag as ExecutionTaskPlanningApply, a1 as ExecutionTaskReturn, _ as ExecutionTaskType, $ as ExecutorContext, ai as GroupedActionDump, s as InsightDump, p as InsightExtractParam, L as LiteUISection, t as PartialInsightDumpFromSDK, z as PlanningAIResponse, y as PlanningAction, N as PlanningActionParamAssert, Q as PlanningActionParamError, J as PlanningActionParamHover, K as PlanningActionParamInputOrKeyPress, G as PlanningActionParamPlan, M as PlanningActionParamScroll, O as PlanningActionParamSleep, H as PlanningActionParamTap, T as PlanningActionParamWaitFor, F as PlanningFurtherPlan, x as PlanningLocateParam, Y as PlaywrightParserOpt, P as Point, X as PuppeteerParserOpt, R as Rect, r as ReportDumpWithAttributes, S as Size, a0 as TaskCacheInfo } from './types-7702c41a.js';
|
|
3
3
|
export { allAIConfig, getAIConfig, overrideAIConfig } from './env.js';
|
|
4
|
-
import { c as callAiFn, r as retrieveElement, a as retrieveSection } from './
|
|
5
|
-
export { p as plan, t as transformElementPositionToId } from './
|
|
4
|
+
import { c as callAiFn, r as retrieveElement, a as retrieveSection } from './automation-1fa20bde.js';
|
|
5
|
+
export { p as plan, t as transformElementPositionToId } from './automation-1fa20bde.js';
|
|
6
6
|
export { getLogDirByType, getVersion, setLogDir } from './utils.js';
|
|
7
7
|
import 'openai/resources';
|
|
8
8
|
|
|
@@ -52,27 +52,16 @@ type AISingleElementResponseByPosition = {
|
|
|
52
52
|
reason: string;
|
|
53
53
|
text: string;
|
|
54
54
|
};
|
|
55
|
-
type AISingleElementResponse = AISingleElementResponseById
|
|
55
|
+
type AISingleElementResponse = AISingleElementResponseById;
|
|
56
56
|
interface AIElementIdResponse {
|
|
57
57
|
elements: {
|
|
58
58
|
id: string;
|
|
59
|
-
reason
|
|
60
|
-
text
|
|
59
|
+
reason?: string;
|
|
60
|
+
text?: string;
|
|
61
61
|
}[];
|
|
62
62
|
errors?: string[];
|
|
63
63
|
}
|
|
64
|
-
|
|
65
|
-
elements: {
|
|
66
|
-
position: {
|
|
67
|
-
x: number;
|
|
68
|
-
y: number;
|
|
69
|
-
};
|
|
70
|
-
reason: string;
|
|
71
|
-
text: string;
|
|
72
|
-
}[];
|
|
73
|
-
errors?: string[];
|
|
74
|
-
}
|
|
75
|
-
type AIElementResponse = AIElementIdResponse | AIElementPositionResponse;
|
|
64
|
+
type AIElementResponse = AIElementIdResponse;
|
|
76
65
|
interface AISectionParseResponse<DataShape> {
|
|
77
66
|
data: DataShape;
|
|
78
67
|
sections?: LiteUISection[];
|
|
@@ -311,4 +300,4 @@ interface GroupedActionDump {
|
|
|
311
300
|
executions: ExecutionDump[];
|
|
312
301
|
}
|
|
313
302
|
|
|
314
|
-
export { type
|
|
303
|
+
export { type ExecutorContext as $, type AISingleElementResponse as A, BaseElement as B, type CallAIFn as C, type DumpSubscriber as D, type ExecutionTask as E, type PlanningFurtherPlan as F, type PlanningActionParamPlan as G, type PlanningActionParamTap as H, type InsightAction as I, type PlanningActionParamHover as J, type PlanningActionParamInputOrKeyPress as K, type LiteUISection as L, type PlanningActionParamScroll as M, type PlanningActionParamAssert as N, type PlanningActionParamSleep as O, type Point as P, type PlanningActionParamError as Q, type Rect as R, type Size as S, type PlanningActionParamWaitFor as T, UIContext as U, type Color as V, type BaseAgentParserOpt as W, type PuppeteerParserOpt as X, type PlaywrightParserOpt as Y, type ExecutionRecorderItem as Z, type ExecutionTaskType as _, type ExecutionTaskProgressOptions as a, type TaskCacheInfo as a0, type ExecutionTaskReturn as a1, type ExecutionTaskInsightLocateParam as a2, type ExecutionTaskInsightLocateOutput as a3, type ExecutionTaskInsightDumpLog as a4, type ExecutionTaskInsightLocateApply as a5, type ExecutionTaskInsightLocate as a6, type ExecutionTaskInsightQueryParam as a7, type ExecutionTaskInsightQueryOutput as a8, type ExecutionTaskInsightQueryApply as a9, type ExecutionTaskInsightQuery as aa, type ExecutionTaskInsightAssertionParam as ab, type ExecutionTaskInsightAssertionApply as ac, type ExecutionTaskInsightAssertion as ad, type ExecutionTaskActionApply as ae, type ExecutionTaskAction as af, type ExecutionTaskPlanningApply as ag, type ExecutionTaskPlanning as ah, type GroupedActionDump as ai, type ExecutionTaskApply as b, type ExecutionDump as c, type InsightOptions as d, type InsightTaskInfo as e, type InsightAssertionResponse as f, type AIUsageInfo as g, AIResponseFormat as h, type AISingleElementResponseById as i, type AISingleElementResponseByPosition as j, type AIElementIdResponse as k, type AIElementResponse as l, type AISectionParseResponse as m, type AIAssertionResponse as n, type EnsureObject as o, type InsightExtractParam as p, type DumpMeta as q, type ReportDumpWithAttributes as r, type InsightDump as s, type PartialInsightDumpFromSDK as t, type ElementById as u, type AgentWaitForOpt as v, type AgentAssertOpt as w, type PlanningLocateParam as x, type PlanningAction as y, type PlanningAIResponse as z };
|
package/dist/lib/utils.js
CHANGED
|
@@ -303,7 +303,7 @@ function stringifyDumpData(data, indents) {
|
|
|
303
303
|
return JSON.stringify(data, replacerForPageObject, indents);
|
|
304
304
|
}
|
|
305
305
|
function getVersion() {
|
|
306
|
-
return "0.8.
|
|
306
|
+
return "0.8.17-beta-20250103010342.0";
|
|
307
307
|
}
|
|
308
308
|
function debugLog(...message) {
|
|
309
309
|
const debugMode = getAIConfig(MIDSCENE_DEBUG_MODE);
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@midscene/core",
|
|
3
|
-
"description": "
|
|
4
|
-
"version": "0.8.
|
|
3
|
+
"description": "Automate browser actions, extract data, and perform assertions using AI. It offers JavaScript SDK, Chrome extension, and support for scripting in YAML. See https://midscenejs.com/ for details.",
|
|
4
|
+
"version": "0.8.17-beta-20250103010342.0",
|
|
5
5
|
"repository": "https://github.com/web-infra-dev/midscene",
|
|
6
6
|
"homepage": "https://midscenejs.com/",
|
|
7
7
|
"jsnext:source": "./src/index.ts",
|
|
@@ -38,11 +38,13 @@
|
|
|
38
38
|
"dependencies": {
|
|
39
39
|
"@anthropic-ai/sdk": "0.33.1",
|
|
40
40
|
"@azure/identity": "4.5.0",
|
|
41
|
+
"@langchain/core": "0.3.26",
|
|
41
42
|
"dirty-json": "0.9.2",
|
|
43
|
+
"langchain": "0.3.8",
|
|
42
44
|
"openai": "4.57.1",
|
|
43
45
|
"optional": "0.1.4",
|
|
44
46
|
"socks-proxy-agent": "8.0.4",
|
|
45
|
-
"@midscene/shared": "0.8.
|
|
47
|
+
"@midscene/shared": "0.8.17-beta-20250103010342.0"
|
|
46
48
|
},
|
|
47
49
|
"devDependencies": {
|
|
48
50
|
"@modern-js/module-tools": "2.60.6",
|
|
@@ -71,6 +73,7 @@
|
|
|
71
73
|
"test:ai": "AITEST=true npm run test",
|
|
72
74
|
"computer": "TEST_COMPUTER=true npm run test:ai -- tests/ai/evaluate/computer.test.ts",
|
|
73
75
|
"evaluate": "npm run test:ai -- tests/ai/evaluate/inspect.test.ts",
|
|
76
|
+
"prompt": "npm run test:ai -- tests/ai/inspect2.test.ts",
|
|
74
77
|
"evaluate:update": "UPDATE_AI_DATA=true npm run test:ai -- tests/ai/evaluate/inspect.test.ts"
|
|
75
78
|
}
|
|
76
79
|
}
|