@midscene/core 0.8.16 → 0.8.17-beta-20250103010342.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,8 +1,8 @@
1
- import { g as AIUsageInfo } from './types-55182ae1.js';
1
+ import { g as AIUsageInfo } from './types-7702c41a.js';
2
2
  import { ChatCompletionMessageParam } from 'openai/resources';
3
3
  export { ChatCompletionMessageParam } from 'openai/resources';
4
- import { A as AIActionType } from './index-43fd19f4.js';
5
- export { f as AiAssert, e as AiExtractElementInfo, b as AiInspectElement, c as callAiFn, d as describeUserPage, p as plan, t as transformElementPositionToId } from './index-43fd19f4.js';
4
+ import { A as AIActionType } from './automation-1fa20bde.js';
5
+ export { f as AiAssert, e as AiExtractElementInfo, b as AiInspectElement, c as callAiFn, d as describeUserPage, p as plan, t as transformElementPositionToId } from './automation-1fa20bde.js';
6
6
 
7
7
  declare function callToGetJSONObject<T>(messages: ChatCompletionMessageParam[], AIActionTypeValue: AIActionType): Promise<{
8
8
  content: T;
@@ -1,4 +1,4 @@
1
- import { g as AIUsageInfo, B as BaseElement, U as UIContext, m as AIElementResponse, A as AISingleElementResponse, i as AISingleElementResponseById, n as AISectionParseResponse, o as AIAssertionResponse, F as PlanningAIResponse } from './types-55182ae1.js';
1
+ import { g as AIUsageInfo, B as BaseElement, U as UIContext, l as AIElementResponse, A as AISingleElementResponse, u as ElementById, m as AISectionParseResponse, n as AIAssertionResponse, z as PlanningAIResponse } from './types-7702c41a.js';
2
2
  import { ChatCompletionSystemMessageParam, ChatCompletionUserMessageParam } from 'openai/resources';
3
3
 
4
4
  type AIArgs = [
@@ -11,10 +11,7 @@ declare enum AIActionType {
11
11
  EXTRACT_DATA = 2,
12
12
  PLAN = 3
13
13
  }
14
- declare function callAiFn<T>(options: {
15
- msgs: AIArgs;
16
- AIActionType: AIActionType;
17
- }): Promise<{
14
+ declare function callAiFn<T>(msgs: AIArgs, AIActionTypeValue: AIActionType): Promise<{
18
15
  content: T;
19
16
  usage?: AIUsageInfo;
20
17
  }>;
@@ -28,7 +25,14 @@ declare function describeUserPage<ElementType extends BaseElement = BaseElement>
28
25
  elementByPosition(position: {
29
26
  x: number;
30
27
  y: number;
28
+ }, size: {
29
+ width: number;
30
+ height: number;
31
31
  }): BaseElement | undefined;
32
+ size: {
33
+ width: number;
34
+ height: number;
35
+ };
32
36
  }>;
33
37
  /**
34
38
  * elements
@@ -41,56 +45,29 @@ declare function retrieveElement(prompt: string, opt?: {
41
45
  */
42
46
  declare function retrieveSection(prompt: string): string;
43
47
 
44
- declare function transformElementPositionToId(aiResult: AIElementResponse, elementsInfo: BaseElement[]): {
48
+ declare function transformElementPositionToId(aiResult: AIElementResponse | [number, number], elementsInfo: BaseElement[], size: {
49
+ width: number;
50
+ height: number;
51
+ }, screenshotBase64: string): Promise<{
45
52
  errors: string[] | undefined;
46
53
  elements: {
47
54
  id: string;
48
- reason: string;
49
- text: string;
55
+ reason?: string | undefined;
56
+ text?: string | undefined;
50
57
  }[];
51
- };
58
+ }>;
52
59
  declare function AiInspectElement<ElementType extends BaseElement = BaseElement>(options: {
53
60
  context: UIContext<ElementType>;
54
61
  multi: boolean;
55
62
  targetElementDescription: string;
56
- callAI?: typeof callAiFn<AIElementResponse>;
63
+ callAI?: typeof callAiFn<AIElementResponse | [number, number]>;
57
64
  useModel?: 'coze' | 'openAI';
58
65
  quickAnswer?: AISingleElementResponse;
59
66
  }): Promise<{
60
- parseResult: {
61
- elements: AISingleElementResponseById[];
62
- errors: never[];
63
- };
64
- elementById: (id: string) => ElementType;
65
- rawResponse?: undefined;
66
- usage?: undefined;
67
- } | {
68
- parseResult: {
69
- errors: string[] | undefined;
70
- elements: {
71
- id: string;
72
- reason: string;
73
- text: string;
74
- }[];
75
- };
76
- elementById: (id: string) => ElementType;
77
- rawResponse?: undefined;
78
- usage?: undefined;
79
- } | {
80
- parseResult: {
81
- errors: string[] | undefined;
82
- elements: {
83
- id: string;
84
- reason: string;
85
- text: string;
86
- }[];
87
- };
88
- rawResponse: {
89
- content: AIElementResponse;
90
- usage?: AIUsageInfo | undefined;
91
- };
92
- elementById: (id: string) => ElementType;
93
- usage: AIUsageInfo | undefined;
67
+ parseResult: AIElementResponse;
68
+ rawResponse: any;
69
+ elementById: ElementById;
70
+ usage?: AIUsageInfo;
94
71
  }>;
95
72
  declare function AiExtractElementInfo<T, ElementType extends BaseElement = BaseElement>(options: {
96
73
  dataQuery: string | Record<string, string>;
@@ -11,6 +11,7 @@ declare const OPENAI_MAX_TOKENS = "OPENAI_MAX_TOKENS";
11
11
  declare const MIDSCENE_MODEL_TEXT_ONLY = "MIDSCENE_MODEL_TEXT_ONLY";
12
12
  declare const MIDSCENE_CACHE = "MIDSCENE_CACHE";
13
13
  declare const MATCH_BY_POSITION = "MATCH_BY_POSITION";
14
+ declare const MIDSCENE_API_TYPE = "MIDSCENE_API_TYPE";
14
15
  declare const MIDSCENE_REPORT_TAG_NAME = "MIDSCENE_REPORT_TAG_NAME";
15
16
  declare const MIDSCENE_USE_AZURE_OPENAI = "MIDSCENE_USE_AZURE_OPENAI";
16
17
  declare const MIDSCENE_AZURE_OPENAI_SCOPE = "MIDSCENE_AZURE_OPENAI_SCOPE";
@@ -79,4 +80,4 @@ declare const allAIConfig: () => {
79
80
  };
80
81
  declare const overrideAIConfig: (newConfig: ReturnType<typeof allConfigFromEnv>, extendMode?: boolean) => void;
81
82
 
82
- export { ANTHROPIC_API_KEY, AZURE_OPENAI_API_VERSION, AZURE_OPENAI_DEPLOYMENT, AZURE_OPENAI_ENDPOINT, AZURE_OPENAI_KEY, MATCH_BY_POSITION, MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_AZURE_OPENAI_SCOPE, MIDSCENE_CACHE, MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG, MIDSCENE_DEBUG_AI_PROFILE, MIDSCENE_DEBUG_MODE, MIDSCENE_LANGSMITH_DEBUG, MIDSCENE_MODEL_NAME, MIDSCENE_MODEL_TEXT_ONLY, MIDSCENE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_OPENAI_SOCKS_PROXY, MIDSCENE_REPORT_TAG_NAME, MIDSCENE_USE_ANTHROPIC_SDK, MIDSCENE_USE_AZURE_OPENAI, OPENAI_API_KEY, OPENAI_BASE_URL, OPENAI_MAX_TOKENS, OPENAI_USE_AZURE, allAIConfig, getAIConfig, getAIConfigInJson, overrideAIConfig };
83
+ export { ANTHROPIC_API_KEY, AZURE_OPENAI_API_VERSION, AZURE_OPENAI_DEPLOYMENT, AZURE_OPENAI_ENDPOINT, AZURE_OPENAI_KEY, MATCH_BY_POSITION, MIDSCENE_API_TYPE, MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_AZURE_OPENAI_SCOPE, MIDSCENE_CACHE, MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG, MIDSCENE_DEBUG_AI_PROFILE, MIDSCENE_DEBUG_MODE, MIDSCENE_LANGSMITH_DEBUG, MIDSCENE_MODEL_NAME, MIDSCENE_MODEL_TEXT_ONLY, MIDSCENE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_OPENAI_SOCKS_PROXY, MIDSCENE_REPORT_TAG_NAME, MIDSCENE_USE_ANTHROPIC_SDK, MIDSCENE_USE_AZURE_OPENAI, OPENAI_API_KEY, OPENAI_BASE_URL, OPENAI_MAX_TOKENS, OPENAI_USE_AZURE, allAIConfig, getAIConfig, getAIConfigInJson, overrideAIConfig };
@@ -1,8 +1,8 @@
1
- import { E as ExecutionTask, a as ExecutionTaskProgressOptions, b as ExecutionTaskApply, c as ExecutionDump, B as BaseElement, U as UIContext, I as InsightAction, D as DumpSubscriber, d as InsightOptions, e as InsightTaskInfo, A as AISingleElementResponse, f as InsightAssertionResponse } from './types-55182ae1.js';
2
- export { o as AIAssertionResponse, k as AIElementIdResponse, l as AIElementPositionResponse, m as AIElementResponse, h as AIResponseFormat, n as AISectionParseResponse, i as AISingleElementResponseById, j as AISingleElementResponseByPosition, g as AIUsageInfo, x as AgentAssertOpt, w as AgentWaitForOpt, X as BaseAgentParserOpt, C as CallAIFn, W as Color, r as DumpMeta, v as ElementById, p as EnsureObject, _ as ExecutionRecorderItem, ag as ExecutionTaskAction, af as ExecutionTaskActionApply, ae as ExecutionTaskInsightAssertion, ad as ExecutionTaskInsightAssertionApply, ac as ExecutionTaskInsightAssertionParam, a5 as ExecutionTaskInsightDumpLog, a7 as ExecutionTaskInsightLocate, a6 as ExecutionTaskInsightLocateApply, a4 as ExecutionTaskInsightLocateOutput, a3 as ExecutionTaskInsightLocateParam, ab as ExecutionTaskInsightQuery, aa as ExecutionTaskInsightQueryApply, a9 as ExecutionTaskInsightQueryOutput, a8 as ExecutionTaskInsightQueryParam, ai as ExecutionTaskPlanning, ah as ExecutionTaskPlanningApply, a2 as ExecutionTaskReturn, $ as ExecutionTaskType, a0 as ExecutorContext, aj as GroupedActionDump, t as InsightDump, q as InsightExtractParam, L as LiteUISection, u as PartialInsightDumpFromSDK, F as PlanningAIResponse, z as PlanningAction, O as PlanningActionParamAssert, T as PlanningActionParamError, K as PlanningActionParamHover, M as PlanningActionParamInputOrKeyPress, H as PlanningActionParamPlan, N as PlanningActionParamScroll, Q as PlanningActionParamSleep, J as PlanningActionParamTap, V as PlanningActionParamWaitFor, G as PlanningFurtherPlan, y as PlanningLocateParam, Z as PlaywrightParserOpt, P as Point, Y as PuppeteerParserOpt, R as Rect, s as ReportDumpWithAttributes, S as Size, a1 as TaskCacheInfo } from './types-55182ae1.js';
1
+ import { E as ExecutionTask, a as ExecutionTaskProgressOptions, b as ExecutionTaskApply, c as ExecutionDump, B as BaseElement, U as UIContext, I as InsightAction, D as DumpSubscriber, d as InsightOptions, e as InsightTaskInfo, A as AISingleElementResponse, f as InsightAssertionResponse } from './types-7702c41a.js';
2
+ export { n as AIAssertionResponse, k as AIElementIdResponse, l as AIElementResponse, h as AIResponseFormat, m as AISectionParseResponse, i as AISingleElementResponseById, j as AISingleElementResponseByPosition, g as AIUsageInfo, w as AgentAssertOpt, v as AgentWaitForOpt, W as BaseAgentParserOpt, C as CallAIFn, V as Color, q as DumpMeta, u as ElementById, o as EnsureObject, Z as ExecutionRecorderItem, af as ExecutionTaskAction, ae as ExecutionTaskActionApply, ad as ExecutionTaskInsightAssertion, ac as ExecutionTaskInsightAssertionApply, ab as ExecutionTaskInsightAssertionParam, a4 as ExecutionTaskInsightDumpLog, a6 as ExecutionTaskInsightLocate, a5 as ExecutionTaskInsightLocateApply, a3 as ExecutionTaskInsightLocateOutput, a2 as ExecutionTaskInsightLocateParam, aa as ExecutionTaskInsightQuery, a9 as ExecutionTaskInsightQueryApply, a8 as ExecutionTaskInsightQueryOutput, a7 as ExecutionTaskInsightQueryParam, ah as ExecutionTaskPlanning, ag as ExecutionTaskPlanningApply, a1 as ExecutionTaskReturn, _ as ExecutionTaskType, $ as ExecutorContext, ai as GroupedActionDump, s as InsightDump, p as InsightExtractParam, L as LiteUISection, t as PartialInsightDumpFromSDK, z as PlanningAIResponse, y as PlanningAction, N as PlanningActionParamAssert, Q as PlanningActionParamError, J as PlanningActionParamHover, K as PlanningActionParamInputOrKeyPress, G as PlanningActionParamPlan, M as PlanningActionParamScroll, O as PlanningActionParamSleep, H as PlanningActionParamTap, T as PlanningActionParamWaitFor, F as PlanningFurtherPlan, x as PlanningLocateParam, Y as PlaywrightParserOpt, P as Point, X as PuppeteerParserOpt, R as Rect, r as ReportDumpWithAttributes, S as Size, a0 as TaskCacheInfo } from './types-7702c41a.js';
3
3
  export { allAIConfig, getAIConfig, overrideAIConfig } from './env.js';
4
- import { c as callAiFn, r as retrieveElement, a as retrieveSection } from './index-43fd19f4.js';
5
- export { p as plan, t as transformElementPositionToId } from './index-43fd19f4.js';
4
+ import { c as callAiFn, r as retrieveElement, a as retrieveSection } from './automation-1fa20bde.js';
5
+ export { p as plan, t as transformElementPositionToId } from './automation-1fa20bde.js';
6
6
  export { getLogDirByType, getVersion, setLogDir } from './utils.js';
7
7
  import 'openai/resources';
8
8
 
@@ -52,27 +52,16 @@ type AISingleElementResponseByPosition = {
52
52
  reason: string;
53
53
  text: string;
54
54
  };
55
- type AISingleElementResponse = AISingleElementResponseById | AISingleElementResponseByPosition;
55
+ type AISingleElementResponse = AISingleElementResponseById;
56
56
  interface AIElementIdResponse {
57
57
  elements: {
58
58
  id: string;
59
- reason: string;
60
- text: string;
59
+ reason?: string;
60
+ text?: string;
61
61
  }[];
62
62
  errors?: string[];
63
63
  }
64
- interface AIElementPositionResponse {
65
- elements: {
66
- position: {
67
- x: number;
68
- y: number;
69
- };
70
- reason: string;
71
- text: string;
72
- }[];
73
- errors?: string[];
74
- }
75
- type AIElementResponse = AIElementIdResponse | AIElementPositionResponse;
64
+ type AIElementResponse = AIElementIdResponse;
76
65
  interface AISectionParseResponse<DataShape> {
77
66
  data: DataShape;
78
67
  sections?: LiteUISection[];
@@ -311,4 +300,4 @@ interface GroupedActionDump {
311
300
  executions: ExecutionDump[];
312
301
  }
313
302
 
314
- export { type ExecutionTaskType as $, type AISingleElementResponse as A, BaseElement as B, type CallAIFn as C, type DumpSubscriber as D, type ExecutionTask as E, type PlanningAIResponse as F, type PlanningFurtherPlan as G, type PlanningActionParamPlan as H, type InsightAction as I, type PlanningActionParamTap as J, type PlanningActionParamHover as K, type LiteUISection as L, type PlanningActionParamInputOrKeyPress as M, type PlanningActionParamScroll as N, type PlanningActionParamAssert as O, type Point as P, type PlanningActionParamSleep as Q, type Rect as R, type Size as S, type PlanningActionParamError as T, UIContext as U, type PlanningActionParamWaitFor as V, type Color as W, type BaseAgentParserOpt as X, type PuppeteerParserOpt as Y, type PlaywrightParserOpt as Z, type ExecutionRecorderItem as _, type ExecutionTaskProgressOptions as a, type ExecutorContext as a0, type TaskCacheInfo as a1, type ExecutionTaskReturn as a2, type ExecutionTaskInsightLocateParam as a3, type ExecutionTaskInsightLocateOutput as a4, type ExecutionTaskInsightDumpLog as a5, type ExecutionTaskInsightLocateApply as a6, type ExecutionTaskInsightLocate as a7, type ExecutionTaskInsightQueryParam as a8, type ExecutionTaskInsightQueryOutput as a9, type ExecutionTaskInsightQueryApply as aa, type ExecutionTaskInsightQuery as ab, type ExecutionTaskInsightAssertionParam as ac, type ExecutionTaskInsightAssertionApply as ad, type ExecutionTaskInsightAssertion as ae, type ExecutionTaskActionApply as af, type ExecutionTaskAction as ag, type ExecutionTaskPlanningApply as ah, type ExecutionTaskPlanning as ai, type GroupedActionDump as aj, type ExecutionTaskApply as b, type ExecutionDump as c, type InsightOptions as d, type InsightTaskInfo as e, type InsightAssertionResponse as f, type AIUsageInfo as g, AIResponseFormat as h, type AISingleElementResponseById as i, type AISingleElementResponseByPosition as j, type AIElementIdResponse as k, type AIElementPositionResponse as l, type AIElementResponse as m, type AISectionParseResponse as n, type AIAssertionResponse as o, type EnsureObject as p, type InsightExtractParam as q, type DumpMeta as r, type ReportDumpWithAttributes as s, type InsightDump as t, type PartialInsightDumpFromSDK as u, type ElementById as v, type AgentWaitForOpt as w, type AgentAssertOpt as x, type PlanningLocateParam as y, type PlanningAction as z };
303
+ export { type ExecutorContext as $, type AISingleElementResponse as A, BaseElement as B, type CallAIFn as C, type DumpSubscriber as D, type ExecutionTask as E, type PlanningFurtherPlan as F, type PlanningActionParamPlan as G, type PlanningActionParamTap as H, type InsightAction as I, type PlanningActionParamHover as J, type PlanningActionParamInputOrKeyPress as K, type LiteUISection as L, type PlanningActionParamScroll as M, type PlanningActionParamAssert as N, type PlanningActionParamSleep as O, type Point as P, type PlanningActionParamError as Q, type Rect as R, type Size as S, type PlanningActionParamWaitFor as T, UIContext as U, type Color as V, type BaseAgentParserOpt as W, type PuppeteerParserOpt as X, type PlaywrightParserOpt as Y, type ExecutionRecorderItem as Z, type ExecutionTaskType as _, type ExecutionTaskProgressOptions as a, type TaskCacheInfo as a0, type ExecutionTaskReturn as a1, type ExecutionTaskInsightLocateParam as a2, type ExecutionTaskInsightLocateOutput as a3, type ExecutionTaskInsightDumpLog as a4, type ExecutionTaskInsightLocateApply as a5, type ExecutionTaskInsightLocate as a6, type ExecutionTaskInsightQueryParam as a7, type ExecutionTaskInsightQueryOutput as a8, type ExecutionTaskInsightQueryApply as a9, type ExecutionTaskInsightQuery as aa, type ExecutionTaskInsightAssertionParam as ab, type ExecutionTaskInsightAssertionApply as ac, type ExecutionTaskInsightAssertion as ad, type ExecutionTaskActionApply as ae, type ExecutionTaskAction as af, type ExecutionTaskPlanningApply as ag, type ExecutionTaskPlanning as ah, type GroupedActionDump as ai, type ExecutionTaskApply as b, type ExecutionDump as c, type InsightOptions as d, type InsightTaskInfo as e, type InsightAssertionResponse as f, type AIUsageInfo as g, AIResponseFormat as h, type AISingleElementResponseById as i, type AISingleElementResponseByPosition as j, type AIElementIdResponse as k, type AIElementResponse as l, type AISectionParseResponse as m, type AIAssertionResponse as n, type EnsureObject as o, type InsightExtractParam as p, type DumpMeta as q, type ReportDumpWithAttributes as r, type InsightDump as s, type PartialInsightDumpFromSDK as t, type ElementById as u, type AgentWaitForOpt as v, type AgentAssertOpt as w, type PlanningLocateParam as x, type PlanningAction as y, type PlanningAIResponse as z };
@@ -1,4 +1,4 @@
1
- import { s as ReportDumpWithAttributes, R as Rect } from './types-55182ae1.js';
1
+ import { r as ReportDumpWithAttributes, R as Rect } from './types-7702c41a.js';
2
2
  import 'openai/resources';
3
3
 
4
4
  declare const insightDumpFileExt = "insight-dump.json";
package/dist/lib/utils.js CHANGED
@@ -303,7 +303,7 @@ function stringifyDumpData(data, indents) {
303
303
  return JSON.stringify(data, replacerForPageObject, indents);
304
304
  }
305
305
  function getVersion() {
306
- return "0.8.16";
306
+ return "0.8.17-beta-20250103010342.0";
307
307
  }
308
308
  function debugLog(...message) {
309
309
  const debugMode = getAIConfig(MIDSCENE_DEBUG_MODE);
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@midscene/core",
3
- "description": "An AI-powered automation SDK can control the page, perform assertions, and extract data in JSON format using natural language. See https://midscenejs.com/ for details.",
4
- "version": "0.8.16",
3
+ "description": "Automate browser actions, extract data, and perform assertions using AI. It offers JavaScript SDK, Chrome extension, and support for scripting in YAML. See https://midscenejs.com/ for details.",
4
+ "version": "0.8.17-beta-20250103010342.0",
5
5
  "repository": "https://github.com/web-infra-dev/midscene",
6
6
  "homepage": "https://midscenejs.com/",
7
7
  "jsnext:source": "./src/index.ts",
@@ -38,11 +38,13 @@
38
38
  "dependencies": {
39
39
  "@anthropic-ai/sdk": "0.33.1",
40
40
  "@azure/identity": "4.5.0",
41
+ "@langchain/core": "0.3.26",
41
42
  "dirty-json": "0.9.2",
43
+ "langchain": "0.3.8",
42
44
  "openai": "4.57.1",
43
45
  "optional": "0.1.4",
44
46
  "socks-proxy-agent": "8.0.4",
45
- "@midscene/shared": "0.8.16"
47
+ "@midscene/shared": "0.8.17-beta-20250103010342.0"
46
48
  },
47
49
  "devDependencies": {
48
50
  "@modern-js/module-tools": "2.60.6",
@@ -71,6 +73,7 @@
71
73
  "test:ai": "AITEST=true npm run test",
72
74
  "computer": "TEST_COMPUTER=true npm run test:ai -- tests/ai/evaluate/computer.test.ts",
73
75
  "evaluate": "npm run test:ai -- tests/ai/evaluate/inspect.test.ts",
76
+ "prompt": "npm run test:ai -- tests/ai/inspect2.test.ts",
74
77
  "evaluate:update": "UPDATE_AI_DATA=true npm run test:ai -- tests/ai/evaluate/inspect.test.ts"
75
78
  }
76
79
  }