@midscene/shared 0.16.8 → 0.16.9-beta-20250507095704.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (17) hide show
  1. package/dist/es/common.js +5 -1
  2. package/dist/es/env.d.ts +6 -1
  3. package/dist/es/env.js +16 -1
  4. package/dist/es/img.d.ts +6 -1
  5. package/dist/es/img.js +21 -13
  6. package/dist/es/logger.js +5 -1
  7. package/dist/lib/common.js +5 -1
  8. package/dist/lib/env.d.ts +6 -1
  9. package/dist/lib/env.js +19 -1
  10. package/dist/lib/img.d.ts +6 -1
  11. package/dist/lib/img.js +21 -13
  12. package/dist/lib/logger.js +5 -1
  13. package/dist/types/env.d.ts +6 -1
  14. package/dist/types/img.d.ts +6 -1
  15. package/package.json +1 -1
  16. package/src/env.ts +17 -0
  17. package/src/img/box-select.ts +31 -17
package/dist/es/common.js CHANGED
@@ -17,6 +17,7 @@ var MIDSCENE_OPENAI_SOCKS_PROXY = "MIDSCENE_OPENAI_SOCKS_PROXY";
17
17
  var OPENAI_API_KEY = "OPENAI_API_KEY";
18
18
  var OPENAI_BASE_URL = "OPENAI_BASE_URL";
19
19
  var OPENAI_MAX_TOKENS = "OPENAI_MAX_TOKENS";
20
+ var MIDSCENE_ADB_PATH = "MIDSCENE_ADB_PATH";
20
21
  var MIDSCENE_CACHE = "MIDSCENE_CACHE";
21
22
  var MIDSCENE_USE_VLM_UI_TARS = "MIDSCENE_USE_VLM_UI_TARS";
22
23
  var MIDSCENE_USE_QWEN_VL = "MIDSCENE_USE_QWEN_VL";
@@ -25,6 +26,7 @@ var MIDSCENE_USE_GEMINI = "MIDSCENE_USE_GEMINI";
25
26
  var MIDSCENE_USE_VL_MODEL = "MIDSCENE_USE_VL_MODEL";
26
27
  var MATCH_BY_POSITION = "MATCH_BY_POSITION";
27
28
  var MIDSCENE_REPORT_TAG_NAME = "MIDSCENE_REPORT_TAG_NAME";
29
+ var MIDSCENE_PREFERRED_LANGUAGE = "MIDSCENE_PREFERRED_LANGUAGE";
28
30
  var MIDSCENE_USE_AZURE_OPENAI = "MIDSCENE_USE_AZURE_OPENAI";
29
31
  var MIDSCENE_AZURE_OPENAI_SCOPE = "MIDSCENE_AZURE_OPENAI_SCOPE";
30
32
  var MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON = "MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON";
@@ -50,6 +52,7 @@ var allConfigFromEnv = () => {
50
52
  [OPENAI_BASE_URL]: process.env[OPENAI_BASE_URL] || void 0,
51
53
  [OPENAI_MAX_TOKENS]: process.env[OPENAI_MAX_TOKENS] || void 0,
52
54
  [OPENAI_USE_AZURE]: process.env[OPENAI_USE_AZURE] || void 0,
55
+ [MIDSCENE_ADB_PATH]: process.env[MIDSCENE_ADB_PATH] || void 0,
53
56
  [MIDSCENE_CACHE]: process.env[MIDSCENE_CACHE] || void 0,
54
57
  [MATCH_BY_POSITION]: process.env[MATCH_BY_POSITION] || void 0,
55
58
  [MIDSCENE_REPORT_TAG_NAME]: process.env[MIDSCENE_REPORT_TAG_NAME] || void 0,
@@ -69,7 +72,8 @@ var allConfigFromEnv = () => {
69
72
  [AZURE_OPENAI_API_VERSION]: process.env[AZURE_OPENAI_API_VERSION] || void 0,
70
73
  [AZURE_OPENAI_DEPLOYMENT]: process.env[AZURE_OPENAI_DEPLOYMENT] || void 0,
71
74
  [MIDSCENE_MCP_USE_PUPPETEER_MODE]: process.env[MIDSCENE_MCP_USE_PUPPETEER_MODE] || void 0,
72
- [MIDSCENE_RUN_DIR]: process.env[MIDSCENE_RUN_DIR] || void 0
75
+ [MIDSCENE_RUN_DIR]: process.env[MIDSCENE_RUN_DIR] || void 0,
76
+ [MIDSCENE_PREFERRED_LANGUAGE]: process.env[MIDSCENE_PREFERRED_LANGUAGE] || void 0
73
77
  };
74
78
  };
75
79
  var globalConfig = null;
package/dist/es/env.d.ts CHANGED
@@ -11,6 +11,7 @@ declare const MIDSCENE_OPENAI_SOCKS_PROXY = "MIDSCENE_OPENAI_SOCKS_PROXY";
11
11
  declare const OPENAI_API_KEY = "OPENAI_API_KEY";
12
12
  declare const OPENAI_BASE_URL = "OPENAI_BASE_URL";
13
13
  declare const OPENAI_MAX_TOKENS = "OPENAI_MAX_TOKENS";
14
+ declare const MIDSCENE_ADB_PATH = "MIDSCENE_ADB_PATH";
14
15
  declare const MIDSCENE_CACHE = "MIDSCENE_CACHE";
15
16
  declare const MIDSCENE_USE_VLM_UI_TARS = "MIDSCENE_USE_VLM_UI_TARS";
16
17
  declare const MIDSCENE_USE_QWEN_VL = "MIDSCENE_USE_QWEN_VL";
@@ -20,6 +21,7 @@ declare const MIDSCENE_USE_VL_MODEL = "MIDSCENE_USE_VL_MODEL";
20
21
  declare const MATCH_BY_POSITION = "MATCH_BY_POSITION";
21
22
  declare const MIDSCENE_API_TYPE = "MIDSCENE-API-TYPE";
22
23
  declare const MIDSCENE_REPORT_TAG_NAME = "MIDSCENE_REPORT_TAG_NAME";
24
+ declare const MIDSCENE_PREFERRED_LANGUAGE = "MIDSCENE_PREFERRED_LANGUAGE";
23
25
  declare const MIDSCENE_USE_AZURE_OPENAI = "MIDSCENE_USE_AZURE_OPENAI";
24
26
  declare const MIDSCENE_AZURE_OPENAI_SCOPE = "MIDSCENE_AZURE_OPENAI_SCOPE";
25
27
  declare const MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON = "MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON";
@@ -44,6 +46,7 @@ declare const allConfigFromEnv: () => {
44
46
  OPENAI_BASE_URL: string | undefined;
45
47
  OPENAI_MAX_TOKENS: string | undefined;
46
48
  OPENAI_USE_AZURE: string | undefined;
49
+ MIDSCENE_ADB_PATH: string | undefined;
47
50
  MIDSCENE_CACHE: string | undefined;
48
51
  MATCH_BY_POSITION: string | undefined;
49
52
  MIDSCENE_REPORT_TAG_NAME: string | undefined;
@@ -64,6 +67,7 @@ declare const allConfigFromEnv: () => {
64
67
  AZURE_OPENAI_DEPLOYMENT: string | undefined;
65
68
  MIDSCENE_MCP_USE_PUPPETEER_MODE: string | undefined;
66
69
  MIDSCENE_RUN_DIR: string | undefined;
70
+ MIDSCENE_PREFERRED_LANGUAGE: string | undefined;
67
71
  };
68
72
  declare enum UITarsModelVersion {
69
73
  V1_0 = "1.0",
@@ -77,5 +81,6 @@ declare const getAIConfig: (configKey: keyof ReturnType<typeof allConfigFromEnv>
77
81
  declare const getAIConfigInBoolean: (configKey: keyof ReturnType<typeof allConfigFromEnv>) => boolean;
78
82
  declare const getAIConfigInJson: (configKey: keyof ReturnType<typeof allConfigFromEnv>) => any;
79
83
  declare const overrideAIConfig: (newConfig: Partial<ReturnType<typeof allConfigFromEnv>>, extendMode?: boolean) => void;
84
+ declare const getPreferredLanguage: () => string | undefined;
80
85
 
81
- export { ANTHROPIC_API_KEY, AZURE_OPENAI_API_VERSION, AZURE_OPENAI_DEPLOYMENT, AZURE_OPENAI_ENDPOINT, AZURE_OPENAI_KEY, MATCH_BY_POSITION, MIDSCENE_API_TYPE, MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_AZURE_OPENAI_SCOPE, MIDSCENE_CACHE, MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG, MIDSCENE_DEBUG_AI_PROFILE, MIDSCENE_DEBUG_AI_RESPONSE, MIDSCENE_DEBUG_MODE, MIDSCENE_FORCE_DEEP_THINK, MIDSCENE_LANGSMITH_DEBUG, MIDSCENE_MCP_USE_PUPPETEER_MODE, MIDSCENE_MODEL_NAME, MIDSCENE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_OPENAI_SOCKS_PROXY, MIDSCENE_REPORT_TAG_NAME, MIDSCENE_RUN_DIR, MIDSCENE_USE_ANTHROPIC_SDK, MIDSCENE_USE_AZURE_OPENAI, MIDSCENE_USE_DOUBAO_VISION, MIDSCENE_USE_GEMINI, MIDSCENE_USE_QWEN_VL, MIDSCENE_USE_VLM_UI_TARS, MIDSCENE_USE_VL_MODEL, OPENAI_API_KEY, OPENAI_BASE_URL, OPENAI_MAX_TOKENS, OPENAI_USE_AZURE, UITarsModelVersion, allConfigFromEnv, getAIConfig, getAIConfigInBoolean, getAIConfigInJson, overrideAIConfig, uiTarsModelVersion, vlLocateMode };
86
+ export { ANTHROPIC_API_KEY, AZURE_OPENAI_API_VERSION, AZURE_OPENAI_DEPLOYMENT, AZURE_OPENAI_ENDPOINT, AZURE_OPENAI_KEY, MATCH_BY_POSITION, MIDSCENE_ADB_PATH, MIDSCENE_API_TYPE, MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_AZURE_OPENAI_SCOPE, MIDSCENE_CACHE, MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG, MIDSCENE_DEBUG_AI_PROFILE, MIDSCENE_DEBUG_AI_RESPONSE, MIDSCENE_DEBUG_MODE, MIDSCENE_FORCE_DEEP_THINK, MIDSCENE_LANGSMITH_DEBUG, MIDSCENE_MCP_USE_PUPPETEER_MODE, MIDSCENE_MODEL_NAME, MIDSCENE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_OPENAI_SOCKS_PROXY, MIDSCENE_PREFERRED_LANGUAGE, MIDSCENE_REPORT_TAG_NAME, MIDSCENE_RUN_DIR, MIDSCENE_USE_ANTHROPIC_SDK, MIDSCENE_USE_AZURE_OPENAI, MIDSCENE_USE_DOUBAO_VISION, MIDSCENE_USE_GEMINI, MIDSCENE_USE_QWEN_VL, MIDSCENE_USE_VLM_UI_TARS, MIDSCENE_USE_VL_MODEL, OPENAI_API_KEY, OPENAI_BASE_URL, OPENAI_MAX_TOKENS, OPENAI_USE_AZURE, UITarsModelVersion, allConfigFromEnv, getAIConfig, getAIConfigInBoolean, getAIConfigInJson, getPreferredLanguage, overrideAIConfig, uiTarsModelVersion, vlLocateMode };
package/dist/es/env.js CHANGED
@@ -12,6 +12,7 @@ var MIDSCENE_OPENAI_SOCKS_PROXY = "MIDSCENE_OPENAI_SOCKS_PROXY";
12
12
  var OPENAI_API_KEY = "OPENAI_API_KEY";
13
13
  var OPENAI_BASE_URL = "OPENAI_BASE_URL";
14
14
  var OPENAI_MAX_TOKENS = "OPENAI_MAX_TOKENS";
15
+ var MIDSCENE_ADB_PATH = "MIDSCENE_ADB_PATH";
15
16
  var MIDSCENE_CACHE = "MIDSCENE_CACHE";
16
17
  var MIDSCENE_USE_VLM_UI_TARS = "MIDSCENE_USE_VLM_UI_TARS";
17
18
  var MIDSCENE_USE_QWEN_VL = "MIDSCENE_USE_QWEN_VL";
@@ -21,6 +22,7 @@ var MIDSCENE_USE_VL_MODEL = "MIDSCENE_USE_VL_MODEL";
21
22
  var MATCH_BY_POSITION = "MATCH_BY_POSITION";
22
23
  var MIDSCENE_API_TYPE = "MIDSCENE-API-TYPE";
23
24
  var MIDSCENE_REPORT_TAG_NAME = "MIDSCENE_REPORT_TAG_NAME";
25
+ var MIDSCENE_PREFERRED_LANGUAGE = "MIDSCENE_PREFERRED_LANGUAGE";
24
26
  var MIDSCENE_USE_AZURE_OPENAI = "MIDSCENE_USE_AZURE_OPENAI";
25
27
  var MIDSCENE_AZURE_OPENAI_SCOPE = "MIDSCENE_AZURE_OPENAI_SCOPE";
26
28
  var MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON = "MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON";
@@ -46,6 +48,7 @@ var allConfigFromEnv = () => {
46
48
  [OPENAI_BASE_URL]: process.env[OPENAI_BASE_URL] || void 0,
47
49
  [OPENAI_MAX_TOKENS]: process.env[OPENAI_MAX_TOKENS] || void 0,
48
50
  [OPENAI_USE_AZURE]: process.env[OPENAI_USE_AZURE] || void 0,
51
+ [MIDSCENE_ADB_PATH]: process.env[MIDSCENE_ADB_PATH] || void 0,
49
52
  [MIDSCENE_CACHE]: process.env[MIDSCENE_CACHE] || void 0,
50
53
  [MATCH_BY_POSITION]: process.env[MATCH_BY_POSITION] || void 0,
51
54
  [MIDSCENE_REPORT_TAG_NAME]: process.env[MIDSCENE_REPORT_TAG_NAME] || void 0,
@@ -65,7 +68,8 @@ var allConfigFromEnv = () => {
65
68
  [AZURE_OPENAI_API_VERSION]: process.env[AZURE_OPENAI_API_VERSION] || void 0,
66
69
  [AZURE_OPENAI_DEPLOYMENT]: process.env[AZURE_OPENAI_DEPLOYMENT] || void 0,
67
70
  [MIDSCENE_MCP_USE_PUPPETEER_MODE]: process.env[MIDSCENE_MCP_USE_PUPPETEER_MODE] || void 0,
68
- [MIDSCENE_RUN_DIR]: process.env[MIDSCENE_RUN_DIR] || void 0
71
+ [MIDSCENE_RUN_DIR]: process.env[MIDSCENE_RUN_DIR] || void 0,
72
+ [MIDSCENE_PREFERRED_LANGUAGE]: process.env[MIDSCENE_PREFERRED_LANGUAGE] || void 0
69
73
  };
70
74
  };
71
75
  var globalConfig = null;
@@ -169,6 +173,14 @@ var overrideAIConfig = (newConfig, extendMode = false) => {
169
173
  const currentConfig = getGlobalConfig();
170
174
  globalConfig = extendMode ? { ...currentConfig, ...newConfig } : { ...newConfig };
171
175
  };
176
+ var getPreferredLanguage = () => {
177
+ if (getAIConfig(MIDSCENE_PREFERRED_LANGUAGE)) {
178
+ return getAIConfig(MIDSCENE_PREFERRED_LANGUAGE);
179
+ }
180
+ const timeZone = Intl.DateTimeFormat().resolvedOptions().timeZone;
181
+ const isChina = timeZone === "Asia/Shanghai";
182
+ return isChina ? "Chinese" : "English";
183
+ };
172
184
  export {
173
185
  ANTHROPIC_API_KEY,
174
186
  AZURE_OPENAI_API_VERSION,
@@ -176,6 +188,7 @@ export {
176
188
  AZURE_OPENAI_ENDPOINT,
177
189
  AZURE_OPENAI_KEY,
178
190
  MATCH_BY_POSITION,
191
+ MIDSCENE_ADB_PATH,
179
192
  MIDSCENE_API_TYPE,
180
193
  MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON,
181
194
  MIDSCENE_AZURE_OPENAI_SCOPE,
@@ -190,6 +203,7 @@ export {
190
203
  MIDSCENE_MODEL_NAME,
191
204
  MIDSCENE_OPENAI_INIT_CONFIG_JSON,
192
205
  MIDSCENE_OPENAI_SOCKS_PROXY,
206
+ MIDSCENE_PREFERRED_LANGUAGE,
193
207
  MIDSCENE_REPORT_TAG_NAME,
194
208
  MIDSCENE_RUN_DIR,
195
209
  MIDSCENE_USE_ANTHROPIC_SDK,
@@ -208,6 +222,7 @@ export {
208
222
  getAIConfig,
209
223
  getAIConfigInBoolean,
210
224
  getAIConfigInJson,
225
+ getPreferredLanguage,
211
226
  overrideAIConfig,
212
227
  uiTarsModelVersion,
213
228
  vlLocateMode
package/dist/es/img.d.ts CHANGED
@@ -114,14 +114,19 @@ declare function paddingToMatchBlockByBase64(imageBase64: string, blockSize?: nu
114
114
  declare function cropByRect(imageBase64: string, rect: Rect, paddingImage: boolean): Promise<string>;
115
115
  declare function jimpToBase64(image: Jimp): Promise<string>;
116
116
 
117
+ interface ElementForOverlay {
118
+ rect: Rect;
119
+ indexId?: number;
120
+ }
117
121
  declare const compositeElementInfoImg: (options: {
118
122
  inputImgBase64: string;
119
- elementsPositionInfo: Array<BaseElement>;
123
+ elementsPositionInfo: Array<ElementForOverlay>;
120
124
  size?: {
121
125
  width: number;
122
126
  height: number;
123
127
  };
124
128
  annotationPadding?: number;
129
+ borderThickness?: number;
125
130
  }) => Promise<string>;
126
131
  declare const processImageElementInfo: (options: {
127
132
  inputImgBase64: string;
package/dist/es/img.js CHANGED
@@ -229,7 +229,7 @@ var loadFonts = async () => {
229
229
  return fonts;
230
230
  }
231
231
  };
232
- var createSvgOverlay = async (elements, imageWidth, imageHeight, boxPadding = 5) => {
232
+ var createSvgOverlay = async (elements, imageWidth, imageHeight, boxPadding = 5, borderThickness = 2) => {
233
233
  const Jimp = await getJimp();
234
234
  const image = new Jimp(imageWidth, imageHeight, 0);
235
235
  const colors = [
@@ -247,17 +247,21 @@ var createSvgOverlay = async (elements, imageWidth, imageHeight, boxPadding = 5)
247
247
  for (let index = 0; index < elements.length; index++) {
248
248
  const element = elements[index];
249
249
  const color = colors[index % colors.length];
250
+ const paddedLeft = Math.max(0, element.rect.left - boxPadding);
251
+ const paddedTop = Math.max(0, element.rect.top - boxPadding);
252
+ const paddedWidth = Math.min(
253
+ imageWidth - paddedLeft,
254
+ element.rect.width + boxPadding * 2
255
+ );
256
+ const paddedHeight = Math.min(
257
+ imageHeight - paddedTop,
258
+ element.rect.height + boxPadding * 2
259
+ );
250
260
  const paddedRect = {
251
- left: Math.max(0, element.rect.left - boxPadding),
252
- top: Math.max(0, element.rect.top - boxPadding),
253
- width: Math.min(
254
- imageWidth - element.rect.left,
255
- element.rect.width + boxPadding * 2
256
- ),
257
- height: Math.min(
258
- imageHeight - element.rect.top,
259
- element.rect.height + boxPadding * 2
260
- )
261
+ left: paddedLeft,
262
+ top: paddedTop,
263
+ width: paddedWidth,
264
+ height: paddedHeight
261
265
  };
262
266
  image.scan(
263
267
  paddedRect.left,
@@ -265,7 +269,10 @@ var createSvgOverlay = async (elements, imageWidth, imageHeight, boxPadding = 5)
265
269
  paddedRect.width,
266
270
  paddedRect.height,
267
271
  (x, y, idx) => {
268
- if (x === paddedRect.left || x === paddedRect.left + paddedRect.width - 1 || y === paddedRect.top || y === paddedRect.top + paddedRect.height - 1) {
272
+ if (x >= paddedRect.left && x < paddedRect.left + borderThickness || // Left border
273
+ x <= paddedRect.left + paddedRect.width - 1 && x > paddedRect.left + paddedRect.width - borderThickness || // Right border
274
+ y >= paddedRect.top && y < paddedRect.top + borderThickness || // Top border
275
+ y <= paddedRect.top + paddedRect.height - 1 && y > paddedRect.top + paddedRect.height - borderThickness) {
269
276
  image.bitmap.data[idx + 0] = color.rect >> 24 & 255;
270
277
  image.bitmap.data[idx + 1] = color.rect >> 16 & 255;
271
278
  image.bitmap.data[idx + 2] = color.rect >> 8 & 255;
@@ -373,7 +380,8 @@ var compositeElementInfoImg = async (options) => {
373
380
  elementsPositionInfo,
374
381
  width,
375
382
  height,
376
- options.annotationPadding
383
+ options.annotationPadding,
384
+ options.borderThickness
377
385
  );
378
386
  const svgImage = await Jimp.read(svgOverlay);
379
387
  const compositeImage = await image.composite(svgImage, 0, 0, {
package/dist/es/logger.js CHANGED
@@ -22,6 +22,7 @@ var MIDSCENE_OPENAI_SOCKS_PROXY = "MIDSCENE_OPENAI_SOCKS_PROXY";
22
22
  var OPENAI_API_KEY = "OPENAI_API_KEY";
23
23
  var OPENAI_BASE_URL = "OPENAI_BASE_URL";
24
24
  var OPENAI_MAX_TOKENS = "OPENAI_MAX_TOKENS";
25
+ var MIDSCENE_ADB_PATH = "MIDSCENE_ADB_PATH";
25
26
  var MIDSCENE_CACHE = "MIDSCENE_CACHE";
26
27
  var MIDSCENE_USE_VLM_UI_TARS = "MIDSCENE_USE_VLM_UI_TARS";
27
28
  var MIDSCENE_USE_QWEN_VL = "MIDSCENE_USE_QWEN_VL";
@@ -30,6 +31,7 @@ var MIDSCENE_USE_GEMINI = "MIDSCENE_USE_GEMINI";
30
31
  var MIDSCENE_USE_VL_MODEL = "MIDSCENE_USE_VL_MODEL";
31
32
  var MATCH_BY_POSITION = "MATCH_BY_POSITION";
32
33
  var MIDSCENE_REPORT_TAG_NAME = "MIDSCENE_REPORT_TAG_NAME";
34
+ var MIDSCENE_PREFERRED_LANGUAGE = "MIDSCENE_PREFERRED_LANGUAGE";
33
35
  var MIDSCENE_USE_AZURE_OPENAI = "MIDSCENE_USE_AZURE_OPENAI";
34
36
  var MIDSCENE_AZURE_OPENAI_SCOPE = "MIDSCENE_AZURE_OPENAI_SCOPE";
35
37
  var MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON = "MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON";
@@ -55,6 +57,7 @@ var allConfigFromEnv = () => {
55
57
  [OPENAI_BASE_URL]: process.env[OPENAI_BASE_URL] || void 0,
56
58
  [OPENAI_MAX_TOKENS]: process.env[OPENAI_MAX_TOKENS] || void 0,
57
59
  [OPENAI_USE_AZURE]: process.env[OPENAI_USE_AZURE] || void 0,
60
+ [MIDSCENE_ADB_PATH]: process.env[MIDSCENE_ADB_PATH] || void 0,
58
61
  [MIDSCENE_CACHE]: process.env[MIDSCENE_CACHE] || void 0,
59
62
  [MATCH_BY_POSITION]: process.env[MATCH_BY_POSITION] || void 0,
60
63
  [MIDSCENE_REPORT_TAG_NAME]: process.env[MIDSCENE_REPORT_TAG_NAME] || void 0,
@@ -74,7 +77,8 @@ var allConfigFromEnv = () => {
74
77
  [AZURE_OPENAI_API_VERSION]: process.env[AZURE_OPENAI_API_VERSION] || void 0,
75
78
  [AZURE_OPENAI_DEPLOYMENT]: process.env[AZURE_OPENAI_DEPLOYMENT] || void 0,
76
79
  [MIDSCENE_MCP_USE_PUPPETEER_MODE]: process.env[MIDSCENE_MCP_USE_PUPPETEER_MODE] || void 0,
77
- [MIDSCENE_RUN_DIR]: process.env[MIDSCENE_RUN_DIR] || void 0
80
+ [MIDSCENE_RUN_DIR]: process.env[MIDSCENE_RUN_DIR] || void 0,
81
+ [MIDSCENE_PREFERRED_LANGUAGE]: process.env[MIDSCENE_PREFERRED_LANGUAGE] || void 0
78
82
  };
79
83
  };
80
84
  var globalConfig = null;
package/dist/lib/common.js CHANGED
@@ -56,6 +56,7 @@ var MIDSCENE_OPENAI_SOCKS_PROXY = "MIDSCENE_OPENAI_SOCKS_PROXY";
56
56
  var OPENAI_API_KEY = "OPENAI_API_KEY";
57
57
  var OPENAI_BASE_URL = "OPENAI_BASE_URL";
58
58
  var OPENAI_MAX_TOKENS = "OPENAI_MAX_TOKENS";
59
+ var MIDSCENE_ADB_PATH = "MIDSCENE_ADB_PATH";
59
60
  var MIDSCENE_CACHE = "MIDSCENE_CACHE";
60
61
  var MIDSCENE_USE_VLM_UI_TARS = "MIDSCENE_USE_VLM_UI_TARS";
61
62
  var MIDSCENE_USE_QWEN_VL = "MIDSCENE_USE_QWEN_VL";
@@ -64,6 +65,7 @@ var MIDSCENE_USE_GEMINI = "MIDSCENE_USE_GEMINI";
64
65
  var MIDSCENE_USE_VL_MODEL = "MIDSCENE_USE_VL_MODEL";
65
66
  var MATCH_BY_POSITION = "MATCH_BY_POSITION";
66
67
  var MIDSCENE_REPORT_TAG_NAME = "MIDSCENE_REPORT_TAG_NAME";
68
+ var MIDSCENE_PREFERRED_LANGUAGE = "MIDSCENE_PREFERRED_LANGUAGE";
67
69
  var MIDSCENE_USE_AZURE_OPENAI = "MIDSCENE_USE_AZURE_OPENAI";
68
70
  var MIDSCENE_AZURE_OPENAI_SCOPE = "MIDSCENE_AZURE_OPENAI_SCOPE";
69
71
  var MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON = "MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON";
@@ -89,6 +91,7 @@ var allConfigFromEnv = () => {
89
91
  [OPENAI_BASE_URL]: process.env[OPENAI_BASE_URL] || void 0,
90
92
  [OPENAI_MAX_TOKENS]: process.env[OPENAI_MAX_TOKENS] || void 0,
91
93
  [OPENAI_USE_AZURE]: process.env[OPENAI_USE_AZURE] || void 0,
94
+ [MIDSCENE_ADB_PATH]: process.env[MIDSCENE_ADB_PATH] || void 0,
92
95
  [MIDSCENE_CACHE]: process.env[MIDSCENE_CACHE] || void 0,
93
96
  [MATCH_BY_POSITION]: process.env[MATCH_BY_POSITION] || void 0,
94
97
  [MIDSCENE_REPORT_TAG_NAME]: process.env[MIDSCENE_REPORT_TAG_NAME] || void 0,
@@ -108,7 +111,8 @@ var allConfigFromEnv = () => {
108
111
  [AZURE_OPENAI_API_VERSION]: process.env[AZURE_OPENAI_API_VERSION] || void 0,
109
112
  [AZURE_OPENAI_DEPLOYMENT]: process.env[AZURE_OPENAI_DEPLOYMENT] || void 0,
110
113
  [MIDSCENE_MCP_USE_PUPPETEER_MODE]: process.env[MIDSCENE_MCP_USE_PUPPETEER_MODE] || void 0,
111
- [MIDSCENE_RUN_DIR]: process.env[MIDSCENE_RUN_DIR] || void 0
114
+ [MIDSCENE_RUN_DIR]: process.env[MIDSCENE_RUN_DIR] || void 0,
115
+ [MIDSCENE_PREFERRED_LANGUAGE]: process.env[MIDSCENE_PREFERRED_LANGUAGE] || void 0
112
116
  };
113
117
  };
114
118
  var globalConfig = null;
package/dist/lib/env.d.ts CHANGED
@@ -11,6 +11,7 @@ declare const MIDSCENE_OPENAI_SOCKS_PROXY = "MIDSCENE_OPENAI_SOCKS_PROXY";
11
11
  declare const OPENAI_API_KEY = "OPENAI_API_KEY";
12
12
  declare const OPENAI_BASE_URL = "OPENAI_BASE_URL";
13
13
  declare const OPENAI_MAX_TOKENS = "OPENAI_MAX_TOKENS";
14
+ declare const MIDSCENE_ADB_PATH = "MIDSCENE_ADB_PATH";
14
15
  declare const MIDSCENE_CACHE = "MIDSCENE_CACHE";
15
16
  declare const MIDSCENE_USE_VLM_UI_TARS = "MIDSCENE_USE_VLM_UI_TARS";
16
17
  declare const MIDSCENE_USE_QWEN_VL = "MIDSCENE_USE_QWEN_VL";
@@ -20,6 +21,7 @@ declare const MIDSCENE_USE_VL_MODEL = "MIDSCENE_USE_VL_MODEL";
20
21
  declare const MATCH_BY_POSITION = "MATCH_BY_POSITION";
21
22
  declare const MIDSCENE_API_TYPE = "MIDSCENE-API-TYPE";
22
23
  declare const MIDSCENE_REPORT_TAG_NAME = "MIDSCENE_REPORT_TAG_NAME";
24
+ declare const MIDSCENE_PREFERRED_LANGUAGE = "MIDSCENE_PREFERRED_LANGUAGE";
23
25
  declare const MIDSCENE_USE_AZURE_OPENAI = "MIDSCENE_USE_AZURE_OPENAI";
24
26
  declare const MIDSCENE_AZURE_OPENAI_SCOPE = "MIDSCENE_AZURE_OPENAI_SCOPE";
25
27
  declare const MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON = "MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON";
@@ -44,6 +46,7 @@ declare const allConfigFromEnv: () => {
44
46
  OPENAI_BASE_URL: string | undefined;
45
47
  OPENAI_MAX_TOKENS: string | undefined;
46
48
  OPENAI_USE_AZURE: string | undefined;
49
+ MIDSCENE_ADB_PATH: string | undefined;
47
50
  MIDSCENE_CACHE: string | undefined;
48
51
  MATCH_BY_POSITION: string | undefined;
49
52
  MIDSCENE_REPORT_TAG_NAME: string | undefined;
@@ -64,6 +67,7 @@ declare const allConfigFromEnv: () => {
64
67
  AZURE_OPENAI_DEPLOYMENT: string | undefined;
65
68
  MIDSCENE_MCP_USE_PUPPETEER_MODE: string | undefined;
66
69
  MIDSCENE_RUN_DIR: string | undefined;
70
+ MIDSCENE_PREFERRED_LANGUAGE: string | undefined;
67
71
  };
68
72
  declare enum UITarsModelVersion {
69
73
  V1_0 = "1.0",
@@ -77,5 +81,6 @@ declare const getAIConfig: (configKey: keyof ReturnType<typeof allConfigFromEnv>
77
81
  declare const getAIConfigInBoolean: (configKey: keyof ReturnType<typeof allConfigFromEnv>) => boolean;
78
82
  declare const getAIConfigInJson: (configKey: keyof ReturnType<typeof allConfigFromEnv>) => any;
79
83
  declare const overrideAIConfig: (newConfig: Partial<ReturnType<typeof allConfigFromEnv>>, extendMode?: boolean) => void;
84
+ declare const getPreferredLanguage: () => string | undefined;
80
85
 
81
- export { ANTHROPIC_API_KEY, AZURE_OPENAI_API_VERSION, AZURE_OPENAI_DEPLOYMENT, AZURE_OPENAI_ENDPOINT, AZURE_OPENAI_KEY, MATCH_BY_POSITION, MIDSCENE_API_TYPE, MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_AZURE_OPENAI_SCOPE, MIDSCENE_CACHE, MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG, MIDSCENE_DEBUG_AI_PROFILE, MIDSCENE_DEBUG_AI_RESPONSE, MIDSCENE_DEBUG_MODE, MIDSCENE_FORCE_DEEP_THINK, MIDSCENE_LANGSMITH_DEBUG, MIDSCENE_MCP_USE_PUPPETEER_MODE, MIDSCENE_MODEL_NAME, MIDSCENE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_OPENAI_SOCKS_PROXY, MIDSCENE_REPORT_TAG_NAME, MIDSCENE_RUN_DIR, MIDSCENE_USE_ANTHROPIC_SDK, MIDSCENE_USE_AZURE_OPENAI, MIDSCENE_USE_DOUBAO_VISION, MIDSCENE_USE_GEMINI, MIDSCENE_USE_QWEN_VL, MIDSCENE_USE_VLM_UI_TARS, MIDSCENE_USE_VL_MODEL, OPENAI_API_KEY, OPENAI_BASE_URL, OPENAI_MAX_TOKENS, OPENAI_USE_AZURE, UITarsModelVersion, allConfigFromEnv, getAIConfig, getAIConfigInBoolean, getAIConfigInJson, overrideAIConfig, uiTarsModelVersion, vlLocateMode };
86
+ export { ANTHROPIC_API_KEY, AZURE_OPENAI_API_VERSION, AZURE_OPENAI_DEPLOYMENT, AZURE_OPENAI_ENDPOINT, AZURE_OPENAI_KEY, MATCH_BY_POSITION, MIDSCENE_ADB_PATH, MIDSCENE_API_TYPE, MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_AZURE_OPENAI_SCOPE, MIDSCENE_CACHE, MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG, MIDSCENE_DEBUG_AI_PROFILE, MIDSCENE_DEBUG_AI_RESPONSE, MIDSCENE_DEBUG_MODE, MIDSCENE_FORCE_DEEP_THINK, MIDSCENE_LANGSMITH_DEBUG, MIDSCENE_MCP_USE_PUPPETEER_MODE, MIDSCENE_MODEL_NAME, MIDSCENE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_OPENAI_SOCKS_PROXY, MIDSCENE_PREFERRED_LANGUAGE, MIDSCENE_REPORT_TAG_NAME, MIDSCENE_RUN_DIR, MIDSCENE_USE_ANTHROPIC_SDK, MIDSCENE_USE_AZURE_OPENAI, MIDSCENE_USE_DOUBAO_VISION, MIDSCENE_USE_GEMINI, MIDSCENE_USE_QWEN_VL, MIDSCENE_USE_VLM_UI_TARS, MIDSCENE_USE_VL_MODEL, OPENAI_API_KEY, OPENAI_BASE_URL, OPENAI_MAX_TOKENS, OPENAI_USE_AZURE, UITarsModelVersion, allConfigFromEnv, getAIConfig, getAIConfigInBoolean, getAIConfigInJson, getPreferredLanguage, overrideAIConfig, uiTarsModelVersion, vlLocateMode };
package/dist/lib/env.js CHANGED
@@ -26,6 +26,7 @@ __export(env_exports, {
26
26
  AZURE_OPENAI_ENDPOINT: () => AZURE_OPENAI_ENDPOINT,
27
27
  AZURE_OPENAI_KEY: () => AZURE_OPENAI_KEY,
28
28
  MATCH_BY_POSITION: () => MATCH_BY_POSITION,
29
+ MIDSCENE_ADB_PATH: () => MIDSCENE_ADB_PATH,
29
30
  MIDSCENE_API_TYPE: () => MIDSCENE_API_TYPE,
30
31
  MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON: () => MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON,
31
32
  MIDSCENE_AZURE_OPENAI_SCOPE: () => MIDSCENE_AZURE_OPENAI_SCOPE,
@@ -40,6 +41,7 @@ __export(env_exports, {
40
41
  MIDSCENE_MODEL_NAME: () => MIDSCENE_MODEL_NAME,
41
42
  MIDSCENE_OPENAI_INIT_CONFIG_JSON: () => MIDSCENE_OPENAI_INIT_CONFIG_JSON,
42
43
  MIDSCENE_OPENAI_SOCKS_PROXY: () => MIDSCENE_OPENAI_SOCKS_PROXY,
44
+ MIDSCENE_PREFERRED_LANGUAGE: () => MIDSCENE_PREFERRED_LANGUAGE,
43
45
  MIDSCENE_REPORT_TAG_NAME: () => MIDSCENE_REPORT_TAG_NAME,
44
46
  MIDSCENE_RUN_DIR: () => MIDSCENE_RUN_DIR,
45
47
  MIDSCENE_USE_ANTHROPIC_SDK: () => MIDSCENE_USE_ANTHROPIC_SDK,
@@ -58,6 +60,7 @@ __export(env_exports, {
58
60
  getAIConfig: () => getAIConfig,
59
61
  getAIConfigInBoolean: () => getAIConfigInBoolean,
60
62
  getAIConfigInJson: () => getAIConfigInJson,
63
+ getPreferredLanguage: () => getPreferredLanguage,
61
64
  overrideAIConfig: () => overrideAIConfig,
62
65
  uiTarsModelVersion: () => uiTarsModelVersion,
63
66
  vlLocateMode: () => vlLocateMode
@@ -76,6 +79,7 @@ var MIDSCENE_OPENAI_SOCKS_PROXY = "MIDSCENE_OPENAI_SOCKS_PROXY";
76
79
  var OPENAI_API_KEY = "OPENAI_API_KEY";
77
80
  var OPENAI_BASE_URL = "OPENAI_BASE_URL";
78
81
  var OPENAI_MAX_TOKENS = "OPENAI_MAX_TOKENS";
82
+ var MIDSCENE_ADB_PATH = "MIDSCENE_ADB_PATH";
79
83
  var MIDSCENE_CACHE = "MIDSCENE_CACHE";
80
84
  var MIDSCENE_USE_VLM_UI_TARS = "MIDSCENE_USE_VLM_UI_TARS";
81
85
  var MIDSCENE_USE_QWEN_VL = "MIDSCENE_USE_QWEN_VL";
@@ -85,6 +89,7 @@ var MIDSCENE_USE_VL_MODEL = "MIDSCENE_USE_VL_MODEL";
85
89
  var MATCH_BY_POSITION = "MATCH_BY_POSITION";
86
90
  var MIDSCENE_API_TYPE = "MIDSCENE-API-TYPE";
87
91
  var MIDSCENE_REPORT_TAG_NAME = "MIDSCENE_REPORT_TAG_NAME";
92
+ var MIDSCENE_PREFERRED_LANGUAGE = "MIDSCENE_PREFERRED_LANGUAGE";
88
93
  var MIDSCENE_USE_AZURE_OPENAI = "MIDSCENE_USE_AZURE_OPENAI";
89
94
  var MIDSCENE_AZURE_OPENAI_SCOPE = "MIDSCENE_AZURE_OPENAI_SCOPE";
90
95
  var MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON = "MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON";
@@ -110,6 +115,7 @@ var allConfigFromEnv = () => {
110
115
  [OPENAI_BASE_URL]: process.env[OPENAI_BASE_URL] || void 0,
111
116
  [OPENAI_MAX_TOKENS]: process.env[OPENAI_MAX_TOKENS] || void 0,
112
117
  [OPENAI_USE_AZURE]: process.env[OPENAI_USE_AZURE] || void 0,
118
+ [MIDSCENE_ADB_PATH]: process.env[MIDSCENE_ADB_PATH] || void 0,
113
119
  [MIDSCENE_CACHE]: process.env[MIDSCENE_CACHE] || void 0,
114
120
  [MATCH_BY_POSITION]: process.env[MATCH_BY_POSITION] || void 0,
115
121
  [MIDSCENE_REPORT_TAG_NAME]: process.env[MIDSCENE_REPORT_TAG_NAME] || void 0,
@@ -129,7 +135,8 @@ var allConfigFromEnv = () => {
129
135
  [AZURE_OPENAI_API_VERSION]: process.env[AZURE_OPENAI_API_VERSION] || void 0,
130
136
  [AZURE_OPENAI_DEPLOYMENT]: process.env[AZURE_OPENAI_DEPLOYMENT] || void 0,
131
137
  [MIDSCENE_MCP_USE_PUPPETEER_MODE]: process.env[MIDSCENE_MCP_USE_PUPPETEER_MODE] || void 0,
132
- [MIDSCENE_RUN_DIR]: process.env[MIDSCENE_RUN_DIR] || void 0
138
+ [MIDSCENE_RUN_DIR]: process.env[MIDSCENE_RUN_DIR] || void 0,
139
+ [MIDSCENE_PREFERRED_LANGUAGE]: process.env[MIDSCENE_PREFERRED_LANGUAGE] || void 0
133
140
  };
134
141
  };
135
142
  var globalConfig = null;
@@ -233,6 +240,14 @@ var overrideAIConfig = (newConfig, extendMode = false) => {
233
240
  const currentConfig = getGlobalConfig();
234
241
  globalConfig = extendMode ? { ...currentConfig, ...newConfig } : { ...newConfig };
235
242
  };
243
+ var getPreferredLanguage = () => {
244
+ if (getAIConfig(MIDSCENE_PREFERRED_LANGUAGE)) {
245
+ return getAIConfig(MIDSCENE_PREFERRED_LANGUAGE);
246
+ }
247
+ const timeZone = Intl.DateTimeFormat().resolvedOptions().timeZone;
248
+ const isChina = timeZone === "Asia/Shanghai";
249
+ return isChina ? "Chinese" : "English";
250
+ };
236
251
  // Annotate the CommonJS export names for ESM import in node:
237
252
  0 && (module.exports = {
238
253
  ANTHROPIC_API_KEY,
@@ -241,6 +256,7 @@ var overrideAIConfig = (newConfig, extendMode = false) => {
241
256
  AZURE_OPENAI_ENDPOINT,
242
257
  AZURE_OPENAI_KEY,
243
258
  MATCH_BY_POSITION,
259
+ MIDSCENE_ADB_PATH,
244
260
  MIDSCENE_API_TYPE,
245
261
  MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON,
246
262
  MIDSCENE_AZURE_OPENAI_SCOPE,
@@ -255,6 +271,7 @@ var overrideAIConfig = (newConfig, extendMode = false) => {
255
271
  MIDSCENE_MODEL_NAME,
256
272
  MIDSCENE_OPENAI_INIT_CONFIG_JSON,
257
273
  MIDSCENE_OPENAI_SOCKS_PROXY,
274
+ MIDSCENE_PREFERRED_LANGUAGE,
258
275
  MIDSCENE_REPORT_TAG_NAME,
259
276
  MIDSCENE_RUN_DIR,
260
277
  MIDSCENE_USE_ANTHROPIC_SDK,
@@ -273,6 +290,7 @@ var overrideAIConfig = (newConfig, extendMode = false) => {
273
290
  getAIConfig,
274
291
  getAIConfigInBoolean,
275
292
  getAIConfigInJson,
293
+ getPreferredLanguage,
276
294
  overrideAIConfig,
277
295
  uiTarsModelVersion,
278
296
  vlLocateMode
package/dist/lib/img.d.ts CHANGED
@@ -114,14 +114,19 @@ declare function paddingToMatchBlockByBase64(imageBase64: string, blockSize?: nu
114
114
  declare function cropByRect(imageBase64: string, rect: Rect, paddingImage: boolean): Promise<string>;
115
115
  declare function jimpToBase64(image: Jimp): Promise<string>;
116
116
 
117
+ interface ElementForOverlay {
118
+ rect: Rect;
119
+ indexId?: number;
120
+ }
117
121
  declare const compositeElementInfoImg: (options: {
118
122
  inputImgBase64: string;
119
- elementsPositionInfo: Array<BaseElement>;
123
+ elementsPositionInfo: Array<ElementForOverlay>;
120
124
  size?: {
121
125
  width: number;
122
126
  height: number;
123
127
  };
124
128
  annotationPadding?: number;
129
+ borderThickness?: number;
125
130
  }) => Promise<string>;
126
131
  declare const processImageElementInfo: (options: {
127
132
  inputImgBase64: string;
package/dist/lib/img.js CHANGED
@@ -284,7 +284,7 @@ var loadFonts = async () => {
284
284
  return fonts;
285
285
  }
286
286
  };
287
- var createSvgOverlay = async (elements, imageWidth, imageHeight, boxPadding = 5) => {
287
+ var createSvgOverlay = async (elements, imageWidth, imageHeight, boxPadding = 5, borderThickness = 2) => {
288
288
  const Jimp = await getJimp();
289
289
  const image = new Jimp(imageWidth, imageHeight, 0);
290
290
  const colors = [
@@ -302,17 +302,21 @@ var createSvgOverlay = async (elements, imageWidth, imageHeight, boxPadding = 5)
302
302
  for (let index = 0; index < elements.length; index++) {
303
303
  const element = elements[index];
304
304
  const color = colors[index % colors.length];
305
+ const paddedLeft = Math.max(0, element.rect.left - boxPadding);
306
+ const paddedTop = Math.max(0, element.rect.top - boxPadding);
307
+ const paddedWidth = Math.min(
308
+ imageWidth - paddedLeft,
309
+ element.rect.width + boxPadding * 2
310
+ );
311
+ const paddedHeight = Math.min(
312
+ imageHeight - paddedTop,
313
+ element.rect.height + boxPadding * 2
314
+ );
305
315
  const paddedRect = {
306
- left: Math.max(0, element.rect.left - boxPadding),
307
- top: Math.max(0, element.rect.top - boxPadding),
308
- width: Math.min(
309
- imageWidth - element.rect.left,
310
- element.rect.width + boxPadding * 2
311
- ),
312
- height: Math.min(
313
- imageHeight - element.rect.top,
314
- element.rect.height + boxPadding * 2
315
- )
316
+ left: paddedLeft,
317
+ top: paddedTop,
318
+ width: paddedWidth,
319
+ height: paddedHeight
316
320
  };
317
321
  image.scan(
318
322
  paddedRect.left,
@@ -320,7 +324,10 @@ var createSvgOverlay = async (elements, imageWidth, imageHeight, boxPadding = 5)
320
324
  paddedRect.width,
321
325
  paddedRect.height,
322
326
  (x, y, idx) => {
323
- if (x === paddedRect.left || x === paddedRect.left + paddedRect.width - 1 || y === paddedRect.top || y === paddedRect.top + paddedRect.height - 1) {
327
+ if (x >= paddedRect.left && x < paddedRect.left + borderThickness || // Left border
328
+ x <= paddedRect.left + paddedRect.width - 1 && x > paddedRect.left + paddedRect.width - borderThickness || // Right border
329
+ y >= paddedRect.top && y < paddedRect.top + borderThickness || // Top border
330
+ y <= paddedRect.top + paddedRect.height - 1 && y > paddedRect.top + paddedRect.height - borderThickness) {
324
331
  image.bitmap.data[idx + 0] = color.rect >> 24 & 255;
325
332
  image.bitmap.data[idx + 1] = color.rect >> 16 & 255;
326
333
  image.bitmap.data[idx + 2] = color.rect >> 8 & 255;
@@ -428,7 +435,8 @@ var compositeElementInfoImg = async (options) => {
428
435
  elementsPositionInfo,
429
436
  width,
430
437
  height,
431
- options.annotationPadding
438
+ options.annotationPadding,
439
+ options.borderThickness
432
440
  );
433
441
  const svgImage = await Jimp.read(svgOverlay);
434
442
  const compositeImage = await image.composite(svgImage, 0, 0, {
package/dist/lib/logger.js CHANGED
@@ -58,6 +58,7 @@ var MIDSCENE_OPENAI_SOCKS_PROXY = "MIDSCENE_OPENAI_SOCKS_PROXY";
58
58
  var OPENAI_API_KEY = "OPENAI_API_KEY";
59
59
  var OPENAI_BASE_URL = "OPENAI_BASE_URL";
60
60
  var OPENAI_MAX_TOKENS = "OPENAI_MAX_TOKENS";
61
+ var MIDSCENE_ADB_PATH = "MIDSCENE_ADB_PATH";
61
62
  var MIDSCENE_CACHE = "MIDSCENE_CACHE";
62
63
  var MIDSCENE_USE_VLM_UI_TARS = "MIDSCENE_USE_VLM_UI_TARS";
63
64
  var MIDSCENE_USE_QWEN_VL = "MIDSCENE_USE_QWEN_VL";
@@ -66,6 +67,7 @@ var MIDSCENE_USE_GEMINI = "MIDSCENE_USE_GEMINI";
66
67
  var MIDSCENE_USE_VL_MODEL = "MIDSCENE_USE_VL_MODEL";
67
68
  var MATCH_BY_POSITION = "MATCH_BY_POSITION";
68
69
  var MIDSCENE_REPORT_TAG_NAME = "MIDSCENE_REPORT_TAG_NAME";
70
+ var MIDSCENE_PREFERRED_LANGUAGE = "MIDSCENE_PREFERRED_LANGUAGE";
69
71
  var MIDSCENE_USE_AZURE_OPENAI = "MIDSCENE_USE_AZURE_OPENAI";
70
72
  var MIDSCENE_AZURE_OPENAI_SCOPE = "MIDSCENE_AZURE_OPENAI_SCOPE";
71
73
  var MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON = "MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON";
@@ -91,6 +93,7 @@ var allConfigFromEnv = () => {
91
93
  [OPENAI_BASE_URL]: process.env[OPENAI_BASE_URL] || void 0,
92
94
  [OPENAI_MAX_TOKENS]: process.env[OPENAI_MAX_TOKENS] || void 0,
93
95
  [OPENAI_USE_AZURE]: process.env[OPENAI_USE_AZURE] || void 0,
96
+ [MIDSCENE_ADB_PATH]: process.env[MIDSCENE_ADB_PATH] || void 0,
94
97
  [MIDSCENE_CACHE]: process.env[MIDSCENE_CACHE] || void 0,
95
98
  [MATCH_BY_POSITION]: process.env[MATCH_BY_POSITION] || void 0,
96
99
  [MIDSCENE_REPORT_TAG_NAME]: process.env[MIDSCENE_REPORT_TAG_NAME] || void 0,
@@ -110,7 +113,8 @@ var allConfigFromEnv = () => {
110
113
  [AZURE_OPENAI_API_VERSION]: process.env[AZURE_OPENAI_API_VERSION] || void 0,
111
114
  [AZURE_OPENAI_DEPLOYMENT]: process.env[AZURE_OPENAI_DEPLOYMENT] || void 0,
112
115
  [MIDSCENE_MCP_USE_PUPPETEER_MODE]: process.env[MIDSCENE_MCP_USE_PUPPETEER_MODE] || void 0,
113
- [MIDSCENE_RUN_DIR]: process.env[MIDSCENE_RUN_DIR] || void 0
116
+ [MIDSCENE_RUN_DIR]: process.env[MIDSCENE_RUN_DIR] || void 0,
117
+ [MIDSCENE_PREFERRED_LANGUAGE]: process.env[MIDSCENE_PREFERRED_LANGUAGE] || void 0
114
118
  };
115
119
  };
116
120
  var globalConfig = null;
package/dist/types/env.d.ts CHANGED
@@ -11,6 +11,7 @@ declare const MIDSCENE_OPENAI_SOCKS_PROXY = "MIDSCENE_OPENAI_SOCKS_PROXY";
11
11
  declare const OPENAI_API_KEY = "OPENAI_API_KEY";
12
12
  declare const OPENAI_BASE_URL = "OPENAI_BASE_URL";
13
13
  declare const OPENAI_MAX_TOKENS = "OPENAI_MAX_TOKENS";
14
+ declare const MIDSCENE_ADB_PATH = "MIDSCENE_ADB_PATH";
14
15
  declare const MIDSCENE_CACHE = "MIDSCENE_CACHE";
15
16
  declare const MIDSCENE_USE_VLM_UI_TARS = "MIDSCENE_USE_VLM_UI_TARS";
16
17
  declare const MIDSCENE_USE_QWEN_VL = "MIDSCENE_USE_QWEN_VL";
@@ -20,6 +21,7 @@ declare const MIDSCENE_USE_VL_MODEL = "MIDSCENE_USE_VL_MODEL";
20
21
  declare const MATCH_BY_POSITION = "MATCH_BY_POSITION";
21
22
  declare const MIDSCENE_API_TYPE = "MIDSCENE-API-TYPE";
22
23
  declare const MIDSCENE_REPORT_TAG_NAME = "MIDSCENE_REPORT_TAG_NAME";
24
+ declare const MIDSCENE_PREFERRED_LANGUAGE = "MIDSCENE_PREFERRED_LANGUAGE";
23
25
  declare const MIDSCENE_USE_AZURE_OPENAI = "MIDSCENE_USE_AZURE_OPENAI";
24
26
  declare const MIDSCENE_AZURE_OPENAI_SCOPE = "MIDSCENE_AZURE_OPENAI_SCOPE";
25
27
  declare const MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON = "MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON";
@@ -44,6 +46,7 @@ declare const allConfigFromEnv: () => {
44
46
  OPENAI_BASE_URL: string | undefined;
45
47
  OPENAI_MAX_TOKENS: string | undefined;
46
48
  OPENAI_USE_AZURE: string | undefined;
49
+ MIDSCENE_ADB_PATH: string | undefined;
47
50
  MIDSCENE_CACHE: string | undefined;
48
51
  MATCH_BY_POSITION: string | undefined;
49
52
  MIDSCENE_REPORT_TAG_NAME: string | undefined;
@@ -64,6 +67,7 @@ declare const allConfigFromEnv: () => {
64
67
  AZURE_OPENAI_DEPLOYMENT: string | undefined;
65
68
  MIDSCENE_MCP_USE_PUPPETEER_MODE: string | undefined;
66
69
  MIDSCENE_RUN_DIR: string | undefined;
70
+ MIDSCENE_PREFERRED_LANGUAGE: string | undefined;
67
71
  };
68
72
  declare enum UITarsModelVersion {
69
73
  V1_0 = "1.0",
@@ -77,5 +81,6 @@ declare const getAIConfig: (configKey: keyof ReturnType<typeof allConfigFromEnv>
77
81
  declare const getAIConfigInBoolean: (configKey: keyof ReturnType<typeof allConfigFromEnv>) => boolean;
78
82
  declare const getAIConfigInJson: (configKey: keyof ReturnType<typeof allConfigFromEnv>) => any;
79
83
  declare const overrideAIConfig: (newConfig: Partial<ReturnType<typeof allConfigFromEnv>>, extendMode?: boolean) => void;
84
+ declare const getPreferredLanguage: () => string | undefined;
80
85
 
81
- export { ANTHROPIC_API_KEY, AZURE_OPENAI_API_VERSION, AZURE_OPENAI_DEPLOYMENT, AZURE_OPENAI_ENDPOINT, AZURE_OPENAI_KEY, MATCH_BY_POSITION, MIDSCENE_API_TYPE, MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_AZURE_OPENAI_SCOPE, MIDSCENE_CACHE, MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG, MIDSCENE_DEBUG_AI_PROFILE, MIDSCENE_DEBUG_AI_RESPONSE, MIDSCENE_DEBUG_MODE, MIDSCENE_FORCE_DEEP_THINK, MIDSCENE_LANGSMITH_DEBUG, MIDSCENE_MCP_USE_PUPPETEER_MODE, MIDSCENE_MODEL_NAME, MIDSCENE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_OPENAI_SOCKS_PROXY, MIDSCENE_REPORT_TAG_NAME, MIDSCENE_RUN_DIR, MIDSCENE_USE_ANTHROPIC_SDK, MIDSCENE_USE_AZURE_OPENAI, MIDSCENE_USE_DOUBAO_VISION, MIDSCENE_USE_GEMINI, MIDSCENE_USE_QWEN_VL, MIDSCENE_USE_VLM_UI_TARS, MIDSCENE_USE_VL_MODEL, OPENAI_API_KEY, OPENAI_BASE_URL, OPENAI_MAX_TOKENS, OPENAI_USE_AZURE, UITarsModelVersion, allConfigFromEnv, getAIConfig, getAIConfigInBoolean, getAIConfigInJson, overrideAIConfig, uiTarsModelVersion, vlLocateMode };
86
+ export { ANTHROPIC_API_KEY, AZURE_OPENAI_API_VERSION, AZURE_OPENAI_DEPLOYMENT, AZURE_OPENAI_ENDPOINT, AZURE_OPENAI_KEY, MATCH_BY_POSITION, MIDSCENE_ADB_PATH, MIDSCENE_API_TYPE, MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_AZURE_OPENAI_SCOPE, MIDSCENE_CACHE, MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG, MIDSCENE_DEBUG_AI_PROFILE, MIDSCENE_DEBUG_AI_RESPONSE, MIDSCENE_DEBUG_MODE, MIDSCENE_FORCE_DEEP_THINK, MIDSCENE_LANGSMITH_DEBUG, MIDSCENE_MCP_USE_PUPPETEER_MODE, MIDSCENE_MODEL_NAME, MIDSCENE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_OPENAI_SOCKS_PROXY, MIDSCENE_PREFERRED_LANGUAGE, MIDSCENE_REPORT_TAG_NAME, MIDSCENE_RUN_DIR, MIDSCENE_USE_ANTHROPIC_SDK, MIDSCENE_USE_AZURE_OPENAI, MIDSCENE_USE_DOUBAO_VISION, MIDSCENE_USE_GEMINI, MIDSCENE_USE_QWEN_VL, MIDSCENE_USE_VLM_UI_TARS, MIDSCENE_USE_VL_MODEL, OPENAI_API_KEY, OPENAI_BASE_URL, OPENAI_MAX_TOKENS, OPENAI_USE_AZURE, UITarsModelVersion, allConfigFromEnv, getAIConfig, getAIConfigInBoolean, getAIConfigInJson, getPreferredLanguage, overrideAIConfig, uiTarsModelVersion, vlLocateMode };
package/dist/types/img.d.ts CHANGED
@@ -114,14 +114,19 @@ declare function paddingToMatchBlockByBase64(imageBase64: string, blockSize?: nu
114
114
  declare function cropByRect(imageBase64: string, rect: Rect, paddingImage: boolean): Promise<string>;
115
115
  declare function jimpToBase64(image: Jimp): Promise<string>;
116
116
 
117
+ interface ElementForOverlay {
118
+ rect: Rect;
119
+ indexId?: number;
120
+ }
117
121
  declare const compositeElementInfoImg: (options: {
118
122
  inputImgBase64: string;
119
- elementsPositionInfo: Array<BaseElement>;
123
+ elementsPositionInfo: Array<ElementForOverlay>;
120
124
  size?: {
121
125
  width: number;
122
126
  height: number;
123
127
  };
124
128
  annotationPadding?: number;
129
+ borderThickness?: number;
125
130
  }) => Promise<string>;
126
131
  declare const processImageElementInfo: (options: {
127
132
  inputImgBase64: string;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@midscene/shared",
3
- "version": "0.16.8",
3
+ "version": "0.16.9-beta-20250507095704.0",
4
4
  "repository": "https://github.com/web-infra-dev/midscene",
5
5
  "homepage": "https://midscenejs.com/",
6
6
  "types": "./dist/types/index.d.ts",
package/src/env.ts CHANGED
@@ -18,6 +18,8 @@ export const OPENAI_API_KEY = 'OPENAI_API_KEY';
18
18
  export const OPENAI_BASE_URL = 'OPENAI_BASE_URL';
19
19
  export const OPENAI_MAX_TOKENS = 'OPENAI_MAX_TOKENS';
20
20
 
21
+ export const MIDSCENE_ADB_PATH = 'MIDSCENE_ADB_PATH';
22
+
21
23
  export const MIDSCENE_CACHE = 'MIDSCENE_CACHE';
22
24
  export const MIDSCENE_USE_VLM_UI_TARS = 'MIDSCENE_USE_VLM_UI_TARS';
23
25
  export const MIDSCENE_USE_QWEN_VL = 'MIDSCENE_USE_QWEN_VL';
@@ -28,6 +30,8 @@ export const MATCH_BY_POSITION = 'MATCH_BY_POSITION';
28
30
  export const MIDSCENE_API_TYPE = 'MIDSCENE-API-TYPE';
29
31
  export const MIDSCENE_REPORT_TAG_NAME = 'MIDSCENE_REPORT_TAG_NAME';
30
32
 
33
+ export const MIDSCENE_PREFERRED_LANGUAGE = 'MIDSCENE_PREFERRED_LANGUAGE';
34
+
31
35
  export const MIDSCENE_USE_AZURE_OPENAI = 'MIDSCENE_USE_AZURE_OPENAI';
32
36
  export const MIDSCENE_AZURE_OPENAI_SCOPE = 'MIDSCENE_AZURE_OPENAI_SCOPE';
33
37
  export const MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON =
@@ -66,6 +70,7 @@ export const allConfigFromEnv = () => {
66
70
  [OPENAI_BASE_URL]: process.env[OPENAI_BASE_URL] || undefined,
67
71
  [OPENAI_MAX_TOKENS]: process.env[OPENAI_MAX_TOKENS] || undefined,
68
72
  [OPENAI_USE_AZURE]: process.env[OPENAI_USE_AZURE] || undefined,
73
+ [MIDSCENE_ADB_PATH]: process.env[MIDSCENE_ADB_PATH] || undefined,
69
74
  [MIDSCENE_CACHE]: process.env[MIDSCENE_CACHE] || undefined,
70
75
  [MATCH_BY_POSITION]: process.env[MATCH_BY_POSITION] || undefined,
71
76
  [MIDSCENE_REPORT_TAG_NAME]:
@@ -97,6 +102,8 @@ export const allConfigFromEnv = () => {
97
102
  [MIDSCENE_MCP_USE_PUPPETEER_MODE]:
98
103
  process.env[MIDSCENE_MCP_USE_PUPPETEER_MODE] || undefined,
99
104
  [MIDSCENE_RUN_DIR]: process.env[MIDSCENE_RUN_DIR] || undefined,
105
+ [MIDSCENE_PREFERRED_LANGUAGE]:
106
+ process.env[MIDSCENE_PREFERRED_LANGUAGE] || undefined,
100
107
  };
101
108
  };
102
109
 
@@ -238,3 +245,13 @@ export const overrideAIConfig = (
238
245
  ? { ...currentConfig, ...newConfig }
239
246
  : { ...newConfig };
240
247
  };
248
+
249
+ export const getPreferredLanguage = () => {
250
+ if (getAIConfig(MIDSCENE_PREFERRED_LANGUAGE)) {
251
+ return getAIConfig(MIDSCENE_PREFERRED_LANGUAGE);
252
+ }
253
+
254
+ const timeZone = Intl.DateTimeFormat().resolvedOptions().timeZone;
255
+ const isChina = timeZone === 'Asia/Shanghai';
256
+ return isChina ? 'Chinese' : 'English';
257
+ };
package/src/img/box-select.ts CHANGED
@@ -1,6 +1,6 @@
1
1
  import assert from 'node:assert';
2
2
  import type Jimp from 'jimp';
3
- import type { BaseElement } from '../types';
3
+ import type { BaseElement, Rect } from '../types';
4
4
  import getJimp from './get-jimp';
5
5
  import { bufferFromBase64, imageInfoOfBase64 } from './index';
6
6
 
@@ -21,11 +21,17 @@ const loadFonts = async () => {
21
21
  }
22
22
  };
23
23
 
24
+ interface ElementForOverlay {
25
+ rect: Rect;
26
+ indexId?: number;
27
+ }
28
+
24
29
  const createSvgOverlay = async (
25
- elements: Array<BaseElement>,
30
+ elements: Array<ElementForOverlay>,
26
31
  imageWidth: number,
27
32
  imageHeight: number,
28
33
  boxPadding = 5,
34
+ borderThickness = 2,
29
35
  ): Promise<Jimp> => {
30
36
  const Jimp = await getJimp();
31
37
  const image = new Jimp(imageWidth, imageHeight, 0x00000000);
@@ -44,17 +50,21 @@ const createSvgOverlay = async (
44
50
  const color = colors[index % colors.length];
45
51
 
46
52
  // Add 5px padding to the rect
53
+ const paddedLeft = Math.max(0, element.rect.left - boxPadding);
54
+ const paddedTop = Math.max(0, element.rect.top - boxPadding);
55
+ const paddedWidth = Math.min(
56
+ imageWidth - paddedLeft,
57
+ element.rect.width + boxPadding * 2,
58
+ );
59
+ const paddedHeight = Math.min(
60
+ imageHeight - paddedTop,
61
+ element.rect.height + boxPadding * 2,
62
+ );
47
63
  const paddedRect = {
48
- left: Math.max(0, element.rect.left - boxPadding),
49
- top: Math.max(0, element.rect.top - boxPadding),
50
- width: Math.min(
51
- imageWidth - element.rect.left,
52
- element.rect.width + boxPadding * 2,
53
- ),
54
- height: Math.min(
55
- imageHeight - element.rect.top,
56
- element.rect.height + boxPadding * 2,
57
- ),
64
+ left: paddedLeft,
65
+ top: paddedTop,
66
+ width: paddedWidth,
67
+ height: paddedHeight,
58
68
  };
59
69
 
60
70
  // Draw rectangle
@@ -65,10 +75,12 @@ const createSvgOverlay = async (
65
75
  paddedRect.height,
66
76
  (x: number, y: number, idx: number): void => {
67
77
  if (
68
- x === paddedRect.left ||
69
- x === paddedRect.left + paddedRect.width - 1 ||
70
- y === paddedRect.top ||
71
- y === paddedRect.top + paddedRect.height - 1
78
+ (x >= paddedRect.left && x < paddedRect.left + borderThickness) || // Left border
79
+ (x <= paddedRect.left + paddedRect.width - 1 &&
80
+ x > paddedRect.left + paddedRect.width - borderThickness) || // Right border
81
+ (y >= paddedRect.top && y < paddedRect.top + borderThickness) || // Top border
82
+ (y <= paddedRect.top + paddedRect.height - 1 &&
83
+ y > paddedRect.top + paddedRect.height - borderThickness) // Bottom border
72
84
  ) {
73
85
  image.bitmap.data[idx + 0] = (color.rect >> 24) & 0xff; // R
74
86
  image.bitmap.data[idx + 1] = (color.rect >> 16) & 0xff; // G
@@ -192,9 +204,10 @@ const createSvgOverlay = async (
192
204
 
193
205
  export const compositeElementInfoImg = async (options: {
194
206
  inputImgBase64: string;
195
- elementsPositionInfo: Array<BaseElement>;
207
+ elementsPositionInfo: Array<ElementForOverlay>;
196
208
  size?: { width: number; height: number };
197
209
  annotationPadding?: number;
210
+ borderThickness?: number;
198
211
  }) => {
199
212
  assert(options.inputImgBase64, 'inputImgBase64 is required');
200
213
  let width = 0;
@@ -237,6 +250,7 @@ export const compositeElementInfoImg = async (options: {
237
250
  width,
238
251
  height,
239
252
  options.annotationPadding,
253
+ options.borderThickness,
240
254
  );
241
255
  const svgImage = await Jimp.read(svgOverlay);
242
256
  const compositeImage = await image.composite(svgImage, 0, 0, {