@midscene/shared 0.16.9-beta-20250507095704.0 → 0.16.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (17) hide show
  1. package/dist/es/common.js +3 -3
  2. package/dist/es/env.d.ts +3 -4
  3. package/dist/es/env.js +4 -13
  4. package/dist/es/img.d.ts +1 -6
  5. package/dist/es/img.js +13 -21
  6. package/dist/es/logger.js +3 -3
  7. package/dist/lib/common.js +3 -3
  8. package/dist/lib/env.d.ts +3 -4
  9. package/dist/lib/env.js +5 -15
  10. package/dist/lib/img.d.ts +1 -6
  11. package/dist/lib/img.js +13 -21
  12. package/dist/lib/logger.js +3 -3
  13. package/dist/types/env.d.ts +3 -4
  14. package/dist/types/img.d.ts +1 -6
  15. package/package.json +1 -1
  16. package/src/env.ts +3 -14
  17. package/src/img/box-select.ts +17 -31
package/dist/es/common.js CHANGED
@@ -14,6 +14,7 @@ var MIDSCENE_DEBUG_MODE = "MIDSCENE_DEBUG_MODE";
14
14
  var MIDSCENE_MCP_USE_PUPPETEER_MODE = "MIDSCENE_MCP_USE_PUPPETEER_MODE";
15
15
  var MIDSCENE_FORCE_DEEP_THINK = "MIDSCENE_FORCE_DEEP_THINK";
16
16
  var MIDSCENE_OPENAI_SOCKS_PROXY = "MIDSCENE_OPENAI_SOCKS_PROXY";
17
+ var MIDSCENE_OPENAI_HTTP_PROXY = "MIDSCENE_OPENAI_HTTP_PROXY";
17
18
  var OPENAI_API_KEY = "OPENAI_API_KEY";
18
19
  var OPENAI_BASE_URL = "OPENAI_BASE_URL";
19
20
  var OPENAI_MAX_TOKENS = "OPENAI_MAX_TOKENS";
@@ -26,7 +27,6 @@ var MIDSCENE_USE_GEMINI = "MIDSCENE_USE_GEMINI";
26
27
  var MIDSCENE_USE_VL_MODEL = "MIDSCENE_USE_VL_MODEL";
27
28
  var MATCH_BY_POSITION = "MATCH_BY_POSITION";
28
29
  var MIDSCENE_REPORT_TAG_NAME = "MIDSCENE_REPORT_TAG_NAME";
29
- var MIDSCENE_PREFERRED_LANGUAGE = "MIDSCENE_PREFERRED_LANGUAGE";
30
30
  var MIDSCENE_USE_AZURE_OPENAI = "MIDSCENE_USE_AZURE_OPENAI";
31
31
  var MIDSCENE_AZURE_OPENAI_SCOPE = "MIDSCENE_AZURE_OPENAI_SCOPE";
32
32
  var MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON = "MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON";
@@ -57,6 +57,7 @@ var allConfigFromEnv = () => {
57
57
  [MATCH_BY_POSITION]: process.env[MATCH_BY_POSITION] || void 0,
58
58
  [MIDSCENE_REPORT_TAG_NAME]: process.env[MIDSCENE_REPORT_TAG_NAME] || void 0,
59
59
  [MIDSCENE_OPENAI_SOCKS_PROXY]: process.env[MIDSCENE_OPENAI_SOCKS_PROXY] || void 0,
60
+ [MIDSCENE_OPENAI_HTTP_PROXY]: process.env[MIDSCENE_OPENAI_HTTP_PROXY] || void 0,
60
61
  [MIDSCENE_USE_AZURE_OPENAI]: process.env[MIDSCENE_USE_AZURE_OPENAI] || void 0,
61
62
  [MIDSCENE_AZURE_OPENAI_SCOPE]: process.env[MIDSCENE_AZURE_OPENAI_SCOPE] || void 0,
62
63
  [MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON]: process.env[MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON] || void 0,
@@ -72,8 +73,7 @@ var allConfigFromEnv = () => {
72
73
  [AZURE_OPENAI_API_VERSION]: process.env[AZURE_OPENAI_API_VERSION] || void 0,
73
74
  [AZURE_OPENAI_DEPLOYMENT]: process.env[AZURE_OPENAI_DEPLOYMENT] || void 0,
74
75
  [MIDSCENE_MCP_USE_PUPPETEER_MODE]: process.env[MIDSCENE_MCP_USE_PUPPETEER_MODE] || void 0,
75
- [MIDSCENE_RUN_DIR]: process.env[MIDSCENE_RUN_DIR] || void 0,
76
- [MIDSCENE_PREFERRED_LANGUAGE]: process.env[MIDSCENE_PREFERRED_LANGUAGE] || void 0
76
+ [MIDSCENE_RUN_DIR]: process.env[MIDSCENE_RUN_DIR] || void 0
77
77
  };
78
78
  };
79
79
  var globalConfig = null;
package/dist/es/env.d.ts CHANGED
@@ -8,6 +8,7 @@ declare const MIDSCENE_DEBUG_MODE = "MIDSCENE_DEBUG_MODE";
8
8
  declare const MIDSCENE_MCP_USE_PUPPETEER_MODE = "MIDSCENE_MCP_USE_PUPPETEER_MODE";
9
9
  declare const MIDSCENE_FORCE_DEEP_THINK = "MIDSCENE_FORCE_DEEP_THINK";
10
10
  declare const MIDSCENE_OPENAI_SOCKS_PROXY = "MIDSCENE_OPENAI_SOCKS_PROXY";
11
+ declare const MIDSCENE_OPENAI_HTTP_PROXY = "MIDSCENE_OPENAI_HTTP_PROXY";
11
12
  declare const OPENAI_API_KEY = "OPENAI_API_KEY";
12
13
  declare const OPENAI_BASE_URL = "OPENAI_BASE_URL";
13
14
  declare const OPENAI_MAX_TOKENS = "OPENAI_MAX_TOKENS";
@@ -21,7 +22,6 @@ declare const MIDSCENE_USE_VL_MODEL = "MIDSCENE_USE_VL_MODEL";
21
22
  declare const MATCH_BY_POSITION = "MATCH_BY_POSITION";
22
23
  declare const MIDSCENE_API_TYPE = "MIDSCENE-API-TYPE";
23
24
  declare const MIDSCENE_REPORT_TAG_NAME = "MIDSCENE_REPORT_TAG_NAME";
24
- declare const MIDSCENE_PREFERRED_LANGUAGE = "MIDSCENE_PREFERRED_LANGUAGE";
25
25
  declare const MIDSCENE_USE_AZURE_OPENAI = "MIDSCENE_USE_AZURE_OPENAI";
26
26
  declare const MIDSCENE_AZURE_OPENAI_SCOPE = "MIDSCENE_AZURE_OPENAI_SCOPE";
27
27
  declare const MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON = "MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON";
@@ -51,6 +51,7 @@ declare const allConfigFromEnv: () => {
51
51
  MATCH_BY_POSITION: string | undefined;
52
52
  MIDSCENE_REPORT_TAG_NAME: string | undefined;
53
53
  MIDSCENE_OPENAI_SOCKS_PROXY: string | undefined;
54
+ MIDSCENE_OPENAI_HTTP_PROXY: string | undefined;
54
55
  MIDSCENE_USE_AZURE_OPENAI: string | undefined;
55
56
  MIDSCENE_AZURE_OPENAI_SCOPE: string | undefined;
56
57
  MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON: string | undefined;
@@ -67,7 +68,6 @@ declare const allConfigFromEnv: () => {
67
68
  AZURE_OPENAI_DEPLOYMENT: string | undefined;
68
69
  MIDSCENE_MCP_USE_PUPPETEER_MODE: string | undefined;
69
70
  MIDSCENE_RUN_DIR: string | undefined;
70
- MIDSCENE_PREFERRED_LANGUAGE: string | undefined;
71
71
  };
72
72
  declare enum UITarsModelVersion {
73
73
  V1_0 = "1.0",
@@ -81,6 +81,5 @@ declare const getAIConfig: (configKey: keyof ReturnType<typeof allConfigFromEnv>
81
81
  declare const getAIConfigInBoolean: (configKey: keyof ReturnType<typeof allConfigFromEnv>) => boolean;
82
82
  declare const getAIConfigInJson: (configKey: keyof ReturnType<typeof allConfigFromEnv>) => any;
83
83
  declare const overrideAIConfig: (newConfig: Partial<ReturnType<typeof allConfigFromEnv>>, extendMode?: boolean) => void;
84
- declare const getPreferredLanguage: () => string | undefined;
85
84
 
86
- export { ANTHROPIC_API_KEY, AZURE_OPENAI_API_VERSION, AZURE_OPENAI_DEPLOYMENT, AZURE_OPENAI_ENDPOINT, AZURE_OPENAI_KEY, MATCH_BY_POSITION, MIDSCENE_ADB_PATH, MIDSCENE_API_TYPE, MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_AZURE_OPENAI_SCOPE, MIDSCENE_CACHE, MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG, MIDSCENE_DEBUG_AI_PROFILE, MIDSCENE_DEBUG_AI_RESPONSE, MIDSCENE_DEBUG_MODE, MIDSCENE_FORCE_DEEP_THINK, MIDSCENE_LANGSMITH_DEBUG, MIDSCENE_MCP_USE_PUPPETEER_MODE, MIDSCENE_MODEL_NAME, MIDSCENE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_OPENAI_SOCKS_PROXY, MIDSCENE_PREFERRED_LANGUAGE, MIDSCENE_REPORT_TAG_NAME, MIDSCENE_RUN_DIR, MIDSCENE_USE_ANTHROPIC_SDK, MIDSCENE_USE_AZURE_OPENAI, MIDSCENE_USE_DOUBAO_VISION, MIDSCENE_USE_GEMINI, MIDSCENE_USE_QWEN_VL, MIDSCENE_USE_VLM_UI_TARS, MIDSCENE_USE_VL_MODEL, OPENAI_API_KEY, OPENAI_BASE_URL, OPENAI_MAX_TOKENS, OPENAI_USE_AZURE, UITarsModelVersion, allConfigFromEnv, getAIConfig, getAIConfigInBoolean, getAIConfigInJson, getPreferredLanguage, overrideAIConfig, uiTarsModelVersion, vlLocateMode };
85
+ export { ANTHROPIC_API_KEY, AZURE_OPENAI_API_VERSION, AZURE_OPENAI_DEPLOYMENT, AZURE_OPENAI_ENDPOINT, AZURE_OPENAI_KEY, MATCH_BY_POSITION, MIDSCENE_ADB_PATH, MIDSCENE_API_TYPE, MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_AZURE_OPENAI_SCOPE, MIDSCENE_CACHE, MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG, MIDSCENE_DEBUG_AI_PROFILE, MIDSCENE_DEBUG_AI_RESPONSE, MIDSCENE_DEBUG_MODE, MIDSCENE_FORCE_DEEP_THINK, MIDSCENE_LANGSMITH_DEBUG, MIDSCENE_MCP_USE_PUPPETEER_MODE, MIDSCENE_MODEL_NAME, MIDSCENE_OPENAI_HTTP_PROXY, MIDSCENE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_OPENAI_SOCKS_PROXY, MIDSCENE_REPORT_TAG_NAME, MIDSCENE_RUN_DIR, MIDSCENE_USE_ANTHROPIC_SDK, MIDSCENE_USE_AZURE_OPENAI, MIDSCENE_USE_DOUBAO_VISION, MIDSCENE_USE_GEMINI, MIDSCENE_USE_QWEN_VL, MIDSCENE_USE_VLM_UI_TARS, MIDSCENE_USE_VL_MODEL, OPENAI_API_KEY, OPENAI_BASE_URL, OPENAI_MAX_TOKENS, OPENAI_USE_AZURE, UITarsModelVersion, allConfigFromEnv, getAIConfig, getAIConfigInBoolean, getAIConfigInJson, overrideAIConfig, uiTarsModelVersion, vlLocateMode };
package/dist/es/env.js CHANGED
@@ -9,6 +9,7 @@ var MIDSCENE_DEBUG_MODE = "MIDSCENE_DEBUG_MODE";
9
9
  var MIDSCENE_MCP_USE_PUPPETEER_MODE = "MIDSCENE_MCP_USE_PUPPETEER_MODE";
10
10
  var MIDSCENE_FORCE_DEEP_THINK = "MIDSCENE_FORCE_DEEP_THINK";
11
11
  var MIDSCENE_OPENAI_SOCKS_PROXY = "MIDSCENE_OPENAI_SOCKS_PROXY";
12
+ var MIDSCENE_OPENAI_HTTP_PROXY = "MIDSCENE_OPENAI_HTTP_PROXY";
12
13
  var OPENAI_API_KEY = "OPENAI_API_KEY";
13
14
  var OPENAI_BASE_URL = "OPENAI_BASE_URL";
14
15
  var OPENAI_MAX_TOKENS = "OPENAI_MAX_TOKENS";
@@ -22,7 +23,6 @@ var MIDSCENE_USE_VL_MODEL = "MIDSCENE_USE_VL_MODEL";
22
23
  var MATCH_BY_POSITION = "MATCH_BY_POSITION";
23
24
  var MIDSCENE_API_TYPE = "MIDSCENE-API-TYPE";
24
25
  var MIDSCENE_REPORT_TAG_NAME = "MIDSCENE_REPORT_TAG_NAME";
25
- var MIDSCENE_PREFERRED_LANGUAGE = "MIDSCENE_PREFERRED_LANGUAGE";
26
26
  var MIDSCENE_USE_AZURE_OPENAI = "MIDSCENE_USE_AZURE_OPENAI";
27
27
  var MIDSCENE_AZURE_OPENAI_SCOPE = "MIDSCENE_AZURE_OPENAI_SCOPE";
28
28
  var MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON = "MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON";
@@ -53,6 +53,7 @@ var allConfigFromEnv = () => {
53
53
  [MATCH_BY_POSITION]: process.env[MATCH_BY_POSITION] || void 0,
54
54
  [MIDSCENE_REPORT_TAG_NAME]: process.env[MIDSCENE_REPORT_TAG_NAME] || void 0,
55
55
  [MIDSCENE_OPENAI_SOCKS_PROXY]: process.env[MIDSCENE_OPENAI_SOCKS_PROXY] || void 0,
56
+ [MIDSCENE_OPENAI_HTTP_PROXY]: process.env[MIDSCENE_OPENAI_HTTP_PROXY] || void 0,
56
57
  [MIDSCENE_USE_AZURE_OPENAI]: process.env[MIDSCENE_USE_AZURE_OPENAI] || void 0,
57
58
  [MIDSCENE_AZURE_OPENAI_SCOPE]: process.env[MIDSCENE_AZURE_OPENAI_SCOPE] || void 0,
58
59
  [MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON]: process.env[MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON] || void 0,
@@ -68,8 +69,7 @@ var allConfigFromEnv = () => {
68
69
  [AZURE_OPENAI_API_VERSION]: process.env[AZURE_OPENAI_API_VERSION] || void 0,
69
70
  [AZURE_OPENAI_DEPLOYMENT]: process.env[AZURE_OPENAI_DEPLOYMENT] || void 0,
70
71
  [MIDSCENE_MCP_USE_PUPPETEER_MODE]: process.env[MIDSCENE_MCP_USE_PUPPETEER_MODE] || void 0,
71
- [MIDSCENE_RUN_DIR]: process.env[MIDSCENE_RUN_DIR] || void 0,
72
- [MIDSCENE_PREFERRED_LANGUAGE]: process.env[MIDSCENE_PREFERRED_LANGUAGE] || void 0
72
+ [MIDSCENE_RUN_DIR]: process.env[MIDSCENE_RUN_DIR] || void 0
73
73
  };
74
74
  };
75
75
  var globalConfig = null;
@@ -173,14 +173,6 @@ var overrideAIConfig = (newConfig, extendMode = false) => {
173
173
  const currentConfig = getGlobalConfig();
174
174
  globalConfig = extendMode ? { ...currentConfig, ...newConfig } : { ...newConfig };
175
175
  };
176
- var getPreferredLanguage = () => {
177
- if (getAIConfig(MIDSCENE_PREFERRED_LANGUAGE)) {
178
- return getAIConfig(MIDSCENE_PREFERRED_LANGUAGE);
179
- }
180
- const timeZone = Intl.DateTimeFormat().resolvedOptions().timeZone;
181
- const isChina = timeZone === "Asia/Shanghai";
182
- return isChina ? "Chinese" : "English";
183
- };
184
176
  export {
185
177
  ANTHROPIC_API_KEY,
186
178
  AZURE_OPENAI_API_VERSION,
@@ -201,9 +193,9 @@ export {
201
193
  MIDSCENE_LANGSMITH_DEBUG,
202
194
  MIDSCENE_MCP_USE_PUPPETEER_MODE,
203
195
  MIDSCENE_MODEL_NAME,
196
+ MIDSCENE_OPENAI_HTTP_PROXY,
204
197
  MIDSCENE_OPENAI_INIT_CONFIG_JSON,
205
198
  MIDSCENE_OPENAI_SOCKS_PROXY,
206
- MIDSCENE_PREFERRED_LANGUAGE,
207
199
  MIDSCENE_REPORT_TAG_NAME,
208
200
  MIDSCENE_RUN_DIR,
209
201
  MIDSCENE_USE_ANTHROPIC_SDK,
@@ -222,7 +214,6 @@ export {
222
214
  getAIConfig,
223
215
  getAIConfigInBoolean,
224
216
  getAIConfigInJson,
225
- getPreferredLanguage,
226
217
  overrideAIConfig,
227
218
  uiTarsModelVersion,
228
219
  vlLocateMode
package/dist/es/img.d.ts CHANGED
@@ -114,19 +114,14 @@ declare function paddingToMatchBlockByBase64(imageBase64: string, blockSize?: nu
114
114
  declare function cropByRect(imageBase64: string, rect: Rect, paddingImage: boolean): Promise<string>;
115
115
  declare function jimpToBase64(image: Jimp): Promise<string>;
116
116
 
117
- interface ElementForOverlay {
118
- rect: Rect;
119
- indexId?: number;
120
- }
121
117
  declare const compositeElementInfoImg: (options: {
122
118
  inputImgBase64: string;
123
- elementsPositionInfo: Array<ElementForOverlay>;
119
+ elementsPositionInfo: Array<BaseElement>;
124
120
  size?: {
125
121
  width: number;
126
122
  height: number;
127
123
  };
128
124
  annotationPadding?: number;
129
- borderThickness?: number;
130
125
  }) => Promise<string>;
131
126
  declare const processImageElementInfo: (options: {
132
127
  inputImgBase64: string;
package/dist/es/img.js CHANGED
@@ -229,7 +229,7 @@ var loadFonts = async () => {
229
229
  return fonts;
230
230
  }
231
231
  };
232
- var createSvgOverlay = async (elements, imageWidth, imageHeight, boxPadding = 5, borderThickness = 2) => {
232
+ var createSvgOverlay = async (elements, imageWidth, imageHeight, boxPadding = 5) => {
233
233
  const Jimp = await getJimp();
234
234
  const image = new Jimp(imageWidth, imageHeight, 0);
235
235
  const colors = [
@@ -247,21 +247,17 @@ var createSvgOverlay = async (elements, imageWidth, imageHeight, boxPadding = 5,
247
247
  for (let index = 0; index < elements.length; index++) {
248
248
  const element = elements[index];
249
249
  const color = colors[index % colors.length];
250
- const paddedLeft = Math.max(0, element.rect.left - boxPadding);
251
- const paddedTop = Math.max(0, element.rect.top - boxPadding);
252
- const paddedWidth = Math.min(
253
- imageWidth - paddedLeft,
254
- element.rect.width + boxPadding * 2
255
- );
256
- const paddedHeight = Math.min(
257
- imageHeight - paddedTop,
258
- element.rect.height + boxPadding * 2
259
- );
260
250
  const paddedRect = {
261
- left: paddedLeft,
262
- top: paddedTop,
263
- width: paddedWidth,
264
- height: paddedHeight
251
+ left: Math.max(0, element.rect.left - boxPadding),
252
+ top: Math.max(0, element.rect.top - boxPadding),
253
+ width: Math.min(
254
+ imageWidth - element.rect.left,
255
+ element.rect.width + boxPadding * 2
256
+ ),
257
+ height: Math.min(
258
+ imageHeight - element.rect.top,
259
+ element.rect.height + boxPadding * 2
260
+ )
265
261
  };
266
262
  image.scan(
267
263
  paddedRect.left,
@@ -269,10 +265,7 @@ var createSvgOverlay = async (elements, imageWidth, imageHeight, boxPadding = 5,
269
265
  paddedRect.width,
270
266
  paddedRect.height,
271
267
  (x, y, idx) => {
272
- if (x >= paddedRect.left && x < paddedRect.left + borderThickness || // Left border
273
- x <= paddedRect.left + paddedRect.width - 1 && x > paddedRect.left + paddedRect.width - borderThickness || // Right border
274
- y >= paddedRect.top && y < paddedRect.top + borderThickness || // Top border
275
- y <= paddedRect.top + paddedRect.height - 1 && y > paddedRect.top + paddedRect.height - borderThickness) {
268
+ if (x === paddedRect.left || x === paddedRect.left + paddedRect.width - 1 || y === paddedRect.top || y === paddedRect.top + paddedRect.height - 1) {
276
269
  image.bitmap.data[idx + 0] = color.rect >> 24 & 255;
277
270
  image.bitmap.data[idx + 1] = color.rect >> 16 & 255;
278
271
  image.bitmap.data[idx + 2] = color.rect >> 8 & 255;
@@ -380,8 +373,7 @@ var compositeElementInfoImg = async (options) => {
380
373
  elementsPositionInfo,
381
374
  width,
382
375
  height,
383
- options.annotationPadding,
384
- options.borderThickness
376
+ options.annotationPadding
385
377
  );
386
378
  const svgImage = await Jimp.read(svgOverlay);
387
379
  const compositeImage = await image.composite(svgImage, 0, 0, {
package/dist/es/logger.js CHANGED
@@ -19,6 +19,7 @@ var MIDSCENE_DEBUG_MODE = "MIDSCENE_DEBUG_MODE";
19
19
  var MIDSCENE_MCP_USE_PUPPETEER_MODE = "MIDSCENE_MCP_USE_PUPPETEER_MODE";
20
20
  var MIDSCENE_FORCE_DEEP_THINK = "MIDSCENE_FORCE_DEEP_THINK";
21
21
  var MIDSCENE_OPENAI_SOCKS_PROXY = "MIDSCENE_OPENAI_SOCKS_PROXY";
22
+ var MIDSCENE_OPENAI_HTTP_PROXY = "MIDSCENE_OPENAI_HTTP_PROXY";
22
23
  var OPENAI_API_KEY = "OPENAI_API_KEY";
23
24
  var OPENAI_BASE_URL = "OPENAI_BASE_URL";
24
25
  var OPENAI_MAX_TOKENS = "OPENAI_MAX_TOKENS";
@@ -31,7 +32,6 @@ var MIDSCENE_USE_GEMINI = "MIDSCENE_USE_GEMINI";
31
32
  var MIDSCENE_USE_VL_MODEL = "MIDSCENE_USE_VL_MODEL";
32
33
  var MATCH_BY_POSITION = "MATCH_BY_POSITION";
33
34
  var MIDSCENE_REPORT_TAG_NAME = "MIDSCENE_REPORT_TAG_NAME";
34
- var MIDSCENE_PREFERRED_LANGUAGE = "MIDSCENE_PREFERRED_LANGUAGE";
35
35
  var MIDSCENE_USE_AZURE_OPENAI = "MIDSCENE_USE_AZURE_OPENAI";
36
36
  var MIDSCENE_AZURE_OPENAI_SCOPE = "MIDSCENE_AZURE_OPENAI_SCOPE";
37
37
  var MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON = "MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON";
@@ -62,6 +62,7 @@ var allConfigFromEnv = () => {
62
62
  [MATCH_BY_POSITION]: process.env[MATCH_BY_POSITION] || void 0,
63
63
  [MIDSCENE_REPORT_TAG_NAME]: process.env[MIDSCENE_REPORT_TAG_NAME] || void 0,
64
64
  [MIDSCENE_OPENAI_SOCKS_PROXY]: process.env[MIDSCENE_OPENAI_SOCKS_PROXY] || void 0,
65
+ [MIDSCENE_OPENAI_HTTP_PROXY]: process.env[MIDSCENE_OPENAI_HTTP_PROXY] || void 0,
65
66
  [MIDSCENE_USE_AZURE_OPENAI]: process.env[MIDSCENE_USE_AZURE_OPENAI] || void 0,
66
67
  [MIDSCENE_AZURE_OPENAI_SCOPE]: process.env[MIDSCENE_AZURE_OPENAI_SCOPE] || void 0,
67
68
  [MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON]: process.env[MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON] || void 0,
@@ -77,8 +78,7 @@ var allConfigFromEnv = () => {
77
78
  [AZURE_OPENAI_API_VERSION]: process.env[AZURE_OPENAI_API_VERSION] || void 0,
78
79
  [AZURE_OPENAI_DEPLOYMENT]: process.env[AZURE_OPENAI_DEPLOYMENT] || void 0,
79
80
  [MIDSCENE_MCP_USE_PUPPETEER_MODE]: process.env[MIDSCENE_MCP_USE_PUPPETEER_MODE] || void 0,
80
- [MIDSCENE_RUN_DIR]: process.env[MIDSCENE_RUN_DIR] || void 0,
81
- [MIDSCENE_PREFERRED_LANGUAGE]: process.env[MIDSCENE_PREFERRED_LANGUAGE] || void 0
81
+ [MIDSCENE_RUN_DIR]: process.env[MIDSCENE_RUN_DIR] || void 0
82
82
  };
83
83
  };
84
84
  var globalConfig = null;
package/dist/lib/common.js CHANGED
@@ -53,6 +53,7 @@ var MIDSCENE_DEBUG_MODE = "MIDSCENE_DEBUG_MODE";
53
53
  var MIDSCENE_MCP_USE_PUPPETEER_MODE = "MIDSCENE_MCP_USE_PUPPETEER_MODE";
54
54
  var MIDSCENE_FORCE_DEEP_THINK = "MIDSCENE_FORCE_DEEP_THINK";
55
55
  var MIDSCENE_OPENAI_SOCKS_PROXY = "MIDSCENE_OPENAI_SOCKS_PROXY";
56
+ var MIDSCENE_OPENAI_HTTP_PROXY = "MIDSCENE_OPENAI_HTTP_PROXY";
56
57
  var OPENAI_API_KEY = "OPENAI_API_KEY";
57
58
  var OPENAI_BASE_URL = "OPENAI_BASE_URL";
58
59
  var OPENAI_MAX_TOKENS = "OPENAI_MAX_TOKENS";
@@ -65,7 +66,6 @@ var MIDSCENE_USE_GEMINI = "MIDSCENE_USE_GEMINI";
65
66
  var MIDSCENE_USE_VL_MODEL = "MIDSCENE_USE_VL_MODEL";
66
67
  var MATCH_BY_POSITION = "MATCH_BY_POSITION";
67
68
  var MIDSCENE_REPORT_TAG_NAME = "MIDSCENE_REPORT_TAG_NAME";
68
- var MIDSCENE_PREFERRED_LANGUAGE = "MIDSCENE_PREFERRED_LANGUAGE";
69
69
  var MIDSCENE_USE_AZURE_OPENAI = "MIDSCENE_USE_AZURE_OPENAI";
70
70
  var MIDSCENE_AZURE_OPENAI_SCOPE = "MIDSCENE_AZURE_OPENAI_SCOPE";
71
71
  var MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON = "MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON";
@@ -96,6 +96,7 @@ var allConfigFromEnv = () => {
96
96
  [MATCH_BY_POSITION]: process.env[MATCH_BY_POSITION] || void 0,
97
97
  [MIDSCENE_REPORT_TAG_NAME]: process.env[MIDSCENE_REPORT_TAG_NAME] || void 0,
98
98
  [MIDSCENE_OPENAI_SOCKS_PROXY]: process.env[MIDSCENE_OPENAI_SOCKS_PROXY] || void 0,
99
+ [MIDSCENE_OPENAI_HTTP_PROXY]: process.env[MIDSCENE_OPENAI_HTTP_PROXY] || void 0,
99
100
  [MIDSCENE_USE_AZURE_OPENAI]: process.env[MIDSCENE_USE_AZURE_OPENAI] || void 0,
100
101
  [MIDSCENE_AZURE_OPENAI_SCOPE]: process.env[MIDSCENE_AZURE_OPENAI_SCOPE] || void 0,
101
102
  [MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON]: process.env[MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON] || void 0,
@@ -111,8 +112,7 @@ var allConfigFromEnv = () => {
111
112
  [AZURE_OPENAI_API_VERSION]: process.env[AZURE_OPENAI_API_VERSION] || void 0,
112
113
  [AZURE_OPENAI_DEPLOYMENT]: process.env[AZURE_OPENAI_DEPLOYMENT] || void 0,
113
114
  [MIDSCENE_MCP_USE_PUPPETEER_MODE]: process.env[MIDSCENE_MCP_USE_PUPPETEER_MODE] || void 0,
114
- [MIDSCENE_RUN_DIR]: process.env[MIDSCENE_RUN_DIR] || void 0,
115
- [MIDSCENE_PREFERRED_LANGUAGE]: process.env[MIDSCENE_PREFERRED_LANGUAGE] || void 0
115
+ [MIDSCENE_RUN_DIR]: process.env[MIDSCENE_RUN_DIR] || void 0
116
116
  };
117
117
  };
118
118
  var globalConfig = null;
package/dist/lib/env.d.ts CHANGED
@@ -8,6 +8,7 @@ declare const MIDSCENE_DEBUG_MODE = "MIDSCENE_DEBUG_MODE";
8
8
  declare const MIDSCENE_MCP_USE_PUPPETEER_MODE = "MIDSCENE_MCP_USE_PUPPETEER_MODE";
9
9
  declare const MIDSCENE_FORCE_DEEP_THINK = "MIDSCENE_FORCE_DEEP_THINK";
10
10
  declare const MIDSCENE_OPENAI_SOCKS_PROXY = "MIDSCENE_OPENAI_SOCKS_PROXY";
11
+ declare const MIDSCENE_OPENAI_HTTP_PROXY = "MIDSCENE_OPENAI_HTTP_PROXY";
11
12
  declare const OPENAI_API_KEY = "OPENAI_API_KEY";
12
13
  declare const OPENAI_BASE_URL = "OPENAI_BASE_URL";
13
14
  declare const OPENAI_MAX_TOKENS = "OPENAI_MAX_TOKENS";
@@ -21,7 +22,6 @@ declare const MIDSCENE_USE_VL_MODEL = "MIDSCENE_USE_VL_MODEL";
21
22
  declare const MATCH_BY_POSITION = "MATCH_BY_POSITION";
22
23
  declare const MIDSCENE_API_TYPE = "MIDSCENE-API-TYPE";
23
24
  declare const MIDSCENE_REPORT_TAG_NAME = "MIDSCENE_REPORT_TAG_NAME";
24
- declare const MIDSCENE_PREFERRED_LANGUAGE = "MIDSCENE_PREFERRED_LANGUAGE";
25
25
  declare const MIDSCENE_USE_AZURE_OPENAI = "MIDSCENE_USE_AZURE_OPENAI";
26
26
  declare const MIDSCENE_AZURE_OPENAI_SCOPE = "MIDSCENE_AZURE_OPENAI_SCOPE";
27
27
  declare const MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON = "MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON";
@@ -51,6 +51,7 @@ declare const allConfigFromEnv: () => {
51
51
  MATCH_BY_POSITION: string | undefined;
52
52
  MIDSCENE_REPORT_TAG_NAME: string | undefined;
53
53
  MIDSCENE_OPENAI_SOCKS_PROXY: string | undefined;
54
+ MIDSCENE_OPENAI_HTTP_PROXY: string | undefined;
54
55
  MIDSCENE_USE_AZURE_OPENAI: string | undefined;
55
56
  MIDSCENE_AZURE_OPENAI_SCOPE: string | undefined;
56
57
  MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON: string | undefined;
@@ -67,7 +68,6 @@ declare const allConfigFromEnv: () => {
67
68
  AZURE_OPENAI_DEPLOYMENT: string | undefined;
68
69
  MIDSCENE_MCP_USE_PUPPETEER_MODE: string | undefined;
69
70
  MIDSCENE_RUN_DIR: string | undefined;
70
- MIDSCENE_PREFERRED_LANGUAGE: string | undefined;
71
71
  };
72
72
  declare enum UITarsModelVersion {
73
73
  V1_0 = "1.0",
@@ -81,6 +81,5 @@ declare const getAIConfig: (configKey: keyof ReturnType<typeof allConfigFromEnv>
81
81
  declare const getAIConfigInBoolean: (configKey: keyof ReturnType<typeof allConfigFromEnv>) => boolean;
82
82
  declare const getAIConfigInJson: (configKey: keyof ReturnType<typeof allConfigFromEnv>) => any;
83
83
  declare const overrideAIConfig: (newConfig: Partial<ReturnType<typeof allConfigFromEnv>>, extendMode?: boolean) => void;
84
- declare const getPreferredLanguage: () => string | undefined;
85
84
 
86
- export { ANTHROPIC_API_KEY, AZURE_OPENAI_API_VERSION, AZURE_OPENAI_DEPLOYMENT, AZURE_OPENAI_ENDPOINT, AZURE_OPENAI_KEY, MATCH_BY_POSITION, MIDSCENE_ADB_PATH, MIDSCENE_API_TYPE, MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_AZURE_OPENAI_SCOPE, MIDSCENE_CACHE, MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG, MIDSCENE_DEBUG_AI_PROFILE, MIDSCENE_DEBUG_AI_RESPONSE, MIDSCENE_DEBUG_MODE, MIDSCENE_FORCE_DEEP_THINK, MIDSCENE_LANGSMITH_DEBUG, MIDSCENE_MCP_USE_PUPPETEER_MODE, MIDSCENE_MODEL_NAME, MIDSCENE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_OPENAI_SOCKS_PROXY, MIDSCENE_PREFERRED_LANGUAGE, MIDSCENE_REPORT_TAG_NAME, MIDSCENE_RUN_DIR, MIDSCENE_USE_ANTHROPIC_SDK, MIDSCENE_USE_AZURE_OPENAI, MIDSCENE_USE_DOUBAO_VISION, MIDSCENE_USE_GEMINI, MIDSCENE_USE_QWEN_VL, MIDSCENE_USE_VLM_UI_TARS, MIDSCENE_USE_VL_MODEL, OPENAI_API_KEY, OPENAI_BASE_URL, OPENAI_MAX_TOKENS, OPENAI_USE_AZURE, UITarsModelVersion, allConfigFromEnv, getAIConfig, getAIConfigInBoolean, getAIConfigInJson, getPreferredLanguage, overrideAIConfig, uiTarsModelVersion, vlLocateMode };
85
+ export { ANTHROPIC_API_KEY, AZURE_OPENAI_API_VERSION, AZURE_OPENAI_DEPLOYMENT, AZURE_OPENAI_ENDPOINT, AZURE_OPENAI_KEY, MATCH_BY_POSITION, MIDSCENE_ADB_PATH, MIDSCENE_API_TYPE, MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_AZURE_OPENAI_SCOPE, MIDSCENE_CACHE, MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG, MIDSCENE_DEBUG_AI_PROFILE, MIDSCENE_DEBUG_AI_RESPONSE, MIDSCENE_DEBUG_MODE, MIDSCENE_FORCE_DEEP_THINK, MIDSCENE_LANGSMITH_DEBUG, MIDSCENE_MCP_USE_PUPPETEER_MODE, MIDSCENE_MODEL_NAME, MIDSCENE_OPENAI_HTTP_PROXY, MIDSCENE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_OPENAI_SOCKS_PROXY, MIDSCENE_REPORT_TAG_NAME, MIDSCENE_RUN_DIR, MIDSCENE_USE_ANTHROPIC_SDK, MIDSCENE_USE_AZURE_OPENAI, MIDSCENE_USE_DOUBAO_VISION, MIDSCENE_USE_GEMINI, MIDSCENE_USE_QWEN_VL, MIDSCENE_USE_VLM_UI_TARS, MIDSCENE_USE_VL_MODEL, OPENAI_API_KEY, OPENAI_BASE_URL, OPENAI_MAX_TOKENS, OPENAI_USE_AZURE, UITarsModelVersion, allConfigFromEnv, getAIConfig, getAIConfigInBoolean, getAIConfigInJson, overrideAIConfig, uiTarsModelVersion, vlLocateMode };
package/dist/lib/env.js CHANGED
@@ -39,9 +39,9 @@ __export(env_exports, {
39
39
  MIDSCENE_LANGSMITH_DEBUG: () => MIDSCENE_LANGSMITH_DEBUG,
40
40
  MIDSCENE_MCP_USE_PUPPETEER_MODE: () => MIDSCENE_MCP_USE_PUPPETEER_MODE,
41
41
  MIDSCENE_MODEL_NAME: () => MIDSCENE_MODEL_NAME,
42
+ MIDSCENE_OPENAI_HTTP_PROXY: () => MIDSCENE_OPENAI_HTTP_PROXY,
42
43
  MIDSCENE_OPENAI_INIT_CONFIG_JSON: () => MIDSCENE_OPENAI_INIT_CONFIG_JSON,
43
44
  MIDSCENE_OPENAI_SOCKS_PROXY: () => MIDSCENE_OPENAI_SOCKS_PROXY,
44
- MIDSCENE_PREFERRED_LANGUAGE: () => MIDSCENE_PREFERRED_LANGUAGE,
45
45
  MIDSCENE_REPORT_TAG_NAME: () => MIDSCENE_REPORT_TAG_NAME,
46
46
  MIDSCENE_RUN_DIR: () => MIDSCENE_RUN_DIR,
47
47
  MIDSCENE_USE_ANTHROPIC_SDK: () => MIDSCENE_USE_ANTHROPIC_SDK,
@@ -60,7 +60,6 @@ __export(env_exports, {
60
60
  getAIConfig: () => getAIConfig,
61
61
  getAIConfigInBoolean: () => getAIConfigInBoolean,
62
62
  getAIConfigInJson: () => getAIConfigInJson,
63
- getPreferredLanguage: () => getPreferredLanguage,
64
63
  overrideAIConfig: () => overrideAIConfig,
65
64
  uiTarsModelVersion: () => uiTarsModelVersion,
66
65
  vlLocateMode: () => vlLocateMode
@@ -76,6 +75,7 @@ var MIDSCENE_DEBUG_MODE = "MIDSCENE_DEBUG_MODE";
76
75
  var MIDSCENE_MCP_USE_PUPPETEER_MODE = "MIDSCENE_MCP_USE_PUPPETEER_MODE";
77
76
  var MIDSCENE_FORCE_DEEP_THINK = "MIDSCENE_FORCE_DEEP_THINK";
78
77
  var MIDSCENE_OPENAI_SOCKS_PROXY = "MIDSCENE_OPENAI_SOCKS_PROXY";
78
+ var MIDSCENE_OPENAI_HTTP_PROXY = "MIDSCENE_OPENAI_HTTP_PROXY";
79
79
  var OPENAI_API_KEY = "OPENAI_API_KEY";
80
80
  var OPENAI_BASE_URL = "OPENAI_BASE_URL";
81
81
  var OPENAI_MAX_TOKENS = "OPENAI_MAX_TOKENS";
@@ -89,7 +89,6 @@ var MIDSCENE_USE_VL_MODEL = "MIDSCENE_USE_VL_MODEL";
89
89
  var MATCH_BY_POSITION = "MATCH_BY_POSITION";
90
90
  var MIDSCENE_API_TYPE = "MIDSCENE-API-TYPE";
91
91
  var MIDSCENE_REPORT_TAG_NAME = "MIDSCENE_REPORT_TAG_NAME";
92
- var MIDSCENE_PREFERRED_LANGUAGE = "MIDSCENE_PREFERRED_LANGUAGE";
93
92
  var MIDSCENE_USE_AZURE_OPENAI = "MIDSCENE_USE_AZURE_OPENAI";
94
93
  var MIDSCENE_AZURE_OPENAI_SCOPE = "MIDSCENE_AZURE_OPENAI_SCOPE";
95
94
  var MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON = "MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON";
@@ -120,6 +119,7 @@ var allConfigFromEnv = () => {
120
119
  [MATCH_BY_POSITION]: process.env[MATCH_BY_POSITION] || void 0,
121
120
  [MIDSCENE_REPORT_TAG_NAME]: process.env[MIDSCENE_REPORT_TAG_NAME] || void 0,
122
121
  [MIDSCENE_OPENAI_SOCKS_PROXY]: process.env[MIDSCENE_OPENAI_SOCKS_PROXY] || void 0,
122
+ [MIDSCENE_OPENAI_HTTP_PROXY]: process.env[MIDSCENE_OPENAI_HTTP_PROXY] || void 0,
123
123
  [MIDSCENE_USE_AZURE_OPENAI]: process.env[MIDSCENE_USE_AZURE_OPENAI] || void 0,
124
124
  [MIDSCENE_AZURE_OPENAI_SCOPE]: process.env[MIDSCENE_AZURE_OPENAI_SCOPE] || void 0,
125
125
  [MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON]: process.env[MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON] || void 0,
@@ -135,8 +135,7 @@ var allConfigFromEnv = () => {
135
135
  [AZURE_OPENAI_API_VERSION]: process.env[AZURE_OPENAI_API_VERSION] || void 0,
136
136
  [AZURE_OPENAI_DEPLOYMENT]: process.env[AZURE_OPENAI_DEPLOYMENT] || void 0,
137
137
  [MIDSCENE_MCP_USE_PUPPETEER_MODE]: process.env[MIDSCENE_MCP_USE_PUPPETEER_MODE] || void 0,
138
- [MIDSCENE_RUN_DIR]: process.env[MIDSCENE_RUN_DIR] || void 0,
139
- [MIDSCENE_PREFERRED_LANGUAGE]: process.env[MIDSCENE_PREFERRED_LANGUAGE] || void 0
138
+ [MIDSCENE_RUN_DIR]: process.env[MIDSCENE_RUN_DIR] || void 0
140
139
  };
141
140
  };
142
141
  var globalConfig = null;
@@ -240,14 +239,6 @@ var overrideAIConfig = (newConfig, extendMode = false) => {
240
239
  const currentConfig = getGlobalConfig();
241
240
  globalConfig = extendMode ? { ...currentConfig, ...newConfig } : { ...newConfig };
242
241
  };
243
- var getPreferredLanguage = () => {
244
- if (getAIConfig(MIDSCENE_PREFERRED_LANGUAGE)) {
245
- return getAIConfig(MIDSCENE_PREFERRED_LANGUAGE);
246
- }
247
- const timeZone = Intl.DateTimeFormat().resolvedOptions().timeZone;
248
- const isChina = timeZone === "Asia/Shanghai";
249
- return isChina ? "Chinese" : "English";
250
- };
251
242
  // Annotate the CommonJS export names for ESM import in node:
252
243
  0 && (module.exports = {
253
244
  ANTHROPIC_API_KEY,
@@ -269,9 +260,9 @@ var getPreferredLanguage = () => {
269
260
  MIDSCENE_LANGSMITH_DEBUG,
270
261
  MIDSCENE_MCP_USE_PUPPETEER_MODE,
271
262
  MIDSCENE_MODEL_NAME,
263
+ MIDSCENE_OPENAI_HTTP_PROXY,
272
264
  MIDSCENE_OPENAI_INIT_CONFIG_JSON,
273
265
  MIDSCENE_OPENAI_SOCKS_PROXY,
274
- MIDSCENE_PREFERRED_LANGUAGE,
275
266
  MIDSCENE_REPORT_TAG_NAME,
276
267
  MIDSCENE_RUN_DIR,
277
268
  MIDSCENE_USE_ANTHROPIC_SDK,
@@ -290,7 +281,6 @@ var getPreferredLanguage = () => {
290
281
  getAIConfig,
291
282
  getAIConfigInBoolean,
292
283
  getAIConfigInJson,
293
- getPreferredLanguage,
294
284
  overrideAIConfig,
295
285
  uiTarsModelVersion,
296
286
  vlLocateMode
package/dist/lib/img.d.ts CHANGED
@@ -114,19 +114,14 @@ declare function paddingToMatchBlockByBase64(imageBase64: string, blockSize?: nu
114
114
  declare function cropByRect(imageBase64: string, rect: Rect, paddingImage: boolean): Promise<string>;
115
115
  declare function jimpToBase64(image: Jimp): Promise<string>;
116
116
 
117
- interface ElementForOverlay {
118
- rect: Rect;
119
- indexId?: number;
120
- }
121
117
  declare const compositeElementInfoImg: (options: {
122
118
  inputImgBase64: string;
123
- elementsPositionInfo: Array<ElementForOverlay>;
119
+ elementsPositionInfo: Array<BaseElement>;
124
120
  size?: {
125
121
  width: number;
126
122
  height: number;
127
123
  };
128
124
  annotationPadding?: number;
129
- borderThickness?: number;
130
125
  }) => Promise<string>;
131
126
  declare const processImageElementInfo: (options: {
132
127
  inputImgBase64: string;
package/dist/lib/img.js CHANGED
@@ -284,7 +284,7 @@ var loadFonts = async () => {
284
284
  return fonts;
285
285
  }
286
286
  };
287
- var createSvgOverlay = async (elements, imageWidth, imageHeight, boxPadding = 5, borderThickness = 2) => {
287
+ var createSvgOverlay = async (elements, imageWidth, imageHeight, boxPadding = 5) => {
288
288
  const Jimp = await getJimp();
289
289
  const image = new Jimp(imageWidth, imageHeight, 0);
290
290
  const colors = [
@@ -302,21 +302,17 @@ var createSvgOverlay = async (elements, imageWidth, imageHeight, boxPadding = 5,
302
302
  for (let index = 0; index < elements.length; index++) {
303
303
  const element = elements[index];
304
304
  const color = colors[index % colors.length];
305
- const paddedLeft = Math.max(0, element.rect.left - boxPadding);
306
- const paddedTop = Math.max(0, element.rect.top - boxPadding);
307
- const paddedWidth = Math.min(
308
- imageWidth - paddedLeft,
309
- element.rect.width + boxPadding * 2
310
- );
311
- const paddedHeight = Math.min(
312
- imageHeight - paddedTop,
313
- element.rect.height + boxPadding * 2
314
- );
315
305
  const paddedRect = {
316
- left: paddedLeft,
317
- top: paddedTop,
318
- width: paddedWidth,
319
- height: paddedHeight
306
+ left: Math.max(0, element.rect.left - boxPadding),
307
+ top: Math.max(0, element.rect.top - boxPadding),
308
+ width: Math.min(
309
+ imageWidth - element.rect.left,
310
+ element.rect.width + boxPadding * 2
311
+ ),
312
+ height: Math.min(
313
+ imageHeight - element.rect.top,
314
+ element.rect.height + boxPadding * 2
315
+ )
320
316
  };
321
317
  image.scan(
322
318
  paddedRect.left,
@@ -324,10 +320,7 @@ var createSvgOverlay = async (elements, imageWidth, imageHeight, boxPadding = 5,
324
320
  paddedRect.width,
325
321
  paddedRect.height,
326
322
  (x, y, idx) => {
327
- if (x >= paddedRect.left && x < paddedRect.left + borderThickness || // Left border
328
- x <= paddedRect.left + paddedRect.width - 1 && x > paddedRect.left + paddedRect.width - borderThickness || // Right border
329
- y >= paddedRect.top && y < paddedRect.top + borderThickness || // Top border
330
- y <= paddedRect.top + paddedRect.height - 1 && y > paddedRect.top + paddedRect.height - borderThickness) {
323
+ if (x === paddedRect.left || x === paddedRect.left + paddedRect.width - 1 || y === paddedRect.top || y === paddedRect.top + paddedRect.height - 1) {
331
324
  image.bitmap.data[idx + 0] = color.rect >> 24 & 255;
332
325
  image.bitmap.data[idx + 1] = color.rect >> 16 & 255;
333
326
  image.bitmap.data[idx + 2] = color.rect >> 8 & 255;
@@ -435,8 +428,7 @@ var compositeElementInfoImg = async (options) => {
435
428
  elementsPositionInfo,
436
429
  width,
437
430
  height,
438
- options.annotationPadding,
439
- options.borderThickness
431
+ options.annotationPadding
440
432
  );
441
433
  const svgImage = await Jimp.read(svgOverlay);
442
434
  const compositeImage = await image.composite(svgImage, 0, 0, {
package/dist/lib/logger.js CHANGED
@@ -55,6 +55,7 @@ var MIDSCENE_DEBUG_MODE = "MIDSCENE_DEBUG_MODE";
55
55
  var MIDSCENE_MCP_USE_PUPPETEER_MODE = "MIDSCENE_MCP_USE_PUPPETEER_MODE";
56
56
  var MIDSCENE_FORCE_DEEP_THINK = "MIDSCENE_FORCE_DEEP_THINK";
57
57
  var MIDSCENE_OPENAI_SOCKS_PROXY = "MIDSCENE_OPENAI_SOCKS_PROXY";
58
+ var MIDSCENE_OPENAI_HTTP_PROXY = "MIDSCENE_OPENAI_HTTP_PROXY";
58
59
  var OPENAI_API_KEY = "OPENAI_API_KEY";
59
60
  var OPENAI_BASE_URL = "OPENAI_BASE_URL";
60
61
  var OPENAI_MAX_TOKENS = "OPENAI_MAX_TOKENS";
@@ -67,7 +68,6 @@ var MIDSCENE_USE_GEMINI = "MIDSCENE_USE_GEMINI";
67
68
  var MIDSCENE_USE_VL_MODEL = "MIDSCENE_USE_VL_MODEL";
68
69
  var MATCH_BY_POSITION = "MATCH_BY_POSITION";
69
70
  var MIDSCENE_REPORT_TAG_NAME = "MIDSCENE_REPORT_TAG_NAME";
70
- var MIDSCENE_PREFERRED_LANGUAGE = "MIDSCENE_PREFERRED_LANGUAGE";
71
71
  var MIDSCENE_USE_AZURE_OPENAI = "MIDSCENE_USE_AZURE_OPENAI";
72
72
  var MIDSCENE_AZURE_OPENAI_SCOPE = "MIDSCENE_AZURE_OPENAI_SCOPE";
73
73
  var MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON = "MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON";
@@ -98,6 +98,7 @@ var allConfigFromEnv = () => {
98
98
  [MATCH_BY_POSITION]: process.env[MATCH_BY_POSITION] || void 0,
99
99
  [MIDSCENE_REPORT_TAG_NAME]: process.env[MIDSCENE_REPORT_TAG_NAME] || void 0,
100
100
  [MIDSCENE_OPENAI_SOCKS_PROXY]: process.env[MIDSCENE_OPENAI_SOCKS_PROXY] || void 0,
101
+ [MIDSCENE_OPENAI_HTTP_PROXY]: process.env[MIDSCENE_OPENAI_HTTP_PROXY] || void 0,
101
102
  [MIDSCENE_USE_AZURE_OPENAI]: process.env[MIDSCENE_USE_AZURE_OPENAI] || void 0,
102
103
  [MIDSCENE_AZURE_OPENAI_SCOPE]: process.env[MIDSCENE_AZURE_OPENAI_SCOPE] || void 0,
103
104
  [MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON]: process.env[MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON] || void 0,
@@ -113,8 +114,7 @@ var allConfigFromEnv = () => {
113
114
  [AZURE_OPENAI_API_VERSION]: process.env[AZURE_OPENAI_API_VERSION] || void 0,
114
115
  [AZURE_OPENAI_DEPLOYMENT]: process.env[AZURE_OPENAI_DEPLOYMENT] || void 0,
115
116
  [MIDSCENE_MCP_USE_PUPPETEER_MODE]: process.env[MIDSCENE_MCP_USE_PUPPETEER_MODE] || void 0,
116
- [MIDSCENE_RUN_DIR]: process.env[MIDSCENE_RUN_DIR] || void 0,
117
- [MIDSCENE_PREFERRED_LANGUAGE]: process.env[MIDSCENE_PREFERRED_LANGUAGE] || void 0
117
+ [MIDSCENE_RUN_DIR]: process.env[MIDSCENE_RUN_DIR] || void 0
118
118
  };
119
119
  };
120
120
  var globalConfig = null;
package/dist/types/env.d.ts CHANGED
@@ -8,6 +8,7 @@ declare const MIDSCENE_DEBUG_MODE = "MIDSCENE_DEBUG_MODE";
8
8
  declare const MIDSCENE_MCP_USE_PUPPETEER_MODE = "MIDSCENE_MCP_USE_PUPPETEER_MODE";
9
9
  declare const MIDSCENE_FORCE_DEEP_THINK = "MIDSCENE_FORCE_DEEP_THINK";
10
10
  declare const MIDSCENE_OPENAI_SOCKS_PROXY = "MIDSCENE_OPENAI_SOCKS_PROXY";
11
+ declare const MIDSCENE_OPENAI_HTTP_PROXY = "MIDSCENE_OPENAI_HTTP_PROXY";
11
12
  declare const OPENAI_API_KEY = "OPENAI_API_KEY";
12
13
  declare const OPENAI_BASE_URL = "OPENAI_BASE_URL";
13
14
  declare const OPENAI_MAX_TOKENS = "OPENAI_MAX_TOKENS";
@@ -21,7 +22,6 @@ declare const MIDSCENE_USE_VL_MODEL = "MIDSCENE_USE_VL_MODEL";
21
22
  declare const MATCH_BY_POSITION = "MATCH_BY_POSITION";
22
23
  declare const MIDSCENE_API_TYPE = "MIDSCENE-API-TYPE";
23
24
  declare const MIDSCENE_REPORT_TAG_NAME = "MIDSCENE_REPORT_TAG_NAME";
24
- declare const MIDSCENE_PREFERRED_LANGUAGE = "MIDSCENE_PREFERRED_LANGUAGE";
25
25
  declare const MIDSCENE_USE_AZURE_OPENAI = "MIDSCENE_USE_AZURE_OPENAI";
26
26
  declare const MIDSCENE_AZURE_OPENAI_SCOPE = "MIDSCENE_AZURE_OPENAI_SCOPE";
27
27
  declare const MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON = "MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON";
@@ -51,6 +51,7 @@ declare const allConfigFromEnv: () => {
51
51
  MATCH_BY_POSITION: string | undefined;
52
52
  MIDSCENE_REPORT_TAG_NAME: string | undefined;
53
53
  MIDSCENE_OPENAI_SOCKS_PROXY: string | undefined;
54
+ MIDSCENE_OPENAI_HTTP_PROXY: string | undefined;
54
55
  MIDSCENE_USE_AZURE_OPENAI: string | undefined;
55
56
  MIDSCENE_AZURE_OPENAI_SCOPE: string | undefined;
56
57
  MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON: string | undefined;
@@ -67,7 +68,6 @@ declare const allConfigFromEnv: () => {
67
68
  AZURE_OPENAI_DEPLOYMENT: string | undefined;
68
69
  MIDSCENE_MCP_USE_PUPPETEER_MODE: string | undefined;
69
70
  MIDSCENE_RUN_DIR: string | undefined;
70
- MIDSCENE_PREFERRED_LANGUAGE: string | undefined;
71
71
  };
72
72
  declare enum UITarsModelVersion {
73
73
  V1_0 = "1.0",
@@ -81,6 +81,5 @@ declare const getAIConfig: (configKey: keyof ReturnType<typeof allConfigFromEnv>
81
81
  declare const getAIConfigInBoolean: (configKey: keyof ReturnType<typeof allConfigFromEnv>) => boolean;
82
82
  declare const getAIConfigInJson: (configKey: keyof ReturnType<typeof allConfigFromEnv>) => any;
83
83
  declare const overrideAIConfig: (newConfig: Partial<ReturnType<typeof allConfigFromEnv>>, extendMode?: boolean) => void;
84
- declare const getPreferredLanguage: () => string | undefined;
85
84
 
86
- export { ANTHROPIC_API_KEY, AZURE_OPENAI_API_VERSION, AZURE_OPENAI_DEPLOYMENT, AZURE_OPENAI_ENDPOINT, AZURE_OPENAI_KEY, MATCH_BY_POSITION, MIDSCENE_ADB_PATH, MIDSCENE_API_TYPE, MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_AZURE_OPENAI_SCOPE, MIDSCENE_CACHE, MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG, MIDSCENE_DEBUG_AI_PROFILE, MIDSCENE_DEBUG_AI_RESPONSE, MIDSCENE_DEBUG_MODE, MIDSCENE_FORCE_DEEP_THINK, MIDSCENE_LANGSMITH_DEBUG, MIDSCENE_MCP_USE_PUPPETEER_MODE, MIDSCENE_MODEL_NAME, MIDSCENE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_OPENAI_SOCKS_PROXY, MIDSCENE_PREFERRED_LANGUAGE, MIDSCENE_REPORT_TAG_NAME, MIDSCENE_RUN_DIR, MIDSCENE_USE_ANTHROPIC_SDK, MIDSCENE_USE_AZURE_OPENAI, MIDSCENE_USE_DOUBAO_VISION, MIDSCENE_USE_GEMINI, MIDSCENE_USE_QWEN_VL, MIDSCENE_USE_VLM_UI_TARS, MIDSCENE_USE_VL_MODEL, OPENAI_API_KEY, OPENAI_BASE_URL, OPENAI_MAX_TOKENS, OPENAI_USE_AZURE, UITarsModelVersion, allConfigFromEnv, getAIConfig, getAIConfigInBoolean, getAIConfigInJson, getPreferredLanguage, overrideAIConfig, uiTarsModelVersion, vlLocateMode };
85
+ export { ANTHROPIC_API_KEY, AZURE_OPENAI_API_VERSION, AZURE_OPENAI_DEPLOYMENT, AZURE_OPENAI_ENDPOINT, AZURE_OPENAI_KEY, MATCH_BY_POSITION, MIDSCENE_ADB_PATH, MIDSCENE_API_TYPE, MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_AZURE_OPENAI_SCOPE, MIDSCENE_CACHE, MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG, MIDSCENE_DEBUG_AI_PROFILE, MIDSCENE_DEBUG_AI_RESPONSE, MIDSCENE_DEBUG_MODE, MIDSCENE_FORCE_DEEP_THINK, MIDSCENE_LANGSMITH_DEBUG, MIDSCENE_MCP_USE_PUPPETEER_MODE, MIDSCENE_MODEL_NAME, MIDSCENE_OPENAI_HTTP_PROXY, MIDSCENE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_OPENAI_SOCKS_PROXY, MIDSCENE_REPORT_TAG_NAME, MIDSCENE_RUN_DIR, MIDSCENE_USE_ANTHROPIC_SDK, MIDSCENE_USE_AZURE_OPENAI, MIDSCENE_USE_DOUBAO_VISION, MIDSCENE_USE_GEMINI, MIDSCENE_USE_QWEN_VL, MIDSCENE_USE_VLM_UI_TARS, MIDSCENE_USE_VL_MODEL, OPENAI_API_KEY, OPENAI_BASE_URL, OPENAI_MAX_TOKENS, OPENAI_USE_AZURE, UITarsModelVersion, allConfigFromEnv, getAIConfig, getAIConfigInBoolean, getAIConfigInJson, overrideAIConfig, uiTarsModelVersion, vlLocateMode };
package/dist/types/img.d.ts CHANGED
@@ -114,19 +114,14 @@ declare function paddingToMatchBlockByBase64(imageBase64: string, blockSize?: nu
114
114
  declare function cropByRect(imageBase64: string, rect: Rect, paddingImage: boolean): Promise<string>;
115
115
  declare function jimpToBase64(image: Jimp): Promise<string>;
116
116
 
117
- interface ElementForOverlay {
118
- rect: Rect;
119
- indexId?: number;
120
- }
121
117
  declare const compositeElementInfoImg: (options: {
122
118
  inputImgBase64: string;
123
- elementsPositionInfo: Array<ElementForOverlay>;
119
+ elementsPositionInfo: Array<BaseElement>;
124
120
  size?: {
125
121
  width: number;
126
122
  height: number;
127
123
  };
128
124
  annotationPadding?: number;
129
- borderThickness?: number;
130
125
  }) => Promise<string>;
131
126
  declare const processImageElementInfo: (options: {
132
127
  inputImgBase64: string;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@midscene/shared",
3
- "version": "0.16.9-beta-20250507095704.0",
3
+ "version": "0.16.9",
4
4
  "repository": "https://github.com/web-infra-dev/midscene",
5
5
  "homepage": "https://midscenejs.com/",
6
6
  "types": "./dist/types/index.d.ts",
package/src/env.ts CHANGED
@@ -14,6 +14,7 @@ export const MIDSCENE_MCP_USE_PUPPETEER_MODE =
14
14
  export const MIDSCENE_FORCE_DEEP_THINK = 'MIDSCENE_FORCE_DEEP_THINK';
15
15
 
16
16
  export const MIDSCENE_OPENAI_SOCKS_PROXY = 'MIDSCENE_OPENAI_SOCKS_PROXY';
17
+ export const MIDSCENE_OPENAI_HTTP_PROXY = 'MIDSCENE_OPENAI_HTTP_PROXY';
17
18
  export const OPENAI_API_KEY = 'OPENAI_API_KEY';
18
19
  export const OPENAI_BASE_URL = 'OPENAI_BASE_URL';
19
20
  export const OPENAI_MAX_TOKENS = 'OPENAI_MAX_TOKENS';
@@ -30,8 +31,6 @@ export const MATCH_BY_POSITION = 'MATCH_BY_POSITION';
30
31
  export const MIDSCENE_API_TYPE = 'MIDSCENE-API-TYPE';
31
32
  export const MIDSCENE_REPORT_TAG_NAME = 'MIDSCENE_REPORT_TAG_NAME';
32
33
 
33
- export const MIDSCENE_PREFERRED_LANGUAGE = 'MIDSCENE_PREFERRED_LANGUAGE';
34
-
35
34
  export const MIDSCENE_USE_AZURE_OPENAI = 'MIDSCENE_USE_AZURE_OPENAI';
36
35
  export const MIDSCENE_AZURE_OPENAI_SCOPE = 'MIDSCENE_AZURE_OPENAI_SCOPE';
37
36
  export const MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON =
@@ -77,6 +76,8 @@ export const allConfigFromEnv = () => {
77
76
  process.env[MIDSCENE_REPORT_TAG_NAME] || undefined,
78
77
  [MIDSCENE_OPENAI_SOCKS_PROXY]:
79
78
  process.env[MIDSCENE_OPENAI_SOCKS_PROXY] || undefined,
79
+ [MIDSCENE_OPENAI_HTTP_PROXY]:
80
+ process.env[MIDSCENE_OPENAI_HTTP_PROXY] || undefined,
80
81
  [MIDSCENE_USE_AZURE_OPENAI]:
81
82
  process.env[MIDSCENE_USE_AZURE_OPENAI] || undefined,
82
83
  [MIDSCENE_AZURE_OPENAI_SCOPE]:
@@ -102,8 +103,6 @@ export const allConfigFromEnv = () => {
102
103
  [MIDSCENE_MCP_USE_PUPPETEER_MODE]:
103
104
  process.env[MIDSCENE_MCP_USE_PUPPETEER_MODE] || undefined,
104
105
  [MIDSCENE_RUN_DIR]: process.env[MIDSCENE_RUN_DIR] || undefined,
105
- [MIDSCENE_PREFERRED_LANGUAGE]:
106
- process.env[MIDSCENE_PREFERRED_LANGUAGE] || undefined,
107
106
  };
108
107
  };
109
108
 
@@ -245,13 +244,3 @@ export const overrideAIConfig = (
245
244
  ? { ...currentConfig, ...newConfig }
246
245
  : { ...newConfig };
247
246
  };
248
-
249
- export const getPreferredLanguage = () => {
250
- if (getAIConfig(MIDSCENE_PREFERRED_LANGUAGE)) {
251
- return getAIConfig(MIDSCENE_PREFERRED_LANGUAGE);
252
- }
253
-
254
- const timeZone = Intl.DateTimeFormat().resolvedOptions().timeZone;
255
- const isChina = timeZone === 'Asia/Shanghai';
256
- return isChina ? 'Chinese' : 'English';
257
- };
package/src/img/box-select.ts CHANGED
@@ -1,6 +1,6 @@
1
1
  import assert from 'node:assert';
2
2
  import type Jimp from 'jimp';
3
- import type { BaseElement, Rect } from '../types';
3
+ import type { BaseElement } from '../types';
4
4
  import getJimp from './get-jimp';
5
5
  import { bufferFromBase64, imageInfoOfBase64 } from './index';
6
6
 
@@ -21,17 +21,11 @@ const loadFonts = async () => {
21
21
  }
22
22
  };
23
23
 
24
- interface ElementForOverlay {
25
- rect: Rect;
26
- indexId?: number;
27
- }
28
-
29
24
  const createSvgOverlay = async (
30
- elements: Array<ElementForOverlay>,
25
+ elements: Array<BaseElement>,
31
26
  imageWidth: number,
32
27
  imageHeight: number,
33
28
  boxPadding = 5,
34
- borderThickness = 2,
35
29
  ): Promise<Jimp> => {
36
30
  const Jimp = await getJimp();
37
31
  const image = new Jimp(imageWidth, imageHeight, 0x00000000);
@@ -50,21 +44,17 @@ const createSvgOverlay = async (
50
44
  const color = colors[index % colors.length];
51
45
 
52
46
  // Add 5px padding to the rect
53
- const paddedLeft = Math.max(0, element.rect.left - boxPadding);
54
- const paddedTop = Math.max(0, element.rect.top - boxPadding);
55
- const paddedWidth = Math.min(
56
- imageWidth - paddedLeft,
57
- element.rect.width + boxPadding * 2,
58
- );
59
- const paddedHeight = Math.min(
60
- imageHeight - paddedTop,
61
- element.rect.height + boxPadding * 2,
62
- );
63
47
  const paddedRect = {
64
- left: paddedLeft,
65
- top: paddedTop,
66
- width: paddedWidth,
67
- height: paddedHeight,
48
+ left: Math.max(0, element.rect.left - boxPadding),
49
+ top: Math.max(0, element.rect.top - boxPadding),
50
+ width: Math.min(
51
+ imageWidth - element.rect.left,
52
+ element.rect.width + boxPadding * 2,
53
+ ),
54
+ height: Math.min(
55
+ imageHeight - element.rect.top,
56
+ element.rect.height + boxPadding * 2,
57
+ ),
68
58
  };
69
59
 
70
60
  // Draw rectangle
@@ -75,12 +65,10 @@ const createSvgOverlay = async (
75
65
  paddedRect.height,
76
66
  (x: number, y: number, idx: number): void => {
77
67
  if (
78
- (x >= paddedRect.left && x < paddedRect.left + borderThickness) || // Left border
79
- (x <= paddedRect.left + paddedRect.width - 1 &&
80
- x > paddedRect.left + paddedRect.width - borderThickness) || // Right border
81
- (y >= paddedRect.top && y < paddedRect.top + borderThickness) || // Top border
82
- (y <= paddedRect.top + paddedRect.height - 1 &&
83
- y > paddedRect.top + paddedRect.height - borderThickness) // Bottom border
68
+ x === paddedRect.left ||
69
+ x === paddedRect.left + paddedRect.width - 1 ||
70
+ y === paddedRect.top ||
71
+ y === paddedRect.top + paddedRect.height - 1
84
72
  ) {
85
73
  image.bitmap.data[idx + 0] = (color.rect >> 24) & 0xff; // R
86
74
  image.bitmap.data[idx + 1] = (color.rect >> 16) & 0xff; // G
@@ -204,10 +192,9 @@ const createSvgOverlay = async (
204
192
 
205
193
  export const compositeElementInfoImg = async (options: {
206
194
  inputImgBase64: string;
207
- elementsPositionInfo: Array<ElementForOverlay>;
195
+ elementsPositionInfo: Array<BaseElement>;
208
196
  size?: { width: number; height: number };
209
197
  annotationPadding?: number;
210
- borderThickness?: number;
211
198
  }) => {
212
199
  assert(options.inputImgBase64, 'inputImgBase64 is required');
213
200
  let width = 0;
@@ -250,7 +237,6 @@ export const compositeElementInfoImg = async (options: {
250
237
  width,
251
238
  height,
252
239
  options.annotationPadding,
253
- options.borderThickness,
254
240
  );
255
241
  const svgImage = await Jimp.read(svgOverlay);
256
242
  const compositeImage = await image.composite(svgImage, 0, 0, {