@aiscene/shared 8.0.2 → 8.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (177) hide show
  1. package/dist/es/agent-tools/agent-behavior-init-args.mjs +44 -0
  2. package/dist/es/agent-tools/base-tools.mjs +163 -0
  3. package/dist/es/agent-tools/chrome-path.mjs +50 -0
  4. package/dist/es/agent-tools/cli-report-session.mjs +78 -0
  5. package/dist/es/agent-tools/error-formatter.mjs +106 -0
  6. package/dist/es/agent-tools/index.mjs +9 -0
  7. package/dist/es/agent-tools/init-arg-utils.mjs +38 -0
  8. package/dist/es/agent-tools/observation-artifact.mjs +5 -0
  9. package/dist/es/agent-tools/observation-record.mjs +248 -0
  10. package/dist/es/agent-tools/tool-defaults.mjs +54 -0
  11. package/dist/es/agent-tools/tool-generator.mjs +529 -0
  12. package/dist/es/agent-tools/types.mjs +3 -0
  13. package/dist/es/agent-tools/user-prompt.mjs +66 -0
  14. package/dist/es/cli/cli-args.mjs +95 -0
  15. package/dist/es/cli/cli-error.mjs +24 -0
  16. package/dist/es/cli/cli-runner.mjs +10 -40
  17. package/dist/es/cli/index.mjs +4 -2
  18. package/dist/es/cli/interrupt.mjs +116 -0
  19. package/dist/es/cli/record-command.mjs +130 -0
  20. package/dist/es/cli/screenshot-file.mjs +24 -0
  21. package/dist/es/cli/verbose-ai-act.mjs +230 -0
  22. package/dist/es/cli/verbose-screenshot.mjs +134 -0
  23. package/dist/es/cli/verbose.mjs +404 -0
  24. package/dist/es/constants/example-code.mjs +2 -2
  25. package/dist/es/constants/index.mjs +6 -1
  26. package/dist/es/env/model-config-manager.mjs +3 -0
  27. package/dist/es/env/parse-model-config.mjs +6 -1
  28. package/dist/es/env/types.mjs +4 -1
  29. package/dist/es/env/utils.mjs +1 -9
  30. package/dist/es/extractor/index.mjs +2 -2
  31. package/dist/es/img/index.mjs +2 -2
  32. package/dist/es/img/info.mjs +58 -17
  33. package/dist/es/img/photon-loader.mjs +4 -0
  34. package/dist/es/key-alias-utils.mjs +19 -0
  35. package/dist/es/mcp/base-server.mjs +10 -9
  36. package/dist/es/mcp/base-tools.mjs +72 -4
  37. package/dist/es/mcp/cli-report-session.mjs +78 -0
  38. package/dist/es/mcp/error-formatter.mjs +19 -0
  39. package/dist/es/mcp/index.mjs +2 -0
  40. package/dist/es/mcp/init-arg-utils.mjs +38 -0
  41. package/dist/es/mcp/inject-report-html-plugin.mjs +1 -1
  42. package/dist/es/mcp/tool-generator.mjs +65 -17
  43. package/dist/es/node/fs.mjs +1 -1
  44. package/dist/es/recorder.mjs +249 -0
  45. package/dist/lib/agent-tools/agent-behavior-init-args.js +87 -0
  46. package/dist/lib/agent-tools/base-tools.js +197 -0
  47. package/dist/lib/agent-tools/chrome-path.js +87 -0
  48. package/dist/lib/agent-tools/cli-report-session.js +121 -0
  49. package/dist/lib/agent-tools/error-formatter.js +149 -0
  50. package/dist/lib/agent-tools/index.js +114 -0
  51. package/dist/lib/agent-tools/init-arg-utils.js +78 -0
  52. package/dist/lib/agent-tools/observation-artifact.js +42 -0
  53. package/dist/lib/agent-tools/observation-record.js +297 -0
  54. package/dist/lib/agent-tools/tool-defaults.js +97 -0
  55. package/dist/lib/agent-tools/tool-generator.js +569 -0
  56. package/dist/lib/agent-tools/types.js +40 -0
  57. package/dist/lib/agent-tools/user-prompt.js +103 -0
  58. package/dist/lib/cli/cli-args.js +138 -0
  59. package/dist/lib/cli/cli-error.js +61 -0
  60. package/dist/lib/cli/cli-runner.js +19 -46
  61. package/dist/lib/cli/index.js +8 -3
  62. package/dist/lib/cli/interrupt.js +156 -0
  63. package/dist/lib/cli/record-command.js +164 -0
  64. package/dist/lib/cli/screenshot-file.js +58 -0
  65. package/dist/lib/cli/verbose-ai-act.js +273 -0
  66. package/dist/lib/cli/verbose-screenshot.js +177 -0
  67. package/dist/lib/cli/verbose.js +465 -0
  68. package/dist/lib/constants/example-code.js +2 -2
  69. package/dist/lib/constants/index.js +23 -3
  70. package/dist/lib/env/model-config-manager.js +3 -0
  71. package/dist/lib/env/parse-model-config.js +6 -1
  72. package/dist/lib/env/types.js +9 -0
  73. package/dist/lib/env/utils.js +0 -11
  74. package/dist/lib/extractor/index.js +3 -0
  75. package/dist/lib/img/index.js +3 -0
  76. package/dist/lib/img/info.js +65 -18
  77. package/dist/lib/img/photon-loader.js +38 -0
  78. package/dist/lib/key-alias-utils.js +62 -0
  79. package/dist/lib/mcp/base-server.js +11 -10
  80. package/dist/lib/mcp/base-tools.js +74 -6
  81. package/dist/lib/mcp/cli-report-session.js +121 -0
  82. package/dist/lib/mcp/error-formatter.js +53 -0
  83. package/dist/lib/mcp/index.js +24 -10
  84. package/dist/lib/mcp/init-arg-utils.js +78 -0
  85. package/dist/lib/mcp/inject-report-html-plugin.js +1 -1
  86. package/dist/lib/mcp/tool-generator.js +74 -26
  87. package/dist/lib/node/fs.js +1 -1
  88. package/dist/lib/recorder.js +307 -0
  89. package/dist/types/agent-tools/agent-behavior-init-args.d.ts +17 -0
  90. package/dist/types/agent-tools/base-tools.d.ts +158 -0
  91. package/dist/types/agent-tools/chrome-path.d.ts +2 -0
  92. package/dist/types/agent-tools/cli-report-session.d.ts +12 -0
  93. package/dist/types/agent-tools/error-formatter.d.ts +30 -0
  94. package/dist/types/agent-tools/index.d.ts +9 -0
  95. package/dist/types/agent-tools/init-arg-utils.d.ts +13 -0
  96. package/dist/types/agent-tools/observation-artifact.d.ts +10 -0
  97. package/dist/types/agent-tools/observation-record.d.ts +38 -0
  98. package/dist/types/agent-tools/tool-defaults.d.ts +63 -0
  99. package/dist/types/agent-tools/tool-generator.d.ts +13 -0
  100. package/dist/types/agent-tools/types.d.ts +213 -0
  101. package/dist/types/agent-tools/user-prompt.d.ts +13 -0
  102. package/dist/types/cli/cli-args.d.ts +8 -0
  103. package/dist/types/cli/cli-error.d.ts +5 -0
  104. package/dist/types/cli/cli-runner.d.ts +4 -7
  105. package/dist/types/cli/index.d.ts +3 -1
  106. package/dist/types/cli/interrupt.d.ts +49 -0
  107. package/dist/types/cli/record-command.d.ts +3 -0
  108. package/dist/types/cli/screenshot-file.d.ts +10 -0
  109. package/dist/types/cli/verbose-ai-act.d.ts +44 -0
  110. package/dist/types/cli/verbose-screenshot.d.ts +10 -0
  111. package/dist/types/cli/verbose.d.ts +40 -0
  112. package/dist/types/constants/example-code.d.ts +1 -1
  113. package/dist/types/constants/index.d.ts +5 -0
  114. package/dist/types/env/types.d.ts +14 -5
  115. package/dist/types/env/utils.d.ts +0 -31
  116. package/dist/types/extractor/index.d.ts +1 -1
  117. package/dist/types/img/index.d.ts +1 -1
  118. package/dist/types/img/info.d.ts +7 -0
  119. package/dist/types/img/photon-loader.d.ts +2 -0
  120. package/dist/types/key-alias-utils.d.ts +9 -0
  121. package/dist/types/mcp/base-tools.d.ts +74 -5
  122. package/dist/types/mcp/cli-report-session.d.ts +12 -0
  123. package/dist/types/mcp/error-formatter.d.ts +12 -0
  124. package/dist/types/mcp/index.d.ts +2 -0
  125. package/dist/types/mcp/init-arg-utils.d.ts +13 -0
  126. package/dist/types/mcp/inject-report-html-plugin.d.ts +2 -2
  127. package/dist/types/mcp/launcher-helper.d.ts +3 -3
  128. package/dist/types/mcp/tool-generator.d.ts +3 -3
  129. package/dist/types/mcp/types.d.ts +10 -0
  130. package/dist/types/recorder.d.ts +113 -0
  131. package/package.json +1 -1
  132. package/src/agent-tools/agent-behavior-init-args.ts +109 -0
  133. package/src/agent-tools/base-tools.ts +399 -0
  134. package/src/agent-tools/chrome-path.ts +74 -0
  135. package/src/agent-tools/cli-report-session.ts +130 -0
  136. package/src/agent-tools/error-formatter.ts +177 -0
  137. package/src/agent-tools/index.ts +9 -0
  138. package/src/agent-tools/init-arg-utils.ts +105 -0
  139. package/src/agent-tools/observation-artifact.ts +29 -0
  140. package/src/agent-tools/observation-record.ts +331 -0
  141. package/src/agent-tools/tool-defaults.ts +119 -0
  142. package/src/agent-tools/tool-generator.ts +866 -0
  143. package/src/agent-tools/types.ts +250 -0
  144. package/src/agent-tools/user-prompt.ts +102 -0
  145. package/src/cli/cli-args.ts +173 -0
  146. package/src/cli/cli-error.ts +24 -0
  147. package/src/cli/cli-runner.ts +37 -56
  148. package/src/cli/index.ts +3 -7
  149. package/src/cli/interrupt.ts +207 -0
  150. package/src/cli/record-command.ts +177 -0
  151. package/src/cli/screenshot-file.ts +61 -0
  152. package/src/cli/verbose-ai-act.ts +387 -0
  153. package/src/cli/verbose-screenshot.ts +269 -0
  154. package/src/cli/verbose.ts +753 -0
  155. package/src/constants/example-code.ts +2 -2
  156. package/src/constants/index.ts +10 -0
  157. package/src/env/model-config-manager.ts +3 -0
  158. package/src/env/parse-model-config.ts +19 -1
  159. package/src/env/types.ts +13 -2
  160. package/src/env/utils.ts +0 -50
  161. package/src/extractor/index.ts +1 -1
  162. package/src/img/index.ts +14 -0
  163. package/src/img/info.ts +97 -45
  164. package/src/img/photon-loader.ts +5 -0
  165. package/src/img/transform.ts +261 -2
  166. package/src/key-alias-utils.ts +23 -0
  167. package/src/mcp/base-server.ts +10 -12
  168. package/src/mcp/base-tools.ts +218 -11
  169. package/src/mcp/cli-report-session.ts +130 -0
  170. package/src/mcp/error-formatter.ts +52 -0
  171. package/src/mcp/index.ts +2 -0
  172. package/src/mcp/init-arg-utils.ts +105 -0
  173. package/src/mcp/inject-report-html-plugin.ts +3 -3
  174. package/src/mcp/launcher-helper.ts +3 -3
  175. package/src/mcp/tool-generator.ts +98 -20
  176. package/src/mcp/types.ts +18 -1
  177. package/src/recorder.ts +625 -0
@@ -5,14 +5,6 @@ const globalModelConfigManager = new ModelConfigManager();
5
5
  const globalConfigManager = new GlobalConfigManager();
6
6
  globalConfigManager.registerModelConfigManager(globalModelConfigManager);
7
7
  globalModelConfigManager.registerGlobalConfigManager(globalConfigManager);
8
- async function getCurrentTime(device, useDeviceTimestamp) {
9
- if (useDeviceTimestamp && device?.getTimestamp) try {
10
- return await device.getTimestamp();
11
- } catch (error) {
12
- console.warn(`Failed to get device time, falling back to system time: ${error}`);
13
- }
14
- return Date.now();
15
- }
16
8
  const getPreferredLanguage = ()=>{
17
9
  const prefer = globalConfigManager.getEnvConfigValue(MIDSCENE_PREFERRED_LANGUAGE);
18
10
  if (prefer) return prefer;
@@ -23,4 +15,4 @@ const getPreferredLanguage = ()=>{
23
15
  const overrideAIConfig = (newConfig, extendMode = false)=>{
24
16
  globalConfigManager.overrideAIConfig(newConfig, extendMode);
25
17
  };
26
- export { getCurrentTime, getPreferredLanguage, globalConfigManager, globalModelConfigManager, overrideAIConfig };
18
+ export { getPreferredLanguage, globalConfigManager, globalModelConfigManager, overrideAIConfig };
@@ -1,5 +1,5 @@
1
1
  import { descriptionOfTree, traverseTree, treeToList, trimAttributes, truncateText } from "./tree.mjs";
2
2
  import { extractTextWithPosition, extractTreeNode, extractTreeNodeAsString } from "./web-extractor.mjs";
3
3
  import { getElementInfoByXpath, getElementXpath, getNodeInfoByXpath, getXpathsById, getXpathsByPoint } from "./locator.mjs";
4
- import { generateElementByRect, isNotContainerElement } from "./dom-util.mjs";
5
- export { descriptionOfTree, generateElementByRect, getElementInfoByXpath, getElementXpath, getNodeInfoByXpath, getXpathsById, getXpathsByPoint, isNotContainerElement, traverseTree, treeToList, trimAttributes, truncateText, extractTreeNode as webExtractNodeTree, extractTreeNodeAsString as webExtractNodeTreeAsString, extractTextWithPosition as webExtractTextWithPosition };
4
+ import { generateElementByPoint, generateElementByRect, isNotContainerElement } from "./dom-util.mjs";
5
+ export { descriptionOfTree, generateElementByPoint, generateElementByRect, getElementInfoByXpath, getElementXpath, getNodeInfoByXpath, getXpathsById, getXpathsByPoint, isNotContainerElement, traverseTree, treeToList, trimAttributes, truncateText, extractTreeNode as webExtractNodeTree, extractTreeNodeAsString as webExtractNodeTreeAsString, extractTextWithPosition as webExtractTextWithPosition };
@@ -1,4 +1,4 @@
1
- import { imageInfoOfBase64, isValidImageBuffer, isValidJPEGImageBuffer, isValidPNGImageBuffer } from "./info.mjs";
1
+ import { imageInfoOfBase64, isValidImageBuffer, isValidJPEGImageBuffer, isValidPNGImageBuffer, validateScreenshotBuffer } from "./info.mjs";
2
2
  import { createImgBase64ByFormat, cropByRect, httpImg2Base64, localImg2Base64, paddingToMatchBlockByBase64, parseBase64, preProcessImageUrl, resizeAndConvertImgBuffer, resizeImgBase64, saveBase64Image, scaleImage, zoomForGPT4o } from "./transform.mjs";
3
3
  import { annotateRects, compositeElementInfoImg, processImageElementInfo } from "./box-select.mjs";
4
- export { annotateRects, compositeElementInfoImg, createImgBase64ByFormat, cropByRect, httpImg2Base64, imageInfoOfBase64, isValidImageBuffer, isValidJPEGImageBuffer, isValidPNGImageBuffer, localImg2Base64, paddingToMatchBlockByBase64, parseBase64, preProcessImageUrl, processImageElementInfo, resizeAndConvertImgBuffer, resizeImgBase64, saveBase64Image, scaleImage, zoomForGPT4o };
4
+ export { annotateRects, compositeElementInfoImg, createImgBase64ByFormat, cropByRect, httpImg2Base64, imageInfoOfBase64, isValidImageBuffer, isValidJPEGImageBuffer, isValidPNGImageBuffer, localImg2Base64, paddingToMatchBlockByBase64, parseBase64, preProcessImageUrl, processImageElementInfo, resizeAndConvertImgBuffer, resizeImgBase64, saveBase64Image, scaleImage, validateScreenshotBuffer, zoomForGPT4o };
@@ -1,27 +1,62 @@
1
1
  import node_assert from "node:assert";
2
2
  import get_photon from "./get-photon.mjs";
3
- async function imageInfoOfBase64(imageBase64) {
4
- if (!imageBase64 || 'string' != typeof imageBase64) throw new Error('Invalid image base64: data is empty or not a string');
5
- const base64Data = imageBase64.replace(/^data:image\/\w+;base64,/, '');
6
- if (!base64Data || 0 === base64Data.length) throw new Error('Invalid image base64: no data after stripping prefix');
7
- if (!/^[A-Za-z0-9+/]*={0,2}$/.test(base64Data)) throw new Error('Invalid image base64: contains invalid characters');
8
- if (base64Data.length < 4) throw new Error(`Invalid image base64: data too short (${base64Data.length} characters)`);
9
- try {
10
- const { PhotonImage } = await get_photon();
11
- const result = PhotonImage.new_from_base64(base64Data);
12
- const image = result instanceof Promise ? await result : result;
13
- const width = image.get_width();
14
- const height = image.get_height();
15
- image.free();
3
+ const isJpegStartOfFrameMarker = (marker)=>marker >= 0xc0 && marker <= 0xcf && 0xc4 !== marker && 0xc8 !== marker && 0xcc !== marker;
4
+ function jpegInfoFromBuffer(imageBuffer) {
5
+ let offset = 2;
6
+ while(offset < imageBuffer.length){
7
+ if (0xff !== imageBuffer[offset]) {
8
+ offset += 1;
9
+ continue;
10
+ }
11
+ while(offset < imageBuffer.length && 0xff === imageBuffer[offset])offset += 1;
12
+ if (offset >= imageBuffer.length) break;
13
+ const marker = imageBuffer[offset];
14
+ offset += 1;
15
+ if (0xd9 === marker || 0xda === marker) break;
16
+ if (0x01 === marker || 0xd8 === marker || marker >= 0xd0 && marker <= 0xd7) continue;
17
+ if (offset + 2 > imageBuffer.length) break;
18
+ const segmentLength = imageBuffer.readUInt16BE(offset);
19
+ if (segmentLength < 2 || offset + segmentLength > imageBuffer.length) break;
20
+ if (isJpegStartOfFrameMarker(marker)) {
21
+ if (segmentLength < 7) break;
22
+ const height = imageBuffer.readUInt16BE(offset + 3);
23
+ const width = imageBuffer.readUInt16BE(offset + 5);
24
+ node_assert(width && height, 'Invalid image: cannot get width or height');
25
+ return {
26
+ width,
27
+ height
28
+ };
29
+ }
30
+ offset += segmentLength;
31
+ }
32
+ throw new Error('Invalid image: cannot get JPEG width or height');
33
+ }
34
+ function encodedImageInfoOfBuffer(imageBuffer) {
35
+ if (isValidPNGImageBuffer(imageBuffer)) {
36
+ const width = imageBuffer.readUInt32BE(16);
37
+ const height = imageBuffer.readUInt32BE(20);
16
38
  node_assert(width && height, 'Invalid image: cannot get width or height');
17
39
  return {
18
40
  width,
19
41
  height
20
42
  };
21
- } catch (error) {
22
- if (error instanceof Error && error.message.includes('unreachable')) throw new Error(`Failed to decode image: invalid or corrupted image data. Original error: ${error.message}`);
23
- throw error;
24
43
  }
44
+ if (isValidJPEGImageBuffer(imageBuffer)) return jpegInfoFromBuffer(imageBuffer);
45
+ throw new Error('Invalid image: unsupported format');
46
+ }
47
+ async function imageInfoOfBase64(imageBase64) {
48
+ const { PhotonImage } = await get_photon();
49
+ const base64Data = imageBase64.replace(/^data:image\/\w+;base64,/, '');
50
+ const result = PhotonImage.new_from_base64(base64Data);
51
+ const image = result instanceof Promise ? await result : result;
52
+ const width = image.get_width();
53
+ const height = image.get_height();
54
+ image.free();
55
+ node_assert(width && height, 'Invalid image: cannot get width or height');
56
+ return {
57
+ width,
58
+ height
59
+ };
25
60
  }
26
61
  function isValidPNGImageBuffer(buffer) {
27
62
  if (!buffer || buffer.length < 8) return false;
@@ -35,4 +70,10 @@ function isValidJPEGImageBuffer(buffer) {
35
70
  function isValidImageBuffer(buffer) {
36
71
  return isValidPNGImageBuffer(buffer) || isValidJPEGImageBuffer(buffer);
37
72
  }
38
- export { imageInfoOfBase64, isValidImageBuffer, isValidJPEGImageBuffer, isValidPNGImageBuffer };
73
+ function validateScreenshotBuffer(screenshotBuffer, { label, minBufferSize = 0 }) {
74
+ const bufferSize = screenshotBuffer?.length ?? 0;
75
+ if (!screenshotBuffer || 0 === bufferSize) throw new Error(`${label} validation failed: buffer size ${bufferSize} bytes`);
76
+ if (!isValidImageBuffer(screenshotBuffer)) throw new Error(`${label} buffer has invalid image format`);
77
+ if (minBufferSize > 0 && bufferSize < minBufferSize) throw new Error(`${label} validation failed: buffer size ${bufferSize} bytes (minimum: ${minBufferSize})`);
78
+ }
79
+ export { encodedImageInfoOfBuffer, imageInfoOfBase64, isValidImageBuffer, isValidJPEGImageBuffer, isValidPNGImageBuffer, validateScreenshotBuffer };
@@ -0,0 +1,4 @@
1
+ async function loadPhotonModule() {
2
+ return import("@silvia-odwyer/photon");
3
+ }
4
+ export { loadPhotonModule };
@@ -0,0 +1,19 @@
1
+ function kebabToCamel(str) {
2
+ return str.replace(/-([a-z])/g, (_, letter)=>letter.toUpperCase());
3
+ }
4
+ function camelToKebab(str) {
5
+ return str.replace(/[A-Z]/g, (letter)=>`-${letter.toLowerCase()}`).replace(/^-/, '');
6
+ }
7
+ function getKeyAliases(key) {
8
+ return [
9
+ ...new Set([
10
+ key,
11
+ kebabToCamel(key),
12
+ camelToKebab(key)
13
+ ])
14
+ ];
15
+ }
16
+ function isRecord(value) {
17
+ return 'object' == typeof value && null !== value && !Array.isArray(value);
18
+ }
19
+ export { camelToKebab, getKeyAliases, isRecord, kebabToCamel };
@@ -1,9 +1,10 @@
1
1
  import { randomUUID } from "node:crypto";
2
+ import { setIsMcp } from "@aiscene/shared/utils";
2
3
  import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
3
4
  import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
4
5
  import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js";
5
6
  import express from "express";
6
- import { setIsMcp } from "../utils.mjs";
7
+ import { getErrorMessage } from "./error-formatter.mjs";
7
8
  function _define_property(obj, key, value) {
8
9
  if (key in obj) Object.defineProperty(obj, key, {
9
10
  value: value,
@@ -45,7 +46,7 @@ class BaseMCPServer {
45
46
  try {
46
47
  await this.toolsManager.initTools();
47
48
  } catch (error) {
48
- const message = error instanceof Error ? error.message : String(error);
49
+ const message = getErrorMessage(error);
49
50
  console.error(`Failed to initialize tools: ${message}`);
50
51
  console.error('Tools will be initialized on first use');
51
52
  }
@@ -71,7 +72,7 @@ class BaseMCPServer {
71
72
  try {
72
73
  await this.mcpServer.connect(transport);
73
74
  } catch (error) {
74
- const message = error instanceof Error ? error.message : String(error);
75
+ const message = getErrorMessage(error);
75
76
  console.error(`Failed to connect MCP stdio transport: ${message}`);
76
77
  throw new Error(`Failed to initialize MCP stdio transport: ${message}`);
77
78
  }
@@ -142,7 +143,7 @@ class BaseMCPServer {
142
143
  });
143
144
  }
144
145
  } catch (error) {
145
- const message = error instanceof Error ? error.message : String(error);
146
+ const message = getErrorMessage(error);
146
147
  const duration = Date.now() - startTime;
147
148
  console.error(`[${new Date().toISOString()}] [${requestId}] MCP request error after ${duration}ms: ${message}`);
148
149
  if (!res.headersSent) res.status(500).json({
@@ -170,7 +171,7 @@ class BaseMCPServer {
170
171
  for (const session of sessions.values())try {
171
172
  await session.transport.close();
172
173
  } catch (error) {
173
- const message = error instanceof Error ? error.message : String(error);
174
+ const message = getErrorMessage(error);
174
175
  console.error(`Failed to close session ${session.transport.sessionId}: ${message}`);
175
176
  }
176
177
  sessions.clear();
@@ -205,7 +206,7 @@ class BaseMCPServer {
205
206
  try {
206
207
  await this.mcpServer.connect(transport);
207
208
  } catch (error) {
208
- const message = error instanceof Error ? error.message : String(error);
209
+ const message = getErrorMessage(error);
209
210
  console.error(`[${new Date().toISOString()}] Failed to connect MCP transport: ${message}`);
210
211
  if (transport.sessionId) sessions.delete(transport.sessionId);
211
212
  throw new Error(`Failed to initialize MCP session: ${message}`);
@@ -224,7 +225,7 @@ class BaseMCPServer {
224
225
  sessions.delete(sid);
225
226
  console.log(`[${new Date().toISOString()}] Session ${sid} cleaned up due to inactivity (remaining: ${sessions.size})`);
226
227
  } catch (error) {
227
- const message = error instanceof Error ? error.message : String(error);
228
+ const message = getErrorMessage(error);
228
229
  console.error(`[${new Date().toISOString()}] Failed to close session ${sid} during cleanup: ${message}`);
229
230
  sessions.delete(sid);
230
231
  }
@@ -237,7 +238,7 @@ class BaseMCPServer {
237
238
  for (const session of sessions.values())try {
238
239
  session.transport.close();
239
240
  } catch (error) {
240
- const message = error instanceof Error ? error.message : String(error);
241
+ const message = getErrorMessage(error);
241
242
  console.error(`Error closing session during shutdown: ${message}`);
242
243
  }
243
244
  sessions.clear();
@@ -250,7 +251,7 @@ class BaseMCPServer {
250
251
  this.performCleanup().finally(()=>process.exit(1));
251
252
  }, 5000);
252
253
  } catch (error) {
253
- const message = error instanceof Error ? error.message : String(error);
254
+ const message = getErrorMessage(error);
254
255
  console.error(`Error closing HTTP server: ${message}`);
255
256
  this.performCleanup().finally(()=>process.exit(1));
256
257
  }
@@ -1,5 +1,8 @@
1
- import { parseBase64 } from "../img/index.mjs";
2
- import { getDebug } from "../logger.mjs";
1
+ import { parseBase64 } from "@aiscene/shared/img";
2
+ import { getDebug } from "@aiscene/shared/logger";
3
+ import { camelToKebab, getKeyAliases } from "../key-alias-utils.mjs";
4
+ import { generateCliReportSession, readCliReportSession, writeCliReportSession } from "./cli-report-session.mjs";
5
+ import { createNamespacedInitArgSchema, extractNamespacedArgs, sanitizeNamespacedArgs } from "./init-arg-utils.mjs";
3
6
  import { generateCommonTools, generateToolsFromActionSpace } from "./tool-generator.mjs";
4
7
  function _define_property(obj, key, value) {
5
8
  if (key in obj) Object.defineProperty(obj, key, {
@@ -13,9 +16,74 @@ function _define_property(obj, key, value) {
13
16
  }
14
17
  const debug = getDebug('mcp:base-tools');
15
18
  class BaseMidsceneTools {
19
+ getInitArgKeys() {
20
+ return this.initArgSpec ? Object.keys(this.initArgSpec.shape) : [];
21
+ }
22
+ extractAgentInitParam(args) {
23
+ if (!this.initArgSpec) return;
24
+ const extracted = extractNamespacedArgs(args, this.initArgSpec.namespace, this.getInitArgKeys());
25
+ if (this.initArgSpec.adapt) return this.initArgSpec.adapt(extracted);
26
+ return extracted;
27
+ }
28
+ sanitizeToolArgs(args) {
29
+ if (!this.initArgSpec) return args;
30
+ return sanitizeNamespacedArgs(args, this.initArgSpec.namespace, this.getInitArgKeys());
31
+ }
32
+ getAgentInitArgSchema() {
33
+ if (!this.initArgSpec) return {};
34
+ return createNamespacedInitArgSchema(this.initArgSpec.namespace, this.initArgSpec.shape);
35
+ }
36
+ getAgentInitArgCliMetadata() {
37
+ if (!this.initArgSpec?.cli) return;
38
+ const options = Object.fromEntries(this.getInitArgKeys().map((key)=>{
39
+ const canonicalKey = `${this.initArgSpec.namespace}.${key}`;
40
+ const preferredName = this.initArgSpec.cli?.preferredNames?.[key] ?? (this.initArgSpec.cli?.preferBareKeys ? camelToKebab(key) : canonicalKey);
41
+ const acceptedNames = new Set([
42
+ preferredName,
43
+ ...this.initArgSpec.cli?.preferBareKeys ? getKeyAliases(key) : getKeyAliases(canonicalKey)
44
+ ]);
45
+ acceptedNames.delete(preferredName);
46
+ return [
47
+ canonicalKey,
48
+ {
49
+ preferredName,
50
+ aliases: [
51
+ ...acceptedNames
52
+ ]
53
+ }
54
+ ];
55
+ }));
56
+ return {
57
+ options
58
+ };
59
+ }
16
60
  preparePlatformTools() {
17
61
  return [];
18
62
  }
63
+ getCliReportSessionName() {}
64
+ createNewCliReportSession(targetIdentity) {
65
+ const sessionName = this.getCliReportSessionName();
66
+ if (!sessionName) return;
67
+ return generateCliReportSession(sessionName, targetIdentity);
68
+ }
69
+ commitCliReportSession(session) {
70
+ if (session) writeCliReportSession(session);
71
+ }
72
+ readCliReportFileName() {
73
+ const sessionName = this.getCliReportSessionName();
74
+ if (!sessionName) return;
75
+ return readCliReportSession(sessionName)?.reportFileName;
76
+ }
77
+ readCliReportAgentOptions() {
78
+ const reportFileName = this.readCliReportFileName();
79
+ if (!reportFileName) return;
80
+ return {
81
+ reportFileName,
82
+ reportAttributes: {
83
+ 'data-group-id': reportFileName
84
+ }
85
+ };
86
+ }
19
87
  async initTools() {
20
88
  this.toolDefinitions = [];
21
89
  const platformTools = this.preparePlatformTools();
@@ -30,8 +98,8 @@ class BaseMidsceneTools {
30
98
  await tempDevice.destroy?.();
31
99
  debug('Action space from temporary device:', actionSpace.map((a)=>a.name).join(', '));
32
100
  }
33
- const actionTools = generateToolsFromActionSpace(actionSpace, ()=>this.ensureAgent());
34
- const commonTools = generateCommonTools(()=>this.ensureAgent());
101
+ const actionTools = generateToolsFromActionSpace(actionSpace, (args = {})=>this.ensureAgent(this.extractAgentInitParam(args)), (args = {})=>this.sanitizeToolArgs(args), this.getAgentInitArgSchema(), this.getAgentInitArgCliMetadata());
102
+ const commonTools = generateCommonTools((args = {})=>this.ensureAgent(this.extractAgentInitParam(args)), this.getAgentInitArgSchema(), this.getAgentInitArgCliMetadata());
35
103
  this.toolDefinitions.push(...actionTools, ...commonTools);
36
104
  debug('Total tools prepared:', this.toolDefinitions.length);
37
105
  }
@@ -0,0 +1,78 @@
1
+ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
2
+ import { join } from "node:path";
3
+ import { getMidsceneRunBaseDir, getMidsceneRunSubDir } from "../common.mjs";
4
+ const sessionDirName = 'cli-report-session';
5
+ function sanitizeSessionName(sessionName) {
6
+ return sessionName.replace(/[^a-zA-Z0-9._-]/g, '_') || 'default';
7
+ }
8
+ function sanitizeFileSegment(segment) {
9
+ const sanitized = segment.replace(/[^a-zA-Z0-9._-]/g, '_') || 'unknown';
10
+ return sanitized.slice(0, 80);
11
+ }
12
+ function ensureHtmlFileName(reportFileName) {
13
+ return reportFileName.endsWith('.html') ? reportFileName : `${reportFileName}.html`;
14
+ }
15
+ function formatDateForFileName(date) {
16
+ const pad = (value)=>String(value).padStart(2, '0');
17
+ const day = [
18
+ date.getFullYear(),
19
+ pad(date.getMonth() + 1),
20
+ pad(date.getDate())
21
+ ].join('-');
22
+ const time = [
23
+ pad(date.getHours()),
24
+ pad(date.getMinutes()),
25
+ pad(date.getSeconds())
26
+ ].join('-');
27
+ return `${day}_${time}`;
28
+ }
29
+ function randomId() {
30
+ return Math.random().toString(36).slice(2, 10);
31
+ }
32
+ function getCliReportSessionDir() {
33
+ const dir = join(getMidsceneRunBaseDir(), sessionDirName);
34
+ if (!existsSync(dir)) mkdirSync(dir, {
35
+ recursive: true
36
+ });
37
+ return dir;
38
+ }
39
+ function getCliReportSessionPath(sessionName) {
40
+ return join(getCliReportSessionDir(), `${sanitizeSessionName(sessionName)}.json`);
41
+ }
42
+ function generateCliReportSession(sessionName, targetIdentity) {
43
+ const identitySegment = targetIdentity ? `-${sanitizeFileSegment(targetIdentity)}` : '';
44
+ const reportFileName = `${sanitizeSessionName(sessionName)}${identitySegment}-${formatDateForFileName(new Date())}-${randomId()}`;
45
+ const reportPath = join(getMidsceneRunSubDir('report'), ensureHtmlFileName(reportFileName));
46
+ const session = {
47
+ version: 1,
48
+ sessionName,
49
+ ...targetIdentity ? {
50
+ targetIdentity
51
+ } : {},
52
+ reportFileName,
53
+ reportPath,
54
+ createdAt: Date.now()
55
+ };
56
+ return session;
57
+ }
58
+ function writeCliReportSession(session) {
59
+ writeFileSync(getCliReportSessionPath(session.sessionName), JSON.stringify(session, null, 2), 'utf-8');
60
+ }
61
+ function createCliReportSession(sessionName, targetIdentity) {
62
+ const session = generateCliReportSession(sessionName, targetIdentity);
63
+ writeCliReportSession(session);
64
+ return session;
65
+ }
66
+ function readCliReportSession(sessionName) {
67
+ const sessionPath = getCliReportSessionPath(sessionName);
68
+ if (!existsSync(sessionPath)) return;
69
+ try {
70
+ const raw = readFileSync(sessionPath, 'utf-8');
71
+ const parsed = JSON.parse(raw);
72
+ if (1 !== parsed.version || parsed.sessionName !== sessionName || 'string' != typeof parsed.reportFileName || !parsed.reportFileName.trim() || /[\\/]/.test(parsed.reportFileName)) return;
73
+ return parsed;
74
+ } catch {
75
+ return;
76
+ }
77
+ }
78
+ export { createCliReportSession, generateCliReportSession, readCliReportSession, writeCliReportSession };
@@ -0,0 +1,19 @@
1
+ function getErrorMessage(error) {
2
+ if (error instanceof Error) return error.message;
3
+ if (null == error) return String(error);
4
+ if ('object' != typeof error) return String(error);
5
+ const candidate = extractStringMessage(error);
6
+ if (candidate) return candidate;
7
+ try {
8
+ return JSON.stringify(error);
9
+ } catch {
10
+ return Object.prototype.toString.call(error);
11
+ }
12
+ }
13
+ function extractStringMessage(error) {
14
+ const anyError = error;
15
+ if ('string' == typeof anyError.message && anyError.message) return anyError.message;
16
+ if (anyError.error && 'string' == typeof anyError.error.message && anyError.error.message) return anyError.error.message;
17
+ if (anyError.cause && 'string' == typeof anyError.cause.message && anyError.cause.message) return anyError.cause.message;
18
+ }
19
+ export { getErrorMessage };
@@ -1,5 +1,7 @@
1
1
  export * from "./base-server.mjs";
2
2
  export * from "./base-tools.mjs";
3
+ export * from "./init-arg-utils.mjs";
4
+ export * from "./error-formatter.mjs";
3
5
  export * from "./tool-generator.mjs";
4
6
  export * from "./types.mjs";
5
7
  export * from "./inject-report-html-plugin.mjs";
@@ -0,0 +1,38 @@
1
+ import { getKeyAliases, isRecord } from "../key-alias-utils.mjs";
2
+ function readAliasedValue(args, key) {
3
+ for (const alias of getKeyAliases(key))if (alias in args) return args[alias];
4
+ }
5
+ function readNamespacedArg(args, namespace, key) {
6
+ const namespacedArgs = readAliasedValue(args, namespace);
7
+ if (isRecord(namespacedArgs)) {
8
+ const nestedValue = readAliasedValue(namespacedArgs, key);
9
+ if (void 0 !== nestedValue) return nestedValue;
10
+ }
11
+ const dottedValue = readAliasedValue(args, `${namespace}.${key}`);
12
+ if (void 0 !== dottedValue) return dottedValue;
13
+ const directValue = readAliasedValue(args, key);
14
+ if (void 0 !== directValue) return directValue;
15
+ }
16
+ function extractNamespacedArgs(args, namespace, keys) {
17
+ const extracted = {};
18
+ for (const key of keys){
19
+ const value = readNamespacedArg(args, namespace, key);
20
+ if (void 0 !== value) extracted[key] = value;
21
+ }
22
+ return Object.keys(extracted).length > 0 ? extracted : void 0;
23
+ }
24
+ function sanitizeNamespacedArgs(args, namespace, keys) {
25
+ const excludedKeys = new Set(getKeyAliases(namespace));
26
+ for (const key of keys){
27
+ for (const alias of getKeyAliases(key))excludedKeys.add(alias);
28
+ for (const alias of getKeyAliases(`${namespace}.${key}`))excludedKeys.add(alias);
29
+ }
30
+ return Object.fromEntries(Object.entries(args).filter(([key])=>!excludedKeys.has(key)));
31
+ }
32
+ function createNamespacedInitArgSchema(namespace, shape) {
33
+ return Object.fromEntries(Object.entries(shape).map(([key, value])=>[
34
+ `${namespace}.${key}`,
35
+ value
36
+ ]));
37
+ }
38
+ export { createNamespacedInitArgSchema, extractNamespacedArgs, sanitizeNamespacedArgs };
@@ -9,7 +9,7 @@ function injectReportHtmlFromCore(packageDir) {
9
9
  setup (api) {
10
10
  api.onAfterBuild(()=>{
11
11
  const coreUtilsPath = node_path.resolve(packageDir, '..', 'core', 'dist', 'lib', 'utils.js');
12
- if (!node_fs.existsSync(coreUtilsPath)) return void console.warn('[inject-report-html] @midscene/core dist not found, skipping');
12
+ if (!node_fs.existsSync(coreUtilsPath)) return void console.warn('[inject-report-html] @aiscene/core dist not found, skipping');
13
13
  const coreContent = node_fs.readFileSync(coreUtilsPath, 'utf-8');
14
14
  if (!coreContent.includes(REPLACED_MARK)) return void console.warn('[inject-report-html] HTML not found in core dist. Ensure report builds first.');
15
15
  const markerIndex = coreContent.indexOf(REPLACED_MARK);
@@ -1,9 +1,7 @@
1
+ import { parseBase64 } from "@aiscene/shared/img";
1
2
  import { z } from "zod";
2
- import { parseBase64 } from "../img/index.mjs";
3
3
  import { getZodDescription, getZodTypeName, isMidsceneLocatorField, unwrapZodField } from "../zod-schema-utils.mjs";
4
- function getErrorMessage(error) {
5
- return error instanceof Error ? error.message : String(error);
6
- }
4
+ import { getErrorMessage } from "./error-formatter.mjs";
7
5
  function describeActionForMCP(action) {
8
6
  const actionDesc = action.description || `Execute ${action.name} action`;
9
7
  if (!action.paramSchema) return `${action.name} action, ${actionDesc}`;
@@ -70,10 +68,13 @@ function transformSchemaField(key, value) {
70
68
  value
71
69
  ];
72
70
  }
73
- function extractActionSchema(paramSchema) {
71
+ function extractActionSchema(paramSchema, actionName) {
74
72
  if (!paramSchema) return {};
75
73
  const shape = getZodObjectShape(paramSchema);
76
- if (!shape) return paramSchema;
74
+ if (!shape) {
75
+ const typeName = paramSchema?._def?.typeName ?? 'unknown';
76
+ throw new Error(`Action "${actionName}" declared a non-object paramSchema (${typeName}). CLI and MCP tool schemas must be a ZodObject (e.g. z.object({ uri: z.string() })) or undefined. Wrap primitive fields in an object schema.`);
77
+ }
77
78
  return Object.fromEntries(Object.entries(shape).map(([key, value])=>transformSchemaField(key, value)));
78
79
  }
79
80
  function getPromptText(prompt) {
@@ -279,17 +280,30 @@ async function captureFailureResult(agent, actionName, errorMessage) {
279
280
  };
280
281
  }
281
282
  }
282
- function generateToolsFromActionSpace(actionSpace, getAgent) {
283
+ function mergeToolCliMetadata(base, extra) {
284
+ const options = {
285
+ ...base?.options ?? {},
286
+ ...extra?.options ?? {}
287
+ };
288
+ return Object.keys(options).length > 0 ? {
289
+ options
290
+ } : void 0;
291
+ }
292
+ function generateToolsFromActionSpace(actionSpace, getAgent, sanitizeArgs = (args)=>args, initArgSchema = {}, initArgCliMetadata) {
283
293
  return actionSpace.map((action)=>{
284
- const schema = extractActionSchema(action.paramSchema);
294
+ const schema = {
295
+ ...extractActionSchema(action.paramSchema, action.name),
296
+ ...initArgSchema
297
+ };
285
298
  return {
286
299
  name: action.name,
287
300
  description: describeActionForMCP(action),
288
301
  schema,
302
+ cli: initArgCliMetadata,
289
303
  handler: async (args)=>{
290
304
  try {
291
- const agent = await getAgent();
292
- const normalizedArgs = normalizeActionArgs(args, action.paramSchema);
305
+ const agent = await getAgent(args);
306
+ const normalizedArgs = normalizeActionArgs(sanitizeArgs(args), action.paramSchema);
293
307
  let actionResult;
294
308
  try {
295
309
  actionResult = await executeAction(agent, action.name, normalizedArgs);
@@ -308,15 +322,18 @@ function generateToolsFromActionSpace(actionSpace, getAgent) {
308
322
  };
309
323
  });
310
324
  }
311
- function generateCommonTools(getAgent) {
325
+ function generateCommonTools(getAgent, initArgSchema = {}, initArgCliMetadata) {
312
326
  return [
313
327
  {
314
328
  name: 'take_screenshot',
315
329
  description: 'Capture screenshot of current page/screen',
316
- schema: {},
317
- handler: async ()=>{
330
+ schema: {
331
+ ...initArgSchema
332
+ },
333
+ cli: initArgCliMetadata,
334
+ handler: async (args = {})=>{
318
335
  try {
319
- const agent = await getAgent();
336
+ const agent = await getAgent(args);
320
337
  const screenshot = await agent.page?.screenshotBase64();
321
338
  if (!screenshot) return createErrorResult('Screenshot not available');
322
339
  const { mimeType, body } = parseBase64(screenshot);
@@ -340,12 +357,14 @@ function generateCommonTools(getAgent) {
340
357
  name: 'act',
341
358
  description: 'Execute a natural language action. The AI will plan and perform multi-step operations in a single invocation, useful for transient UI interactions (e.g., Spotlight, dropdown menus) that disappear between separate commands.',
342
359
  schema: {
343
- prompt: z.string().describe('Natural language description of the action to perform, e.g. "press Command+Space, type Safari, press Enter"')
360
+ prompt: z.string().describe('Natural language description of the action to perform, e.g. "press Command+Space, type Safari, press Enter"'),
361
+ ...initArgSchema
344
362
  },
345
- handler: async (args)=>{
363
+ cli: mergeToolCliMetadata(void 0, initArgCliMetadata),
364
+ handler: async (args = {})=>{
346
365
  const prompt = args.prompt;
347
366
  try {
348
- const agent = await getAgent();
367
+ const agent = await getAgent(args);
349
368
  if (!agent.aiAction) return createErrorResult('act is not supported by this agent');
350
369
  const result = await agent.aiAction(prompt, {
351
370
  deepThink: false
@@ -365,6 +384,35 @@ function generateCommonTools(getAgent) {
365
384
  return createErrorResult(`Failed to execute act: ${errorMessage}`);
366
385
  }
367
386
  }
387
+ },
388
+ {
389
+ name: 'assert',
390
+ description: 'Assert a natural language statement against the current page/screen.',
391
+ schema: {
392
+ prompt: z.string().describe('Natural language assertion to verify, e.g. "there is a login button visible"'),
393
+ ...initArgSchema
394
+ },
395
+ cli: mergeToolCliMetadata(void 0, initArgCliMetadata),
396
+ handler: async (args = {})=>{
397
+ const prompt = args.prompt;
398
+ try {
399
+ const agent = await getAgent(args);
400
+ if (!agent.aiAssert) return createErrorResult('assert is not supported by this agent');
401
+ await agent.aiAssert(prompt);
402
+ return {
403
+ content: [
404
+ {
405
+ type: 'text',
406
+ text: 'Assertion passed.'
407
+ }
408
+ ]
409
+ };
410
+ } catch (error) {
411
+ const errorMessage = getErrorMessage(error);
412
+ console.error('Error executing assert:', errorMessage);
413
+ return createErrorResult(`Failed to execute assert: ${errorMessage}`);
414
+ }
415
+ }
368
416
  }
369
417
  ];
370
418
  }