@aiscene/shared 8.0.2 → 8.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/agent-tools/agent-behavior-init-args.mjs +44 -0
- package/dist/es/agent-tools/base-tools.mjs +163 -0
- package/dist/es/agent-tools/chrome-path.mjs +50 -0
- package/dist/es/agent-tools/cli-report-session.mjs +78 -0
- package/dist/es/agent-tools/error-formatter.mjs +106 -0
- package/dist/es/agent-tools/index.mjs +9 -0
- package/dist/es/agent-tools/init-arg-utils.mjs +38 -0
- package/dist/es/agent-tools/observation-artifact.mjs +5 -0
- package/dist/es/agent-tools/observation-record.mjs +248 -0
- package/dist/es/agent-tools/tool-defaults.mjs +54 -0
- package/dist/es/agent-tools/tool-generator.mjs +529 -0
- package/dist/es/agent-tools/types.mjs +3 -0
- package/dist/es/agent-tools/user-prompt.mjs +66 -0
- package/dist/es/cli/cli-args.mjs +95 -0
- package/dist/es/cli/cli-error.mjs +24 -0
- package/dist/es/cli/cli-runner.mjs +10 -40
- package/dist/es/cli/index.mjs +4 -2
- package/dist/es/cli/interrupt.mjs +116 -0
- package/dist/es/cli/record-command.mjs +130 -0
- package/dist/es/cli/screenshot-file.mjs +24 -0
- package/dist/es/cli/verbose-ai-act.mjs +230 -0
- package/dist/es/cli/verbose-screenshot.mjs +134 -0
- package/dist/es/cli/verbose.mjs +404 -0
- package/dist/es/constants/example-code.mjs +2 -2
- package/dist/es/constants/index.mjs +6 -1
- package/dist/es/env/model-config-manager.mjs +3 -0
- package/dist/es/env/parse-model-config.mjs +6 -1
- package/dist/es/env/types.mjs +4 -1
- package/dist/es/env/utils.mjs +1 -9
- package/dist/es/extractor/index.mjs +2 -2
- package/dist/es/img/index.mjs +2 -2
- package/dist/es/img/info.mjs +58 -17
- package/dist/es/img/photon-loader.mjs +4 -0
- package/dist/es/key-alias-utils.mjs +19 -0
- package/dist/es/mcp/base-server.mjs +10 -9
- package/dist/es/mcp/base-tools.mjs +72 -4
- package/dist/es/mcp/cli-report-session.mjs +78 -0
- package/dist/es/mcp/error-formatter.mjs +19 -0
- package/dist/es/mcp/index.mjs +2 -0
- package/dist/es/mcp/init-arg-utils.mjs +38 -0
- package/dist/es/mcp/inject-report-html-plugin.mjs +1 -1
- package/dist/es/mcp/tool-generator.mjs +65 -17
- package/dist/es/node/fs.mjs +1 -1
- package/dist/es/recorder.mjs +249 -0
- package/dist/lib/agent-tools/agent-behavior-init-args.js +87 -0
- package/dist/lib/agent-tools/base-tools.js +197 -0
- package/dist/lib/agent-tools/chrome-path.js +87 -0
- package/dist/lib/agent-tools/cli-report-session.js +121 -0
- package/dist/lib/agent-tools/error-formatter.js +149 -0
- package/dist/lib/agent-tools/index.js +114 -0
- package/dist/lib/agent-tools/init-arg-utils.js +78 -0
- package/dist/lib/agent-tools/observation-artifact.js +42 -0
- package/dist/lib/agent-tools/observation-record.js +297 -0
- package/dist/lib/agent-tools/tool-defaults.js +97 -0
- package/dist/lib/agent-tools/tool-generator.js +569 -0
- package/dist/lib/agent-tools/types.js +40 -0
- package/dist/lib/agent-tools/user-prompt.js +103 -0
- package/dist/lib/cli/cli-args.js +138 -0
- package/dist/lib/cli/cli-error.js +61 -0
- package/dist/lib/cli/cli-runner.js +19 -46
- package/dist/lib/cli/index.js +8 -3
- package/dist/lib/cli/interrupt.js +156 -0
- package/dist/lib/cli/record-command.js +164 -0
- package/dist/lib/cli/screenshot-file.js +58 -0
- package/dist/lib/cli/verbose-ai-act.js +273 -0
- package/dist/lib/cli/verbose-screenshot.js +177 -0
- package/dist/lib/cli/verbose.js +465 -0
- package/dist/lib/constants/example-code.js +2 -2
- package/dist/lib/constants/index.js +23 -3
- package/dist/lib/env/model-config-manager.js +3 -0
- package/dist/lib/env/parse-model-config.js +6 -1
- package/dist/lib/env/types.js +9 -0
- package/dist/lib/env/utils.js +0 -11
- package/dist/lib/extractor/index.js +3 -0
- package/dist/lib/img/index.js +3 -0
- package/dist/lib/img/info.js +65 -18
- package/dist/lib/img/photon-loader.js +38 -0
- package/dist/lib/key-alias-utils.js +62 -0
- package/dist/lib/mcp/base-server.js +11 -10
- package/dist/lib/mcp/base-tools.js +74 -6
- package/dist/lib/mcp/cli-report-session.js +121 -0
- package/dist/lib/mcp/error-formatter.js +53 -0
- package/dist/lib/mcp/index.js +24 -10
- package/dist/lib/mcp/init-arg-utils.js +78 -0
- package/dist/lib/mcp/inject-report-html-plugin.js +1 -1
- package/dist/lib/mcp/tool-generator.js +74 -26
- package/dist/lib/node/fs.js +1 -1
- package/dist/lib/recorder.js +307 -0
- package/dist/types/agent-tools/agent-behavior-init-args.d.ts +17 -0
- package/dist/types/agent-tools/base-tools.d.ts +158 -0
- package/dist/types/agent-tools/chrome-path.d.ts +2 -0
- package/dist/types/agent-tools/cli-report-session.d.ts +12 -0
- package/dist/types/agent-tools/error-formatter.d.ts +30 -0
- package/dist/types/agent-tools/index.d.ts +9 -0
- package/dist/types/agent-tools/init-arg-utils.d.ts +13 -0
- package/dist/types/agent-tools/observation-artifact.d.ts +10 -0
- package/dist/types/agent-tools/observation-record.d.ts +38 -0
- package/dist/types/agent-tools/tool-defaults.d.ts +63 -0
- package/dist/types/agent-tools/tool-generator.d.ts +13 -0
- package/dist/types/agent-tools/types.d.ts +213 -0
- package/dist/types/agent-tools/user-prompt.d.ts +13 -0
- package/dist/types/cli/cli-args.d.ts +8 -0
- package/dist/types/cli/cli-error.d.ts +5 -0
- package/dist/types/cli/cli-runner.d.ts +4 -7
- package/dist/types/cli/index.d.ts +3 -1
- package/dist/types/cli/interrupt.d.ts +49 -0
- package/dist/types/cli/record-command.d.ts +3 -0
- package/dist/types/cli/screenshot-file.d.ts +10 -0
- package/dist/types/cli/verbose-ai-act.d.ts +44 -0
- package/dist/types/cli/verbose-screenshot.d.ts +10 -0
- package/dist/types/cli/verbose.d.ts +40 -0
- package/dist/types/constants/example-code.d.ts +1 -1
- package/dist/types/constants/index.d.ts +5 -0
- package/dist/types/env/types.d.ts +14 -5
- package/dist/types/env/utils.d.ts +0 -31
- package/dist/types/extractor/index.d.ts +1 -1
- package/dist/types/img/index.d.ts +1 -1
- package/dist/types/img/info.d.ts +7 -0
- package/dist/types/img/photon-loader.d.ts +2 -0
- package/dist/types/key-alias-utils.d.ts +9 -0
- package/dist/types/mcp/base-tools.d.ts +74 -5
- package/dist/types/mcp/cli-report-session.d.ts +12 -0
- package/dist/types/mcp/error-formatter.d.ts +12 -0
- package/dist/types/mcp/index.d.ts +2 -0
- package/dist/types/mcp/init-arg-utils.d.ts +13 -0
- package/dist/types/mcp/inject-report-html-plugin.d.ts +2 -2
- package/dist/types/mcp/launcher-helper.d.ts +3 -3
- package/dist/types/mcp/tool-generator.d.ts +3 -3
- package/dist/types/mcp/types.d.ts +10 -0
- package/dist/types/recorder.d.ts +113 -0
- package/package.json +1 -1
- package/src/agent-tools/agent-behavior-init-args.ts +109 -0
- package/src/agent-tools/base-tools.ts +399 -0
- package/src/agent-tools/chrome-path.ts +74 -0
- package/src/agent-tools/cli-report-session.ts +130 -0
- package/src/agent-tools/error-formatter.ts +177 -0
- package/src/agent-tools/index.ts +9 -0
- package/src/agent-tools/init-arg-utils.ts +105 -0
- package/src/agent-tools/observation-artifact.ts +29 -0
- package/src/agent-tools/observation-record.ts +331 -0
- package/src/agent-tools/tool-defaults.ts +119 -0
- package/src/agent-tools/tool-generator.ts +866 -0
- package/src/agent-tools/types.ts +250 -0
- package/src/agent-tools/user-prompt.ts +102 -0
- package/src/cli/cli-args.ts +173 -0
- package/src/cli/cli-error.ts +24 -0
- package/src/cli/cli-runner.ts +37 -56
- package/src/cli/index.ts +3 -7
- package/src/cli/interrupt.ts +207 -0
- package/src/cli/record-command.ts +177 -0
- package/src/cli/screenshot-file.ts +61 -0
- package/src/cli/verbose-ai-act.ts +387 -0
- package/src/cli/verbose-screenshot.ts +269 -0
- package/src/cli/verbose.ts +753 -0
- package/src/constants/example-code.ts +2 -2
- package/src/constants/index.ts +10 -0
- package/src/env/model-config-manager.ts +3 -0
- package/src/env/parse-model-config.ts +19 -1
- package/src/env/types.ts +13 -2
- package/src/env/utils.ts +0 -50
- package/src/extractor/index.ts +1 -1
- package/src/img/index.ts +14 -0
- package/src/img/info.ts +97 -45
- package/src/img/photon-loader.ts +5 -0
- package/src/img/transform.ts +261 -2
- package/src/key-alias-utils.ts +23 -0
- package/src/mcp/base-server.ts +10 -12
- package/src/mcp/base-tools.ts +218 -11
- package/src/mcp/cli-report-session.ts +130 -0
- package/src/mcp/error-formatter.ts +52 -0
- package/src/mcp/index.ts +2 -0
- package/src/mcp/init-arg-utils.ts +105 -0
- package/src/mcp/inject-report-html-plugin.ts +3 -3
- package/src/mcp/launcher-helper.ts +3 -3
- package/src/mcp/tool-generator.ts +98 -20
- package/src/mcp/types.ts +18 -1
- package/src/recorder.ts +625 -0
package/dist/es/env/utils.mjs
CHANGED
|
@@ -5,14 +5,6 @@ const globalModelConfigManager = new ModelConfigManager();
|
|
|
5
5
|
const globalConfigManager = new GlobalConfigManager();
|
|
6
6
|
globalConfigManager.registerModelConfigManager(globalModelConfigManager);
|
|
7
7
|
globalModelConfigManager.registerGlobalConfigManager(globalConfigManager);
|
|
8
|
-
async function getCurrentTime(device, useDeviceTimestamp) {
|
|
9
|
-
if (useDeviceTimestamp && device?.getTimestamp) try {
|
|
10
|
-
return await device.getTimestamp();
|
|
11
|
-
} catch (error) {
|
|
12
|
-
console.warn(`Failed to get device time, falling back to system time: ${error}`);
|
|
13
|
-
}
|
|
14
|
-
return Date.now();
|
|
15
|
-
}
|
|
16
8
|
const getPreferredLanguage = ()=>{
|
|
17
9
|
const prefer = globalConfigManager.getEnvConfigValue(MIDSCENE_PREFERRED_LANGUAGE);
|
|
18
10
|
if (prefer) return prefer;
|
|
@@ -23,4 +15,4 @@ const getPreferredLanguage = ()=>{
|
|
|
23
15
|
const overrideAIConfig = (newConfig, extendMode = false)=>{
|
|
24
16
|
globalConfigManager.overrideAIConfig(newConfig, extendMode);
|
|
25
17
|
};
|
|
26
|
-
export {
|
|
18
|
+
export { getPreferredLanguage, globalConfigManager, globalModelConfigManager, overrideAIConfig };
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { descriptionOfTree, traverseTree, treeToList, trimAttributes, truncateText } from "./tree.mjs";
|
|
2
2
|
import { extractTextWithPosition, extractTreeNode, extractTreeNodeAsString } from "./web-extractor.mjs";
|
|
3
3
|
import { getElementInfoByXpath, getElementXpath, getNodeInfoByXpath, getXpathsById, getXpathsByPoint } from "./locator.mjs";
|
|
4
|
-
import { generateElementByRect, isNotContainerElement } from "./dom-util.mjs";
|
|
5
|
-
export { descriptionOfTree, generateElementByRect, getElementInfoByXpath, getElementXpath, getNodeInfoByXpath, getXpathsById, getXpathsByPoint, isNotContainerElement, traverseTree, treeToList, trimAttributes, truncateText, extractTreeNode as webExtractNodeTree, extractTreeNodeAsString as webExtractNodeTreeAsString, extractTextWithPosition as webExtractTextWithPosition };
|
|
4
|
+
import { generateElementByPoint, generateElementByRect, isNotContainerElement } from "./dom-util.mjs";
|
|
5
|
+
export { descriptionOfTree, generateElementByPoint, generateElementByRect, getElementInfoByXpath, getElementXpath, getNodeInfoByXpath, getXpathsById, getXpathsByPoint, isNotContainerElement, traverseTree, treeToList, trimAttributes, truncateText, extractTreeNode as webExtractNodeTree, extractTreeNodeAsString as webExtractNodeTreeAsString, extractTextWithPosition as webExtractTextWithPosition };
|
package/dist/es/img/index.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { imageInfoOfBase64, isValidImageBuffer, isValidJPEGImageBuffer, isValidPNGImageBuffer } from "./info.mjs";
|
|
1
|
+
import { imageInfoOfBase64, isValidImageBuffer, isValidJPEGImageBuffer, isValidPNGImageBuffer, validateScreenshotBuffer } from "./info.mjs";
|
|
2
2
|
import { createImgBase64ByFormat, cropByRect, httpImg2Base64, localImg2Base64, paddingToMatchBlockByBase64, parseBase64, preProcessImageUrl, resizeAndConvertImgBuffer, resizeImgBase64, saveBase64Image, scaleImage, zoomForGPT4o } from "./transform.mjs";
|
|
3
3
|
import { annotateRects, compositeElementInfoImg, processImageElementInfo } from "./box-select.mjs";
|
|
4
|
-
export { annotateRects, compositeElementInfoImg, createImgBase64ByFormat, cropByRect, httpImg2Base64, imageInfoOfBase64, isValidImageBuffer, isValidJPEGImageBuffer, isValidPNGImageBuffer, localImg2Base64, paddingToMatchBlockByBase64, parseBase64, preProcessImageUrl, processImageElementInfo, resizeAndConvertImgBuffer, resizeImgBase64, saveBase64Image, scaleImage, zoomForGPT4o };
|
|
4
|
+
export { annotateRects, compositeElementInfoImg, createImgBase64ByFormat, cropByRect, httpImg2Base64, imageInfoOfBase64, isValidImageBuffer, isValidJPEGImageBuffer, isValidPNGImageBuffer, localImg2Base64, paddingToMatchBlockByBase64, parseBase64, preProcessImageUrl, processImageElementInfo, resizeAndConvertImgBuffer, resizeImgBase64, saveBase64Image, scaleImage, validateScreenshotBuffer, zoomForGPT4o };
|
package/dist/es/img/info.mjs
CHANGED
|
@@ -1,27 +1,62 @@
|
|
|
1
1
|
import node_assert from "node:assert";
|
|
2
2
|
import get_photon from "./get-photon.mjs";
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
const
|
|
14
|
-
|
|
15
|
-
|
|
3
|
+
const isJpegStartOfFrameMarker = (marker)=>marker >= 0xc0 && marker <= 0xcf && 0xc4 !== marker && 0xc8 !== marker && 0xcc !== marker;
|
|
4
|
+
function jpegInfoFromBuffer(imageBuffer) {
|
|
5
|
+
let offset = 2;
|
|
6
|
+
while(offset < imageBuffer.length){
|
|
7
|
+
if (0xff !== imageBuffer[offset]) {
|
|
8
|
+
offset += 1;
|
|
9
|
+
continue;
|
|
10
|
+
}
|
|
11
|
+
while(offset < imageBuffer.length && 0xff === imageBuffer[offset])offset += 1;
|
|
12
|
+
if (offset >= imageBuffer.length) break;
|
|
13
|
+
const marker = imageBuffer[offset];
|
|
14
|
+
offset += 1;
|
|
15
|
+
if (0xd9 === marker || 0xda === marker) break;
|
|
16
|
+
if (0x01 === marker || 0xd8 === marker || marker >= 0xd0 && marker <= 0xd7) continue;
|
|
17
|
+
if (offset + 2 > imageBuffer.length) break;
|
|
18
|
+
const segmentLength = imageBuffer.readUInt16BE(offset);
|
|
19
|
+
if (segmentLength < 2 || offset + segmentLength > imageBuffer.length) break;
|
|
20
|
+
if (isJpegStartOfFrameMarker(marker)) {
|
|
21
|
+
if (segmentLength < 7) break;
|
|
22
|
+
const height = imageBuffer.readUInt16BE(offset + 3);
|
|
23
|
+
const width = imageBuffer.readUInt16BE(offset + 5);
|
|
24
|
+
node_assert(width && height, 'Invalid image: cannot get width or height');
|
|
25
|
+
return {
|
|
26
|
+
width,
|
|
27
|
+
height
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
offset += segmentLength;
|
|
31
|
+
}
|
|
32
|
+
throw new Error('Invalid image: cannot get JPEG width or height');
|
|
33
|
+
}
|
|
34
|
+
function encodedImageInfoOfBuffer(imageBuffer) {
|
|
35
|
+
if (isValidPNGImageBuffer(imageBuffer)) {
|
|
36
|
+
const width = imageBuffer.readUInt32BE(16);
|
|
37
|
+
const height = imageBuffer.readUInt32BE(20);
|
|
16
38
|
node_assert(width && height, 'Invalid image: cannot get width or height');
|
|
17
39
|
return {
|
|
18
40
|
width,
|
|
19
41
|
height
|
|
20
42
|
};
|
|
21
|
-
} catch (error) {
|
|
22
|
-
if (error instanceof Error && error.message.includes('unreachable')) throw new Error(`Failed to decode image: invalid or corrupted image data. Original error: ${error.message}`);
|
|
23
|
-
throw error;
|
|
24
43
|
}
|
|
44
|
+
if (isValidJPEGImageBuffer(imageBuffer)) return jpegInfoFromBuffer(imageBuffer);
|
|
45
|
+
throw new Error('Invalid image: unsupported format');
|
|
46
|
+
}
|
|
47
|
+
async function imageInfoOfBase64(imageBase64) {
|
|
48
|
+
const { PhotonImage } = await get_photon();
|
|
49
|
+
const base64Data = imageBase64.replace(/^data:image\/\w+;base64,/, '');
|
|
50
|
+
const result = PhotonImage.new_from_base64(base64Data);
|
|
51
|
+
const image = result instanceof Promise ? await result : result;
|
|
52
|
+
const width = image.get_width();
|
|
53
|
+
const height = image.get_height();
|
|
54
|
+
image.free();
|
|
55
|
+
node_assert(width && height, 'Invalid image: cannot get width or height');
|
|
56
|
+
return {
|
|
57
|
+
width,
|
|
58
|
+
height
|
|
59
|
+
};
|
|
25
60
|
}
|
|
26
61
|
function isValidPNGImageBuffer(buffer) {
|
|
27
62
|
if (!buffer || buffer.length < 8) return false;
|
|
@@ -35,4 +70,10 @@ function isValidJPEGImageBuffer(buffer) {
|
|
|
35
70
|
function isValidImageBuffer(buffer) {
|
|
36
71
|
return isValidPNGImageBuffer(buffer) || isValidJPEGImageBuffer(buffer);
|
|
37
72
|
}
|
|
38
|
-
|
|
73
|
+
function validateScreenshotBuffer(screenshotBuffer, { label, minBufferSize = 0 }) {
|
|
74
|
+
const bufferSize = screenshotBuffer?.length ?? 0;
|
|
75
|
+
if (!screenshotBuffer || 0 === bufferSize) throw new Error(`${label} validation failed: buffer size ${bufferSize} bytes`);
|
|
76
|
+
if (!isValidImageBuffer(screenshotBuffer)) throw new Error(`${label} buffer has invalid image format`);
|
|
77
|
+
if (minBufferSize > 0 && bufferSize < minBufferSize) throw new Error(`${label} validation failed: buffer size ${bufferSize} bytes (minimum: ${minBufferSize})`);
|
|
78
|
+
}
|
|
79
|
+
export { encodedImageInfoOfBuffer, imageInfoOfBase64, isValidImageBuffer, isValidJPEGImageBuffer, isValidPNGImageBuffer, validateScreenshotBuffer };
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
function kebabToCamel(str) {
|
|
2
|
+
return str.replace(/-([a-z])/g, (_, letter)=>letter.toUpperCase());
|
|
3
|
+
}
|
|
4
|
+
function camelToKebab(str) {
|
|
5
|
+
return str.replace(/[A-Z]/g, (letter)=>`-${letter.toLowerCase()}`).replace(/^-/, '');
|
|
6
|
+
}
|
|
7
|
+
function getKeyAliases(key) {
|
|
8
|
+
return [
|
|
9
|
+
...new Set([
|
|
10
|
+
key,
|
|
11
|
+
kebabToCamel(key),
|
|
12
|
+
camelToKebab(key)
|
|
13
|
+
])
|
|
14
|
+
];
|
|
15
|
+
}
|
|
16
|
+
function isRecord(value) {
|
|
17
|
+
return 'object' == typeof value && null !== value && !Array.isArray(value);
|
|
18
|
+
}
|
|
19
|
+
export { camelToKebab, getKeyAliases, isRecord, kebabToCamel };
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { setIsMcp } from "@aiscene/shared/utils";
|
|
2
3
|
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
3
4
|
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
4
5
|
import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js";
|
|
5
6
|
import express from "express";
|
|
6
|
-
import {
|
|
7
|
+
import { getErrorMessage } from "./error-formatter.mjs";
|
|
7
8
|
function _define_property(obj, key, value) {
|
|
8
9
|
if (key in obj) Object.defineProperty(obj, key, {
|
|
9
10
|
value: value,
|
|
@@ -45,7 +46,7 @@ class BaseMCPServer {
|
|
|
45
46
|
try {
|
|
46
47
|
await this.toolsManager.initTools();
|
|
47
48
|
} catch (error) {
|
|
48
|
-
const message =
|
|
49
|
+
const message = getErrorMessage(error);
|
|
49
50
|
console.error(`Failed to initialize tools: ${message}`);
|
|
50
51
|
console.error('Tools will be initialized on first use');
|
|
51
52
|
}
|
|
@@ -71,7 +72,7 @@ class BaseMCPServer {
|
|
|
71
72
|
try {
|
|
72
73
|
await this.mcpServer.connect(transport);
|
|
73
74
|
} catch (error) {
|
|
74
|
-
const message =
|
|
75
|
+
const message = getErrorMessage(error);
|
|
75
76
|
console.error(`Failed to connect MCP stdio transport: ${message}`);
|
|
76
77
|
throw new Error(`Failed to initialize MCP stdio transport: ${message}`);
|
|
77
78
|
}
|
|
@@ -142,7 +143,7 @@ class BaseMCPServer {
|
|
|
142
143
|
});
|
|
143
144
|
}
|
|
144
145
|
} catch (error) {
|
|
145
|
-
const message =
|
|
146
|
+
const message = getErrorMessage(error);
|
|
146
147
|
const duration = Date.now() - startTime;
|
|
147
148
|
console.error(`[${new Date().toISOString()}] [${requestId}] MCP request error after ${duration}ms: ${message}`);
|
|
148
149
|
if (!res.headersSent) res.status(500).json({
|
|
@@ -170,7 +171,7 @@ class BaseMCPServer {
|
|
|
170
171
|
for (const session of sessions.values())try {
|
|
171
172
|
await session.transport.close();
|
|
172
173
|
} catch (error) {
|
|
173
|
-
const message =
|
|
174
|
+
const message = getErrorMessage(error);
|
|
174
175
|
console.error(`Failed to close session ${session.transport.sessionId}: ${message}`);
|
|
175
176
|
}
|
|
176
177
|
sessions.clear();
|
|
@@ -205,7 +206,7 @@ class BaseMCPServer {
|
|
|
205
206
|
try {
|
|
206
207
|
await this.mcpServer.connect(transport);
|
|
207
208
|
} catch (error) {
|
|
208
|
-
const message =
|
|
209
|
+
const message = getErrorMessage(error);
|
|
209
210
|
console.error(`[${new Date().toISOString()}] Failed to connect MCP transport: ${message}`);
|
|
210
211
|
if (transport.sessionId) sessions.delete(transport.sessionId);
|
|
211
212
|
throw new Error(`Failed to initialize MCP session: ${message}`);
|
|
@@ -224,7 +225,7 @@ class BaseMCPServer {
|
|
|
224
225
|
sessions.delete(sid);
|
|
225
226
|
console.log(`[${new Date().toISOString()}] Session ${sid} cleaned up due to inactivity (remaining: ${sessions.size})`);
|
|
226
227
|
} catch (error) {
|
|
227
|
-
const message =
|
|
228
|
+
const message = getErrorMessage(error);
|
|
228
229
|
console.error(`[${new Date().toISOString()}] Failed to close session ${sid} during cleanup: ${message}`);
|
|
229
230
|
sessions.delete(sid);
|
|
230
231
|
}
|
|
@@ -237,7 +238,7 @@ class BaseMCPServer {
|
|
|
237
238
|
for (const session of sessions.values())try {
|
|
238
239
|
session.transport.close();
|
|
239
240
|
} catch (error) {
|
|
240
|
-
const message =
|
|
241
|
+
const message = getErrorMessage(error);
|
|
241
242
|
console.error(`Error closing session during shutdown: ${message}`);
|
|
242
243
|
}
|
|
243
244
|
sessions.clear();
|
|
@@ -250,7 +251,7 @@ class BaseMCPServer {
|
|
|
250
251
|
this.performCleanup().finally(()=>process.exit(1));
|
|
251
252
|
}, 5000);
|
|
252
253
|
} catch (error) {
|
|
253
|
-
const message =
|
|
254
|
+
const message = getErrorMessage(error);
|
|
254
255
|
console.error(`Error closing HTTP server: ${message}`);
|
|
255
256
|
this.performCleanup().finally(()=>process.exit(1));
|
|
256
257
|
}
|
|
@@ -1,5 +1,8 @@
|
|
|
1
|
-
import { parseBase64 } from "
|
|
2
|
-
import { getDebug } from "
|
|
1
|
+
import { parseBase64 } from "@aiscene/shared/img";
|
|
2
|
+
import { getDebug } from "@aiscene/shared/logger";
|
|
3
|
+
import { camelToKebab, getKeyAliases } from "../key-alias-utils.mjs";
|
|
4
|
+
import { generateCliReportSession, readCliReportSession, writeCliReportSession } from "./cli-report-session.mjs";
|
|
5
|
+
import { createNamespacedInitArgSchema, extractNamespacedArgs, sanitizeNamespacedArgs } from "./init-arg-utils.mjs";
|
|
3
6
|
import { generateCommonTools, generateToolsFromActionSpace } from "./tool-generator.mjs";
|
|
4
7
|
function _define_property(obj, key, value) {
|
|
5
8
|
if (key in obj) Object.defineProperty(obj, key, {
|
|
@@ -13,9 +16,74 @@ function _define_property(obj, key, value) {
|
|
|
13
16
|
}
|
|
14
17
|
const debug = getDebug('mcp:base-tools');
|
|
15
18
|
class BaseMidsceneTools {
|
|
19
|
+
getInitArgKeys() {
|
|
20
|
+
return this.initArgSpec ? Object.keys(this.initArgSpec.shape) : [];
|
|
21
|
+
}
|
|
22
|
+
extractAgentInitParam(args) {
|
|
23
|
+
if (!this.initArgSpec) return;
|
|
24
|
+
const extracted = extractNamespacedArgs(args, this.initArgSpec.namespace, this.getInitArgKeys());
|
|
25
|
+
if (this.initArgSpec.adapt) return this.initArgSpec.adapt(extracted);
|
|
26
|
+
return extracted;
|
|
27
|
+
}
|
|
28
|
+
sanitizeToolArgs(args) {
|
|
29
|
+
if (!this.initArgSpec) return args;
|
|
30
|
+
return sanitizeNamespacedArgs(args, this.initArgSpec.namespace, this.getInitArgKeys());
|
|
31
|
+
}
|
|
32
|
+
getAgentInitArgSchema() {
|
|
33
|
+
if (!this.initArgSpec) return {};
|
|
34
|
+
return createNamespacedInitArgSchema(this.initArgSpec.namespace, this.initArgSpec.shape);
|
|
35
|
+
}
|
|
36
|
+
getAgentInitArgCliMetadata() {
|
|
37
|
+
if (!this.initArgSpec?.cli) return;
|
|
38
|
+
const options = Object.fromEntries(this.getInitArgKeys().map((key)=>{
|
|
39
|
+
const canonicalKey = `${this.initArgSpec.namespace}.${key}`;
|
|
40
|
+
const preferredName = this.initArgSpec.cli?.preferredNames?.[key] ?? (this.initArgSpec.cli?.preferBareKeys ? camelToKebab(key) : canonicalKey);
|
|
41
|
+
const acceptedNames = new Set([
|
|
42
|
+
preferredName,
|
|
43
|
+
...this.initArgSpec.cli?.preferBareKeys ? getKeyAliases(key) : getKeyAliases(canonicalKey)
|
|
44
|
+
]);
|
|
45
|
+
acceptedNames.delete(preferredName);
|
|
46
|
+
return [
|
|
47
|
+
canonicalKey,
|
|
48
|
+
{
|
|
49
|
+
preferredName,
|
|
50
|
+
aliases: [
|
|
51
|
+
...acceptedNames
|
|
52
|
+
]
|
|
53
|
+
}
|
|
54
|
+
];
|
|
55
|
+
}));
|
|
56
|
+
return {
|
|
57
|
+
options
|
|
58
|
+
};
|
|
59
|
+
}
|
|
16
60
|
preparePlatformTools() {
|
|
17
61
|
return [];
|
|
18
62
|
}
|
|
63
|
+
getCliReportSessionName() {}
|
|
64
|
+
createNewCliReportSession(targetIdentity) {
|
|
65
|
+
const sessionName = this.getCliReportSessionName();
|
|
66
|
+
if (!sessionName) return;
|
|
67
|
+
return generateCliReportSession(sessionName, targetIdentity);
|
|
68
|
+
}
|
|
69
|
+
commitCliReportSession(session) {
|
|
70
|
+
if (session) writeCliReportSession(session);
|
|
71
|
+
}
|
|
72
|
+
readCliReportFileName() {
|
|
73
|
+
const sessionName = this.getCliReportSessionName();
|
|
74
|
+
if (!sessionName) return;
|
|
75
|
+
return readCliReportSession(sessionName)?.reportFileName;
|
|
76
|
+
}
|
|
77
|
+
readCliReportAgentOptions() {
|
|
78
|
+
const reportFileName = this.readCliReportFileName();
|
|
79
|
+
if (!reportFileName) return;
|
|
80
|
+
return {
|
|
81
|
+
reportFileName,
|
|
82
|
+
reportAttributes: {
|
|
83
|
+
'data-group-id': reportFileName
|
|
84
|
+
}
|
|
85
|
+
};
|
|
86
|
+
}
|
|
19
87
|
async initTools() {
|
|
20
88
|
this.toolDefinitions = [];
|
|
21
89
|
const platformTools = this.preparePlatformTools();
|
|
@@ -30,8 +98,8 @@ class BaseMidsceneTools {
|
|
|
30
98
|
await tempDevice.destroy?.();
|
|
31
99
|
debug('Action space from temporary device:', actionSpace.map((a)=>a.name).join(', '));
|
|
32
100
|
}
|
|
33
|
-
const actionTools = generateToolsFromActionSpace(actionSpace, ()=>this.ensureAgent());
|
|
34
|
-
const commonTools = generateCommonTools(()=>this.ensureAgent());
|
|
101
|
+
const actionTools = generateToolsFromActionSpace(actionSpace, (args = {})=>this.ensureAgent(this.extractAgentInitParam(args)), (args = {})=>this.sanitizeToolArgs(args), this.getAgentInitArgSchema(), this.getAgentInitArgCliMetadata());
|
|
102
|
+
const commonTools = generateCommonTools((args = {})=>this.ensureAgent(this.extractAgentInitParam(args)), this.getAgentInitArgSchema(), this.getAgentInitArgCliMetadata());
|
|
35
103
|
this.toolDefinitions.push(...actionTools, ...commonTools);
|
|
36
104
|
debug('Total tools prepared:', this.toolDefinitions.length);
|
|
37
105
|
}
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { getMidsceneRunBaseDir, getMidsceneRunSubDir } from "../common.mjs";
|
|
4
|
+
const sessionDirName = 'cli-report-session';
|
|
5
|
+
function sanitizeSessionName(sessionName) {
|
|
6
|
+
return sessionName.replace(/[^a-zA-Z0-9._-]/g, '_') || 'default';
|
|
7
|
+
}
|
|
8
|
+
function sanitizeFileSegment(segment) {
|
|
9
|
+
const sanitized = segment.replace(/[^a-zA-Z0-9._-]/g, '_') || 'unknown';
|
|
10
|
+
return sanitized.slice(0, 80);
|
|
11
|
+
}
|
|
12
|
+
function ensureHtmlFileName(reportFileName) {
|
|
13
|
+
return reportFileName.endsWith('.html') ? reportFileName : `${reportFileName}.html`;
|
|
14
|
+
}
|
|
15
|
+
function formatDateForFileName(date) {
|
|
16
|
+
const pad = (value)=>String(value).padStart(2, '0');
|
|
17
|
+
const day = [
|
|
18
|
+
date.getFullYear(),
|
|
19
|
+
pad(date.getMonth() + 1),
|
|
20
|
+
pad(date.getDate())
|
|
21
|
+
].join('-');
|
|
22
|
+
const time = [
|
|
23
|
+
pad(date.getHours()),
|
|
24
|
+
pad(date.getMinutes()),
|
|
25
|
+
pad(date.getSeconds())
|
|
26
|
+
].join('-');
|
|
27
|
+
return `${day}_${time}`;
|
|
28
|
+
}
|
|
29
|
+
function randomId() {
|
|
30
|
+
return Math.random().toString(36).slice(2, 10);
|
|
31
|
+
}
|
|
32
|
+
function getCliReportSessionDir() {
|
|
33
|
+
const dir = join(getMidsceneRunBaseDir(), sessionDirName);
|
|
34
|
+
if (!existsSync(dir)) mkdirSync(dir, {
|
|
35
|
+
recursive: true
|
|
36
|
+
});
|
|
37
|
+
return dir;
|
|
38
|
+
}
|
|
39
|
+
function getCliReportSessionPath(sessionName) {
|
|
40
|
+
return join(getCliReportSessionDir(), `${sanitizeSessionName(sessionName)}.json`);
|
|
41
|
+
}
|
|
42
|
+
function generateCliReportSession(sessionName, targetIdentity) {
|
|
43
|
+
const identitySegment = targetIdentity ? `-${sanitizeFileSegment(targetIdentity)}` : '';
|
|
44
|
+
const reportFileName = `${sanitizeSessionName(sessionName)}${identitySegment}-${formatDateForFileName(new Date())}-${randomId()}`;
|
|
45
|
+
const reportPath = join(getMidsceneRunSubDir('report'), ensureHtmlFileName(reportFileName));
|
|
46
|
+
const session = {
|
|
47
|
+
version: 1,
|
|
48
|
+
sessionName,
|
|
49
|
+
...targetIdentity ? {
|
|
50
|
+
targetIdentity
|
|
51
|
+
} : {},
|
|
52
|
+
reportFileName,
|
|
53
|
+
reportPath,
|
|
54
|
+
createdAt: Date.now()
|
|
55
|
+
};
|
|
56
|
+
return session;
|
|
57
|
+
}
|
|
58
|
+
function writeCliReportSession(session) {
|
|
59
|
+
writeFileSync(getCliReportSessionPath(session.sessionName), JSON.stringify(session, null, 2), 'utf-8');
|
|
60
|
+
}
|
|
61
|
+
function createCliReportSession(sessionName, targetIdentity) {
|
|
62
|
+
const session = generateCliReportSession(sessionName, targetIdentity);
|
|
63
|
+
writeCliReportSession(session);
|
|
64
|
+
return session;
|
|
65
|
+
}
|
|
66
|
+
function readCliReportSession(sessionName) {
|
|
67
|
+
const sessionPath = getCliReportSessionPath(sessionName);
|
|
68
|
+
if (!existsSync(sessionPath)) return;
|
|
69
|
+
try {
|
|
70
|
+
const raw = readFileSync(sessionPath, 'utf-8');
|
|
71
|
+
const parsed = JSON.parse(raw);
|
|
72
|
+
if (1 !== parsed.version || parsed.sessionName !== sessionName || 'string' != typeof parsed.reportFileName || !parsed.reportFileName.trim() || /[\\/]/.test(parsed.reportFileName)) return;
|
|
73
|
+
return parsed;
|
|
74
|
+
} catch {
|
|
75
|
+
return;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
export { createCliReportSession, generateCliReportSession, readCliReportSession, writeCliReportSession };
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
function getErrorMessage(error) {
|
|
2
|
+
if (error instanceof Error) return error.message;
|
|
3
|
+
if (null == error) return String(error);
|
|
4
|
+
if ('object' != typeof error) return String(error);
|
|
5
|
+
const candidate = extractStringMessage(error);
|
|
6
|
+
if (candidate) return candidate;
|
|
7
|
+
try {
|
|
8
|
+
return JSON.stringify(error);
|
|
9
|
+
} catch {
|
|
10
|
+
return Object.prototype.toString.call(error);
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
function extractStringMessage(error) {
|
|
14
|
+
const anyError = error;
|
|
15
|
+
if ('string' == typeof anyError.message && anyError.message) return anyError.message;
|
|
16
|
+
if (anyError.error && 'string' == typeof anyError.error.message && anyError.error.message) return anyError.error.message;
|
|
17
|
+
if (anyError.cause && 'string' == typeof anyError.cause.message && anyError.cause.message) return anyError.cause.message;
|
|
18
|
+
}
|
|
19
|
+
export { getErrorMessage };
|
package/dist/es/mcp/index.mjs
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
export * from "./base-server.mjs";
|
|
2
2
|
export * from "./base-tools.mjs";
|
|
3
|
+
export * from "./init-arg-utils.mjs";
|
|
4
|
+
export * from "./error-formatter.mjs";
|
|
3
5
|
export * from "./tool-generator.mjs";
|
|
4
6
|
export * from "./types.mjs";
|
|
5
7
|
export * from "./inject-report-html-plugin.mjs";
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { getKeyAliases, isRecord } from "../key-alias-utils.mjs";
|
|
2
|
+
function readAliasedValue(args, key) {
|
|
3
|
+
for (const alias of getKeyAliases(key))if (alias in args) return args[alias];
|
|
4
|
+
}
|
|
5
|
+
function readNamespacedArg(args, namespace, key) {
|
|
6
|
+
const namespacedArgs = readAliasedValue(args, namespace);
|
|
7
|
+
if (isRecord(namespacedArgs)) {
|
|
8
|
+
const nestedValue = readAliasedValue(namespacedArgs, key);
|
|
9
|
+
if (void 0 !== nestedValue) return nestedValue;
|
|
10
|
+
}
|
|
11
|
+
const dottedValue = readAliasedValue(args, `${namespace}.${key}`);
|
|
12
|
+
if (void 0 !== dottedValue) return dottedValue;
|
|
13
|
+
const directValue = readAliasedValue(args, key);
|
|
14
|
+
if (void 0 !== directValue) return directValue;
|
|
15
|
+
}
|
|
16
|
+
function extractNamespacedArgs(args, namespace, keys) {
|
|
17
|
+
const extracted = {};
|
|
18
|
+
for (const key of keys){
|
|
19
|
+
const value = readNamespacedArg(args, namespace, key);
|
|
20
|
+
if (void 0 !== value) extracted[key] = value;
|
|
21
|
+
}
|
|
22
|
+
return Object.keys(extracted).length > 0 ? extracted : void 0;
|
|
23
|
+
}
|
|
24
|
+
function sanitizeNamespacedArgs(args, namespace, keys) {
|
|
25
|
+
const excludedKeys = new Set(getKeyAliases(namespace));
|
|
26
|
+
for (const key of keys){
|
|
27
|
+
for (const alias of getKeyAliases(key))excludedKeys.add(alias);
|
|
28
|
+
for (const alias of getKeyAliases(`${namespace}.${key}`))excludedKeys.add(alias);
|
|
29
|
+
}
|
|
30
|
+
return Object.fromEntries(Object.entries(args).filter(([key])=>!excludedKeys.has(key)));
|
|
31
|
+
}
|
|
32
|
+
function createNamespacedInitArgSchema(namespace, shape) {
|
|
33
|
+
return Object.fromEntries(Object.entries(shape).map(([key, value])=>[
|
|
34
|
+
`${namespace}.${key}`,
|
|
35
|
+
value
|
|
36
|
+
]));
|
|
37
|
+
}
|
|
38
|
+
export { createNamespacedInitArgSchema, extractNamespacedArgs, sanitizeNamespacedArgs };
|
|
@@ -9,7 +9,7 @@ function injectReportHtmlFromCore(packageDir) {
|
|
|
9
9
|
setup (api) {
|
|
10
10
|
api.onAfterBuild(()=>{
|
|
11
11
|
const coreUtilsPath = node_path.resolve(packageDir, '..', 'core', 'dist', 'lib', 'utils.js');
|
|
12
|
-
if (!node_fs.existsSync(coreUtilsPath)) return void console.warn('[inject-report-html] @
|
|
12
|
+
if (!node_fs.existsSync(coreUtilsPath)) return void console.warn('[inject-report-html] @aiscene/core dist not found, skipping');
|
|
13
13
|
const coreContent = node_fs.readFileSync(coreUtilsPath, 'utf-8');
|
|
14
14
|
if (!coreContent.includes(REPLACED_MARK)) return void console.warn('[inject-report-html] HTML not found in core dist. Ensure report builds first.');
|
|
15
15
|
const markerIndex = coreContent.indexOf(REPLACED_MARK);
|
|
@@ -1,9 +1,7 @@
|
|
|
1
|
+
import { parseBase64 } from "@aiscene/shared/img";
|
|
1
2
|
import { z } from "zod";
|
|
2
|
-
import { parseBase64 } from "../img/index.mjs";
|
|
3
3
|
import { getZodDescription, getZodTypeName, isMidsceneLocatorField, unwrapZodField } from "../zod-schema-utils.mjs";
|
|
4
|
-
|
|
5
|
-
return error instanceof Error ? error.message : String(error);
|
|
6
|
-
}
|
|
4
|
+
import { getErrorMessage } from "./error-formatter.mjs";
|
|
7
5
|
function describeActionForMCP(action) {
|
|
8
6
|
const actionDesc = action.description || `Execute ${action.name} action`;
|
|
9
7
|
if (!action.paramSchema) return `${action.name} action, ${actionDesc}`;
|
|
@@ -70,10 +68,13 @@ function transformSchemaField(key, value) {
|
|
|
70
68
|
value
|
|
71
69
|
];
|
|
72
70
|
}
|
|
73
|
-
function extractActionSchema(paramSchema) {
|
|
71
|
+
function extractActionSchema(paramSchema, actionName) {
|
|
74
72
|
if (!paramSchema) return {};
|
|
75
73
|
const shape = getZodObjectShape(paramSchema);
|
|
76
|
-
if (!shape)
|
|
74
|
+
if (!shape) {
|
|
75
|
+
const typeName = paramSchema?._def?.typeName ?? 'unknown';
|
|
76
|
+
throw new Error(`Action "${actionName}" declared a non-object paramSchema (${typeName}). CLI and MCP tool schemas must be a ZodObject (e.g. z.object({ uri: z.string() })) or undefined. Wrap primitive fields in an object schema.`);
|
|
77
|
+
}
|
|
77
78
|
return Object.fromEntries(Object.entries(shape).map(([key, value])=>transformSchemaField(key, value)));
|
|
78
79
|
}
|
|
79
80
|
function getPromptText(prompt) {
|
|
@@ -279,17 +280,30 @@ async function captureFailureResult(agent, actionName, errorMessage) {
|
|
|
279
280
|
};
|
|
280
281
|
}
|
|
281
282
|
}
|
|
282
|
-
function
|
|
283
|
+
function mergeToolCliMetadata(base, extra) {
|
|
284
|
+
const options = {
|
|
285
|
+
...base?.options ?? {},
|
|
286
|
+
...extra?.options ?? {}
|
|
287
|
+
};
|
|
288
|
+
return Object.keys(options).length > 0 ? {
|
|
289
|
+
options
|
|
290
|
+
} : void 0;
|
|
291
|
+
}
|
|
292
|
+
function generateToolsFromActionSpace(actionSpace, getAgent, sanitizeArgs = (args)=>args, initArgSchema = {}, initArgCliMetadata) {
|
|
283
293
|
return actionSpace.map((action)=>{
|
|
284
|
-
const schema =
|
|
294
|
+
const schema = {
|
|
295
|
+
...extractActionSchema(action.paramSchema, action.name),
|
|
296
|
+
...initArgSchema
|
|
297
|
+
};
|
|
285
298
|
return {
|
|
286
299
|
name: action.name,
|
|
287
300
|
description: describeActionForMCP(action),
|
|
288
301
|
schema,
|
|
302
|
+
cli: initArgCliMetadata,
|
|
289
303
|
handler: async (args)=>{
|
|
290
304
|
try {
|
|
291
|
-
const agent = await getAgent();
|
|
292
|
-
const normalizedArgs = normalizeActionArgs(args, action.paramSchema);
|
|
305
|
+
const agent = await getAgent(args);
|
|
306
|
+
const normalizedArgs = normalizeActionArgs(sanitizeArgs(args), action.paramSchema);
|
|
293
307
|
let actionResult;
|
|
294
308
|
try {
|
|
295
309
|
actionResult = await executeAction(agent, action.name, normalizedArgs);
|
|
@@ -308,15 +322,18 @@ function generateToolsFromActionSpace(actionSpace, getAgent) {
|
|
|
308
322
|
};
|
|
309
323
|
});
|
|
310
324
|
}
|
|
311
|
-
function generateCommonTools(getAgent) {
|
|
325
|
+
function generateCommonTools(getAgent, initArgSchema = {}, initArgCliMetadata) {
|
|
312
326
|
return [
|
|
313
327
|
{
|
|
314
328
|
name: 'take_screenshot',
|
|
315
329
|
description: 'Capture screenshot of current page/screen',
|
|
316
|
-
schema: {
|
|
317
|
-
|
|
330
|
+
schema: {
|
|
331
|
+
...initArgSchema
|
|
332
|
+
},
|
|
333
|
+
cli: initArgCliMetadata,
|
|
334
|
+
handler: async (args = {})=>{
|
|
318
335
|
try {
|
|
319
|
-
const agent = await getAgent();
|
|
336
|
+
const agent = await getAgent(args);
|
|
320
337
|
const screenshot = await agent.page?.screenshotBase64();
|
|
321
338
|
if (!screenshot) return createErrorResult('Screenshot not available');
|
|
322
339
|
const { mimeType, body } = parseBase64(screenshot);
|
|
@@ -340,12 +357,14 @@ function generateCommonTools(getAgent) {
|
|
|
340
357
|
name: 'act',
|
|
341
358
|
description: 'Execute a natural language action. The AI will plan and perform multi-step operations in a single invocation, useful for transient UI interactions (e.g., Spotlight, dropdown menus) that disappear between separate commands.',
|
|
342
359
|
schema: {
|
|
343
|
-
prompt: z.string().describe('Natural language description of the action to perform, e.g. "press Command+Space, type Safari, press Enter"')
|
|
360
|
+
prompt: z.string().describe('Natural language description of the action to perform, e.g. "press Command+Space, type Safari, press Enter"'),
|
|
361
|
+
...initArgSchema
|
|
344
362
|
},
|
|
345
|
-
|
|
363
|
+
cli: mergeToolCliMetadata(void 0, initArgCliMetadata),
|
|
364
|
+
handler: async (args = {})=>{
|
|
346
365
|
const prompt = args.prompt;
|
|
347
366
|
try {
|
|
348
|
-
const agent = await getAgent();
|
|
367
|
+
const agent = await getAgent(args);
|
|
349
368
|
if (!agent.aiAction) return createErrorResult('act is not supported by this agent');
|
|
350
369
|
const result = await agent.aiAction(prompt, {
|
|
351
370
|
deepThink: false
|
|
@@ -365,6 +384,35 @@ function generateCommonTools(getAgent) {
|
|
|
365
384
|
return createErrorResult(`Failed to execute act: ${errorMessage}`);
|
|
366
385
|
}
|
|
367
386
|
}
|
|
387
|
+
},
|
|
388
|
+
{
|
|
389
|
+
name: 'assert',
|
|
390
|
+
description: 'Assert a natural language statement against the current page/screen.',
|
|
391
|
+
schema: {
|
|
392
|
+
prompt: z.string().describe('Natural language assertion to verify, e.g. "there is a login button visible"'),
|
|
393
|
+
...initArgSchema
|
|
394
|
+
},
|
|
395
|
+
cli: mergeToolCliMetadata(void 0, initArgCliMetadata),
|
|
396
|
+
handler: async (args = {})=>{
|
|
397
|
+
const prompt = args.prompt;
|
|
398
|
+
try {
|
|
399
|
+
const agent = await getAgent(args);
|
|
400
|
+
if (!agent.aiAssert) return createErrorResult('assert is not supported by this agent');
|
|
401
|
+
await agent.aiAssert(prompt);
|
|
402
|
+
return {
|
|
403
|
+
content: [
|
|
404
|
+
{
|
|
405
|
+
type: 'text',
|
|
406
|
+
text: 'Assertion passed.'
|
|
407
|
+
}
|
|
408
|
+
]
|
|
409
|
+
};
|
|
410
|
+
} catch (error) {
|
|
411
|
+
const errorMessage = getErrorMessage(error);
|
|
412
|
+
console.error('Error executing assert:', errorMessage);
|
|
413
|
+
return createErrorResult(`Failed to execute assert: ${errorMessage}`);
|
|
414
|
+
}
|
|
415
|
+
}
|
|
368
416
|
}
|
|
369
417
|
];
|
|
370
418
|
}
|