@aiscene/shared 8.0.2 → 8.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/agent-tools/agent-behavior-init-args.mjs +44 -0
- package/dist/es/agent-tools/base-tools.mjs +163 -0
- package/dist/es/agent-tools/chrome-path.mjs +50 -0
- package/dist/es/agent-tools/cli-report-session.mjs +78 -0
- package/dist/es/agent-tools/error-formatter.mjs +106 -0
- package/dist/es/agent-tools/index.mjs +9 -0
- package/dist/es/agent-tools/init-arg-utils.mjs +38 -0
- package/dist/es/agent-tools/observation-artifact.mjs +5 -0
- package/dist/es/agent-tools/observation-record.mjs +248 -0
- package/dist/es/agent-tools/tool-defaults.mjs +54 -0
- package/dist/es/agent-tools/tool-generator.mjs +529 -0
- package/dist/es/agent-tools/types.mjs +3 -0
- package/dist/es/agent-tools/user-prompt.mjs +66 -0
- package/dist/es/cli/cli-args.mjs +95 -0
- package/dist/es/cli/cli-error.mjs +24 -0
- package/dist/es/cli/cli-runner.mjs +10 -40
- package/dist/es/cli/index.mjs +4 -2
- package/dist/es/cli/interrupt.mjs +116 -0
- package/dist/es/cli/record-command.mjs +130 -0
- package/dist/es/cli/screenshot-file.mjs +24 -0
- package/dist/es/cli/verbose-ai-act.mjs +230 -0
- package/dist/es/cli/verbose-screenshot.mjs +134 -0
- package/dist/es/cli/verbose.mjs +404 -0
- package/dist/es/constants/example-code.mjs +2 -2
- package/dist/es/constants/index.mjs +6 -1
- package/dist/es/env/model-config-manager.mjs +3 -0
- package/dist/es/env/parse-model-config.mjs +6 -1
- package/dist/es/env/types.mjs +4 -1
- package/dist/es/env/utils.mjs +1 -9
- package/dist/es/extractor/index.mjs +2 -2
- package/dist/es/img/index.mjs +2 -2
- package/dist/es/img/info.mjs +58 -17
- package/dist/es/img/photon-loader.mjs +4 -0
- package/dist/es/key-alias-utils.mjs +19 -0
- package/dist/es/mcp/base-server.mjs +10 -9
- package/dist/es/mcp/base-tools.mjs +72 -4
- package/dist/es/mcp/cli-report-session.mjs +78 -0
- package/dist/es/mcp/error-formatter.mjs +19 -0
- package/dist/es/mcp/index.mjs +2 -0
- package/dist/es/mcp/init-arg-utils.mjs +38 -0
- package/dist/es/mcp/inject-report-html-plugin.mjs +1 -1
- package/dist/es/mcp/tool-generator.mjs +65 -17
- package/dist/es/node/fs.mjs +1 -1
- package/dist/es/recorder.mjs +249 -0
- package/dist/lib/agent-tools/agent-behavior-init-args.js +87 -0
- package/dist/lib/agent-tools/base-tools.js +197 -0
- package/dist/lib/agent-tools/chrome-path.js +87 -0
- package/dist/lib/agent-tools/cli-report-session.js +121 -0
- package/dist/lib/agent-tools/error-formatter.js +149 -0
- package/dist/lib/agent-tools/index.js +114 -0
- package/dist/lib/agent-tools/init-arg-utils.js +78 -0
- package/dist/lib/agent-tools/observation-artifact.js +42 -0
- package/dist/lib/agent-tools/observation-record.js +297 -0
- package/dist/lib/agent-tools/tool-defaults.js +97 -0
- package/dist/lib/agent-tools/tool-generator.js +569 -0
- package/dist/lib/agent-tools/types.js +40 -0
- package/dist/lib/agent-tools/user-prompt.js +103 -0
- package/dist/lib/cli/cli-args.js +138 -0
- package/dist/lib/cli/cli-error.js +61 -0
- package/dist/lib/cli/cli-runner.js +19 -46
- package/dist/lib/cli/index.js +8 -3
- package/dist/lib/cli/interrupt.js +156 -0
- package/dist/lib/cli/record-command.js +164 -0
- package/dist/lib/cli/screenshot-file.js +58 -0
- package/dist/lib/cli/verbose-ai-act.js +273 -0
- package/dist/lib/cli/verbose-screenshot.js +177 -0
- package/dist/lib/cli/verbose.js +465 -0
- package/dist/lib/constants/example-code.js +2 -2
- package/dist/lib/constants/index.js +23 -3
- package/dist/lib/env/model-config-manager.js +3 -0
- package/dist/lib/env/parse-model-config.js +6 -1
- package/dist/lib/env/types.js +9 -0
- package/dist/lib/env/utils.js +0 -11
- package/dist/lib/extractor/index.js +3 -0
- package/dist/lib/img/index.js +3 -0
- package/dist/lib/img/info.js +65 -18
- package/dist/lib/img/photon-loader.js +38 -0
- package/dist/lib/key-alias-utils.js +62 -0
- package/dist/lib/mcp/base-server.js +11 -10
- package/dist/lib/mcp/base-tools.js +74 -6
- package/dist/lib/mcp/cli-report-session.js +121 -0
- package/dist/lib/mcp/error-formatter.js +53 -0
- package/dist/lib/mcp/index.js +24 -10
- package/dist/lib/mcp/init-arg-utils.js +78 -0
- package/dist/lib/mcp/inject-report-html-plugin.js +1 -1
- package/dist/lib/mcp/tool-generator.js +74 -26
- package/dist/lib/node/fs.js +1 -1
- package/dist/lib/recorder.js +307 -0
- package/dist/types/agent-tools/agent-behavior-init-args.d.ts +17 -0
- package/dist/types/agent-tools/base-tools.d.ts +158 -0
- package/dist/types/agent-tools/chrome-path.d.ts +2 -0
- package/dist/types/agent-tools/cli-report-session.d.ts +12 -0
- package/dist/types/agent-tools/error-formatter.d.ts +30 -0
- package/dist/types/agent-tools/index.d.ts +9 -0
- package/dist/types/agent-tools/init-arg-utils.d.ts +13 -0
- package/dist/types/agent-tools/observation-artifact.d.ts +10 -0
- package/dist/types/agent-tools/observation-record.d.ts +38 -0
- package/dist/types/agent-tools/tool-defaults.d.ts +63 -0
- package/dist/types/agent-tools/tool-generator.d.ts +13 -0
- package/dist/types/agent-tools/types.d.ts +213 -0
- package/dist/types/agent-tools/user-prompt.d.ts +13 -0
- package/dist/types/cli/cli-args.d.ts +8 -0
- package/dist/types/cli/cli-error.d.ts +5 -0
- package/dist/types/cli/cli-runner.d.ts +4 -7
- package/dist/types/cli/index.d.ts +3 -1
- package/dist/types/cli/interrupt.d.ts +49 -0
- package/dist/types/cli/record-command.d.ts +3 -0
- package/dist/types/cli/screenshot-file.d.ts +10 -0
- package/dist/types/cli/verbose-ai-act.d.ts +44 -0
- package/dist/types/cli/verbose-screenshot.d.ts +10 -0
- package/dist/types/cli/verbose.d.ts +40 -0
- package/dist/types/constants/example-code.d.ts +1 -1
- package/dist/types/constants/index.d.ts +5 -0
- package/dist/types/env/types.d.ts +14 -5
- package/dist/types/env/utils.d.ts +0 -31
- package/dist/types/extractor/index.d.ts +1 -1
- package/dist/types/img/index.d.ts +1 -1
- package/dist/types/img/info.d.ts +7 -0
- package/dist/types/img/photon-loader.d.ts +2 -0
- package/dist/types/key-alias-utils.d.ts +9 -0
- package/dist/types/mcp/base-tools.d.ts +74 -5
- package/dist/types/mcp/cli-report-session.d.ts +12 -0
- package/dist/types/mcp/error-formatter.d.ts +12 -0
- package/dist/types/mcp/index.d.ts +2 -0
- package/dist/types/mcp/init-arg-utils.d.ts +13 -0
- package/dist/types/mcp/inject-report-html-plugin.d.ts +2 -2
- package/dist/types/mcp/launcher-helper.d.ts +3 -3
- package/dist/types/mcp/tool-generator.d.ts +3 -3
- package/dist/types/mcp/types.d.ts +10 -0
- package/dist/types/recorder.d.ts +113 -0
- package/package.json +1 -1
- package/src/agent-tools/agent-behavior-init-args.ts +109 -0
- package/src/agent-tools/base-tools.ts +399 -0
- package/src/agent-tools/chrome-path.ts +74 -0
- package/src/agent-tools/cli-report-session.ts +130 -0
- package/src/agent-tools/error-formatter.ts +177 -0
- package/src/agent-tools/index.ts +9 -0
- package/src/agent-tools/init-arg-utils.ts +105 -0
- package/src/agent-tools/observation-artifact.ts +29 -0
- package/src/agent-tools/observation-record.ts +331 -0
- package/src/agent-tools/tool-defaults.ts +119 -0
- package/src/agent-tools/tool-generator.ts +866 -0
- package/src/agent-tools/types.ts +250 -0
- package/src/agent-tools/user-prompt.ts +102 -0
- package/src/cli/cli-args.ts +173 -0
- package/src/cli/cli-error.ts +24 -0
- package/src/cli/cli-runner.ts +37 -56
- package/src/cli/index.ts +3 -7
- package/src/cli/interrupt.ts +207 -0
- package/src/cli/record-command.ts +177 -0
- package/src/cli/screenshot-file.ts +61 -0
- package/src/cli/verbose-ai-act.ts +387 -0
- package/src/cli/verbose-screenshot.ts +269 -0
- package/src/cli/verbose.ts +753 -0
- package/src/constants/example-code.ts +2 -2
- package/src/constants/index.ts +10 -0
- package/src/env/model-config-manager.ts +3 -0
- package/src/env/parse-model-config.ts +19 -1
- package/src/env/types.ts +13 -2
- package/src/env/utils.ts +0 -50
- package/src/extractor/index.ts +1 -1
- package/src/img/index.ts +14 -0
- package/src/img/info.ts +97 -45
- package/src/img/photon-loader.ts +5 -0
- package/src/img/transform.ts +261 -2
- package/src/key-alias-utils.ts +23 -0
- package/src/mcp/base-server.ts +10 -12
- package/src/mcp/base-tools.ts +218 -11
- package/src/mcp/cli-report-session.ts +130 -0
- package/src/mcp/error-formatter.ts +52 -0
- package/src/mcp/index.ts +2 -0
- package/src/mcp/init-arg-utils.ts +105 -0
- package/src/mcp/inject-report-html-plugin.ts +3 -3
- package/src/mcp/launcher-helper.ts +3 -3
- package/src/mcp/tool-generator.ts +98 -20
- package/src/mcp/types.ts +18 -1
- package/src/recorder.ts +625 -0
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
import { getMidsceneRunBaseDir, getMidsceneRunSubDir } from '../common';
|
|
4
|
+
|
|
5
|
+
export interface CliReportSession {
|
|
6
|
+
version: 1;
|
|
7
|
+
sessionName: string;
|
|
8
|
+
targetIdentity?: string;
|
|
9
|
+
reportFileName: string;
|
|
10
|
+
reportPath: string;
|
|
11
|
+
createdAt: number;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
const sessionDirName = 'cli-report-session';
|
|
15
|
+
|
|
16
|
+
function sanitizeSessionName(sessionName: string): string {
|
|
17
|
+
return sessionName.replace(/[^a-zA-Z0-9._-]/g, '_') || 'default';
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
function sanitizeFileSegment(segment: string): string {
|
|
21
|
+
const sanitized = segment.replace(/[^a-zA-Z0-9._-]/g, '_') || 'unknown';
|
|
22
|
+
return sanitized.slice(0, 80);
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function ensureHtmlFileName(reportFileName: string): string {
|
|
26
|
+
return reportFileName.endsWith('.html')
|
|
27
|
+
? reportFileName
|
|
28
|
+
: `${reportFileName}.html`;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function formatDateForFileName(date: Date): string {
|
|
32
|
+
const pad = (value: number) => String(value).padStart(2, '0');
|
|
33
|
+
const day = [
|
|
34
|
+
date.getFullYear(),
|
|
35
|
+
pad(date.getMonth() + 1),
|
|
36
|
+
pad(date.getDate()),
|
|
37
|
+
].join('-');
|
|
38
|
+
const time = [
|
|
39
|
+
pad(date.getHours()),
|
|
40
|
+
pad(date.getMinutes()),
|
|
41
|
+
pad(date.getSeconds()),
|
|
42
|
+
].join('-');
|
|
43
|
+
return `${day}_${time}`;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function randomId(): string {
|
|
47
|
+
return Math.random().toString(36).slice(2, 10);
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function getCliReportSessionDir(): string {
|
|
51
|
+
const dir = join(getMidsceneRunBaseDir(), sessionDirName);
|
|
52
|
+
if (!existsSync(dir)) {
|
|
53
|
+
mkdirSync(dir, { recursive: true });
|
|
54
|
+
}
|
|
55
|
+
return dir;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function getCliReportSessionPath(sessionName: string): string {
|
|
59
|
+
return join(
|
|
60
|
+
getCliReportSessionDir(),
|
|
61
|
+
`${sanitizeSessionName(sessionName)}.json`,
|
|
62
|
+
);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export function generateCliReportSession(
|
|
66
|
+
sessionName: string,
|
|
67
|
+
targetIdentity?: string,
|
|
68
|
+
): CliReportSession {
|
|
69
|
+
const identitySegment = targetIdentity
|
|
70
|
+
? `-${sanitizeFileSegment(targetIdentity)}`
|
|
71
|
+
: '';
|
|
72
|
+
const reportFileName = `${sanitizeSessionName(sessionName)}${identitySegment}-${formatDateForFileName(new Date())}-${randomId()}`;
|
|
73
|
+
const reportPath = join(
|
|
74
|
+
getMidsceneRunSubDir('report'),
|
|
75
|
+
ensureHtmlFileName(reportFileName),
|
|
76
|
+
);
|
|
77
|
+
const session: CliReportSession = {
|
|
78
|
+
version: 1,
|
|
79
|
+
sessionName,
|
|
80
|
+
...(targetIdentity ? { targetIdentity } : {}),
|
|
81
|
+
reportFileName,
|
|
82
|
+
reportPath,
|
|
83
|
+
createdAt: Date.now(),
|
|
84
|
+
};
|
|
85
|
+
return session;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
export function writeCliReportSession(session: CliReportSession): void {
|
|
89
|
+
writeFileSync(
|
|
90
|
+
getCliReportSessionPath(session.sessionName),
|
|
91
|
+
JSON.stringify(session, null, 2),
|
|
92
|
+
'utf-8',
|
|
93
|
+
);
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
export function createCliReportSession(
|
|
97
|
+
sessionName: string,
|
|
98
|
+
targetIdentity?: string,
|
|
99
|
+
): CliReportSession {
|
|
100
|
+
const session = generateCliReportSession(sessionName, targetIdentity);
|
|
101
|
+
writeCliReportSession(session);
|
|
102
|
+
return session;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
export function readCliReportSession(
|
|
106
|
+
sessionName: string,
|
|
107
|
+
): CliReportSession | undefined {
|
|
108
|
+
const sessionPath = getCliReportSessionPath(sessionName);
|
|
109
|
+
if (!existsSync(sessionPath)) {
|
|
110
|
+
return undefined;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
try {
|
|
114
|
+
const raw = readFileSync(sessionPath, 'utf-8');
|
|
115
|
+
const parsed = JSON.parse(raw) as Partial<CliReportSession>;
|
|
116
|
+
if (
|
|
117
|
+
parsed.version !== 1 ||
|
|
118
|
+
parsed.sessionName !== sessionName ||
|
|
119
|
+
typeof parsed.reportFileName !== 'string' ||
|
|
120
|
+
!parsed.reportFileName.trim() ||
|
|
121
|
+
/[\\/]/.test(parsed.reportFileName)
|
|
122
|
+
) {
|
|
123
|
+
return undefined;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
return parsed as CliReportSession;
|
|
127
|
+
} catch {
|
|
128
|
+
return undefined;
|
|
129
|
+
}
|
|
130
|
+
}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Extract a human-readable message from an unknown thrown value.
|
|
3
|
+
*
|
|
4
|
+
* Many SDK/transport layers reject with structured objects (e.g.
|
|
5
|
+
* `{ code, message }`, `{ error: { message } }`, `{ cause: { message } }`)
|
|
6
|
+
* rather than `Error` instances. `String(obj)` collapses those to
|
|
7
|
+
* `"[object Object]"`, which is useless for diagnostics. This helper walks
|
|
8
|
+
* the common shapes, falls back to `JSON.stringify`, and finally to
|
|
9
|
+
* `Object.prototype.toString.call` so that callers always get something
|
|
10
|
+
* actionable in logs and surfaced tool results.
|
|
11
|
+
*/
|
|
12
|
+
export function getErrorMessage(error: unknown): string {
|
|
13
|
+
if (error instanceof Error) return error.message;
|
|
14
|
+
if (error === null || error === undefined) return String(error);
|
|
15
|
+
if (typeof error !== 'object') return String(error);
|
|
16
|
+
|
|
17
|
+
const candidate = extractStringMessage(error);
|
|
18
|
+
if (candidate) return candidate;
|
|
19
|
+
|
|
20
|
+
try {
|
|
21
|
+
return JSON.stringify(error);
|
|
22
|
+
} catch {
|
|
23
|
+
return Object.prototype.toString.call(error);
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
function extractStringMessage(error: object): string | undefined {
|
|
28
|
+
const anyError = error as {
|
|
29
|
+
message?: unknown;
|
|
30
|
+
error?: { message?: unknown };
|
|
31
|
+
cause?: { message?: unknown };
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
if (typeof anyError.message === 'string' && anyError.message) {
|
|
35
|
+
return anyError.message;
|
|
36
|
+
}
|
|
37
|
+
if (
|
|
38
|
+
anyError.error &&
|
|
39
|
+
typeof anyError.error.message === 'string' &&
|
|
40
|
+
anyError.error.message
|
|
41
|
+
) {
|
|
42
|
+
return anyError.error.message;
|
|
43
|
+
}
|
|
44
|
+
if (
|
|
45
|
+
anyError.cause &&
|
|
46
|
+
typeof anyError.cause.message === 'string' &&
|
|
47
|
+
anyError.cause.message
|
|
48
|
+
) {
|
|
49
|
+
return anyError.cause.message;
|
|
50
|
+
}
|
|
51
|
+
return undefined;
|
|
52
|
+
}
|
package/src/mcp/index.ts
CHANGED
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import type { z } from 'zod';
|
|
2
|
+
import { getKeyAliases, isRecord } from '../key-alias-utils';
|
|
3
|
+
import type { ToolSchema } from './types';
|
|
4
|
+
|
|
5
|
+
function readAliasedValue(
|
|
6
|
+
args: Record<string, unknown>,
|
|
7
|
+
key: string,
|
|
8
|
+
): unknown | undefined {
|
|
9
|
+
for (const alias of getKeyAliases(key)) {
|
|
10
|
+
if (alias in args) {
|
|
11
|
+
return args[alias];
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
return undefined;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
function readNamespacedArg(
|
|
19
|
+
args: Record<string, unknown>,
|
|
20
|
+
namespace: string,
|
|
21
|
+
key: string,
|
|
22
|
+
): unknown | undefined {
|
|
23
|
+
// Lookup order: namespace object first, then flat dotted form, then bare key
|
|
24
|
+
// fallback. Namespace-aware inputs win so multi-platform callers cannot be
|
|
25
|
+
// cross-contaminated by a top-level bare `deviceId` leaking into the wrong
|
|
26
|
+
// platform.
|
|
27
|
+
const namespacedArgs = readAliasedValue(args, namespace);
|
|
28
|
+
if (isRecord(namespacedArgs)) {
|
|
29
|
+
const nestedValue = readAliasedValue(namespacedArgs, key);
|
|
30
|
+
if (nestedValue !== undefined) {
|
|
31
|
+
return nestedValue;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
const dottedValue = readAliasedValue(args, `${namespace}.${key}`);
|
|
36
|
+
if (dottedValue !== undefined) {
|
|
37
|
+
return dottedValue;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
const directValue = readAliasedValue(args, key);
|
|
41
|
+
if (directValue !== undefined) {
|
|
42
|
+
return directValue;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
return undefined;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export function extractNamespacedArgs<
|
|
49
|
+
TFieldName extends string,
|
|
50
|
+
TArgs extends Record<string, unknown> = Record<string, unknown>,
|
|
51
|
+
>(
|
|
52
|
+
args: Record<string, unknown>,
|
|
53
|
+
namespace: string,
|
|
54
|
+
keys: readonly TFieldName[],
|
|
55
|
+
): TArgs | undefined {
|
|
56
|
+
const extracted: Record<string, unknown> = {};
|
|
57
|
+
|
|
58
|
+
for (const key of keys) {
|
|
59
|
+
const value = readNamespacedArg(args, namespace, key);
|
|
60
|
+
if (value !== undefined) {
|
|
61
|
+
extracted[key] = value;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
return Object.keys(extracted).length > 0 ? (extracted as TArgs) : undefined;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
export function sanitizeNamespacedArgs(
|
|
69
|
+
args: Record<string, unknown>,
|
|
70
|
+
namespace: string,
|
|
71
|
+
keys: readonly string[],
|
|
72
|
+
): Record<string, unknown> {
|
|
73
|
+
const excludedKeys = new Set<string>(getKeyAliases(namespace));
|
|
74
|
+
|
|
75
|
+
for (const key of keys) {
|
|
76
|
+
for (const alias of getKeyAliases(key)) {
|
|
77
|
+
excludedKeys.add(alias);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
for (const alias of getKeyAliases(`${namespace}.${key}`)) {
|
|
81
|
+
excludedKeys.add(alias);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
return Object.fromEntries(
|
|
86
|
+
Object.entries(args).filter(([key]) => !excludedKeys.has(key)),
|
|
87
|
+
);
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* Build a flat MCP tool schema whose keys are dotted `"<namespace>.<field>"`.
|
|
92
|
+
*
|
|
93
|
+
* We intentionally stay flat (rather than `{ namespace: z.object({...}) }`) so
|
|
94
|
+
* that CLI (`--android.device-id`), MCP clients, and `--help` output all share
|
|
95
|
+
* the same spelling. `readNamespacedArg` understands all three input shapes:
|
|
96
|
+
* nested namespace object, dotted flat key, and bare key fallback.
|
|
97
|
+
*/
|
|
98
|
+
export function createNamespacedInitArgSchema(
|
|
99
|
+
namespace: string,
|
|
100
|
+
shape: Record<string, z.ZodTypeAny>,
|
|
101
|
+
): ToolSchema {
|
|
102
|
+
return Object.fromEntries(
|
|
103
|
+
Object.entries(shape).map(([key, value]) => [`${namespace}.${key}`, value]),
|
|
104
|
+
);
|
|
105
|
+
}
|
|
@@ -10,12 +10,12 @@ interface RslibPluginApi {
|
|
|
10
10
|
}
|
|
11
11
|
|
|
12
12
|
/**
|
|
13
|
-
* Rslib plugin to inject report HTML from @
|
|
13
|
+
* Rslib plugin to inject report HTML from @aiscene/core dist into MCP bundle.
|
|
14
14
|
* This runs after build and reads the already-injected HTML from core.
|
|
15
15
|
*
|
|
16
16
|
* Prerequisites:
|
|
17
17
|
* - @midscene/report must be in devDependencies to ensure correct build order
|
|
18
|
-
* - @
|
|
18
|
+
* - @aiscene/core dist must exist with injected HTML
|
|
19
19
|
*
|
|
20
20
|
* @param packageDir - The directory of the MCP package (use __dirname)
|
|
21
21
|
*/
|
|
@@ -35,7 +35,7 @@ export function injectReportHtmlFromCore(packageDir: string) {
|
|
|
35
35
|
|
|
36
36
|
if (!fs.existsSync(coreUtilsPath)) {
|
|
37
37
|
console.warn(
|
|
38
|
-
'[inject-report-html] @
|
|
38
|
+
'[inject-report-html] @aiscene/core dist not found, skipping',
|
|
39
39
|
);
|
|
40
40
|
return;
|
|
41
41
|
}
|
|
@@ -11,7 +11,7 @@ export interface LaunchMCPServerOptions extends HttpLaunchOptions {
|
|
|
11
11
|
}
|
|
12
12
|
|
|
13
13
|
/**
|
|
14
|
-
* Generic agent type (avoid importing from @
|
|
14
|
+
* Generic agent type (avoid importing from @aiscene/core to prevent circular deps)
|
|
15
15
|
*/
|
|
16
16
|
export interface GenericAgent<TDevice = any> {
|
|
17
17
|
interface: TDevice;
|
|
@@ -64,8 +64,8 @@ export interface MCPServerLauncherConfig<
|
|
|
64
64
|
*
|
|
65
65
|
* @example
|
|
66
66
|
* ```typescript
|
|
67
|
-
* import { createMCPServerLauncher } from '@
|
|
68
|
-
* import { Agent } from '@
|
|
67
|
+
* import { createMCPServerLauncher } from '@aiscene/shared/mcp';
|
|
68
|
+
* import { Agent } from '@aiscene/core/agent';
|
|
69
69
|
* import { WebMidsceneTools } from './web-tools';
|
|
70
70
|
* import { WebMCPServer } from './server';
|
|
71
71
|
*
|
|
@@ -1,25 +1,21 @@
|
|
|
1
|
+
import { parseBase64 } from '@aiscene/shared/img';
|
|
1
2
|
import { z } from 'zod';
|
|
2
|
-
import { parseBase64 } from '../img';
|
|
3
3
|
import {
|
|
4
4
|
getZodDescription,
|
|
5
5
|
getZodTypeName,
|
|
6
6
|
isMidsceneLocatorField,
|
|
7
7
|
unwrapZodField,
|
|
8
8
|
} from '../zod-schema-utils';
|
|
9
|
+
import { getErrorMessage } from './error-formatter';
|
|
9
10
|
import type {
|
|
10
11
|
ActionSpaceItem,
|
|
11
12
|
BaseAgent,
|
|
13
|
+
ToolCliMetadata,
|
|
12
14
|
ToolDefinition,
|
|
13
15
|
ToolResult,
|
|
16
|
+
ToolSchema,
|
|
14
17
|
} from './types';
|
|
15
18
|
|
|
16
|
-
/**
|
|
17
|
-
* Extract error message from unknown error type
|
|
18
|
-
*/
|
|
19
|
-
function getErrorMessage(error: unknown): string {
|
|
20
|
-
return error instanceof Error ? error.message : String(error);
|
|
21
|
-
}
|
|
22
|
-
|
|
23
19
|
/**
|
|
24
20
|
* Generate MCP tool description from ActionSpaceItem
|
|
25
21
|
* Format: "actionName action, description. Parameters: param1 (type) - desc; param2 (type) - desc"
|
|
@@ -157,10 +153,19 @@ function transformSchemaField(
|
|
|
157
153
|
}
|
|
158
154
|
|
|
159
155
|
/**
|
|
160
|
-
* Extract and transform schema from action's paramSchema
|
|
156
|
+
* Extract and transform schema from action's paramSchema.
|
|
157
|
+
*
|
|
158
|
+
* CLI and MCP both expose parameters as named fields, so the only schema
|
|
159
|
+
* shapes we can surface are ZodObject (any number of fields) or undefined
|
|
160
|
+
* (the action takes no parameters). A primitive schema like `z.string()`
|
|
161
|
+
* silently degraded to leaking the ZodString instance's prototype methods
|
|
162
|
+
* as CLI flags — see https://github.com/web-infra-dev/midscene/issues/2313.
|
|
163
|
+
* Reject such schemas up front so the next author gets a loud error
|
|
164
|
+
* instead of a silent misconfiguration at runtime.
|
|
161
165
|
*/
|
|
162
166
|
function extractActionSchema(
|
|
163
167
|
paramSchema: z.ZodTypeAny | undefined,
|
|
168
|
+
actionName: string,
|
|
164
169
|
): Record<string, z.ZodTypeAny> {
|
|
165
170
|
if (!paramSchema) {
|
|
166
171
|
return {};
|
|
@@ -168,7 +173,12 @@ function extractActionSchema(
|
|
|
168
173
|
|
|
169
174
|
const shape = getZodObjectShape(paramSchema);
|
|
170
175
|
if (!shape) {
|
|
171
|
-
|
|
176
|
+
const typeName =
|
|
177
|
+
(paramSchema as unknown as { _def?: { typeName?: string } })?._def
|
|
178
|
+
?.typeName ?? 'unknown';
|
|
179
|
+
throw new Error(
|
|
180
|
+
`Action "${actionName}" declared a non-object paramSchema (${typeName}). CLI and MCP tool schemas must be a ZodObject (e.g. z.object({ uri: z.string() })) or undefined. Wrap primitive fields in an object schema.`,
|
|
181
|
+
);
|
|
172
182
|
}
|
|
173
183
|
|
|
174
184
|
return Object.fromEntries(
|
|
@@ -453,25 +463,49 @@ async function captureFailureResult(
|
|
|
453
463
|
}
|
|
454
464
|
}
|
|
455
465
|
|
|
466
|
+
function mergeToolCliMetadata(
|
|
467
|
+
base?: ToolCliMetadata,
|
|
468
|
+
extra?: ToolCliMetadata,
|
|
469
|
+
): ToolCliMetadata | undefined {
|
|
470
|
+
const options = {
|
|
471
|
+
...(base?.options ?? {}),
|
|
472
|
+
...(extra?.options ?? {}),
|
|
473
|
+
};
|
|
474
|
+
|
|
475
|
+
return Object.keys(options).length > 0 ? { options } : undefined;
|
|
476
|
+
}
|
|
477
|
+
|
|
456
478
|
/**
|
|
457
479
|
* Converts DeviceAction from actionSpace into MCP ToolDefinition
|
|
458
480
|
* This is the core logic that removes need for hardcoded tool definitions
|
|
459
481
|
*/
|
|
460
482
|
export function generateToolsFromActionSpace(
|
|
461
483
|
actionSpace: ActionSpaceItem[],
|
|
462
|
-
getAgent: () => Promise<BaseAgent>,
|
|
484
|
+
getAgent: (args?: Record<string, unknown>) => Promise<BaseAgent>,
|
|
485
|
+
sanitizeArgs: (args: Record<string, unknown>) => Record<string, unknown> = (
|
|
486
|
+
args,
|
|
487
|
+
) => args,
|
|
488
|
+
initArgSchema: ToolSchema = {},
|
|
489
|
+
initArgCliMetadata?: ToolCliMetadata,
|
|
463
490
|
): ToolDefinition[] {
|
|
464
491
|
return actionSpace.map((action) => {
|
|
465
|
-
const schema =
|
|
492
|
+
const schema = {
|
|
493
|
+
...extractActionSchema(action.paramSchema as z.ZodTypeAny, action.name),
|
|
494
|
+
...initArgSchema,
|
|
495
|
+
};
|
|
466
496
|
|
|
467
497
|
return {
|
|
468
498
|
name: action.name,
|
|
469
499
|
description: describeActionForMCP(action),
|
|
470
500
|
schema,
|
|
501
|
+
cli: initArgCliMetadata,
|
|
471
502
|
handler: async (args: Record<string, unknown>) => {
|
|
472
503
|
try {
|
|
473
|
-
const agent = await getAgent();
|
|
474
|
-
const normalizedArgs = normalizeActionArgs(
|
|
504
|
+
const agent = await getAgent(args);
|
|
505
|
+
const normalizedArgs = normalizeActionArgs(
|
|
506
|
+
sanitizeArgs(args),
|
|
507
|
+
action.paramSchema,
|
|
508
|
+
);
|
|
475
509
|
let actionResult: unknown;
|
|
476
510
|
|
|
477
511
|
try {
|
|
@@ -513,16 +547,23 @@ export function generateToolsFromActionSpace(
|
|
|
513
547
|
* Generate common tools (screenshot, act)
|
|
514
548
|
*/
|
|
515
549
|
export function generateCommonTools(
|
|
516
|
-
getAgent: () => Promise<BaseAgent>,
|
|
550
|
+
getAgent: (args?: Record<string, unknown>) => Promise<BaseAgent>,
|
|
551
|
+
initArgSchema: ToolSchema = {},
|
|
552
|
+
initArgCliMetadata?: ToolCliMetadata,
|
|
517
553
|
): ToolDefinition[] {
|
|
518
554
|
return [
|
|
519
555
|
{
|
|
520
556
|
name: 'take_screenshot',
|
|
521
557
|
description: 'Capture screenshot of current page/screen',
|
|
522
|
-
schema: {
|
|
523
|
-
|
|
558
|
+
schema: {
|
|
559
|
+
...initArgSchema,
|
|
560
|
+
},
|
|
561
|
+
cli: initArgCliMetadata,
|
|
562
|
+
handler: async (
|
|
563
|
+
args: Record<string, unknown> = {},
|
|
564
|
+
): Promise<ToolResult> => {
|
|
524
565
|
try {
|
|
525
|
-
const agent = await getAgent();
|
|
566
|
+
const agent = await getAgent(args);
|
|
526
567
|
const screenshot = await agent.page?.screenshotBase64();
|
|
527
568
|
if (!screenshot) {
|
|
528
569
|
return createErrorResult('Screenshot not available');
|
|
@@ -550,11 +591,15 @@ export function generateCommonTools(
|
|
|
550
591
|
.describe(
|
|
551
592
|
'Natural language description of the action to perform, e.g. "press Command+Space, type Safari, press Enter"',
|
|
552
593
|
),
|
|
594
|
+
...initArgSchema,
|
|
553
595
|
},
|
|
554
|
-
|
|
596
|
+
cli: mergeToolCliMetadata(undefined, initArgCliMetadata),
|
|
597
|
+
handler: async (
|
|
598
|
+
args: Record<string, unknown> = {},
|
|
599
|
+
): Promise<ToolResult> => {
|
|
555
600
|
const prompt = args.prompt as string;
|
|
556
601
|
try {
|
|
557
|
-
const agent = await getAgent();
|
|
602
|
+
const agent = await getAgent(args);
|
|
558
603
|
if (!agent.aiAction) {
|
|
559
604
|
return createErrorResult('act is not supported by this agent');
|
|
560
605
|
}
|
|
@@ -576,5 +621,38 @@ export function generateCommonTools(
|
|
|
576
621
|
}
|
|
577
622
|
},
|
|
578
623
|
},
|
|
624
|
+
{
|
|
625
|
+
name: 'assert',
|
|
626
|
+
description:
|
|
627
|
+
'Assert a natural language statement against the current page/screen.',
|
|
628
|
+
schema: {
|
|
629
|
+
prompt: z
|
|
630
|
+
.string()
|
|
631
|
+
.describe(
|
|
632
|
+
'Natural language assertion to verify, e.g. "there is a login button visible"',
|
|
633
|
+
),
|
|
634
|
+
...initArgSchema,
|
|
635
|
+
},
|
|
636
|
+
cli: mergeToolCliMetadata(undefined, initArgCliMetadata),
|
|
637
|
+
handler: async (
|
|
638
|
+
args: Record<string, unknown> = {},
|
|
639
|
+
): Promise<ToolResult> => {
|
|
640
|
+
const prompt = args.prompt as string;
|
|
641
|
+
try {
|
|
642
|
+
const agent = await getAgent(args);
|
|
643
|
+
if (!agent.aiAssert) {
|
|
644
|
+
return createErrorResult('assert is not supported by this agent');
|
|
645
|
+
}
|
|
646
|
+
await agent.aiAssert(prompt);
|
|
647
|
+
return {
|
|
648
|
+
content: [{ type: 'text', text: 'Assertion passed.' }],
|
|
649
|
+
};
|
|
650
|
+
} catch (error: unknown) {
|
|
651
|
+
const errorMessage = getErrorMessage(error);
|
|
652
|
+
console.error('Error executing assert:', errorMessage);
|
|
653
|
+
return createErrorResult(`Failed to execute assert: ${errorMessage}`);
|
|
654
|
+
}
|
|
655
|
+
},
|
|
656
|
+
},
|
|
579
657
|
];
|
|
580
658
|
}
|
package/src/mcp/types.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
|
2
2
|
import type { z } from 'zod';
|
|
3
3
|
|
|
4
|
-
// Avoid circular dependency: don't import from @
|
|
4
|
+
// Avoid circular dependency: don't import from @aiscene/core
|
|
5
5
|
// Instead, use generic types that will be provided by implementation
|
|
6
6
|
|
|
7
7
|
/**
|
|
@@ -47,6 +47,15 @@ export type ToolHandler<T = Record<string, unknown>> = (
|
|
|
47
47
|
*/
|
|
48
48
|
export type ToolSchema = Record<string, z.ZodTypeAny>;
|
|
49
49
|
|
|
50
|
+
export interface ToolCliOption {
|
|
51
|
+
preferredName?: string;
|
|
52
|
+
aliases?: string[];
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export interface ToolCliMetadata {
|
|
56
|
+
options?: Record<string, ToolCliOption>;
|
|
57
|
+
}
|
|
58
|
+
|
|
50
59
|
/**
|
|
51
60
|
* Tool definition for MCP server
|
|
52
61
|
*/
|
|
@@ -55,6 +64,7 @@ export interface ToolDefinition<T = Record<string, unknown>> {
|
|
|
55
64
|
description: string;
|
|
56
65
|
schema: ToolSchema;
|
|
57
66
|
handler: ToolHandler<T>;
|
|
67
|
+
cli?: ToolCliMetadata;
|
|
58
68
|
}
|
|
59
69
|
|
|
60
70
|
/**
|
|
@@ -96,6 +106,13 @@ export interface BaseAgent {
|
|
|
96
106
|
assertion: string,
|
|
97
107
|
options: Record<string, unknown>,
|
|
98
108
|
) => Promise<unknown>;
|
|
109
|
+
aiAssert?: (
|
|
110
|
+
assertion: string,
|
|
111
|
+
msg?: string,
|
|
112
|
+
options?: Record<string, unknown>,
|
|
113
|
+
) => Promise<unknown>;
|
|
114
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
115
|
+
aiQuery?: (demand: any, opt?: any) => Promise<any>;
|
|
99
116
|
}
|
|
100
117
|
|
|
101
118
|
/**
|