@aiscene/shared 8.0.2 → 8.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (177) hide show
  1. package/dist/es/agent-tools/agent-behavior-init-args.mjs +44 -0
  2. package/dist/es/agent-tools/base-tools.mjs +163 -0
  3. package/dist/es/agent-tools/chrome-path.mjs +50 -0
  4. package/dist/es/agent-tools/cli-report-session.mjs +78 -0
  5. package/dist/es/agent-tools/error-formatter.mjs +106 -0
  6. package/dist/es/agent-tools/index.mjs +9 -0
  7. package/dist/es/agent-tools/init-arg-utils.mjs +38 -0
  8. package/dist/es/agent-tools/observation-artifact.mjs +5 -0
  9. package/dist/es/agent-tools/observation-record.mjs +248 -0
  10. package/dist/es/agent-tools/tool-defaults.mjs +54 -0
  11. package/dist/es/agent-tools/tool-generator.mjs +529 -0
  12. package/dist/es/agent-tools/types.mjs +3 -0
  13. package/dist/es/agent-tools/user-prompt.mjs +66 -0
  14. package/dist/es/cli/cli-args.mjs +95 -0
  15. package/dist/es/cli/cli-error.mjs +24 -0
  16. package/dist/es/cli/cli-runner.mjs +10 -40
  17. package/dist/es/cli/index.mjs +4 -2
  18. package/dist/es/cli/interrupt.mjs +116 -0
  19. package/dist/es/cli/record-command.mjs +130 -0
  20. package/dist/es/cli/screenshot-file.mjs +24 -0
  21. package/dist/es/cli/verbose-ai-act.mjs +230 -0
  22. package/dist/es/cli/verbose-screenshot.mjs +134 -0
  23. package/dist/es/cli/verbose.mjs +404 -0
  24. package/dist/es/constants/example-code.mjs +2 -2
  25. package/dist/es/constants/index.mjs +6 -1
  26. package/dist/es/env/model-config-manager.mjs +3 -0
  27. package/dist/es/env/parse-model-config.mjs +6 -1
  28. package/dist/es/env/types.mjs +4 -1
  29. package/dist/es/env/utils.mjs +1 -9
  30. package/dist/es/extractor/index.mjs +2 -2
  31. package/dist/es/img/index.mjs +2 -2
  32. package/dist/es/img/info.mjs +58 -17
  33. package/dist/es/img/photon-loader.mjs +4 -0
  34. package/dist/es/key-alias-utils.mjs +19 -0
  35. package/dist/es/mcp/base-server.mjs +10 -9
  36. package/dist/es/mcp/base-tools.mjs +72 -4
  37. package/dist/es/mcp/cli-report-session.mjs +78 -0
  38. package/dist/es/mcp/error-formatter.mjs +19 -0
  39. package/dist/es/mcp/index.mjs +2 -0
  40. package/dist/es/mcp/init-arg-utils.mjs +38 -0
  41. package/dist/es/mcp/inject-report-html-plugin.mjs +1 -1
  42. package/dist/es/mcp/tool-generator.mjs +65 -17
  43. package/dist/es/node/fs.mjs +1 -1
  44. package/dist/es/recorder.mjs +249 -0
  45. package/dist/lib/agent-tools/agent-behavior-init-args.js +87 -0
  46. package/dist/lib/agent-tools/base-tools.js +197 -0
  47. package/dist/lib/agent-tools/chrome-path.js +87 -0
  48. package/dist/lib/agent-tools/cli-report-session.js +121 -0
  49. package/dist/lib/agent-tools/error-formatter.js +149 -0
  50. package/dist/lib/agent-tools/index.js +114 -0
  51. package/dist/lib/agent-tools/init-arg-utils.js +78 -0
  52. package/dist/lib/agent-tools/observation-artifact.js +42 -0
  53. package/dist/lib/agent-tools/observation-record.js +297 -0
  54. package/dist/lib/agent-tools/tool-defaults.js +97 -0
  55. package/dist/lib/agent-tools/tool-generator.js +569 -0
  56. package/dist/lib/agent-tools/types.js +40 -0
  57. package/dist/lib/agent-tools/user-prompt.js +103 -0
  58. package/dist/lib/cli/cli-args.js +138 -0
  59. package/dist/lib/cli/cli-error.js +61 -0
  60. package/dist/lib/cli/cli-runner.js +19 -46
  61. package/dist/lib/cli/index.js +8 -3
  62. package/dist/lib/cli/interrupt.js +156 -0
  63. package/dist/lib/cli/record-command.js +164 -0
  64. package/dist/lib/cli/screenshot-file.js +58 -0
  65. package/dist/lib/cli/verbose-ai-act.js +273 -0
  66. package/dist/lib/cli/verbose-screenshot.js +177 -0
  67. package/dist/lib/cli/verbose.js +465 -0
  68. package/dist/lib/constants/example-code.js +2 -2
  69. package/dist/lib/constants/index.js +23 -3
  70. package/dist/lib/env/model-config-manager.js +3 -0
  71. package/dist/lib/env/parse-model-config.js +6 -1
  72. package/dist/lib/env/types.js +9 -0
  73. package/dist/lib/env/utils.js +0 -11
  74. package/dist/lib/extractor/index.js +3 -0
  75. package/dist/lib/img/index.js +3 -0
  76. package/dist/lib/img/info.js +65 -18
  77. package/dist/lib/img/photon-loader.js +38 -0
  78. package/dist/lib/key-alias-utils.js +62 -0
  79. package/dist/lib/mcp/base-server.js +11 -10
  80. package/dist/lib/mcp/base-tools.js +74 -6
  81. package/dist/lib/mcp/cli-report-session.js +121 -0
  82. package/dist/lib/mcp/error-formatter.js +53 -0
  83. package/dist/lib/mcp/index.js +24 -10
  84. package/dist/lib/mcp/init-arg-utils.js +78 -0
  85. package/dist/lib/mcp/inject-report-html-plugin.js +1 -1
  86. package/dist/lib/mcp/tool-generator.js +74 -26
  87. package/dist/lib/node/fs.js +1 -1
  88. package/dist/lib/recorder.js +307 -0
  89. package/dist/types/agent-tools/agent-behavior-init-args.d.ts +17 -0
  90. package/dist/types/agent-tools/base-tools.d.ts +158 -0
  91. package/dist/types/agent-tools/chrome-path.d.ts +2 -0
  92. package/dist/types/agent-tools/cli-report-session.d.ts +12 -0
  93. package/dist/types/agent-tools/error-formatter.d.ts +30 -0
  94. package/dist/types/agent-tools/index.d.ts +9 -0
  95. package/dist/types/agent-tools/init-arg-utils.d.ts +13 -0
  96. package/dist/types/agent-tools/observation-artifact.d.ts +10 -0
  97. package/dist/types/agent-tools/observation-record.d.ts +38 -0
  98. package/dist/types/agent-tools/tool-defaults.d.ts +63 -0
  99. package/dist/types/agent-tools/tool-generator.d.ts +13 -0
  100. package/dist/types/agent-tools/types.d.ts +213 -0
  101. package/dist/types/agent-tools/user-prompt.d.ts +13 -0
  102. package/dist/types/cli/cli-args.d.ts +8 -0
  103. package/dist/types/cli/cli-error.d.ts +5 -0
  104. package/dist/types/cli/cli-runner.d.ts +4 -7
  105. package/dist/types/cli/index.d.ts +3 -1
  106. package/dist/types/cli/interrupt.d.ts +49 -0
  107. package/dist/types/cli/record-command.d.ts +3 -0
  108. package/dist/types/cli/screenshot-file.d.ts +10 -0
  109. package/dist/types/cli/verbose-ai-act.d.ts +44 -0
  110. package/dist/types/cli/verbose-screenshot.d.ts +10 -0
  111. package/dist/types/cli/verbose.d.ts +40 -0
  112. package/dist/types/constants/example-code.d.ts +1 -1
  113. package/dist/types/constants/index.d.ts +5 -0
  114. package/dist/types/env/types.d.ts +14 -5
  115. package/dist/types/env/utils.d.ts +0 -31
  116. package/dist/types/extractor/index.d.ts +1 -1
  117. package/dist/types/img/index.d.ts +1 -1
  118. package/dist/types/img/info.d.ts +7 -0
  119. package/dist/types/img/photon-loader.d.ts +2 -0
  120. package/dist/types/key-alias-utils.d.ts +9 -0
  121. package/dist/types/mcp/base-tools.d.ts +74 -5
  122. package/dist/types/mcp/cli-report-session.d.ts +12 -0
  123. package/dist/types/mcp/error-formatter.d.ts +12 -0
  124. package/dist/types/mcp/index.d.ts +2 -0
  125. package/dist/types/mcp/init-arg-utils.d.ts +13 -0
  126. package/dist/types/mcp/inject-report-html-plugin.d.ts +2 -2
  127. package/dist/types/mcp/launcher-helper.d.ts +3 -3
  128. package/dist/types/mcp/tool-generator.d.ts +3 -3
  129. package/dist/types/mcp/types.d.ts +10 -0
  130. package/dist/types/recorder.d.ts +113 -0
  131. package/package.json +1 -1
  132. package/src/agent-tools/agent-behavior-init-args.ts +109 -0
  133. package/src/agent-tools/base-tools.ts +399 -0
  134. package/src/agent-tools/chrome-path.ts +74 -0
  135. package/src/agent-tools/cli-report-session.ts +130 -0
  136. package/src/agent-tools/error-formatter.ts +177 -0
  137. package/src/agent-tools/index.ts +9 -0
  138. package/src/agent-tools/init-arg-utils.ts +105 -0
  139. package/src/agent-tools/observation-artifact.ts +29 -0
  140. package/src/agent-tools/observation-record.ts +331 -0
  141. package/src/agent-tools/tool-defaults.ts +119 -0
  142. package/src/agent-tools/tool-generator.ts +866 -0
  143. package/src/agent-tools/types.ts +250 -0
  144. package/src/agent-tools/user-prompt.ts +102 -0
  145. package/src/cli/cli-args.ts +173 -0
  146. package/src/cli/cli-error.ts +24 -0
  147. package/src/cli/cli-runner.ts +37 -56
  148. package/src/cli/index.ts +3 -7
  149. package/src/cli/interrupt.ts +207 -0
  150. package/src/cli/record-command.ts +177 -0
  151. package/src/cli/screenshot-file.ts +61 -0
  152. package/src/cli/verbose-ai-act.ts +387 -0
  153. package/src/cli/verbose-screenshot.ts +269 -0
  154. package/src/cli/verbose.ts +753 -0
  155. package/src/constants/example-code.ts +2 -2
  156. package/src/constants/index.ts +10 -0
  157. package/src/env/model-config-manager.ts +3 -0
  158. package/src/env/parse-model-config.ts +19 -1
  159. package/src/env/types.ts +13 -2
  160. package/src/env/utils.ts +0 -50
  161. package/src/extractor/index.ts +1 -1
  162. package/src/img/index.ts +14 -0
  163. package/src/img/info.ts +97 -45
  164. package/src/img/photon-loader.ts +5 -0
  165. package/src/img/transform.ts +261 -2
  166. package/src/key-alias-utils.ts +23 -0
  167. package/src/mcp/base-server.ts +10 -12
  168. package/src/mcp/base-tools.ts +218 -11
  169. package/src/mcp/cli-report-session.ts +130 -0
  170. package/src/mcp/error-formatter.ts +52 -0
  171. package/src/mcp/index.ts +2 -0
  172. package/src/mcp/init-arg-utils.ts +105 -0
  173. package/src/mcp/inject-report-html-plugin.ts +3 -3
  174. package/src/mcp/launcher-helper.ts +3 -3
  175. package/src/mcp/tool-generator.ts +98 -20
  176. package/src/mcp/types.ts +18 -1
  177. package/src/recorder.ts +625 -0
@@ -0,0 +1,130 @@
1
+ import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
2
+ import { join } from 'node:path';
3
+ import { getMidsceneRunBaseDir, getMidsceneRunSubDir } from '../common';
4
+
5
+ export interface CliReportSession {
6
+ version: 1;
7
+ sessionName: string;
8
+ targetIdentity?: string;
9
+ reportFileName: string;
10
+ reportPath: string;
11
+ createdAt: number;
12
+ }
13
+
14
+ const sessionDirName = 'cli-report-session';
15
+
16
+ function sanitizeSessionName(sessionName: string): string {
17
+ return sessionName.replace(/[^a-zA-Z0-9._-]/g, '_') || 'default';
18
+ }
19
+
20
+ function sanitizeFileSegment(segment: string): string {
21
+ const sanitized = segment.replace(/[^a-zA-Z0-9._-]/g, '_') || 'unknown';
22
+ return sanitized.slice(0, 80);
23
+ }
24
+
25
+ function ensureHtmlFileName(reportFileName: string): string {
26
+ return reportFileName.endsWith('.html')
27
+ ? reportFileName
28
+ : `${reportFileName}.html`;
29
+ }
30
+
31
+ function formatDateForFileName(date: Date): string {
32
+ const pad = (value: number) => String(value).padStart(2, '0');
33
+ const day = [
34
+ date.getFullYear(),
35
+ pad(date.getMonth() + 1),
36
+ pad(date.getDate()),
37
+ ].join('-');
38
+ const time = [
39
+ pad(date.getHours()),
40
+ pad(date.getMinutes()),
41
+ pad(date.getSeconds()),
42
+ ].join('-');
43
+ return `${day}_${time}`;
44
+ }
45
+
46
+ function randomId(): string {
47
+ return Math.random().toString(36).slice(2, 10);
48
+ }
49
+
50
+ function getCliReportSessionDir(): string {
51
+ const dir = join(getMidsceneRunBaseDir(), sessionDirName);
52
+ if (!existsSync(dir)) {
53
+ mkdirSync(dir, { recursive: true });
54
+ }
55
+ return dir;
56
+ }
57
+
58
+ function getCliReportSessionPath(sessionName: string): string {
59
+ return join(
60
+ getCliReportSessionDir(),
61
+ `${sanitizeSessionName(sessionName)}.json`,
62
+ );
63
+ }
64
+
65
+ export function generateCliReportSession(
66
+ sessionName: string,
67
+ targetIdentity?: string,
68
+ ): CliReportSession {
69
+ const identitySegment = targetIdentity
70
+ ? `-${sanitizeFileSegment(targetIdentity)}`
71
+ : '';
72
+ const reportFileName = `${sanitizeSessionName(sessionName)}${identitySegment}-${formatDateForFileName(new Date())}-${randomId()}`;
73
+ const reportPath = join(
74
+ getMidsceneRunSubDir('report'),
75
+ ensureHtmlFileName(reportFileName),
76
+ );
77
+ const session: CliReportSession = {
78
+ version: 1,
79
+ sessionName,
80
+ ...(targetIdentity ? { targetIdentity } : {}),
81
+ reportFileName,
82
+ reportPath,
83
+ createdAt: Date.now(),
84
+ };
85
+ return session;
86
+ }
87
+
88
+ export function writeCliReportSession(session: CliReportSession): void {
89
+ writeFileSync(
90
+ getCliReportSessionPath(session.sessionName),
91
+ JSON.stringify(session, null, 2),
92
+ 'utf-8',
93
+ );
94
+ }
95
+
96
+ export function createCliReportSession(
97
+ sessionName: string,
98
+ targetIdentity?: string,
99
+ ): CliReportSession {
100
+ const session = generateCliReportSession(sessionName, targetIdentity);
101
+ writeCliReportSession(session);
102
+ return session;
103
+ }
104
+
105
+ export function readCliReportSession(
106
+ sessionName: string,
107
+ ): CliReportSession | undefined {
108
+ const sessionPath = getCliReportSessionPath(sessionName);
109
+ if (!existsSync(sessionPath)) {
110
+ return undefined;
111
+ }
112
+
113
+ try {
114
+ const raw = readFileSync(sessionPath, 'utf-8');
115
+ const parsed = JSON.parse(raw) as Partial<CliReportSession>;
116
+ if (
117
+ parsed.version !== 1 ||
118
+ parsed.sessionName !== sessionName ||
119
+ typeof parsed.reportFileName !== 'string' ||
120
+ !parsed.reportFileName.trim() ||
121
+ /[\\/]/.test(parsed.reportFileName)
122
+ ) {
123
+ return undefined;
124
+ }
125
+
126
+ return parsed as CliReportSession;
127
+ } catch {
128
+ return undefined;
129
+ }
130
+ }
@@ -0,0 +1,52 @@
1
+ /**
2
+ * Extract a human-readable message from an unknown thrown value.
3
+ *
4
+ * Many SDK/transport layers reject with structured objects (e.g.
5
+ * `{ code, message }`, `{ error: { message } }`, `{ cause: { message } }`)
6
+ * rather than `Error` instances. `String(obj)` collapses those to
7
+ * `"[object Object]"`, which is useless for diagnostics. This helper walks
8
+ * the common shapes, falls back to `JSON.stringify`, and finally to
9
+ * `Object.prototype.toString.call` so that callers always get something
10
+ * actionable in logs and surfaced tool results.
11
+ */
12
+ export function getErrorMessage(error: unknown): string {
13
+ if (error instanceof Error) return error.message;
14
+ if (error === null || error === undefined) return String(error);
15
+ if (typeof error !== 'object') return String(error);
16
+
17
+ const candidate = extractStringMessage(error);
18
+ if (candidate) return candidate;
19
+
20
+ try {
21
+ return JSON.stringify(error);
22
+ } catch {
23
+ return Object.prototype.toString.call(error);
24
+ }
25
+ }
26
+
27
+ function extractStringMessage(error: object): string | undefined {
28
+ const anyError = error as {
29
+ message?: unknown;
30
+ error?: { message?: unknown };
31
+ cause?: { message?: unknown };
32
+ };
33
+
34
+ if (typeof anyError.message === 'string' && anyError.message) {
35
+ return anyError.message;
36
+ }
37
+ if (
38
+ anyError.error &&
39
+ typeof anyError.error.message === 'string' &&
40
+ anyError.error.message
41
+ ) {
42
+ return anyError.error.message;
43
+ }
44
+ if (
45
+ anyError.cause &&
46
+ typeof anyError.cause.message === 'string' &&
47
+ anyError.cause.message
48
+ ) {
49
+ return anyError.cause.message;
50
+ }
51
+ return undefined;
52
+ }
package/src/mcp/index.ts CHANGED
@@ -1,5 +1,7 @@
1
1
  export * from './base-server';
2
2
  export * from './base-tools';
3
+ export * from './init-arg-utils';
4
+ export * from './error-formatter';
3
5
  export * from './tool-generator';
4
6
  export * from './types';
5
7
  export * from './inject-report-html-plugin';
@@ -0,0 +1,105 @@
1
+ import type { z } from 'zod';
2
+ import { getKeyAliases, isRecord } from '../key-alias-utils';
3
+ import type { ToolSchema } from './types';
4
+
5
+ function readAliasedValue(
6
+ args: Record<string, unknown>,
7
+ key: string,
8
+ ): unknown | undefined {
9
+ for (const alias of getKeyAliases(key)) {
10
+ if (alias in args) {
11
+ return args[alias];
12
+ }
13
+ }
14
+
15
+ return undefined;
16
+ }
17
+
18
+ function readNamespacedArg(
19
+ args: Record<string, unknown>,
20
+ namespace: string,
21
+ key: string,
22
+ ): unknown | undefined {
23
+ // Lookup order: namespace object first, then flat dotted form, then bare key
24
+ // fallback. Namespace-aware inputs win so multi-platform callers cannot be
25
+ // cross-contaminated by a top-level bare `deviceId` leaking into the wrong
26
+ // platform.
27
+ const namespacedArgs = readAliasedValue(args, namespace);
28
+ if (isRecord(namespacedArgs)) {
29
+ const nestedValue = readAliasedValue(namespacedArgs, key);
30
+ if (nestedValue !== undefined) {
31
+ return nestedValue;
32
+ }
33
+ }
34
+
35
+ const dottedValue = readAliasedValue(args, `${namespace}.${key}`);
36
+ if (dottedValue !== undefined) {
37
+ return dottedValue;
38
+ }
39
+
40
+ const directValue = readAliasedValue(args, key);
41
+ if (directValue !== undefined) {
42
+ return directValue;
43
+ }
44
+
45
+ return undefined;
46
+ }
47
+
48
+ export function extractNamespacedArgs<
49
+ TFieldName extends string,
50
+ TArgs extends Record<string, unknown> = Record<string, unknown>,
51
+ >(
52
+ args: Record<string, unknown>,
53
+ namespace: string,
54
+ keys: readonly TFieldName[],
55
+ ): TArgs | undefined {
56
+ const extracted: Record<string, unknown> = {};
57
+
58
+ for (const key of keys) {
59
+ const value = readNamespacedArg(args, namespace, key);
60
+ if (value !== undefined) {
61
+ extracted[key] = value;
62
+ }
63
+ }
64
+
65
+ return Object.keys(extracted).length > 0 ? (extracted as TArgs) : undefined;
66
+ }
67
+
68
+ export function sanitizeNamespacedArgs(
69
+ args: Record<string, unknown>,
70
+ namespace: string,
71
+ keys: readonly string[],
72
+ ): Record<string, unknown> {
73
+ const excludedKeys = new Set<string>(getKeyAliases(namespace));
74
+
75
+ for (const key of keys) {
76
+ for (const alias of getKeyAliases(key)) {
77
+ excludedKeys.add(alias);
78
+ }
79
+
80
+ for (const alias of getKeyAliases(`${namespace}.${key}`)) {
81
+ excludedKeys.add(alias);
82
+ }
83
+ }
84
+
85
+ return Object.fromEntries(
86
+ Object.entries(args).filter(([key]) => !excludedKeys.has(key)),
87
+ );
88
+ }
89
+
90
+ /**
91
+ * Build a flat MCP tool schema whose keys are dotted `"<namespace>.<field>"`.
92
+ *
93
+ * We intentionally stay flat (rather than `{ namespace: z.object({...}) }`) so
94
+ * that CLI (`--android.device-id`), MCP clients, and `--help` output all share
95
+ * the same spelling. `readNamespacedArg` understands all three input shapes:
96
+ * nested namespace object, dotted flat key, and bare key fallback.
97
+ */
98
+ export function createNamespacedInitArgSchema(
99
+ namespace: string,
100
+ shape: Record<string, z.ZodTypeAny>,
101
+ ): ToolSchema {
102
+ return Object.fromEntries(
103
+ Object.entries(shape).map(([key, value]) => [`${namespace}.${key}`, value]),
104
+ );
105
+ }
@@ -10,12 +10,12 @@ interface RslibPluginApi {
10
10
  }
11
11
 
12
12
  /**
13
- * Rslib plugin to inject report HTML from @midscene/core dist into MCP bundle.
13
+ * Rslib plugin to inject report HTML from @aiscene/core dist into MCP bundle.
14
14
  * This runs after build and reads the already-injected HTML from core.
15
15
  *
16
16
  * Prerequisites:
17
17
  * - @midscene/report must be in devDependencies to ensure correct build order
18
- * - @midscene/core dist must exist with injected HTML
18
+ * - @aiscene/core dist must exist with injected HTML
19
19
  *
20
20
  * @param packageDir - The directory of the MCP package (use __dirname)
21
21
  */
@@ -35,7 +35,7 @@ export function injectReportHtmlFromCore(packageDir: string) {
35
35
 
36
36
  if (!fs.existsSync(coreUtilsPath)) {
37
37
  console.warn(
38
- '[inject-report-html] @midscene/core dist not found, skipping',
38
+ '[inject-report-html] @aiscene/core dist not found, skipping',
39
39
  );
40
40
  return;
41
41
  }
@@ -11,7 +11,7 @@ export interface LaunchMCPServerOptions extends HttpLaunchOptions {
11
11
  }
12
12
 
13
13
  /**
14
- * Generic agent type (avoid importing from @midscene/core to prevent circular deps)
14
+ * Generic agent type (avoid importing from @aiscene/core to prevent circular deps)
15
15
  */
16
16
  export interface GenericAgent<TDevice = any> {
17
17
  interface: TDevice;
@@ -64,8 +64,8 @@ export interface MCPServerLauncherConfig<
64
64
  *
65
65
  * @example
66
66
  * ```typescript
67
- * import { createMCPServerLauncher } from '@midscene/shared/mcp';
68
- * import { Agent } from '@midscene/core/agent';
67
+ * import { createMCPServerLauncher } from '@aiscene/shared/mcp';
68
+ * import { Agent } from '@aiscene/core/agent';
69
69
  * import { WebMidsceneTools } from './web-tools';
70
70
  * import { WebMCPServer } from './server';
71
71
  *
@@ -1,25 +1,21 @@
1
+ import { parseBase64 } from '@aiscene/shared/img';
1
2
  import { z } from 'zod';
2
- import { parseBase64 } from '../img';
3
3
  import {
4
4
  getZodDescription,
5
5
  getZodTypeName,
6
6
  isMidsceneLocatorField,
7
7
  unwrapZodField,
8
8
  } from '../zod-schema-utils';
9
+ import { getErrorMessage } from './error-formatter';
9
10
  import type {
10
11
  ActionSpaceItem,
11
12
  BaseAgent,
13
+ ToolCliMetadata,
12
14
  ToolDefinition,
13
15
  ToolResult,
16
+ ToolSchema,
14
17
  } from './types';
15
18
 
16
- /**
17
- * Extract error message from unknown error type
18
- */
19
- function getErrorMessage(error: unknown): string {
20
- return error instanceof Error ? error.message : String(error);
21
- }
22
-
23
19
  /**
24
20
  * Generate MCP tool description from ActionSpaceItem
25
21
  * Format: "actionName action, description. Parameters: param1 (type) - desc; param2 (type) - desc"
@@ -157,10 +153,19 @@ function transformSchemaField(
157
153
  }
158
154
 
159
155
  /**
160
- * Extract and transform schema from action's paramSchema
156
+ * Extract and transform schema from action's paramSchema.
157
+ *
158
+ * CLI and MCP both expose parameters as named fields, so the only schema
159
+ * shapes we can surface are ZodObject (any number of fields) or undefined
160
+ * (the action takes no parameters). A primitive schema like `z.string()`
161
+ * silently degraded to leaking the ZodString instance's prototype methods
162
+ * as CLI flags — see https://github.com/web-infra-dev/midscene/issues/2313.
163
+ * Reject such schemas up front so the next author gets a loud error
164
+ * instead of a silent misconfiguration at runtime.
161
165
  */
162
166
  function extractActionSchema(
163
167
  paramSchema: z.ZodTypeAny | undefined,
168
+ actionName: string,
164
169
  ): Record<string, z.ZodTypeAny> {
165
170
  if (!paramSchema) {
166
171
  return {};
@@ -168,7 +173,12 @@ function extractActionSchema(
168
173
 
169
174
  const shape = getZodObjectShape(paramSchema);
170
175
  if (!shape) {
171
- return paramSchema as unknown as Record<string, z.ZodTypeAny>;
176
+ const typeName =
177
+ (paramSchema as unknown as { _def?: { typeName?: string } })?._def
178
+ ?.typeName ?? 'unknown';
179
+ throw new Error(
180
+ `Action "${actionName}" declared a non-object paramSchema (${typeName}). CLI and MCP tool schemas must be a ZodObject (e.g. z.object({ uri: z.string() })) or undefined. Wrap primitive fields in an object schema.`,
181
+ );
172
182
  }
173
183
 
174
184
  return Object.fromEntries(
@@ -453,25 +463,49 @@ async function captureFailureResult(
453
463
  }
454
464
  }
455
465
 
466
+ function mergeToolCliMetadata(
467
+ base?: ToolCliMetadata,
468
+ extra?: ToolCliMetadata,
469
+ ): ToolCliMetadata | undefined {
470
+ const options = {
471
+ ...(base?.options ?? {}),
472
+ ...(extra?.options ?? {}),
473
+ };
474
+
475
+ return Object.keys(options).length > 0 ? { options } : undefined;
476
+ }
477
+
456
478
  /**
457
479
  * Converts DeviceAction from actionSpace into MCP ToolDefinition
458
480
  * This is the core logic that removes need for hardcoded tool definitions
459
481
  */
460
482
  export function generateToolsFromActionSpace(
461
483
  actionSpace: ActionSpaceItem[],
462
- getAgent: () => Promise<BaseAgent>,
484
+ getAgent: (args?: Record<string, unknown>) => Promise<BaseAgent>,
485
+ sanitizeArgs: (args: Record<string, unknown>) => Record<string, unknown> = (
486
+ args,
487
+ ) => args,
488
+ initArgSchema: ToolSchema = {},
489
+ initArgCliMetadata?: ToolCliMetadata,
463
490
  ): ToolDefinition[] {
464
491
  return actionSpace.map((action) => {
465
- const schema = extractActionSchema(action.paramSchema as z.ZodTypeAny);
492
+ const schema = {
493
+ ...extractActionSchema(action.paramSchema as z.ZodTypeAny, action.name),
494
+ ...initArgSchema,
495
+ };
466
496
 
467
497
  return {
468
498
  name: action.name,
469
499
  description: describeActionForMCP(action),
470
500
  schema,
501
+ cli: initArgCliMetadata,
471
502
  handler: async (args: Record<string, unknown>) => {
472
503
  try {
473
- const agent = await getAgent();
474
- const normalizedArgs = normalizeActionArgs(args, action.paramSchema);
504
+ const agent = await getAgent(args);
505
+ const normalizedArgs = normalizeActionArgs(
506
+ sanitizeArgs(args),
507
+ action.paramSchema,
508
+ );
475
509
  let actionResult: unknown;
476
510
 
477
511
  try {
@@ -513,16 +547,23 @@ export function generateToolsFromActionSpace(
513
547
  * Generate common tools (screenshot, act)
514
548
  */
515
549
  export function generateCommonTools(
516
- getAgent: () => Promise<BaseAgent>,
550
+ getAgent: (args?: Record<string, unknown>) => Promise<BaseAgent>,
551
+ initArgSchema: ToolSchema = {},
552
+ initArgCliMetadata?: ToolCliMetadata,
517
553
  ): ToolDefinition[] {
518
554
  return [
519
555
  {
520
556
  name: 'take_screenshot',
521
557
  description: 'Capture screenshot of current page/screen',
522
- schema: {},
523
- handler: async (): Promise<ToolResult> => {
558
+ schema: {
559
+ ...initArgSchema,
560
+ },
561
+ cli: initArgCliMetadata,
562
+ handler: async (
563
+ args: Record<string, unknown> = {},
564
+ ): Promise<ToolResult> => {
524
565
  try {
525
- const agent = await getAgent();
566
+ const agent = await getAgent(args);
526
567
  const screenshot = await agent.page?.screenshotBase64();
527
568
  if (!screenshot) {
528
569
  return createErrorResult('Screenshot not available');
@@ -550,11 +591,15 @@ export function generateCommonTools(
550
591
  .describe(
551
592
  'Natural language description of the action to perform, e.g. "press Command+Space, type Safari, press Enter"',
552
593
  ),
594
+ ...initArgSchema,
553
595
  },
554
- handler: async (args: Record<string, unknown>): Promise<ToolResult> => {
596
+ cli: mergeToolCliMetadata(undefined, initArgCliMetadata),
597
+ handler: async (
598
+ args: Record<string, unknown> = {},
599
+ ): Promise<ToolResult> => {
555
600
  const prompt = args.prompt as string;
556
601
  try {
557
- const agent = await getAgent();
602
+ const agent = await getAgent(args);
558
603
  if (!agent.aiAction) {
559
604
  return createErrorResult('act is not supported by this agent');
560
605
  }
@@ -576,5 +621,38 @@ export function generateCommonTools(
576
621
  }
577
622
  },
578
623
  },
624
+ {
625
+ name: 'assert',
626
+ description:
627
+ 'Assert a natural language statement against the current page/screen.',
628
+ schema: {
629
+ prompt: z
630
+ .string()
631
+ .describe(
632
+ 'Natural language assertion to verify, e.g. "there is a login button visible"',
633
+ ),
634
+ ...initArgSchema,
635
+ },
636
+ cli: mergeToolCliMetadata(undefined, initArgCliMetadata),
637
+ handler: async (
638
+ args: Record<string, unknown> = {},
639
+ ): Promise<ToolResult> => {
640
+ const prompt = args.prompt as string;
641
+ try {
642
+ const agent = await getAgent(args);
643
+ if (!agent.aiAssert) {
644
+ return createErrorResult('assert is not supported by this agent');
645
+ }
646
+ await agent.aiAssert(prompt);
647
+ return {
648
+ content: [{ type: 'text', text: 'Assertion passed.' }],
649
+ };
650
+ } catch (error: unknown) {
651
+ const errorMessage = getErrorMessage(error);
652
+ console.error('Error executing assert:', errorMessage);
653
+ return createErrorResult(`Failed to execute assert: ${errorMessage}`);
654
+ }
655
+ },
656
+ },
579
657
  ];
580
658
  }
package/src/mcp/types.ts CHANGED
@@ -1,7 +1,7 @@
1
1
  import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
2
2
  import type { z } from 'zod';
3
3
 
4
- // Avoid circular dependency: don't import from @midscene/core
4
+ // Avoid circular dependency: don't import from @aiscene/core
5
5
  // Instead, use generic types that will be provided by implementation
6
6
 
7
7
  /**
@@ -47,6 +47,15 @@ export type ToolHandler<T = Record<string, unknown>> = (
47
47
  */
48
48
  export type ToolSchema = Record<string, z.ZodTypeAny>;
49
49
 
50
+ export interface ToolCliOption {
51
+ preferredName?: string;
52
+ aliases?: string[];
53
+ }
54
+
55
+ export interface ToolCliMetadata {
56
+ options?: Record<string, ToolCliOption>;
57
+ }
58
+
50
59
  /**
51
60
  * Tool definition for MCP server
52
61
  */
@@ -55,6 +64,7 @@ export interface ToolDefinition<T = Record<string, unknown>> {
55
64
  description: string;
56
65
  schema: ToolSchema;
57
66
  handler: ToolHandler<T>;
67
+ cli?: ToolCliMetadata;
58
68
  }
59
69
 
60
70
  /**
@@ -96,6 +106,13 @@ export interface BaseAgent {
96
106
  assertion: string,
97
107
  options: Record<string, unknown>,
98
108
  ) => Promise<unknown>;
109
+ aiAssert?: (
110
+ assertion: string,
111
+ msg?: string,
112
+ options?: Record<string, unknown>,
113
+ ) => Promise<unknown>;
114
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
115
+ aiQuery?: (demand: any, opt?: any) => Promise<any>;
99
116
  }
100
117
 
101
118
  /**