@aiscene/shared 8.0.3 → 8.0.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (105) hide show
  1. package/dist/es/agent-tools/agent-behavior-init-args.mjs +44 -0
  2. package/dist/es/agent-tools/base-tools.mjs +163 -0
  3. package/dist/es/agent-tools/chrome-path.mjs +50 -0
  4. package/dist/es/agent-tools/cli-report-session.mjs +78 -0
  5. package/dist/es/agent-tools/error-formatter.mjs +106 -0
  6. package/dist/es/agent-tools/index.mjs +9 -0
  7. package/dist/es/agent-tools/init-arg-utils.mjs +38 -0
  8. package/dist/es/agent-tools/observation-artifact.mjs +5 -0
  9. package/dist/es/agent-tools/observation-record.mjs +248 -0
  10. package/dist/es/agent-tools/tool-defaults.mjs +54 -0
  11. package/dist/es/agent-tools/tool-generator.mjs +529 -0
  12. package/dist/es/agent-tools/types.mjs +3 -0
  13. package/dist/es/agent-tools/user-prompt.mjs +66 -0
  14. package/dist/es/cli/interrupt.mjs +116 -0
  15. package/dist/es/cli/record-command.mjs +130 -0
  16. package/dist/es/cli/screenshot-file.mjs +24 -0
  17. package/dist/es/cli/verbose-ai-act.mjs +230 -0
  18. package/dist/es/cli/verbose-screenshot.mjs +134 -0
  19. package/dist/es/cli/verbose.mjs +404 -0
  20. package/dist/es/env/parse-model-config.mjs +1 -1
  21. package/dist/es/env/types.mjs +18 -3
  22. package/dist/es/img/index.mjs +3 -3
  23. package/dist/es/img/info.mjs +45 -1
  24. package/dist/es/img/photon-loader.mjs +4 -0
  25. package/dist/es/img/transform.mjs +113 -3
  26. package/dist/es/recorder.mjs +249 -0
  27. package/dist/lib/agent-tools/agent-behavior-init-args.js +87 -0
  28. package/dist/lib/agent-tools/base-tools.js +197 -0
  29. package/dist/lib/agent-tools/chrome-path.js +87 -0
  30. package/dist/lib/agent-tools/cli-report-session.js +121 -0
  31. package/dist/lib/agent-tools/error-formatter.js +149 -0
  32. package/dist/lib/agent-tools/index.js +114 -0
  33. package/dist/lib/agent-tools/init-arg-utils.js +78 -0
  34. package/dist/lib/agent-tools/observation-artifact.js +42 -0
  35. package/dist/lib/agent-tools/observation-record.js +297 -0
  36. package/dist/lib/agent-tools/tool-defaults.js +97 -0
  37. package/dist/lib/agent-tools/tool-generator.js +569 -0
  38. package/dist/lib/agent-tools/types.js +40 -0
  39. package/dist/lib/agent-tools/user-prompt.js +103 -0
  40. package/dist/lib/cli/interrupt.js +156 -0
  41. package/dist/lib/cli/record-command.js +164 -0
  42. package/dist/lib/cli/screenshot-file.js +58 -0
  43. package/dist/lib/cli/verbose-ai-act.js +273 -0
  44. package/dist/lib/cli/verbose-screenshot.js +177 -0
  45. package/dist/lib/cli/verbose.js +465 -0
  46. package/dist/lib/env/parse-model-config.js +1 -1
  47. package/dist/lib/env/types.js +32 -2
  48. package/dist/lib/img/index.js +29 -5
  49. package/dist/lib/img/info.js +48 -1
  50. package/dist/lib/img/photon-loader.js +38 -0
  51. package/dist/lib/img/transform.js +135 -4
  52. package/dist/lib/recorder.js +307 -0
  53. package/dist/types/agent-tools/agent-behavior-init-args.d.ts +17 -0
  54. package/dist/types/agent-tools/base-tools.d.ts +158 -0
  55. package/dist/types/agent-tools/chrome-path.d.ts +2 -0
  56. package/dist/types/agent-tools/cli-report-session.d.ts +12 -0
  57. package/dist/types/agent-tools/error-formatter.d.ts +30 -0
  58. package/dist/types/agent-tools/index.d.ts +9 -0
  59. package/dist/types/agent-tools/init-arg-utils.d.ts +13 -0
  60. package/dist/types/agent-tools/observation-artifact.d.ts +10 -0
  61. package/dist/types/agent-tools/observation-record.d.ts +38 -0
  62. package/dist/types/agent-tools/tool-defaults.d.ts +63 -0
  63. package/dist/types/agent-tools/tool-generator.d.ts +13 -0
  64. package/dist/types/agent-tools/types.d.ts +213 -0
  65. package/dist/types/agent-tools/user-prompt.d.ts +13 -0
  66. package/dist/types/cli/interrupt.d.ts +49 -0
  67. package/dist/types/cli/record-command.d.ts +3 -0
  68. package/dist/types/cli/screenshot-file.d.ts +10 -0
  69. package/dist/types/cli/verbose-ai-act.d.ts +44 -0
  70. package/dist/types/cli/verbose-screenshot.d.ts +10 -0
  71. package/dist/types/cli/verbose.d.ts +40 -0
  72. package/dist/types/env/types.d.ts +16 -7
  73. package/dist/types/img/index.d.ts +2 -2
  74. package/dist/types/img/info.d.ts +2 -0
  75. package/dist/types/img/photon-loader.d.ts +2 -0
  76. package/dist/types/img/transform.d.ts +22 -2
  77. package/dist/types/mcp/types.d.ts +1 -0
  78. package/dist/types/recorder.d.ts +113 -0
  79. package/package.json +1 -1
  80. package/src/agent-tools/agent-behavior-init-args.ts +109 -0
  81. package/src/agent-tools/base-tools.ts +399 -0
  82. package/src/agent-tools/chrome-path.ts +74 -0
  83. package/src/agent-tools/cli-report-session.ts +130 -0
  84. package/src/agent-tools/error-formatter.ts +177 -0
  85. package/src/agent-tools/index.ts +9 -0
  86. package/src/agent-tools/init-arg-utils.ts +105 -0
  87. package/src/agent-tools/observation-artifact.ts +29 -0
  88. package/src/agent-tools/observation-record.ts +331 -0
  89. package/src/agent-tools/tool-defaults.ts +119 -0
  90. package/src/agent-tools/tool-generator.ts +866 -0
  91. package/src/agent-tools/types.ts +250 -0
  92. package/src/agent-tools/user-prompt.ts +102 -0
  93. package/src/cli/interrupt.ts +207 -0
  94. package/src/cli/record-command.ts +177 -0
  95. package/src/cli/screenshot-file.ts +61 -0
  96. package/src/cli/verbose-ai-act.ts +387 -0
  97. package/src/cli/verbose-screenshot.ts +269 -0
  98. package/src/cli/verbose.ts +753 -0
  99. package/src/env/types.ts +30 -2
  100. package/src/img/index.ts +12 -0
  101. package/src/img/info.ts +61 -0
  102. package/src/img/photon-loader.ts +5 -0
  103. package/src/img/transform.ts +262 -3
  104. package/src/mcp/types.ts +2 -0
  105. package/src/recorder.ts +625 -0
@@ -0,0 +1,113 @@
1
+ export type MidsceneRecorderEventType = 'click' | 'drag' | 'scroll' | 'input' | 'navigation' | 'setViewport' | 'keydown';
2
+ export type MidsceneRecorderSourceKind = 'studio-preview' | 'unsupported' | (string & {});
3
+ export type MidsceneRecorderPlatformId = 'web' | 'android' | 'ios' | 'computer' | 'harmony' | (string & {});
4
+ export interface MidsceneRecorderElementRect {
5
+ left?: number;
6
+ top?: number;
7
+ width?: number;
8
+ height?: number;
9
+ x?: number;
10
+ y?: number;
11
+ }
12
+ export interface MidsceneRecorderPageInfo {
13
+ width: number;
14
+ height: number;
15
+ }
16
+ /**
17
+ * A screenshot stored outside the recording event payload.
18
+ *
19
+ * Recorder events are persisted in the Studio renderer. Keeping full data
20
+ * URLs there makes long recordings retain every screenshot in the renderer
21
+ * heap, so screenshot bytes live in the Playground run directory instead.
22
+ */
23
+ export interface MidsceneRecorderScreenshotAssetRef {
24
+ id: string;
25
+ mimeType: string;
26
+ bytes: number;
27
+ }
28
+ export interface MidsceneRecorderEvent {
29
+ type: MidsceneRecorderEventType;
30
+ source?: MidsceneRecorderSourceKind;
31
+ actionType?: string;
32
+ rawPayload?: Record<string, unknown>;
33
+ url?: string;
34
+ title?: string;
35
+ value?: string;
36
+ elementRect?: MidsceneRecorderElementRect;
37
+ pageInfo: MidsceneRecorderPageInfo;
38
+ screenshotBefore?: string;
39
+ screenshotAfter?: string;
40
+ /** The single screenshot retained for AI description and Markdown export. */
41
+ screenshotAsset?: MidsceneRecorderScreenshotAssetRef;
42
+ semantic?: MidsceneRecorderSemantic;
43
+ elementDescription?: string;
44
+ descriptionLoading?: boolean;
45
+ screenshotWithBox?: string;
46
+ timestamp: number;
47
+ hashId: string;
48
+ mergedHashIds?: string[];
49
+ }
50
+ export type MidsceneRecorderSemanticSource = 'aiDescribe' | 'recorderAI' | 'heuristic';
51
+ export type MidsceneRecorderSemanticStatus = 'pending' | 'ready' | 'failed';
52
+ export type MidsceneRecorderSemanticConfidence = 'high' | 'medium' | 'low';
53
+ export interface MidsceneRecorderSemanticAiDescribe {
54
+ verifyPrompt: boolean;
55
+ verifyPassed?: boolean;
56
+ deepLocate?: boolean;
57
+ centerDistance?: number;
58
+ expectedCenter?: [number, number];
59
+ actualCenter?: [number, number];
60
+ annotatedScreenshotPath?: string;
61
+ }
62
+ export interface MidsceneRecorderSemantic {
63
+ source: MidsceneRecorderSemanticSource;
64
+ status: MidsceneRecorderSemanticStatus;
65
+ elementDescription?: string;
66
+ replayInstruction?: string;
67
+ actionSummary?: string;
68
+ confidence?: MidsceneRecorderSemanticConfidence;
69
+ error?: string;
70
+ aiDescribe?: MidsceneRecorderSemanticAiDescribe;
71
+ fallbackFrom?: MidsceneRecorderSemantic;
72
+ }
73
+ export interface MidsceneRecorderSemanticAction {
74
+ type: MidsceneRecorderEventType;
75
+ actionType?: string;
76
+ value?: string;
77
+ url?: string;
78
+ scrollDestinationDescription?: string;
79
+ }
80
+ export interface MidsceneRecorderTarget {
81
+ platformId: MidsceneRecorderPlatformId;
82
+ deviceId?: string;
83
+ label?: string;
84
+ values: Record<string, string | number | boolean>;
85
+ }
86
+ export interface MidsceneRecorderGeneratedCode {
87
+ markdown?: string;
88
+ yaml?: string;
89
+ playwright?: string;
90
+ updatedAt?: number;
91
+ }
92
+ export interface MidsceneRecorderMarkdownScreenshotAsset {
93
+ eventIndex: number;
94
+ eventHashId: string;
95
+ eventType: MidsceneRecorderEventType;
96
+ relativePath: string;
97
+ dataUrl: string;
98
+ base64Data: string;
99
+ mimeType: string;
100
+ }
101
+ export interface MidsceneRecorderMarkdownScreenshotOptions {
102
+ baseDir?: string;
103
+ maxScreenshots?: number;
104
+ }
105
+ export declare const DEFAULT_MIDSCENE_RECORDER_MARKDOWN_MAX_SCREENSHOTS = 20;
106
+ export declare function getMidsceneRecorderSemantic(event: Pick<MidsceneRecorderEvent, 'semantic'>): MidsceneRecorderSemantic | undefined;
107
+ export declare function buildMidsceneRecorderReplayInstruction(event: MidsceneRecorderSemanticAction, elementDescription: string): string;
108
+ export declare function buildMidsceneRecorderActionSummary(event: MidsceneRecorderSemanticAction, elementDescription: string): string;
109
+ export declare function getMidsceneRecorderEventDescription(event: MidsceneRecorderEvent): string;
110
+ export declare function getMidsceneRecorderScreenshotsForLLM(events: MidsceneRecorderEvent[], maxScreenshots?: number): string[];
111
+ export declare function sanitizeMidsceneRecorderFileName(value: string): string;
112
+ export declare function createMidsceneRecorderMarkdownScreenshotAssets(events: MidsceneRecorderEvent[], options?: MidsceneRecorderMarkdownScreenshotOptions): MidsceneRecorderMarkdownScreenshotAsset[];
113
+ export declare function stringifyMidsceneRecorderTargetBlock(target: MidsceneRecorderTarget): string;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@aiscene/shared",
3
- "version": "8.0.3",
3
+ "version": "8.0.5",
4
4
  "repository": "https://github.com/web-infra-dev/midscene",
5
5
  "homepage": "https://midscenejs.com/",
6
6
  "types": "./dist/types/index.d.ts",
@@ -0,0 +1,109 @@
1
+ import { z } from 'zod';
2
+
3
+ export interface AgentBehaviorInitArgs {
4
+ aiActContext?: string;
5
+ aiActionContext?: string;
6
+ replanningCycleLimit?: number;
7
+ waitAfterAction?: number;
8
+ screenshotShrinkFactor?: number;
9
+ }
10
+
11
+ type ExposedAgentBehaviorInitArgKey = Exclude<
12
+ keyof AgentBehaviorInitArgs,
13
+ 'aiActionContext'
14
+ >;
15
+
16
+ export const agentBehaviorInitArgShape = {
17
+ aiActContext: z
18
+ .string()
19
+ .optional()
20
+ .describe(
21
+ 'Background knowledge passed to aiAct. Default: no extra context.',
22
+ ),
23
+ replanningCycleLimit: z
24
+ .number()
25
+ .int()
26
+ .nonnegative()
27
+ .optional()
28
+ .describe(
29
+ 'Maximum number of replanning cycles for aiAct. Default: model adapter default.',
30
+ ),
31
+ waitAfterAction: z
32
+ .number()
33
+ .nonnegative()
34
+ .optional()
35
+ .describe(
36
+ 'Wait time in milliseconds after each action execution. Default: 300ms.',
37
+ ),
38
+ screenshotShrinkFactor: z
39
+ .number()
40
+ .min(1)
41
+ .optional()
42
+ .describe(
43
+ 'Screenshot shrink factor before sending images to AI. Default: 1; high values may reduce recognition quality, especially on mobile.',
44
+ ),
45
+ } satisfies Record<ExposedAgentBehaviorInitArgKey, z.ZodTypeAny>;
46
+
47
+ export function extractAgentBehaviorInitArgs(
48
+ extracted: Partial<AgentBehaviorInitArgs> | undefined,
49
+ ): AgentBehaviorInitArgs | undefined {
50
+ if (!extracted) {
51
+ return undefined;
52
+ }
53
+
54
+ const agentOptions: AgentBehaviorInitArgs = {
55
+ ...(typeof extracted.aiActContext === 'string'
56
+ ? { aiActContext: extracted.aiActContext }
57
+ : {}),
58
+ ...(typeof extracted.aiActionContext === 'string'
59
+ ? { aiActionContext: extracted.aiActionContext }
60
+ : {}),
61
+ ...(typeof extracted.replanningCycleLimit === 'number'
62
+ ? { replanningCycleLimit: extracted.replanningCycleLimit }
63
+ : {}),
64
+ ...(typeof extracted.waitAfterAction === 'number'
65
+ ? { waitAfterAction: extracted.waitAfterAction }
66
+ : {}),
67
+ ...(typeof extracted.screenshotShrinkFactor === 'number'
68
+ ? { screenshotShrinkFactor: extracted.screenshotShrinkFactor }
69
+ : {}),
70
+ };
71
+
72
+ return Object.keys(agentOptions).length > 0 ? agentOptions : undefined;
73
+ }
74
+
75
+ function stableJsonValue(value: unknown): unknown {
76
+ if (Array.isArray(value)) {
77
+ return value.map(stableJsonValue);
78
+ }
79
+
80
+ if (value && typeof value === 'object') {
81
+ return Object.fromEntries(
82
+ Object.entries(value as Record<string, unknown>)
83
+ .sort(([left], [right]) => left.localeCompare(right))
84
+ .map(([key, nestedValue]) => [key, stableJsonValue(nestedValue)]),
85
+ );
86
+ }
87
+
88
+ return value;
89
+ }
90
+
91
+ export function getAgentInitArgsSignature(
92
+ initArgs: object | undefined,
93
+ ): string | undefined {
94
+ if (!initArgs || Object.keys(initArgs).length === 0) {
95
+ return undefined;
96
+ }
97
+
98
+ return JSON.stringify(stableJsonValue(initArgs));
99
+ }
100
+
101
+ export function shouldRebuildAgentForInitArgs(
102
+ currentSignature: string | undefined,
103
+ nextSignature: string | undefined,
104
+ ): boolean {
105
+ return (
106
+ currentSignature !== nextSignature &&
107
+ (currentSignature !== undefined || nextSignature !== undefined)
108
+ );
109
+ }
@@ -0,0 +1,399 @@
1
+ import { parseBase64 } from '@aiscene/shared/img';
2
+ import { getDebug } from '@aiscene/shared/logger';
3
+ import type { z } from 'zod';
4
+ import { createRecordCliCommand } from '../cli/record-command';
5
+ import { camelToKebab, getKeyAliases } from '../key-alias-utils';
6
+ import {
7
+ type CliReportSession,
8
+ generateCliReportSession,
9
+ readCliReportSession,
10
+ writeCliReportSession,
11
+ } from './cli-report-session';
12
+ import {
13
+ createNamespacedInitArgSchema,
14
+ extractNamespacedArgs,
15
+ sanitizeNamespacedArgs,
16
+ } from './init-arg-utils';
17
+ import { type ToolDefaults, mergeToolDefaults } from './tool-defaults';
18
+ import {
19
+ generateCommonTools,
20
+ generateToolsFromActionSpace,
21
+ } from './tool-generator';
22
+ import type {
23
+ ActionSpaceItem,
24
+ BaseAgent,
25
+ BaseDevice,
26
+ IMidsceneTools,
27
+ ToolCliMetadata,
28
+ ToolDefinition,
29
+ ToolSchema,
30
+ } from './types';
31
+
32
+ const debug = getDebug('agent-tools:base-tools');
33
+
34
+ /**
35
+ * Declarative description of a platform's agent init args.
36
+ * Collapses the `extractAgentInitParam` / `sanitizeToolArgs` /
37
+ * `getAgentInitArgSchema` trio into a single data declaration.
38
+ */
39
+ export interface InitArgSpec<TInitParam> {
40
+ /** Arg namespace, e.g. `android`, `ios`. */
41
+ namespace: string;
42
+ /** Zod shape describing the init args. Field names drive the tool schema. */
43
+ shape: Record<string, z.ZodTypeAny>;
44
+ /**
45
+ * Optional CLI presentation hints. These affect `--help` output for
46
+ * single-platform CLIs but do not alter YAML protocol keys.
47
+ */
48
+ cli?: {
49
+ /** Prefer bare `--device-id`-style options in platform CLI help output. */
50
+ preferBareKeys?: boolean;
51
+ /** Override the displayed option name for specific init arg fields. */
52
+ preferredNames?: Record<string, string>;
53
+ };
54
+ /**
55
+ * Adapt extracted namespaced args into the concrete `TInitParam` passed to
56
+ * `ensureAgent`. Defaults to returning the raw extracted record.
57
+ */
58
+ adapt?: (
59
+ extracted: Record<string, unknown> | undefined,
60
+ ) => TInitParam | undefined;
61
+ }
62
+
63
+ /**
64
+ * Base class for platform-specific Midscene tools.
65
+ * @typeParam TAgent - Platform-specific agent type.
66
+ * @typeParam TInitParam - Platform-specific init parameter consumed by
67
+ * `ensureAgent`. Defaults to `undefined` for platforms that take no args.
68
+ */
69
+ export abstract class BaseMidsceneTools<
70
+ TAgent extends BaseAgent = BaseAgent,
71
+ TInitParam = unknown,
72
+ > implements IMidsceneTools
73
+ {
74
+ protected agent?: TAgent;
75
+ protected toolDefinitions: ToolDefinition[] = [];
76
+
77
+ /**
78
+ * Default options injected into every generated tool call (e.g. forced deep
79
+ * locate / deep think). Set from startup/CLI behavior flags before
80
+ * `initTools()` so they are baked into the generated tool handlers.
81
+ * See https://github.com/web-infra-dev/midscene/issues/2446.
82
+ */
83
+ protected toolDefaults: ToolDefaults = {};
84
+
85
+ /**
86
+ * Declarative init-arg spec. Subclasses that accept CLI init args should
87
+ * set this once and get `extractAgentInitParam` / `sanitizeToolArgs` /
88
+ * `getAgentInitArgSchema` auto-implemented.
89
+ *
90
+ * Declared with `declare` so that TS doesn't emit an `Object.defineProperty`
91
+ * for this field on the base constructor, which would otherwise overwrite
92
+ * a subclass field initializer under `useDefineForClassFields`.
93
+ */
94
+ protected declare readonly initArgSpec?: InitArgSpec<TInitParam>;
95
+
96
+ /**
97
+ * Ensure agent is initialized and ready for use.
98
+ * Must be implemented by subclasses to create platform-specific agent.
99
+ * @param initParam Optional initialization parameter (platform-specific, e.g., URL, device ID)
100
+ * @returns Promise resolving to initialized agent instance
101
+ * @throws Error if agent initialization fails
102
+ */
103
+ protected abstract ensureAgent(initParam?: TInitParam): Promise<TAgent>;
104
+
105
+ private getInitArgKeys(): readonly string[] {
106
+ return this.initArgSpec ? Object.keys(this.initArgSpec.shape) : [];
107
+ }
108
+
109
+ /**
110
+ * Extract a platform-specific agent init parameter from CLI tool args.
111
+ */
112
+ protected extractAgentInitParam(
113
+ args: Record<string, unknown>,
114
+ ): TInitParam | undefined {
115
+ if (!this.initArgSpec) {
116
+ return undefined;
117
+ }
118
+ const extracted = extractNamespacedArgs(
119
+ args,
120
+ this.initArgSpec.namespace,
121
+ this.getInitArgKeys(),
122
+ );
123
+ if (this.initArgSpec.adapt) {
124
+ return this.initArgSpec.adapt(extracted);
125
+ }
126
+ return extracted as TInitParam | undefined;
127
+ }
128
+
129
+ /**
130
+ * Remove platform-specific init args before dispatching a tool payload to the action itself.
131
+ */
132
+ protected sanitizeToolArgs(
133
+ args: Record<string, unknown>,
134
+ ): Record<string, unknown> {
135
+ if (!this.initArgSpec) {
136
+ return args;
137
+ }
138
+ return sanitizeNamespacedArgs(
139
+ args,
140
+ this.initArgSpec.namespace,
141
+ this.getInitArgKeys(),
142
+ );
143
+ }
144
+
145
+ /**
146
+ * Expose platform-specific init args on action/common tool schemas.
147
+ */
148
+ protected getAgentInitArgSchema(): ToolSchema {
149
+ if (!this.initArgSpec) {
150
+ return {};
151
+ }
152
+ return createNamespacedInitArgSchema(
153
+ this.initArgSpec.namespace,
154
+ this.initArgSpec.shape,
155
+ );
156
+ }
157
+
158
+ /**
159
+ * Expose CLI-only metadata for platform init args so single-platform help can
160
+ * show ergonomic bare flags while the underlying schema stays namespaced.
161
+ * When `preferBareKeys` is enabled, single-platform CLIs only accept the
162
+ * bare spellings; namespaced dotted spellings remain available through the
163
+ * YAML schema instead of the platform CLI surface.
164
+ */
165
+ protected getAgentInitArgCliMetadata(): ToolCliMetadata | undefined {
166
+ if (!this.initArgSpec?.cli) {
167
+ return undefined;
168
+ }
169
+
170
+ const options = Object.fromEntries(
171
+ this.getInitArgKeys().map((key) => {
172
+ const canonicalKey = `${this.initArgSpec!.namespace}.${key}`;
173
+ const preferredName =
174
+ this.initArgSpec!.cli?.preferredNames?.[key] ??
175
+ (this.initArgSpec!.cli?.preferBareKeys
176
+ ? camelToKebab(key)
177
+ : canonicalKey);
178
+
179
+ const acceptedNames = new Set<string>([
180
+ preferredName,
181
+ ...(this.initArgSpec!.cli?.preferBareKeys
182
+ ? getKeyAliases(key)
183
+ : getKeyAliases(canonicalKey)),
184
+ ]);
185
+ acceptedNames.delete(preferredName);
186
+
187
+ return [
188
+ canonicalKey,
189
+ {
190
+ preferredName,
191
+ aliases: [...acceptedNames],
192
+ },
193
+ ];
194
+ }),
195
+ );
196
+
197
+ return { options };
198
+ }
199
+
200
+ /**
201
+ * Optional: prepare platform-specific tools (e.g., device connection)
202
+ */
203
+ protected preparePlatformTools(): ToolDefinition[] {
204
+ return [];
205
+ }
206
+
207
+ protected getCliReportSessionName(): string | undefined {
208
+ return undefined;
209
+ }
210
+
211
+ protected createNewCliReportSession(
212
+ targetIdentity?: string,
213
+ ): CliReportSession | undefined {
214
+ const sessionName = this.getCliReportSessionName();
215
+ if (!sessionName) {
216
+ return undefined;
217
+ }
218
+ return generateCliReportSession(sessionName, targetIdentity);
219
+ }
220
+
221
+ protected commitCliReportSession(session?: CliReportSession): void {
222
+ if (session) {
223
+ writeCliReportSession(session);
224
+ }
225
+ }
226
+
227
+ protected readCliReportFileName(): string | undefined {
228
+ const sessionName = this.getCliReportSessionName();
229
+ if (!sessionName) {
230
+ return undefined;
231
+ }
232
+ return readCliReportSession(sessionName)?.reportFileName;
233
+ }
234
+
235
+ protected readCliReportAgentOptions():
236
+ | {
237
+ reportFileName: string;
238
+ reportAttributes: Record<string, string>;
239
+ }
240
+ | undefined {
241
+ const reportFileName = this.readCliReportFileName();
242
+ if (!reportFileName) {
243
+ return undefined;
244
+ }
245
+ return {
246
+ reportFileName,
247
+ reportAttributes: {
248
+ 'data-group-id': reportFileName,
249
+ },
250
+ };
251
+ }
252
+
253
+ /**
254
+ * Must be implemented by subclasses to create a temporary device instance
255
+ * This allows getting real actionSpace without connecting to device
256
+ */
257
+ protected abstract createTemporaryDevice(): BaseDevice;
258
+
259
+ /**
260
+ * Initialize all tools by querying actionSpace
261
+ * Uses two-layer fallback strategy:
262
+ * 1. Try to get actionSpace from connected agent (if available)
263
+ * 2. Create temporary device instance to read actionSpace (always succeeds)
264
+ */
265
+ public async initTools(): Promise<void> {
266
+ this.toolDefinitions = [];
267
+
268
+ // 1. Add platform-specific tools first (device connection, etc.)
269
+ // These don't require an agent and should always be available
270
+ const platformTools = this.preparePlatformTools();
271
+ this.toolDefinitions.push(...platformTools);
272
+
273
+ // 2. Get action space: use pre-set agent if available, otherwise temp device.
274
+ // For CLI usage, agent is deferred to the first real command.
275
+ let actionSpace: ActionSpaceItem[];
276
+ if (this.agent) {
277
+ actionSpace = await this.agent.getActionSpace();
278
+ debug(
279
+ 'Action space from agent:',
280
+ actionSpace.map((a) => a.name).join(', '),
281
+ );
282
+ } else {
283
+ const tempDevice = this.createTemporaryDevice();
284
+ actionSpace = tempDevice.actionSpace();
285
+ await tempDevice.destroy?.();
286
+ debug(
287
+ 'Action space from temporary device:',
288
+ actionSpace.map((a) => a.name).join(', '),
289
+ );
290
+ }
291
+
292
+ // 3. Generate tools from action space (core innovation)
293
+ const actionTools = generateToolsFromActionSpace(
294
+ actionSpace,
295
+ (args = {}) => this.ensureAgent(this.extractAgentInitParam(args)),
296
+ (args = {}) => this.sanitizeToolArgs(args),
297
+ this.getAgentInitArgSchema(),
298
+ this.getAgentInitArgCliMetadata(),
299
+ this.toolDefaults,
300
+ );
301
+
302
+ // 4. Add common tools (screenshot, act, assert)
303
+ const commonTools = generateCommonTools(
304
+ (args = {}) => this.ensureAgent(this.extractAgentInitParam(args)),
305
+ this.getAgentInitArgSchema(),
306
+ this.getAgentInitArgCliMetadata(),
307
+ this.toolDefaults,
308
+ );
309
+ this.toolDefinitions.push(...actionTools, ...commonTools);
310
+
311
+ debug('Total tools prepared:', this.toolDefinitions.length);
312
+ }
313
+
314
+ /**
315
+ * Cleanup method - destroy agent and release resources
316
+ */
317
+ public async destroy(): Promise<void> {
318
+ await this.agent?.destroy?.();
319
+ }
320
+
321
+ /**
322
+ * Get tool definitions
323
+ */
324
+ public getToolDefinitions(): ToolDefinition[] {
325
+ return this.toolDefinitions;
326
+ }
327
+
328
+ /** Commands that exist only on the foreground CLI surface. */
329
+ public getCliToolDefinitions(): ToolDefinition[] {
330
+ return [
331
+ createRecordCliCommand(
332
+ (args = {}) => this.ensureAgent(this.extractAgentInitParam(args)),
333
+ this.getAgentInitArgSchema(),
334
+ this.getAgentInitArgCliMetadata(),
335
+ ),
336
+ ];
337
+ }
338
+
339
+ /**
340
+ * Set agent for the tools manager
341
+ */
342
+ public setAgent(agent: TAgent): void {
343
+ this.agent = agent;
344
+ }
345
+
346
+ /**
347
+ * Set the default options injected into generated tool calls. Must be called
348
+ * before `initTools()` because the values are captured into the generated
349
+ * tool handlers. Merges with any previously set defaults.
350
+ */
351
+ public setToolDefaults(toolDefaults: ToolDefaults): void {
352
+ this.toolDefaults = mergeToolDefaults(this.toolDefaults, toolDefaults);
353
+ }
354
+
355
+ /**
356
+ * Helper: Convert base64 screenshot to image content array
357
+ */
358
+ protected buildScreenshotContent(screenshot: string) {
359
+ const { mimeType, body } = parseBase64(screenshot);
360
+ return [
361
+ {
362
+ type: 'image' as const,
363
+ data: body,
364
+ mimeType,
365
+ },
366
+ ];
367
+ }
368
+
369
+ /**
370
+ * Helper: Build a simple text result for tool responses
371
+ */
372
+ protected buildTextResult(text: string) {
373
+ return {
374
+ content: [{ type: 'text' as const, text }],
375
+ };
376
+ }
377
+
378
+ /**
379
+ * Create a disconnect handler for releasing platform resources
380
+ * @param platformName Human-readable platform name for the response message
381
+ * @returns Handler function that destroys the agent and returns appropriate response
382
+ */
383
+ protected createDisconnectHandler(platformName: string) {
384
+ return async () => {
385
+ if (!this.agent) {
386
+ return this.buildTextResult('No active connection to disconnect');
387
+ }
388
+
389
+ try {
390
+ await this.agent.destroy?.();
391
+ } catch (error) {
392
+ debug('Failed to destroy agent during disconnect:', error);
393
+ }
394
+ this.agent = undefined;
395
+
396
+ return this.buildTextResult(`Disconnected from ${platformName}`);
397
+ };
398
+ }
399
+ }
@@ -0,0 +1,74 @@
1
+ import { existsSync } from 'node:fs';
2
+ import {
3
+ MIDSCENE_CHROME_PATH,
4
+ MIDSCENE_MCP_CHROME_PATH,
5
+ globalConfigManager,
6
+ } from '../env';
7
+ import { getDebug } from '../logger';
8
+
9
+ const warnChromePath = getDebug('agent-tools:chrome-path', { console: true });
10
+ let hasWarnedLegacyChromePath = false;
11
+ let cachedSystemChromePath: string | undefined;
12
+
13
+ export function getSystemChromePath(): string | undefined {
14
+ if (cachedSystemChromePath !== undefined) {
15
+ return cachedSystemChromePath;
16
+ }
17
+
18
+ const platform = process.platform;
19
+
20
+ const chromePaths: Record<string, string[]> = {
21
+ darwin: [
22
+ '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
23
+ '/Applications/Chromium.app/Contents/MacOS/Chromium',
24
+ ],
25
+ win32: [
26
+ 'C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe',
27
+ 'C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe',
28
+ `C:\\Users\\${process.env.USERNAME ?? process.env.USER}\\AppData\\Local\\Google\\Chrome\\Application\\chrome.exe`,
29
+ ],
30
+ linux: [
31
+ // Prefer actual binaries over wrapper scripts.
32
+ // Wrappers in /usr/bin may strip --user-data-dir, causing
33
+ // "DevTools remote debugging requires a non-default data directory" errors.
34
+ '/opt/google/chrome/chrome',
35
+ '/opt/google/chrome/google-chrome',
36
+ '/usr/bin/google-chrome-stable',
37
+ '/usr/bin/google-chrome',
38
+ '/usr/bin/chromium-browser',
39
+ '/usr/bin/chromium',
40
+ '/snap/bin/chromium',
41
+ ],
42
+ };
43
+
44
+ const paths = chromePaths[platform] ?? [];
45
+ const foundPath = paths.find((p) => existsSync(p));
46
+ if (foundPath) {
47
+ cachedSystemChromePath = foundPath;
48
+ }
49
+ return foundPath;
50
+ }
51
+
52
+ export function resolveChromePath(): string {
53
+ const primaryEnvPath =
54
+ globalConfigManager.getEnvConfigValue(MIDSCENE_CHROME_PATH);
55
+ const legacyEnvPath = globalConfigManager.getEnvConfigValue(
56
+ MIDSCENE_MCP_CHROME_PATH,
57
+ );
58
+ const envPath = primaryEnvPath || legacyEnvPath;
59
+ if (!primaryEnvPath && legacyEnvPath && !hasWarnedLegacyChromePath) {
60
+ warnChromePath(
61
+ 'MIDSCENE_MCP_CHROME_PATH is deprecated. Use MIDSCENE_CHROME_PATH instead.',
62
+ );
63
+ hasWarnedLegacyChromePath = true;
64
+ }
65
+ if (envPath && envPath !== 'auto' && existsSync(envPath)) {
66
+ return envPath;
67
+ }
68
+ const systemPath = getSystemChromePath();
69
+ if (systemPath) return systemPath;
70
+
71
+ throw new Error(
72
+ 'Chrome not found. Install Google Chrome or set MIDSCENE_CHROME_PATH environment variable.',
73
+ );
74
+ }