@aiscene/shared 8.0.3 → 8.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. package/dist/es/agent-tools/agent-behavior-init-args.mjs +44 -0
  2. package/dist/es/agent-tools/base-tools.mjs +163 -0
  3. package/dist/es/agent-tools/chrome-path.mjs +50 -0
  4. package/dist/es/agent-tools/cli-report-session.mjs +78 -0
  5. package/dist/es/agent-tools/error-formatter.mjs +106 -0
  6. package/dist/es/agent-tools/index.mjs +9 -0
  7. package/dist/es/agent-tools/init-arg-utils.mjs +38 -0
  8. package/dist/es/agent-tools/observation-artifact.mjs +5 -0
  9. package/dist/es/agent-tools/observation-record.mjs +248 -0
  10. package/dist/es/agent-tools/tool-defaults.mjs +54 -0
  11. package/dist/es/agent-tools/tool-generator.mjs +529 -0
  12. package/dist/es/agent-tools/types.mjs +3 -0
  13. package/dist/es/agent-tools/user-prompt.mjs +66 -0
  14. package/dist/es/cli/interrupt.mjs +116 -0
  15. package/dist/es/cli/record-command.mjs +130 -0
  16. package/dist/es/cli/screenshot-file.mjs +24 -0
  17. package/dist/es/cli/verbose-ai-act.mjs +230 -0
  18. package/dist/es/cli/verbose-screenshot.mjs +134 -0
  19. package/dist/es/cli/verbose.mjs +404 -0
  20. package/dist/es/env/parse-model-config.mjs +1 -1
  21. package/dist/es/env/types.mjs +3 -1
  22. package/dist/es/img/info.mjs +45 -1
  23. package/dist/es/img/photon-loader.mjs +4 -0
  24. package/dist/es/recorder.mjs +249 -0
  25. package/dist/lib/agent-tools/agent-behavior-init-args.js +87 -0
  26. package/dist/lib/agent-tools/base-tools.js +197 -0
  27. package/dist/lib/agent-tools/chrome-path.js +87 -0
  28. package/dist/lib/agent-tools/cli-report-session.js +121 -0
  29. package/dist/lib/agent-tools/error-formatter.js +149 -0
  30. package/dist/lib/agent-tools/index.js +114 -0
  31. package/dist/lib/agent-tools/init-arg-utils.js +78 -0
  32. package/dist/lib/agent-tools/observation-artifact.js +42 -0
  33. package/dist/lib/agent-tools/observation-record.js +297 -0
  34. package/dist/lib/agent-tools/tool-defaults.js +97 -0
  35. package/dist/lib/agent-tools/tool-generator.js +569 -0
  36. package/dist/lib/agent-tools/types.js +40 -0
  37. package/dist/lib/agent-tools/user-prompt.js +103 -0
  38. package/dist/lib/cli/interrupt.js +156 -0
  39. package/dist/lib/cli/record-command.js +164 -0
  40. package/dist/lib/cli/screenshot-file.js +58 -0
  41. package/dist/lib/cli/verbose-ai-act.js +273 -0
  42. package/dist/lib/cli/verbose-screenshot.js +177 -0
  43. package/dist/lib/cli/verbose.js +465 -0
  44. package/dist/lib/env/parse-model-config.js +1 -1
  45. package/dist/lib/env/types.js +5 -0
  46. package/dist/lib/img/info.js +48 -1
  47. package/dist/lib/img/photon-loader.js +38 -0
  48. package/dist/lib/recorder.js +307 -0
  49. package/dist/types/agent-tools/agent-behavior-init-args.d.ts +17 -0
  50. package/dist/types/agent-tools/base-tools.d.ts +158 -0
  51. package/dist/types/agent-tools/chrome-path.d.ts +2 -0
  52. package/dist/types/agent-tools/cli-report-session.d.ts +12 -0
  53. package/dist/types/agent-tools/error-formatter.d.ts +30 -0
  54. package/dist/types/agent-tools/index.d.ts +9 -0
  55. package/dist/types/agent-tools/init-arg-utils.d.ts +13 -0
  56. package/dist/types/agent-tools/observation-artifact.d.ts +10 -0
  57. package/dist/types/agent-tools/observation-record.d.ts +38 -0
  58. package/dist/types/agent-tools/tool-defaults.d.ts +63 -0
  59. package/dist/types/agent-tools/tool-generator.d.ts +13 -0
  60. package/dist/types/agent-tools/types.d.ts +213 -0
  61. package/dist/types/agent-tools/user-prompt.d.ts +13 -0
  62. package/dist/types/cli/interrupt.d.ts +49 -0
  63. package/dist/types/cli/record-command.d.ts +3 -0
  64. package/dist/types/cli/screenshot-file.d.ts +10 -0
  65. package/dist/types/cli/verbose-ai-act.d.ts +44 -0
  66. package/dist/types/cli/verbose-screenshot.d.ts +10 -0
  67. package/dist/types/cli/verbose.d.ts +40 -0
  68. package/dist/types/env/types.d.ts +4 -3
  69. package/dist/types/img/info.d.ts +2 -0
  70. package/dist/types/img/photon-loader.d.ts +2 -0
  71. package/dist/types/mcp/types.d.ts +1 -0
  72. package/dist/types/recorder.d.ts +113 -0
  73. package/package.json +1 -1
  74. package/src/agent-tools/agent-behavior-init-args.ts +109 -0
  75. package/src/agent-tools/base-tools.ts +399 -0
  76. package/src/agent-tools/chrome-path.ts +74 -0
  77. package/src/agent-tools/cli-report-session.ts +130 -0
  78. package/src/agent-tools/error-formatter.ts +177 -0
  79. package/src/agent-tools/index.ts +9 -0
  80. package/src/agent-tools/init-arg-utils.ts +105 -0
  81. package/src/agent-tools/observation-artifact.ts +29 -0
  82. package/src/agent-tools/observation-record.ts +331 -0
  83. package/src/agent-tools/tool-defaults.ts +119 -0
  84. package/src/agent-tools/tool-generator.ts +866 -0
  85. package/src/agent-tools/types.ts +250 -0
  86. package/src/agent-tools/user-prompt.ts +102 -0
  87. package/src/cli/interrupt.ts +207 -0
  88. package/src/cli/record-command.ts +177 -0
  89. package/src/cli/screenshot-file.ts +61 -0
  90. package/src/cli/verbose-ai-act.ts +387 -0
  91. package/src/cli/verbose-screenshot.ts +269 -0
  92. package/src/cli/verbose.ts +753 -0
  93. package/src/env/types.ts +2 -0
  94. package/src/img/index.ts +12 -0
  95. package/src/img/info.ts +61 -0
  96. package/src/img/photon-loader.ts +5 -0
  97. package/src/img/transform.ts +261 -2
  98. package/src/mcp/types.ts +2 -0
  99. package/src/recorder.ts +625 -0
@@ -0,0 +1,569 @@
1
+ "use strict";
2
+ var __webpack_require__ = {};
3
+ (()=>{
4
+ __webpack_require__.d = (exports1, definition)=>{
5
+ for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
6
+ enumerable: true,
7
+ get: definition[key]
8
+ });
9
+ };
10
+ })();
11
+ (()=>{
12
+ __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
13
+ })();
14
+ (()=>{
15
+ __webpack_require__.r = (exports1)=>{
16
+ if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
17
+ value: 'Module'
18
+ });
19
+ Object.defineProperty(exports1, '__esModule', {
20
+ value: true
21
+ });
22
+ };
23
+ })();
24
+ var __webpack_exports__ = {};
25
+ __webpack_require__.r(__webpack_exports__);
26
+ __webpack_require__.d(__webpack_exports__, {
27
+ generateCommonTools: ()=>generateCommonTools,
28
+ generateToolsFromActionSpace: ()=>generateToolsFromActionSpace,
29
+ composeUserPrompt: ()=>external_user_prompt_js_namespaceObject.composeUserPrompt
30
+ });
31
+ const img_namespaceObject = require("@aiscene/shared/img");
32
+ const external_zod_namespaceObject = require("zod");
33
+ const verbose_js_namespaceObject = require("../cli/verbose.js");
34
+ const external_zod_schema_utils_js_namespaceObject = require("../zod-schema-utils.js");
35
+ const external_error_formatter_js_namespaceObject = require("./error-formatter.js");
36
+ const external_observation_artifact_js_namespaceObject = require("./observation-artifact.js");
37
+ const external_observation_record_js_namespaceObject = require("./observation-record.js");
38
+ const external_user_prompt_js_namespaceObject = require("./user-prompt.js");
39
+ function describeActionForTool(action) {
40
+ const actionDesc = action.description || `Execute ${action.name} action`;
41
+ if (!action.paramSchema) return `${action.name} action, ${actionDesc}`;
42
+ const shape = getZodObjectShape(action.paramSchema);
43
+ if (!shape) {
44
+ const typeName = (0, external_zod_schema_utils_js_namespaceObject.getZodTypeName)(action.paramSchema);
45
+ const description = (0, external_zod_schema_utils_js_namespaceObject.getZodDescription)(action.paramSchema);
46
+ const paramDesc = description ? `${typeName} - ${description}` : typeName;
47
+ return `${action.name} action, ${actionDesc}. Parameter: ${paramDesc}`;
48
+ }
49
+ const paramDescriptions = [];
50
+ for (const [key, field] of Object.entries(shape))if (field && 'object' == typeof field) {
51
+ const isFieldOptional = 'function' == typeof field.isOptional && field.isOptional();
52
+ const typeName = (0, external_zod_schema_utils_js_namespaceObject.getZodTypeName)(field);
53
+ const description = (0, external_zod_schema_utils_js_namespaceObject.getZodDescription)(field);
54
+ let paramStr = `${key}${isFieldOptional ? '?' : ''} (${typeName})`;
55
+ if (description) paramStr += ` - ${description}`;
56
+ paramDescriptions.push(paramStr);
57
+ }
58
+ if (0 === paramDescriptions.length) return `${action.name} action, ${actionDesc}`;
59
+ return `${action.name} action, ${actionDesc}. Parameters: ${paramDescriptions.join('; ')}`;
60
+ }
61
+ function isZodOptional(value) {
62
+ return '_def' in value && value._def?.typeName === 'ZodOptional';
63
+ }
64
+ function unwrapOptional(value) {
65
+ if (isZodOptional(value)) return {
66
+ innerValue: value._def.innerType,
67
+ isOptional: true
68
+ };
69
+ return {
70
+ innerValue: value,
71
+ isOptional: false
72
+ };
73
+ }
74
+ function getZodObjectShape(value) {
75
+ if (!value) return;
76
+ const actualValue = (0, external_zod_schema_utils_js_namespaceObject.unwrapZodField)(value);
77
+ if (actualValue._def?.typeName !== 'ZodObject') return;
78
+ if ('function' == typeof actualValue._def.shape) return actualValue._def.shape();
79
+ return actualValue.shape;
80
+ }
81
+ function isRecord(value) {
82
+ return 'object' == typeof value && null !== value && !Array.isArray(value);
83
+ }
84
+ function makePromptOptional(shape, wrapInOptional, description) {
85
+ const newShape = {
86
+ ...shape
87
+ };
88
+ newShape.prompt = shape.prompt.optional();
89
+ let newSchema = external_zod_namespaceObject.z.object(newShape).passthrough();
90
+ if (wrapInOptional) newSchema = newSchema.optional();
91
+ if (description) newSchema = newSchema.describe(description);
92
+ return newSchema;
93
+ }
94
+ function transformSchemaField(key, value) {
95
+ const { innerValue, isOptional } = unwrapOptional(value);
96
+ const shape = getZodObjectShape(innerValue);
97
+ if (shape && (0, external_zod_schema_utils_js_namespaceObject.isMidsceneLocatorField)(innerValue)) return [
98
+ key,
99
+ makePromptOptional(shape, isOptional, (0, external_zod_schema_utils_js_namespaceObject.getZodDescription)(value))
100
+ ];
101
+ return [
102
+ key,
103
+ value
104
+ ];
105
+ }
106
+ function extractActionSchema(paramSchema, actionName) {
107
+ if (!paramSchema) return {};
108
+ const shape = getZodObjectShape(paramSchema);
109
+ if (!shape) {
110
+ const typeName = paramSchema?._def?.typeName ?? 'unknown';
111
+ throw new Error(`Action "${actionName}" declared a non-object paramSchema (${typeName}). CLI tool schemas must be a ZodObject (e.g. z.object({ uri: z.string() })) or undefined. Wrap primitive fields in an object schema.`);
112
+ }
113
+ return Object.fromEntries(Object.entries(shape).map(([key, value])=>transformSchemaField(key, value)));
114
+ }
115
+ function getPromptText(prompt) {
116
+ if ('string' == typeof prompt) return prompt;
117
+ if (isRecord(prompt) && 'string' == typeof prompt.prompt) return prompt.prompt;
118
+ }
119
+ function moveLocateExtrasIntoPrompt(value, locateFieldKeys) {
120
+ const promptText = getPromptText(value.prompt);
121
+ if (!promptText) return value;
122
+ const normalizedPrompt = isRecord(value.prompt) ? {
123
+ ...value.prompt
124
+ } : {
125
+ prompt: promptText
126
+ };
127
+ const normalizedLocate = {};
128
+ let movedExtraField = false;
129
+ for (const [key, fieldValue] of Object.entries(value))if ('prompt' !== key) {
130
+ if (locateFieldKeys.has(key)) {
131
+ normalizedLocate[key] = fieldValue;
132
+ continue;
133
+ }
134
+ movedExtraField = true;
135
+ if (!(key in normalizedPrompt)) normalizedPrompt[key] = fieldValue;
136
+ }
137
+ if (!movedExtraField) return value;
138
+ return {
139
+ ...normalizedLocate,
140
+ prompt: normalizedPrompt
141
+ };
142
+ }
143
+ function normalizeLocateLikeArg(value, fieldSchema) {
144
+ if ('string' == typeof value) return {
145
+ prompt: value
146
+ };
147
+ if (!isRecord(value)) return value;
148
+ const shape = getZodObjectShape(fieldSchema);
149
+ if (!shape) return value;
150
+ return moveLocateExtrasIntoPrompt(value, new Set(Object.keys(shape)));
151
+ }
152
+ function normalizeActionArgs(args, paramSchema) {
153
+ if (!paramSchema) return args;
154
+ const shape = getZodObjectShape(paramSchema);
155
+ if (!shape) return args;
156
+ return Object.fromEntries(Object.entries(args).map(([key, value])=>{
157
+ const fieldSchema = shape[key];
158
+ if (!fieldSchema) return [
159
+ key,
160
+ value
161
+ ];
162
+ if ((0, external_zod_schema_utils_js_namespaceObject.isMidsceneLocatorField)(fieldSchema)) return [
163
+ key,
164
+ normalizeLocateLikeArg(value, fieldSchema)
165
+ ];
166
+ return [
167
+ key,
168
+ value
169
+ ];
170
+ }));
171
+ }
172
+ function mergeLocateDefaults(locate, defaults) {
173
+ let merged;
174
+ for (const [key, value] of Object.entries(defaults))if (void 0 === locate[key]) {
175
+ if ('deepLocate' !== key || void 0 === locate.deepThink) {
176
+ merged = merged ?? {
177
+ ...locate
178
+ };
179
+ merged[key] = value;
180
+ }
181
+ }
182
+ return merged ?? locate;
183
+ }
184
+ function applyLocateDefaults(args, paramSchema, locateDefaults) {
185
+ if (!paramSchema || 0 === Object.keys(locateDefaults).length) return args;
186
+ const shape = getZodObjectShape(paramSchema);
187
+ if (!shape) return args;
188
+ return Object.fromEntries(Object.entries(args).map(([key, value])=>{
189
+ const fieldSchema = shape[key];
190
+ if (fieldSchema && (0, external_zod_schema_utils_js_namespaceObject.isMidsceneLocatorField)(fieldSchema) && isRecord(value)) return [
191
+ key,
192
+ mergeLocateDefaults(value, locateDefaults)
193
+ ];
194
+ return [
195
+ key,
196
+ value
197
+ ];
198
+ }));
199
+ }
200
+ function serializeArgsToDescription(args) {
201
+ try {
202
+ return Object.entries(args).map(([key, value])=>{
203
+ if ('object' == typeof value && null !== value) try {
204
+ return `${key}: ${JSON.stringify(value)}`;
205
+ } catch {
206
+ return `${key}: [object]`;
207
+ }
208
+ return `${key}: "${value}"`;
209
+ }).join(', ');
210
+ } catch (error) {
211
+ const errorMessage = (0, external_error_formatter_js_namespaceObject.getErrorMessage)(error);
212
+ console.error('Error serializing args:', errorMessage);
213
+ return `[args serialization failed: ${errorMessage}]`;
214
+ }
215
+ }
216
+ function buildActionInstruction(actionName, args) {
217
+ const locatePrompt = isRecord(args.locate) ? getPromptText(args.locate.prompt) : void 0;
218
+ switch(actionName){
219
+ case 'Tap':
220
+ return locatePrompt ? `Tap on "${locatePrompt}"` : 'Tap';
221
+ case 'Input':
222
+ {
223
+ const value = args.value ?? args.content ?? '';
224
+ return locatePrompt ? `Input "${value}" into "${locatePrompt}"` : `Input "${value}"`;
225
+ }
226
+ case 'Scroll':
227
+ {
228
+ const direction = args.direction ?? 'down';
229
+ return locatePrompt ? `Scroll ${direction} on "${locatePrompt}"` : `Scroll ${direction}`;
230
+ }
231
+ case 'Hover':
232
+ return locatePrompt ? `Hover over "${locatePrompt}"` : 'Hover';
233
+ case 'KeyboardPress':
234
+ {
235
+ const key = args.value ?? args.key ?? '';
236
+ return `Press key "${key}"`;
237
+ }
238
+ default:
239
+ {
240
+ const argsDescription = serializeArgsToDescription(args);
241
+ return argsDescription ? `${actionName}: ${argsDescription}` : actionName;
242
+ }
243
+ }
244
+ }
245
+ async function executeAction(agent, actionName, args) {
246
+ if (agent.callActionInActionSpace) return agent.callActionInActionSpace(actionName, args);
247
+ if (agent.aiAction) {
248
+ const instruction = buildActionInstruction(actionName, args);
249
+ return agent.aiAction(instruction);
250
+ }
251
+ throw new Error(`Action "${actionName}" is not supported by this agent`);
252
+ }
253
+ async function captureScreenshotResult(agent, actionName, actionResult) {
254
+ const content = [
255
+ {
256
+ type: 'text',
257
+ text: `Action "${actionName}" completed.`
258
+ }
259
+ ];
260
+ if (void 0 !== actionResult) content.push({
261
+ type: 'text',
262
+ text: `Result: ${serializeActionResult(actionResult)}`
263
+ });
264
+ try {
265
+ const screenshot = await agent.page?.screenshotBase64();
266
+ if (!screenshot) return {
267
+ content
268
+ };
269
+ const { mimeType, body } = (0, img_namespaceObject.parseBase64)(screenshot);
270
+ content.push({
271
+ type: 'image',
272
+ data: body,
273
+ mimeType
274
+ });
275
+ return {
276
+ content
277
+ };
278
+ } catch (error) {
279
+ const errorMessage = (0, external_error_formatter_js_namespaceObject.getErrorMessage)(error);
280
+ console.error('Error capturing screenshot:', errorMessage);
281
+ content[0] = {
282
+ type: 'text',
283
+ text: `Action "${actionName}" completed (screenshot unavailable: ${errorMessage})`
284
+ };
285
+ return {
286
+ content
287
+ };
288
+ }
289
+ }
290
+ function serializeActionResult(actionResult) {
291
+ if ('string' == typeof actionResult) return actionResult;
292
+ try {
293
+ return JSON.stringify(actionResult);
294
+ } catch {
295
+ return String(actionResult);
296
+ }
297
+ }
298
+ function createErrorResult(message) {
299
+ return {
300
+ content: [
301
+ {
302
+ type: 'text',
303
+ text: message
304
+ }
305
+ ],
306
+ isError: true
307
+ };
308
+ }
309
+ async function captureFailureResult(agent, actionName, errorMessage) {
310
+ const warningText = `Warning: Action "${actionName}" failed: ${errorMessage}. Check the screenshot below for the current page state and decide how to proceed.`;
311
+ try {
312
+ const screenshot = await agent.page?.screenshotBase64();
313
+ if (!screenshot) return {
314
+ content: [
315
+ {
316
+ type: 'text',
317
+ text: warningText
318
+ }
319
+ ]
320
+ };
321
+ const { mimeType, body } = (0, img_namespaceObject.parseBase64)(screenshot);
322
+ return {
323
+ content: [
324
+ {
325
+ type: 'text',
326
+ text: warningText
327
+ },
328
+ {
329
+ type: 'image',
330
+ data: body,
331
+ mimeType
332
+ }
333
+ ]
334
+ };
335
+ } catch {
336
+ return {
337
+ content: [
338
+ {
339
+ type: 'text',
340
+ text: warningText
341
+ }
342
+ ]
343
+ };
344
+ }
345
+ }
346
+ function mergeToolCliMetadata(base, extra) {
347
+ const options = {
348
+ ...base?.options ?? {},
349
+ ...extra?.options ?? {}
350
+ };
351
+ const positionals = base?.positionals ?? extra?.positionals;
352
+ return Object.keys(options).length > 0 || positionals ? {
353
+ options,
354
+ positionals
355
+ } : void 0;
356
+ }
357
+ function generateToolsFromActionSpace(actionSpace, getAgent, sanitizeArgs = (args)=>args, initArgSchema = {}, initArgCliMetadata, toolDefaults = {}) {
358
+ return actionSpace.map((action)=>{
359
+ const schema = {
360
+ ...extractActionSchema(action.paramSchema, action.name),
361
+ ...initArgSchema
362
+ };
363
+ return {
364
+ name: action.name,
365
+ description: describeActionForTool(action),
366
+ schema,
367
+ cli: initArgCliMetadata,
368
+ handler: async (args)=>{
369
+ try {
370
+ const agent = await getAgent(args);
371
+ (0, verbose_js_namespaceObject.emitCliVerboseEvent)({
372
+ event: 'agent_ready',
373
+ tool: action.name
374
+ });
375
+ const unsubscribeVerbose = (0, verbose_js_namespaceObject.attachCliVerboseDumpListener)(agent, {
376
+ toolName: action.name
377
+ });
378
+ try {
379
+ let normalizedArgs = normalizeActionArgs(sanitizeArgs(args), action.paramSchema);
380
+ if (toolDefaults.locate) normalizedArgs = applyLocateDefaults(normalizedArgs, action.paramSchema, toolDefaults.locate);
381
+ let actionResult;
382
+ try {
383
+ actionResult = await executeAction(agent, action.name, normalizedArgs);
384
+ } catch (error) {
385
+ const errorMessage = (0, external_error_formatter_js_namespaceObject.getErrorMessage)(error);
386
+ console.error(`Error executing action "${action.name}":`, errorMessage);
387
+ return await captureFailureResult(agent, action.name, errorMessage);
388
+ }
389
+ return await captureScreenshotResult(agent, action.name, actionResult);
390
+ } finally{
391
+ unsubscribeVerbose();
392
+ }
393
+ } catch (error) {
394
+ const errorMessage = (0, external_error_formatter_js_namespaceObject.getErrorMessage)(error);
395
+ console.error(`Error in handler for "${action.name}":`, errorMessage);
396
+ return createErrorResult(`Failed to get agent or execute action "${action.name}": ${errorMessage}`);
397
+ }
398
+ }
399
+ };
400
+ });
401
+ }
402
+ function generateCommonTools(getAgent, initArgSchema = {}, initArgCliMetadata, toolDefaults = {}) {
403
+ return [
404
+ {
405
+ name: 'take_screenshot',
406
+ description: 'Capture screenshot of current page/screen',
407
+ schema: {
408
+ ...initArgSchema
409
+ },
410
+ cli: initArgCliMetadata,
411
+ handler: async (args = {})=>{
412
+ try {
413
+ const agent = await getAgent(args);
414
+ (0, verbose_js_namespaceObject.emitCliVerboseEvent)({
415
+ event: 'agent_ready',
416
+ tool: 'take_screenshot'
417
+ });
418
+ const unsubscribeVerbose = (0, verbose_js_namespaceObject.attachCliVerboseDumpListener)(agent, {
419
+ toolName: 'take_screenshot'
420
+ });
421
+ try {
422
+ const screenshot = await agent.page?.screenshotBase64();
423
+ if (!screenshot) return createErrorResult('Screenshot not available');
424
+ await agent.recordToReport?.('take_screenshot', {
425
+ screenshotBase64: screenshot
426
+ });
427
+ const { mimeType, body } = (0, img_namespaceObject.parseBase64)(screenshot);
428
+ return {
429
+ content: [
430
+ {
431
+ type: 'image',
432
+ data: body,
433
+ mimeType
434
+ }
435
+ ]
436
+ };
437
+ } finally{
438
+ unsubscribeVerbose();
439
+ }
440
+ } catch (error) {
441
+ const errorMessage = (0, external_error_formatter_js_namespaceObject.getErrorMessage)(error);
442
+ console.error('Error taking screenshot:', errorMessage);
443
+ return createErrorResult(`Failed to capture screenshot: ${errorMessage}`);
444
+ }
445
+ }
446
+ },
447
+ {
448
+ name: 'act',
449
+ description: 'Execute a natural language action. The AI will plan and perform multi-step operations in a single invocation, useful for transient UI interactions (e.g., Spotlight, dropdown menus) that disappear between separate commands.',
450
+ schema: {
451
+ prompt: external_zod_namespaceObject.z.string().describe('Natural language description of the action to perform, e.g. "press Command+Space, type Safari, press Enter"'),
452
+ deepLocate: external_zod_namespaceObject.z.boolean().optional().describe('Use deep locate for every element this action targets. Improves precision for small or ambiguous targets at the cost of speed. Defaults to the server --deep-locate setting.'),
453
+ deepThink: external_zod_namespaceObject.z.boolean().optional().describe('Plan this action with deep thinking (richer context and sub-goal decomposition). Helps with complex multi-step instructions at the cost of speed. Defaults to the server --deep-think setting.'),
454
+ fileChooserAllowedDir: external_zod_namespaceObject.z.string().optional().describe('Directory that model-planned file uploads may access. Required when the prompt asks the model to upload files.'),
455
+ ...external_user_prompt_js_namespaceObject.promptInputExtraSchema,
456
+ ...initArgSchema
457
+ },
458
+ cli: mergeToolCliMetadata(void 0, initArgCliMetadata),
459
+ handler: async (args = {})=>{
460
+ const prompt = args.prompt;
461
+ try {
462
+ const agent = await getAgent(args);
463
+ (0, verbose_js_namespaceObject.emitCliVerboseEvent)({
464
+ event: 'agent_ready',
465
+ tool: 'act'
466
+ });
467
+ if (!agent.aiAction) return createErrorResult('act is not supported by this agent');
468
+ const unsubscribeVerbose = (0, verbose_js_namespaceObject.attachCliVerboseDumpListener)(agent, {
469
+ toolName: 'act'
470
+ });
471
+ try {
472
+ const actOptions = {
473
+ deepThink: false,
474
+ ...toolDefaults.act
475
+ };
476
+ if (void 0 !== args.deepLocate) actOptions.deepLocate = args.deepLocate;
477
+ if (void 0 !== args.deepThink) actOptions.deepThink = args.deepThink;
478
+ if (void 0 !== args.fileChooserAllowedDir) actOptions.fileChooserAllowedDir = args.fileChooserAllowedDir;
479
+ const userPrompt = (0, external_user_prompt_js_namespaceObject.composeUserPrompt)({
480
+ prompt,
481
+ image: args.image,
482
+ imageName: args.imageName,
483
+ convertHttpImage2Base64: args.convertHttpImage2Base64
484
+ });
485
+ const result = await agent.aiAction(userPrompt, actOptions);
486
+ return await captureScreenshotResult(agent, 'act', result);
487
+ } finally{
488
+ unsubscribeVerbose();
489
+ }
490
+ } catch (error) {
491
+ const errorMessage = (0, external_error_formatter_js_namespaceObject.getErrorMessage)(error);
492
+ console.error('Error executing act:', errorMessage);
493
+ return createErrorResult(`Failed to execute act: ${errorMessage}`);
494
+ }
495
+ }
496
+ },
497
+ {
498
+ name: 'assert',
499
+ description: 'Assert a natural language statement against the current page/screen or a saved UI observation record.',
500
+ schema: {
501
+ prompt: external_zod_namespaceObject.z.string().describe('Natural language assertion to verify, e.g. "there is a login button visible"'),
502
+ message: external_zod_namespaceObject.z.string().optional().describe('Custom error message to throw when the assertion fails, e.g. "the login button should be visible".'),
503
+ record: external_zod_namespaceObject.z.string().optional().describe('Path to a JSON observation manifest created by the record command. Keep its adjacent image directory with it. When set, the assertion uses that ordered frame window instead of the current screen.'),
504
+ ...external_user_prompt_js_namespaceObject.promptInputExtraSchema,
505
+ ...initArgSchema
506
+ },
507
+ cli: mergeToolCliMetadata(void 0, initArgCliMetadata),
508
+ handler: async (args = {})=>{
509
+ const prompt = args.prompt;
510
+ const message = args.message;
511
+ try {
512
+ const agent = await getAgent(args);
513
+ (0, verbose_js_namespaceObject.emitCliVerboseEvent)({
514
+ event: 'agent_ready',
515
+ tool: 'assert'
516
+ });
517
+ if (!agent.aiAssert) return createErrorResult('assert is not supported by this agent');
518
+ const unsubscribeVerbose = (0, verbose_js_namespaceObject.attachCliVerboseDumpListener)(agent, {
519
+ toolName: 'assert'
520
+ });
521
+ try {
522
+ const observationRecord = 'string' == typeof args.record ? (0, external_observation_record_js_namespaceObject.readUIObservationRecord)(args.record) : void 0;
523
+ const userPrompt = (0, external_user_prompt_js_namespaceObject.composeUserPrompt)({
524
+ prompt,
525
+ image: args.image,
526
+ imageName: args.imageName,
527
+ convertHttpImage2Base64: args.convertHttpImage2Base64
528
+ });
529
+ if (observationRecord) {
530
+ const observationArtifacts = (0, external_observation_artifact_js_namespaceObject.resolveObservationArtifactAdapter)(agent);
531
+ if (!observationArtifacts) throw new Error('assert --record is not supported because this agent cannot load UI observations');
532
+ const observation = observationArtifacts.loadRecord(observationRecord);
533
+ try {
534
+ await observation.aiAssert(userPrompt, message);
535
+ } finally{
536
+ await observation.dispose?.();
537
+ }
538
+ } else await agent.aiAssert(userPrompt, message);
539
+ return {
540
+ content: [
541
+ {
542
+ type: 'text',
543
+ text: 'Assertion passed.'
544
+ }
545
+ ]
546
+ };
547
+ } finally{
548
+ unsubscribeVerbose();
549
+ }
550
+ } catch (error) {
551
+ const errorMessage = (0, external_error_formatter_js_namespaceObject.getErrorMessage)(error);
552
+ console.error('Error executing assert:', errorMessage);
553
+ return createErrorResult(`Failed to execute assert: ${errorMessage}`);
554
+ }
555
+ }
556
+ }
557
+ ];
558
+ }
559
+ exports.composeUserPrompt = __webpack_exports__.composeUserPrompt;
560
+ exports.generateCommonTools = __webpack_exports__.generateCommonTools;
561
+ exports.generateToolsFromActionSpace = __webpack_exports__.generateToolsFromActionSpace;
562
+ for(var __rspack_i in __webpack_exports__)if (-1 === [
563
+ "composeUserPrompt",
564
+ "generateCommonTools",
565
+ "generateToolsFromActionSpace"
566
+ ].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
567
+ Object.defineProperty(exports, '__esModule', {
568
+ value: true
569
+ });
@@ -0,0 +1,40 @@
1
+ "use strict";
2
+ var __webpack_require__ = {};
3
+ (()=>{
4
+ __webpack_require__.d = (exports1, definition)=>{
5
+ for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
6
+ enumerable: true,
7
+ get: definition[key]
8
+ });
9
+ };
10
+ })();
11
+ (()=>{
12
+ __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
13
+ })();
14
+ (()=>{
15
+ __webpack_require__.r = (exports1)=>{
16
+ if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
17
+ value: 'Module'
18
+ });
19
+ Object.defineProperty(exports1, '__esModule', {
20
+ value: true
21
+ });
22
+ };
23
+ })();
24
+ var __webpack_exports__ = {};
25
+ __webpack_require__.r(__webpack_exports__);
26
+ __webpack_require__.d(__webpack_exports__, {
27
+ defaultAppLoadingCheckIntervalMs: ()=>defaultAppLoadingCheckIntervalMs,
28
+ defaultAppLoadingTimeoutMs: ()=>defaultAppLoadingTimeoutMs
29
+ });
30
+ const defaultAppLoadingTimeoutMs = 10000;
31
+ const defaultAppLoadingCheckIntervalMs = 2000;
32
+ exports.defaultAppLoadingCheckIntervalMs = __webpack_exports__.defaultAppLoadingCheckIntervalMs;
33
+ exports.defaultAppLoadingTimeoutMs = __webpack_exports__.defaultAppLoadingTimeoutMs;
34
+ for(var __rspack_i in __webpack_exports__)if (-1 === [
35
+ "defaultAppLoadingCheckIntervalMs",
36
+ "defaultAppLoadingTimeoutMs"
37
+ ].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
38
+ Object.defineProperty(exports, '__esModule', {
39
+ value: true
40
+ });