@allternit/computer-driver 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/README.md +104 -2
- package/dist/anthropic.d.ts +43 -0
- package/dist/anthropic.js +157 -0
- package/dist/client.d.ts +157 -0
- package/dist/client.js +173 -0
- package/dist/gemini.d.ts +56 -0
- package/dist/gemini.js +107 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.js +7 -0
- package/dist/openai.d.ts +98 -0
- package/dist/openai.js +113 -0
- package/dist/v2-generated.d.ts +1445 -0
- package/dist/v2-generated.js +1266 -0
- package/dist/v2.d.ts +279 -0
- package/dist/v2.js +154 -0
- package/package.json +58 -4
|
@@ -0,0 +1,1266 @@
|
|
|
1
|
+
// Generated by contracts/computer-toolset/generate.mjs from contracts/computer-toolset/*.json. Do not edit.
|
|
2
|
+
/* eslint-disable */
|
|
3
|
+
// allternit.computer.v2 (computer_toolset_20260801): every member input type.
|
|
4
|
+
export const COMPUTER_V2_MEMBER_NAMES = ["screenshot", "zoom", "left_click", "right_click", "middle_click", "double_click", "triple_click", "mouse_move", "left_click_drag", "left_mouse_down", "left_mouse_up", "scroll", "type", "key", "hold_key", "wait", "cursor_position", "read_ui", "act", "run_batch", "verify", "request_human", "use_credential", "run_subtask", "run_parallel", "run_skill", "skills"];
|
|
5
|
+
/** The ten driver-backed structured members (read_ui … skills). */
|
|
6
|
+
export const COMPUTER_V2_STRUCTURED_MEMBER_NAMES = ["read_ui", "act", "run_batch", "verify", "request_human", "use_credential", "run_subtask", "run_parallel", "run_skill", "skills"];
|
|
7
|
+
/** The per-member JSON Schemas, for building provider function tools. */
|
|
8
|
+
export const COMPUTER_V2_TOOL_SCHEMAS = {
|
|
9
|
+
"screenshot": {
|
|
10
|
+
"type": "object",
|
|
11
|
+
"properties": {},
|
|
12
|
+
"required": [],
|
|
13
|
+
"additionalProperties": false
|
|
14
|
+
},
|
|
15
|
+
"zoom": {
|
|
16
|
+
"type": "object",
|
|
17
|
+
"properties": {
|
|
18
|
+
"region": {
|
|
19
|
+
"type": "array",
|
|
20
|
+
"items": {
|
|
21
|
+
"type": "number"
|
|
22
|
+
},
|
|
23
|
+
"minItems": 4,
|
|
24
|
+
"maxItems": 4,
|
|
25
|
+
"description": "(x0, y0, x1, y1): The region to capture."
|
|
26
|
+
}
|
|
27
|
+
},
|
|
28
|
+
"required": [
|
|
29
|
+
"region"
|
|
30
|
+
],
|
|
31
|
+
"additionalProperties": false
|
|
32
|
+
},
|
|
33
|
+
"left_click": {
|
|
34
|
+
"type": "object",
|
|
35
|
+
"properties": {
|
|
36
|
+
"coordinate": {
|
|
37
|
+
"type": "array",
|
|
38
|
+
"items": {
|
|
39
|
+
"type": "number"
|
|
40
|
+
},
|
|
41
|
+
"minItems": 2,
|
|
42
|
+
"maxItems": 2,
|
|
43
|
+
"description": "(x, y): x pixels from the left edge, y pixels from the top edge."
|
|
44
|
+
},
|
|
45
|
+
"text": {
|
|
46
|
+
"type": "string",
|
|
47
|
+
"description": "Optional key combination to hold down during this action (e.g. \"ctrl\", \"shift\", \"ctrl+shift\")."
|
|
48
|
+
}
|
|
49
|
+
},
|
|
50
|
+
"required": [],
|
|
51
|
+
"additionalProperties": false
|
|
52
|
+
},
|
|
53
|
+
"right_click": {
|
|
54
|
+
"type": "object",
|
|
55
|
+
"properties": {
|
|
56
|
+
"coordinate": {
|
|
57
|
+
"type": "array",
|
|
58
|
+
"items": {
|
|
59
|
+
"type": "number"
|
|
60
|
+
},
|
|
61
|
+
"minItems": 2,
|
|
62
|
+
"maxItems": 2,
|
|
63
|
+
"description": "(x, y): x pixels from the left edge, y pixels from the top edge."
|
|
64
|
+
},
|
|
65
|
+
"text": {
|
|
66
|
+
"type": "string",
|
|
67
|
+
"description": "Optional key combination to hold down during this action (e.g. \"ctrl\", \"shift\", \"ctrl+shift\")."
|
|
68
|
+
}
|
|
69
|
+
},
|
|
70
|
+
"required": [],
|
|
71
|
+
"additionalProperties": false
|
|
72
|
+
},
|
|
73
|
+
"middle_click": {
|
|
74
|
+
"type": "object",
|
|
75
|
+
"properties": {
|
|
76
|
+
"coordinate": {
|
|
77
|
+
"type": "array",
|
|
78
|
+
"items": {
|
|
79
|
+
"type": "number"
|
|
80
|
+
},
|
|
81
|
+
"minItems": 2,
|
|
82
|
+
"maxItems": 2,
|
|
83
|
+
"description": "(x, y): x pixels from the left edge, y pixels from the top edge."
|
|
84
|
+
},
|
|
85
|
+
"text": {
|
|
86
|
+
"type": "string",
|
|
87
|
+
"description": "Optional key combination to hold down during this action (e.g. \"ctrl\", \"shift\", \"ctrl+shift\")."
|
|
88
|
+
}
|
|
89
|
+
},
|
|
90
|
+
"required": [],
|
|
91
|
+
"additionalProperties": false
|
|
92
|
+
},
|
|
93
|
+
"double_click": {
|
|
94
|
+
"type": "object",
|
|
95
|
+
"properties": {
|
|
96
|
+
"coordinate": {
|
|
97
|
+
"type": "array",
|
|
98
|
+
"items": {
|
|
99
|
+
"type": "number"
|
|
100
|
+
},
|
|
101
|
+
"minItems": 2,
|
|
102
|
+
"maxItems": 2,
|
|
103
|
+
"description": "(x, y): x pixels from the left edge, y pixels from the top edge."
|
|
104
|
+
},
|
|
105
|
+
"text": {
|
|
106
|
+
"type": "string",
|
|
107
|
+
"description": "Optional key combination to hold down during this action (e.g. \"ctrl\", \"shift\", \"ctrl+shift\")."
|
|
108
|
+
}
|
|
109
|
+
},
|
|
110
|
+
"required": [],
|
|
111
|
+
"additionalProperties": false
|
|
112
|
+
},
|
|
113
|
+
"triple_click": {
|
|
114
|
+
"type": "object",
|
|
115
|
+
"properties": {
|
|
116
|
+
"coordinate": {
|
|
117
|
+
"type": "array",
|
|
118
|
+
"items": {
|
|
119
|
+
"type": "number"
|
|
120
|
+
},
|
|
121
|
+
"minItems": 2,
|
|
122
|
+
"maxItems": 2,
|
|
123
|
+
"description": "(x, y): x pixels from the left edge, y pixels from the top edge."
|
|
124
|
+
},
|
|
125
|
+
"text": {
|
|
126
|
+
"type": "string",
|
|
127
|
+
"description": "Optional key combination to hold down during this action (e.g. \"ctrl\", \"shift\", \"ctrl+shift\")."
|
|
128
|
+
}
|
|
129
|
+
},
|
|
130
|
+
"required": [],
|
|
131
|
+
"additionalProperties": false
|
|
132
|
+
},
|
|
133
|
+
"mouse_move": {
|
|
134
|
+
"type": "object",
|
|
135
|
+
"properties": {
|
|
136
|
+
"coordinate": {
|
|
137
|
+
"type": "array",
|
|
138
|
+
"items": {
|
|
139
|
+
"type": "number"
|
|
140
|
+
},
|
|
141
|
+
"minItems": 2,
|
|
142
|
+
"maxItems": 2,
|
|
143
|
+
"description": "(x, y): x pixels from the left edge, y pixels from the top edge."
|
|
144
|
+
}
|
|
145
|
+
},
|
|
146
|
+
"required": [
|
|
147
|
+
"coordinate"
|
|
148
|
+
],
|
|
149
|
+
"additionalProperties": false
|
|
150
|
+
},
|
|
151
|
+
"left_click_drag": {
|
|
152
|
+
"type": "object",
|
|
153
|
+
"properties": {
|
|
154
|
+
"coordinate": {
|
|
155
|
+
"type": "array",
|
|
156
|
+
"items": {
|
|
157
|
+
"type": "number"
|
|
158
|
+
},
|
|
159
|
+
"minItems": 2,
|
|
160
|
+
"maxItems": 2,
|
|
161
|
+
"description": "(x, y): x pixels from the left edge, y pixels from the top edge."
|
|
162
|
+
},
|
|
163
|
+
"start_coordinate": {
|
|
164
|
+
"type": "array",
|
|
165
|
+
"items": {
|
|
166
|
+
"type": "number"
|
|
167
|
+
},
|
|
168
|
+
"minItems": 2,
|
|
169
|
+
"maxItems": 2,
|
|
170
|
+
"description": "(x, y): x pixels from the left edge, y pixels from the top edge."
|
|
171
|
+
},
|
|
172
|
+
"text": {
|
|
173
|
+
"type": "string",
|
|
174
|
+
"description": "Optional key combination to hold down during this action (e.g. \"ctrl\", \"shift\", \"ctrl+shift\")."
|
|
175
|
+
}
|
|
176
|
+
},
|
|
177
|
+
"required": [
|
|
178
|
+
"coordinate",
|
|
179
|
+
"start_coordinate"
|
|
180
|
+
],
|
|
181
|
+
"additionalProperties": false
|
|
182
|
+
},
|
|
183
|
+
"left_mouse_down": {
|
|
184
|
+
"type": "object",
|
|
185
|
+
"properties": {},
|
|
186
|
+
"required": [],
|
|
187
|
+
"additionalProperties": false
|
|
188
|
+
},
|
|
189
|
+
"left_mouse_up": {
|
|
190
|
+
"type": "object",
|
|
191
|
+
"properties": {},
|
|
192
|
+
"required": [],
|
|
193
|
+
"additionalProperties": false
|
|
194
|
+
},
|
|
195
|
+
"scroll": {
|
|
196
|
+
"type": "object",
|
|
197
|
+
"properties": {
|
|
198
|
+
"scroll_amount": {
|
|
199
|
+
"type": "number",
|
|
200
|
+
"description": "Number of 'clicks' of the scroll wheel."
|
|
201
|
+
},
|
|
202
|
+
"scroll_direction": {
|
|
203
|
+
"type": "string",
|
|
204
|
+
"enum": [
|
|
205
|
+
"up",
|
|
206
|
+
"down",
|
|
207
|
+
"left",
|
|
208
|
+
"right"
|
|
209
|
+
]
|
|
210
|
+
},
|
|
211
|
+
"coordinate": {
|
|
212
|
+
"type": "array",
|
|
213
|
+
"items": {
|
|
214
|
+
"type": "number"
|
|
215
|
+
},
|
|
216
|
+
"minItems": 2,
|
|
217
|
+
"maxItems": 2,
|
|
218
|
+
"description": "(x, y): x pixels from the left edge, y pixels from the top edge."
|
|
219
|
+
},
|
|
220
|
+
"text": {
|
|
221
|
+
"type": "string",
|
|
222
|
+
"description": "Optional key combination to hold down during this action (e.g. \"ctrl\", \"shift\", \"ctrl+shift\")."
|
|
223
|
+
}
|
|
224
|
+
},
|
|
225
|
+
"required": [
|
|
226
|
+
"scroll_amount",
|
|
227
|
+
"scroll_direction"
|
|
228
|
+
],
|
|
229
|
+
"additionalProperties": false
|
|
230
|
+
},
|
|
231
|
+
"type": {
|
|
232
|
+
"type": "object",
|
|
233
|
+
"properties": {
|
|
234
|
+
"text": {
|
|
235
|
+
"type": "string",
|
|
236
|
+
"description": "The text to type."
|
|
237
|
+
}
|
|
238
|
+
},
|
|
239
|
+
"required": [
|
|
240
|
+
"text"
|
|
241
|
+
],
|
|
242
|
+
"additionalProperties": false
|
|
243
|
+
},
|
|
244
|
+
"key": {
|
|
245
|
+
"type": "object",
|
|
246
|
+
"properties": {
|
|
247
|
+
"text": {
|
|
248
|
+
"type": "string",
|
|
249
|
+
"description": "The key or key-combination to press."
|
|
250
|
+
},
|
|
251
|
+
"repeat": {
|
|
252
|
+
"type": "number",
|
|
253
|
+
"description": "Number of times to repeat the key press. Default is 1."
|
|
254
|
+
}
|
|
255
|
+
},
|
|
256
|
+
"required": [
|
|
257
|
+
"text"
|
|
258
|
+
],
|
|
259
|
+
"additionalProperties": false
|
|
260
|
+
},
|
|
261
|
+
"hold_key": {
|
|
262
|
+
"type": "object",
|
|
263
|
+
"properties": {
|
|
264
|
+
"duration": {
|
|
265
|
+
"type": "number",
|
|
266
|
+
"description": "Duration to hold the key, in seconds."
|
|
267
|
+
},
|
|
268
|
+
"text": {
|
|
269
|
+
"type": "string",
|
|
270
|
+
"description": "The key or key-combination to hold."
|
|
271
|
+
}
|
|
272
|
+
},
|
|
273
|
+
"required": [
|
|
274
|
+
"duration",
|
|
275
|
+
"text"
|
|
276
|
+
],
|
|
277
|
+
"additionalProperties": false
|
|
278
|
+
},
|
|
279
|
+
"wait": {
|
|
280
|
+
"type": "object",
|
|
281
|
+
"properties": {
|
|
282
|
+
"duration": {
|
|
283
|
+
"type": "number",
|
|
284
|
+
"description": "Duration to wait, in seconds."
|
|
285
|
+
}
|
|
286
|
+
},
|
|
287
|
+
"required": [
|
|
288
|
+
"duration"
|
|
289
|
+
],
|
|
290
|
+
"additionalProperties": false
|
|
291
|
+
},
|
|
292
|
+
"cursor_position": {
|
|
293
|
+
"type": "object",
|
|
294
|
+
"properties": {},
|
|
295
|
+
"required": [],
|
|
296
|
+
"additionalProperties": false
|
|
297
|
+
},
|
|
298
|
+
"read_ui": {
|
|
299
|
+
"type": "object",
|
|
300
|
+
"properties": {
|
|
301
|
+
"app": {
|
|
302
|
+
"type": "string",
|
|
303
|
+
"description": "App name or bundle id (macOS); default: the frontmost app."
|
|
304
|
+
},
|
|
305
|
+
"pid": {
|
|
306
|
+
"type": "integer"
|
|
307
|
+
},
|
|
308
|
+
"window_id": {
|
|
309
|
+
"type": "integer"
|
|
310
|
+
},
|
|
311
|
+
"query": {
|
|
312
|
+
"type": "string",
|
|
313
|
+
"description": "Only elements whose name or value contains this text."
|
|
314
|
+
},
|
|
315
|
+
"max_elements": {
|
|
316
|
+
"type": "integer",
|
|
317
|
+
"description": "Cap the element list (default 200)."
|
|
318
|
+
},
|
|
319
|
+
"since": {
|
|
320
|
+
"type": "integer",
|
|
321
|
+
"description": "A map version from an earlier read_ui: answer only the changes since then."
|
|
322
|
+
},
|
|
323
|
+
"paths": {
|
|
324
|
+
"type": "boolean",
|
|
325
|
+
"description": "Add each element's tree path (role:ordinal chain from the window). Replay locators use it; you rarely need it."
|
|
326
|
+
},
|
|
327
|
+
"crops": {
|
|
328
|
+
"type": "boolean",
|
|
329
|
+
"description": "Add an 8x8 pixel hash per element (macOS). Slower: forces a fresh walk of the window. Replay's pixel fallback uses it."
|
|
330
|
+
},
|
|
331
|
+
"vision": {
|
|
332
|
+
"type": "string",
|
|
333
|
+
"enum": [
|
|
334
|
+
"auto",
|
|
335
|
+
"off",
|
|
336
|
+
"only"
|
|
337
|
+
],
|
|
338
|
+
"description": "The vision fallback: auto (default; used when the tree is empty or can't see into part of the window), off (tree only), only (vision only)."
|
|
339
|
+
},
|
|
340
|
+
"target": {
|
|
341
|
+
"type": "string",
|
|
342
|
+
"description": "Ground this description to one element with the local grounder; the answer's `grounded` has its id, point and confidence."
|
|
343
|
+
},
|
|
344
|
+
"zoom": {
|
|
345
|
+
"type": "string",
|
|
346
|
+
"enum": [
|
|
347
|
+
"auto",
|
|
348
|
+
"always",
|
|
349
|
+
"never"
|
|
350
|
+
],
|
|
351
|
+
"description": "target: zoom into a crop around a low-confidence answer and ground again (default auto)."
|
|
352
|
+
}
|
|
353
|
+
},
|
|
354
|
+
"required": [],
|
|
355
|
+
"additionalProperties": false
|
|
356
|
+
},
|
|
357
|
+
"act": {
|
|
358
|
+
"type": "object",
|
|
359
|
+
"properties": {
|
|
360
|
+
"id": {
|
|
361
|
+
"type": "string",
|
|
362
|
+
"description": "The element id from read_ui."
|
|
363
|
+
},
|
|
364
|
+
"op": {
|
|
365
|
+
"type": "string",
|
|
366
|
+
"enum": [
|
|
367
|
+
"click",
|
|
368
|
+
"set_value",
|
|
369
|
+
"select",
|
|
370
|
+
"press",
|
|
371
|
+
"menu",
|
|
372
|
+
"focus"
|
|
373
|
+
],
|
|
374
|
+
"description": "What to do to the element."
|
|
375
|
+
},
|
|
376
|
+
"value": {
|
|
377
|
+
"type": "string",
|
|
378
|
+
"description": "set_value: the text to type into the element; select: the child's name to pick."
|
|
379
|
+
},
|
|
380
|
+
"key": {
|
|
381
|
+
"type": "string",
|
|
382
|
+
"description": "press: the key name (Return, Tab, Escape, ...)."
|
|
383
|
+
},
|
|
384
|
+
"path": {
|
|
385
|
+
"anyOf": [
|
|
386
|
+
{
|
|
387
|
+
"type": "string"
|
|
388
|
+
},
|
|
389
|
+
{
|
|
390
|
+
"type": "array",
|
|
391
|
+
"items": {
|
|
392
|
+
"type": "string"
|
|
393
|
+
}
|
|
394
|
+
}
|
|
395
|
+
],
|
|
396
|
+
"description": "menu: the menu path, 'File > Save' or ['File', 'Save']."
|
|
397
|
+
},
|
|
398
|
+
"version": {
|
|
399
|
+
"type": "integer",
|
|
400
|
+
"description": "The read_ui map version this action was planned on; a stale version is refused with the fresh map."
|
|
401
|
+
},
|
|
402
|
+
"app": {
|
|
403
|
+
"type": "string",
|
|
404
|
+
"description": "App name or bundle id (macOS); default: the frontmost app."
|
|
405
|
+
},
|
|
406
|
+
"pid": {
|
|
407
|
+
"type": "integer"
|
|
408
|
+
},
|
|
409
|
+
"window_id": {
|
|
410
|
+
"type": "integer"
|
|
411
|
+
}
|
|
412
|
+
},
|
|
413
|
+
"required": [
|
|
414
|
+
"id",
|
|
415
|
+
"op"
|
|
416
|
+
],
|
|
417
|
+
"additionalProperties": false
|
|
418
|
+
},
|
|
419
|
+
"run_batch": {
|
|
420
|
+
"type": "object",
|
|
421
|
+
"properties": {
|
|
422
|
+
"app": {
|
|
423
|
+
"type": "string",
|
|
424
|
+
"description": "App name or bundle id (macOS); default: the frontmost app."
|
|
425
|
+
},
|
|
426
|
+
"pid": {
|
|
427
|
+
"type": "integer"
|
|
428
|
+
},
|
|
429
|
+
"window_id": {
|
|
430
|
+
"type": "integer"
|
|
431
|
+
},
|
|
432
|
+
"version": {
|
|
433
|
+
"type": "integer",
|
|
434
|
+
"description": "Pin the read_ui map version the steps were planned on; stale is refused with the fresh map."
|
|
435
|
+
},
|
|
436
|
+
"steps": {
|
|
437
|
+
"type": "array",
|
|
438
|
+
"minItems": 1,
|
|
439
|
+
"maxItems": 50,
|
|
440
|
+
"items": {
|
|
441
|
+
"type": "object",
|
|
442
|
+
"properties": {
|
|
443
|
+
"act": {
|
|
444
|
+
"type": "object",
|
|
445
|
+
"properties": {
|
|
446
|
+
"id": {
|
|
447
|
+
"type": "string"
|
|
448
|
+
},
|
|
449
|
+
"op": {
|
|
450
|
+
"type": "string",
|
|
451
|
+
"enum": [
|
|
452
|
+
"click",
|
|
453
|
+
"set_value",
|
|
454
|
+
"select",
|
|
455
|
+
"press",
|
|
456
|
+
"menu",
|
|
457
|
+
"focus"
|
|
458
|
+
]
|
|
459
|
+
},
|
|
460
|
+
"value": {
|
|
461
|
+
"type": "string"
|
|
462
|
+
},
|
|
463
|
+
"key": {
|
|
464
|
+
"type": "string"
|
|
465
|
+
}
|
|
466
|
+
},
|
|
467
|
+
"required": [
|
|
468
|
+
"id",
|
|
469
|
+
"op"
|
|
470
|
+
],
|
|
471
|
+
"additionalProperties": false
|
|
472
|
+
},
|
|
473
|
+
"menu": {
|
|
474
|
+
"anyOf": [
|
|
475
|
+
{
|
|
476
|
+
"type": "string"
|
|
477
|
+
},
|
|
478
|
+
{
|
|
479
|
+
"type": "array",
|
|
480
|
+
"items": {
|
|
481
|
+
"type": "string"
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
]
|
|
485
|
+
},
|
|
486
|
+
"pixel": {
|
|
487
|
+
"type": "object",
|
|
488
|
+
"properties": {
|
|
489
|
+
"tool": {
|
|
490
|
+
"type": "string"
|
|
491
|
+
},
|
|
492
|
+
"args": {
|
|
493
|
+
"type": "object"
|
|
494
|
+
}
|
|
495
|
+
},
|
|
496
|
+
"required": [
|
|
497
|
+
"tool"
|
|
498
|
+
],
|
|
499
|
+
"additionalProperties": false
|
|
500
|
+
},
|
|
501
|
+
"wait": {
|
|
502
|
+
"type": "number",
|
|
503
|
+
"description": "Wait this many milliseconds before the next step."
|
|
504
|
+
},
|
|
505
|
+
"wait_for": {
|
|
506
|
+
"type": "object",
|
|
507
|
+
"properties": {
|
|
508
|
+
"id": {
|
|
509
|
+
"type": "string",
|
|
510
|
+
"description": "An element id from read_ui."
|
|
511
|
+
},
|
|
512
|
+
"role": {
|
|
513
|
+
"type": "string",
|
|
514
|
+
"description": "An accessibility role, e.g. button, textfield, checkbox."
|
|
515
|
+
},
|
|
516
|
+
"name": {
|
|
517
|
+
"type": "string",
|
|
518
|
+
"description": "Substring match against the element's name/label."
|
|
519
|
+
},
|
|
520
|
+
"text": {
|
|
521
|
+
"type": "string",
|
|
522
|
+
"description": "Substring match against the element's displayed text or value."
|
|
523
|
+
},
|
|
524
|
+
"gone": {
|
|
525
|
+
"type": "boolean",
|
|
526
|
+
"description": "True: no element may match; false: at least one must."
|
|
527
|
+
},
|
|
528
|
+
"value": {
|
|
529
|
+
"type": "string",
|
|
530
|
+
"description": "The element's value must equal this exactly."
|
|
531
|
+
},
|
|
532
|
+
"enabled": {
|
|
533
|
+
"type": "boolean",
|
|
534
|
+
"description": "The element's enabled state must equal this."
|
|
535
|
+
}
|
|
536
|
+
},
|
|
537
|
+
"required": [],
|
|
538
|
+
"additionalProperties": false,
|
|
539
|
+
"description": "Precondition: checked before the step; failure stops the batch."
|
|
540
|
+
},
|
|
541
|
+
"expect": {
|
|
542
|
+
"type": "object",
|
|
543
|
+
"properties": {
|
|
544
|
+
"id": {
|
|
545
|
+
"type": "string",
|
|
546
|
+
"description": "An element id from read_ui."
|
|
547
|
+
},
|
|
548
|
+
"role": {
|
|
549
|
+
"type": "string",
|
|
550
|
+
"description": "An accessibility role, e.g. button, textfield, checkbox."
|
|
551
|
+
},
|
|
552
|
+
"name": {
|
|
553
|
+
"type": "string",
|
|
554
|
+
"description": "Substring match against the element's name/label."
|
|
555
|
+
},
|
|
556
|
+
"text": {
|
|
557
|
+
"type": "string",
|
|
558
|
+
"description": "Substring match against the element's displayed text or value."
|
|
559
|
+
},
|
|
560
|
+
"gone": {
|
|
561
|
+
"type": "boolean",
|
|
562
|
+
"description": "True: no element may match; false: at least one must."
|
|
563
|
+
},
|
|
564
|
+
"value": {
|
|
565
|
+
"type": "string",
|
|
566
|
+
"description": "The element's value must equal this exactly."
|
|
567
|
+
},
|
|
568
|
+
"enabled": {
|
|
569
|
+
"type": "boolean",
|
|
570
|
+
"description": "The element's enabled state must equal this."
|
|
571
|
+
}
|
|
572
|
+
},
|
|
573
|
+
"required": [],
|
|
574
|
+
"additionalProperties": false,
|
|
575
|
+
"description": "Postcondition: checked after the step; failure stops the batch."
|
|
576
|
+
},
|
|
577
|
+
"timeout_ms": {
|
|
578
|
+
"type": "integer",
|
|
579
|
+
"description": "How long to wait for this step's checks (default 3000)."
|
|
580
|
+
}
|
|
581
|
+
},
|
|
582
|
+
"required": [],
|
|
583
|
+
"additionalProperties": false
|
|
584
|
+
}
|
|
585
|
+
}
|
|
586
|
+
},
|
|
587
|
+
"required": [
|
|
588
|
+
"steps"
|
|
589
|
+
],
|
|
590
|
+
"additionalProperties": false
|
|
591
|
+
},
|
|
592
|
+
"verify": {
|
|
593
|
+
"type": "object",
|
|
594
|
+
"properties": {
|
|
595
|
+
"app": {
|
|
596
|
+
"type": "string",
|
|
597
|
+
"description": "App name or bundle id (macOS); default: the frontmost app."
|
|
598
|
+
},
|
|
599
|
+
"pid": {
|
|
600
|
+
"type": "integer"
|
|
601
|
+
},
|
|
602
|
+
"window_id": {
|
|
603
|
+
"type": "integer"
|
|
604
|
+
},
|
|
605
|
+
"checks": {
|
|
606
|
+
"type": "array",
|
|
607
|
+
"minItems": 1,
|
|
608
|
+
"maxItems": 20,
|
|
609
|
+
"items": {
|
|
610
|
+
"type": "object",
|
|
611
|
+
"properties": {
|
|
612
|
+
"id": {
|
|
613
|
+
"type": "string",
|
|
614
|
+
"description": "An element id from read_ui."
|
|
615
|
+
},
|
|
616
|
+
"role": {
|
|
617
|
+
"type": "string",
|
|
618
|
+
"description": "An accessibility role, e.g. button, textfield, checkbox."
|
|
619
|
+
},
|
|
620
|
+
"name": {
|
|
621
|
+
"type": "string",
|
|
622
|
+
"description": "Substring match against the element's name/label."
|
|
623
|
+
},
|
|
624
|
+
"text": {
|
|
625
|
+
"type": "string",
|
|
626
|
+
"description": "Substring match against the element's displayed text or value."
|
|
627
|
+
},
|
|
628
|
+
"gone": {
|
|
629
|
+
"type": "boolean",
|
|
630
|
+
"description": "True: no element may match; false: at least one must."
|
|
631
|
+
},
|
|
632
|
+
"value": {
|
|
633
|
+
"type": "string",
|
|
634
|
+
"description": "The element's value must equal this exactly."
|
|
635
|
+
},
|
|
636
|
+
"enabled": {
|
|
637
|
+
"type": "boolean",
|
|
638
|
+
"description": "The element's enabled state must equal this."
|
|
639
|
+
}
|
|
640
|
+
},
|
|
641
|
+
"required": [],
|
|
642
|
+
"additionalProperties": false
|
|
643
|
+
}
|
|
644
|
+
},
|
|
645
|
+
"timeout_ms": {
|
|
646
|
+
"type": "integer",
|
|
647
|
+
"description": "Wait up to this long for every check to hold (default 0: one fresh read)."
|
|
648
|
+
}
|
|
649
|
+
},
|
|
650
|
+
"required": [
|
|
651
|
+
"checks"
|
|
652
|
+
],
|
|
653
|
+
"additionalProperties": false
|
|
654
|
+
},
|
|
655
|
+
"request_human": {
|
|
656
|
+
"type": "object",
|
|
657
|
+
"properties": {
|
|
658
|
+
"reason": {
|
|
659
|
+
"type": "string",
|
|
660
|
+
"description": "What the person should do, shown in the live view (e.g. 'Complete the CAPTCHA')."
|
|
661
|
+
},
|
|
662
|
+
"timeout_ms": {
|
|
663
|
+
"type": "integer",
|
|
664
|
+
"description": "How long to wait for the person (default 300000, max 1800000)."
|
|
665
|
+
}
|
|
666
|
+
},
|
|
667
|
+
"required": [],
|
|
668
|
+
"additionalProperties": false
|
|
669
|
+
},
|
|
670
|
+
"use_credential": {
|
|
671
|
+
"type": "object",
|
|
672
|
+
"properties": {
|
|
673
|
+
"name": {
|
|
674
|
+
"type": "string",
|
|
675
|
+
"description": "The credential's name in the vault."
|
|
676
|
+
},
|
|
677
|
+
"kind": {
|
|
678
|
+
"type": "string",
|
|
679
|
+
"enum": [
|
|
680
|
+
"secret",
|
|
681
|
+
"totp"
|
|
682
|
+
],
|
|
683
|
+
"description": "secret: type the stored value; totp: generate and type a fresh TOTP code."
|
|
684
|
+
},
|
|
685
|
+
"source": {
|
|
686
|
+
"type": "string",
|
|
687
|
+
"enum": [
|
|
688
|
+
"vault",
|
|
689
|
+
"keychain",
|
|
690
|
+
"bitwarden",
|
|
691
|
+
"onepassword"
|
|
692
|
+
],
|
|
693
|
+
"description": "Where to resolve the credential from (default: the sealed vault, then this Mac's Keychain)."
|
|
694
|
+
},
|
|
695
|
+
"domain": {
|
|
696
|
+
"type": "string",
|
|
697
|
+
"description": "The app or domain the credential is being typed into; checked against the credential's binding."
|
|
698
|
+
}
|
|
699
|
+
},
|
|
700
|
+
"required": [
|
|
701
|
+
"name"
|
|
702
|
+
],
|
|
703
|
+
"additionalProperties": false
|
|
704
|
+
},
|
|
705
|
+
"run_subtask": {
|
|
706
|
+
"type": "object",
|
|
707
|
+
"properties": {
|
|
708
|
+
"goal": {
|
|
709
|
+
"type": "string",
|
|
710
|
+
"description": "What the subtask must achieve, in one or two sentences (e.g. 'Fill the sign-up form and submit it')."
|
|
711
|
+
},
|
|
712
|
+
"inputs": {
|
|
713
|
+
"type": "array",
|
|
714
|
+
"maxItems": 20,
|
|
715
|
+
"items": {
|
|
716
|
+
"type": "object",
|
|
717
|
+
"properties": {
|
|
718
|
+
"name": {
|
|
719
|
+
"type": "string",
|
|
720
|
+
"description": "What the value is (e.g. 'email', 'full name')."
|
|
721
|
+
},
|
|
722
|
+
"value": {
|
|
723
|
+
"type": "string",
|
|
724
|
+
"description": "The literal text to type."
|
|
725
|
+
}
|
|
726
|
+
},
|
|
727
|
+
"required": [
|
|
728
|
+
"name",
|
|
729
|
+
"value"
|
|
730
|
+
],
|
|
731
|
+
"additionalProperties": false
|
|
732
|
+
},
|
|
733
|
+
"description": "The only text the subtask may type. Not for secrets: use use_credential."
|
|
734
|
+
},
|
|
735
|
+
"success": {
|
|
736
|
+
"type": "array",
|
|
737
|
+
"minItems": 1,
|
|
738
|
+
"maxItems": 10,
|
|
739
|
+
"items": {
|
|
740
|
+
"type": "object",
|
|
741
|
+
"properties": {
|
|
742
|
+
"id": {
|
|
743
|
+
"type": "string",
|
|
744
|
+
"description": "An element id from read_ui."
|
|
745
|
+
},
|
|
746
|
+
"role": {
|
|
747
|
+
"type": "string",
|
|
748
|
+
"description": "An accessibility role, e.g. button, textfield, checkbox."
|
|
749
|
+
},
|
|
750
|
+
"name": {
|
|
751
|
+
"type": "string",
|
|
752
|
+
"description": "Substring match against the element's name/label."
|
|
753
|
+
},
|
|
754
|
+
"text": {
|
|
755
|
+
"type": "string",
|
|
756
|
+
"description": "Substring match against the element's displayed text or value."
|
|
757
|
+
},
|
|
758
|
+
"gone": {
|
|
759
|
+
"type": "boolean",
|
|
760
|
+
"description": "True: no element may match; false: at least one must."
|
|
761
|
+
},
|
|
762
|
+
"value": {
|
|
763
|
+
"type": "string",
|
|
764
|
+
"description": "The element's value must equal this exactly."
|
|
765
|
+
},
|
|
766
|
+
"enabled": {
|
|
767
|
+
"type": "boolean",
|
|
768
|
+
"description": "The element's enabled state must equal this."
|
|
769
|
+
},
|
|
770
|
+
"ask": {
|
|
771
|
+
"type": "string",
|
|
772
|
+
"description": "A fuzzy yes/no check in plain words (\"Is the confirmation message showing?\"), answered by a typed verify decision over the screen. Use only when role/name/text/value checks can't express it."
|
|
773
|
+
}
|
|
774
|
+
},
|
|
775
|
+
"required": [],
|
|
776
|
+
"additionalProperties": false
|
|
777
|
+
},
|
|
778
|
+
"description": "Checks that all hold when the goal is met, in verify syntax ({role?, name?, text?, value?, gone?, enabled?}) or {ask: '...'} for a fuzzy yes/no. Without checks the loop stops when a decision picks done."
|
|
779
|
+
},
|
|
780
|
+
"constraints": {
|
|
781
|
+
"type": "object",
|
|
782
|
+
"properties": {
|
|
783
|
+
"ops": {
|
|
784
|
+
"type": "array",
|
|
785
|
+
"items": {
|
|
786
|
+
"type": "string",
|
|
787
|
+
"enum": [
|
|
788
|
+
"click",
|
|
789
|
+
"set_value",
|
|
790
|
+
"select",
|
|
791
|
+
"press",
|
|
792
|
+
"focus"
|
|
793
|
+
]
|
|
794
|
+
},
|
|
795
|
+
"description": "Ops the subtask may use (default click, set_value, select, press)."
|
|
796
|
+
},
|
|
797
|
+
"avoid": {
|
|
798
|
+
"type": "array",
|
|
799
|
+
"maxItems": 20,
|
|
800
|
+
"items": {
|
|
801
|
+
"type": "string"
|
|
802
|
+
},
|
|
803
|
+
"description": "Never act on elements whose name contains any of these."
|
|
804
|
+
},
|
|
805
|
+
"allow_irreversible": {
|
|
806
|
+
"type": "boolean",
|
|
807
|
+
"description": "Let the subtask click buttons named like send, pay, buy, delete, publish (default false: those hand back to you)."
|
|
808
|
+
}
|
|
809
|
+
},
|
|
810
|
+
"required": [],
|
|
811
|
+
"additionalProperties": false
|
|
812
|
+
},
|
|
813
|
+
"max_steps": {
|
|
814
|
+
"type": "integer",
|
|
815
|
+
"description": "Most actions the subtask may take (default 12, at most 40)."
|
|
816
|
+
},
|
|
817
|
+
"budget_ms": {
|
|
818
|
+
"type": "integer",
|
|
819
|
+
"description": "Wall-clock budget in milliseconds (default 60000, at most 300000)."
|
|
820
|
+
},
|
|
821
|
+
"app": {
|
|
822
|
+
"type": "string",
|
|
823
|
+
"description": "App name or bundle id (macOS); default: the frontmost app."
|
|
824
|
+
},
|
|
825
|
+
"pid": {
|
|
826
|
+
"type": "integer"
|
|
827
|
+
},
|
|
828
|
+
"window_id": {
|
|
829
|
+
"type": "integer"
|
|
830
|
+
},
|
|
831
|
+
"query": {
|
|
832
|
+
"type": "string",
|
|
833
|
+
"description": "Only consider elements whose name or value contains this text."
|
|
834
|
+
},
|
|
835
|
+
"max_elements": {
|
|
836
|
+
"type": "integer",
|
|
837
|
+
"description": "Cap the elements read per step (default 120)."
|
|
838
|
+
},
|
|
839
|
+
"cache": {
|
|
840
|
+
"type": "string",
|
|
841
|
+
"enum": [
|
|
842
|
+
"auto",
|
|
843
|
+
"off",
|
|
844
|
+
"refresh"
|
|
845
|
+
],
|
|
846
|
+
"description": "Replay cache (default auto). auto: a subtask seen before (same goal, app and window, same input names) replays its recorded steps with no decisions, checking each with verify, and re-infers only a step whose element moved; a new one runs the decision loop and is recorded once its success checks hold. refresh: ignore the cached steps and record anew. off: neither read nor write the cache."
|
|
847
|
+
},
|
|
848
|
+
"save_as": {
|
|
849
|
+
"type": "string",
|
|
850
|
+
"description": "Save this subtask's recording as a named skill once it succeeds (overwrites a skill of that name). Run it later with run_skill and new input values."
|
|
851
|
+
},
|
|
852
|
+
"api_options": {
|
|
853
|
+
"type": "array",
|
|
854
|
+
"maxItems": 10,
|
|
855
|
+
"items": {
|
|
856
|
+
"type": "object",
|
|
857
|
+
"properties": {
|
|
858
|
+
"tool": {
|
|
859
|
+
"type": "string",
|
|
860
|
+
"description": "The MCP tool or API action you can call yourself (e.g. 'github.create_issue')."
|
|
861
|
+
},
|
|
862
|
+
"description": {
|
|
863
|
+
"type": "string",
|
|
864
|
+
"description": "What it does, in one line."
|
|
865
|
+
}
|
|
866
|
+
},
|
|
867
|
+
"required": [
|
|
868
|
+
"tool"
|
|
869
|
+
],
|
|
870
|
+
"additionalProperties": false
|
|
871
|
+
},
|
|
872
|
+
"description": "MCP tools or API actions you hold that could do this goal without the GUI. They are offered ahead of every GUI option on the first step; if the decision picks one, the subtask stops with status use_api and names it, and you make that call yourself. An API call beats GUI clicks when both exist."
|
|
873
|
+
},
|
|
874
|
+
"best_of": {
|
|
875
|
+
"type": "integer",
|
|
876
|
+
"description": "Behavior Best-of-N for high-value subtasks: run N rollouts (2-5) at once on N sandbox computers listed in computers, then a judge decision picks one by the rollouts' step narratives. Sandbox/cloud computers only, never the person's own machine. Returns the chosen rollout, why, and every rollout's narrative."
|
|
877
|
+
},
|
|
878
|
+
"computers": {
|
|
879
|
+
"type": "array",
|
|
880
|
+
"maxItems": 5,
|
|
881
|
+
"items": {
|
|
882
|
+
"type": "string"
|
|
883
|
+
},
|
|
884
|
+
"description": "With best_of: the sandbox computer ids the rollouts run on, one rollout per computer (at least best_of distinct ids)."
|
|
885
|
+
},
|
|
886
|
+
"max_cost_usd": {
|
|
887
|
+
"type": "number",
|
|
888
|
+
"description": "With best_of: the cap on decision spend for the whole call in USD (default 0.25, at most 5). The judge may use the planner-class tier only while under it."
|
|
889
|
+
}
|
|
890
|
+
},
|
|
891
|
+
"required": [
|
|
892
|
+
"goal"
|
|
893
|
+
],
|
|
894
|
+
"additionalProperties": false
|
|
895
|
+
},
|
|
896
|
+
"run_parallel": {
|
|
897
|
+
"type": "object",
|
|
898
|
+
"properties": {
|
|
899
|
+
"subtasks": {
|
|
900
|
+
"type": "array",
|
|
901
|
+
"minItems": 1,
|
|
902
|
+
"maxItems": 8,
|
|
903
|
+
"items": {
|
|
904
|
+
"type": "object",
|
|
905
|
+
"properties": {
|
|
906
|
+
"computer": {
|
|
907
|
+
"type": "string",
|
|
908
|
+
"description": "The computer id this subtask runs on (default: the computer this call is made on)."
|
|
909
|
+
},
|
|
910
|
+
"goal": {
|
|
911
|
+
"type": "string",
|
|
912
|
+
"description": "What the subtask must achieve, in one or two sentences (e.g. 'Fill the sign-up form and submit it')."
|
|
913
|
+
},
|
|
914
|
+
"inputs": {
|
|
915
|
+
"type": "array",
|
|
916
|
+
"maxItems": 20,
|
|
917
|
+
"items": {
|
|
918
|
+
"type": "object",
|
|
919
|
+
"properties": {
|
|
920
|
+
"name": {
|
|
921
|
+
"type": "string",
|
|
922
|
+
"description": "What the value is (e.g. 'email', 'full name')."
|
|
923
|
+
},
|
|
924
|
+
"value": {
|
|
925
|
+
"type": "string",
|
|
926
|
+
"description": "The literal text to type."
|
|
927
|
+
}
|
|
928
|
+
},
|
|
929
|
+
"required": [
|
|
930
|
+
"name",
|
|
931
|
+
"value"
|
|
932
|
+
],
|
|
933
|
+
"additionalProperties": false
|
|
934
|
+
},
|
|
935
|
+
"description": "The only text the subtask may type. Not for secrets: use use_credential."
|
|
936
|
+
},
|
|
937
|
+
"success": {
|
|
938
|
+
"type": "array",
|
|
939
|
+
"minItems": 1,
|
|
940
|
+
"maxItems": 10,
|
|
941
|
+
"items": {
|
|
942
|
+
"type": "object",
|
|
943
|
+
"properties": {
|
|
944
|
+
"id": {
|
|
945
|
+
"type": "string",
|
|
946
|
+
"description": "An element id from read_ui."
|
|
947
|
+
},
|
|
948
|
+
"role": {
|
|
949
|
+
"type": "string",
|
|
950
|
+
"description": "An accessibility role, e.g. button, textfield, checkbox."
|
|
951
|
+
},
|
|
952
|
+
"name": {
|
|
953
|
+
"type": "string",
|
|
954
|
+
"description": "Substring match against the element's name/label."
|
|
955
|
+
},
|
|
956
|
+
"text": {
|
|
957
|
+
"type": "string",
|
|
958
|
+
"description": "Substring match against the element's displayed text or value."
|
|
959
|
+
},
|
|
960
|
+
"gone": {
|
|
961
|
+
"type": "boolean",
|
|
962
|
+
"description": "True: no element may match; false: at least one must."
|
|
963
|
+
},
|
|
964
|
+
"value": {
|
|
965
|
+
"type": "string",
|
|
966
|
+
"description": "The element's value must equal this exactly."
|
|
967
|
+
},
|
|
968
|
+
"enabled": {
|
|
969
|
+
"type": "boolean",
|
|
970
|
+
"description": "The element's enabled state must equal this."
|
|
971
|
+
},
|
|
972
|
+
"ask": {
|
|
973
|
+
"type": "string",
|
|
974
|
+
"description": "A fuzzy yes/no check in plain words (\"Is the confirmation message showing?\"), answered by a typed verify decision over the screen. Use only when role/name/text/value checks can't express it."
|
|
975
|
+
}
|
|
976
|
+
},
|
|
977
|
+
"required": [],
|
|
978
|
+
"additionalProperties": false
|
|
979
|
+
},
|
|
980
|
+
"description": "Checks that all hold when the goal is met, in verify syntax ({role?, name?, text?, value?, gone?, enabled?}) or {ask: '...'} for a fuzzy yes/no. Without checks the loop stops when a decision picks done."
|
|
981
|
+
},
|
|
982
|
+
"constraints": {
|
|
983
|
+
"type": "object",
|
|
984
|
+
"properties": {
|
|
985
|
+
"ops": {
|
|
986
|
+
"type": "array",
|
|
987
|
+
"items": {
|
|
988
|
+
"type": "string",
|
|
989
|
+
"enum": [
|
|
990
|
+
"click",
|
|
991
|
+
"set_value",
|
|
992
|
+
"select",
|
|
993
|
+
"press",
|
|
994
|
+
"focus"
|
|
995
|
+
]
|
|
996
|
+
},
|
|
997
|
+
"description": "Ops the subtask may use (default click, set_value, select, press)."
|
|
998
|
+
},
|
|
999
|
+
"avoid": {
|
|
1000
|
+
"type": "array",
|
|
1001
|
+
"maxItems": 20,
|
|
1002
|
+
"items": {
|
|
1003
|
+
"type": "string"
|
|
1004
|
+
},
|
|
1005
|
+
"description": "Never act on elements whose name contains any of these."
|
|
1006
|
+
},
|
|
1007
|
+
"allow_irreversible": {
|
|
1008
|
+
"type": "boolean",
|
|
1009
|
+
"description": "Let the subtask click buttons named like send, pay, buy, delete, publish (default false: those hand back to you)."
|
|
1010
|
+
}
|
|
1011
|
+
},
|
|
1012
|
+
"required": [],
|
|
1013
|
+
"additionalProperties": false
|
|
1014
|
+
},
|
|
1015
|
+
"max_steps": {
|
|
1016
|
+
"type": "integer",
|
|
1017
|
+
"description": "Most actions the subtask may take (default 12, at most 40)."
|
|
1018
|
+
},
|
|
1019
|
+
"budget_ms": {
|
|
1020
|
+
"type": "integer",
|
|
1021
|
+
"description": "Wall-clock budget in milliseconds (default 60000, at most 300000)."
|
|
1022
|
+
},
|
|
1023
|
+
"app": {
|
|
1024
|
+
"type": "string",
|
|
1025
|
+
"description": "App name or bundle id (macOS); default: the frontmost app."
|
|
1026
|
+
},
|
|
1027
|
+
"pid": {
|
|
1028
|
+
"type": "integer"
|
|
1029
|
+
},
|
|
1030
|
+
"window_id": {
|
|
1031
|
+
"type": "integer"
|
|
1032
|
+
},
|
|
1033
|
+
"query": {
|
|
1034
|
+
"type": "string",
|
|
1035
|
+
"description": "Only consider elements whose name or value contains this text."
|
|
1036
|
+
},
|
|
1037
|
+
"max_elements": {
|
|
1038
|
+
"type": "integer",
|
|
1039
|
+
"description": "Cap the elements read per step (default 120)."
|
|
1040
|
+
},
|
|
1041
|
+
"api_options": {
|
|
1042
|
+
"type": "array",
|
|
1043
|
+
"maxItems": 10,
|
|
1044
|
+
"items": {
|
|
1045
|
+
"type": "object",
|
|
1046
|
+
"properties": {
|
|
1047
|
+
"tool": {
|
|
1048
|
+
"type": "string",
|
|
1049
|
+
"description": "The MCP tool or API action you can call yourself (e.g. 'github.create_issue')."
|
|
1050
|
+
},
|
|
1051
|
+
"description": {
|
|
1052
|
+
"type": "string",
|
|
1053
|
+
"description": "What it does, in one line."
|
|
1054
|
+
}
|
|
1055
|
+
},
|
|
1056
|
+
"required": [
|
|
1057
|
+
"tool"
|
|
1058
|
+
],
|
|
1059
|
+
"additionalProperties": false
|
|
1060
|
+
},
|
|
1061
|
+
"description": "MCP tools or API actions you hold that could do this goal without the GUI. They are offered ahead of every GUI option on the first step; if the decision picks one, the subtask stops with status use_api and names it, and you make that call yourself. An API call beats GUI clicks when both exist."
|
|
1062
|
+
}
|
|
1063
|
+
},
|
|
1064
|
+
"required": [
|
|
1065
|
+
"goal"
|
|
1066
|
+
],
|
|
1067
|
+
"additionalProperties": false
|
|
1068
|
+
},
|
|
1069
|
+
"description": "Independent subtasks, one per computer. Same fields as run_subtask plus computer."
|
|
1070
|
+
},
|
|
1071
|
+
"max_parallel": {
|
|
1072
|
+
"type": "integer",
|
|
1073
|
+
"description": "Most subtasks running at once (default 4, at most 8)."
|
|
1074
|
+
},
|
|
1075
|
+
"budget_ms": {
|
|
1076
|
+
"type": "integer",
|
|
1077
|
+
"description": "Wall-clock budget for the whole call in milliseconds (default 120000, at most 600000). Each subtask's own budget_ms is cut to what is left when it starts."
|
|
1078
|
+
},
|
|
1079
|
+
"max_cost_usd": {
|
|
1080
|
+
"type": "number",
|
|
1081
|
+
"description": "Cap on decision spend for the whole call in USD (default 0.25, at most 5). Once reached, subtasks not yet started come back skipped."
|
|
1082
|
+
}
|
|
1083
|
+
},
|
|
1084
|
+
"required": [
|
|
1085
|
+
"subtasks"
|
|
1086
|
+
],
|
|
1087
|
+
"additionalProperties": false
|
|
1088
|
+
},
|
|
1089
|
+
"run_skill": {
|
|
1090
|
+
"type": "object",
|
|
1091
|
+
"properties": {
|
|
1092
|
+
"name": {
|
|
1093
|
+
"type": "string",
|
|
1094
|
+
"description": "The skill's name (from skills)."
|
|
1095
|
+
},
|
|
1096
|
+
"inputs": {
|
|
1097
|
+
"type": "array",
|
|
1098
|
+
"maxItems": 20,
|
|
1099
|
+
"items": {
|
|
1100
|
+
"type": "object",
|
|
1101
|
+
"properties": {
|
|
1102
|
+
"name": {
|
|
1103
|
+
"type": "string",
|
|
1104
|
+
"description": "What the value is (e.g. 'email', 'full name')."
|
|
1105
|
+
},
|
|
1106
|
+
"value": {
|
|
1107
|
+
"type": "string",
|
|
1108
|
+
"description": "The literal text to type."
|
|
1109
|
+
}
|
|
1110
|
+
},
|
|
1111
|
+
"required": [
|
|
1112
|
+
"name",
|
|
1113
|
+
"value"
|
|
1114
|
+
],
|
|
1115
|
+
"additionalProperties": false
|
|
1116
|
+
},
|
|
1117
|
+
"description": "A value for each of the skill's inputs (skills lists their names). Not for secrets: use use_credential."
|
|
1118
|
+
},
|
|
1119
|
+
"app": {
|
|
1120
|
+
"type": "string",
|
|
1121
|
+
"description": "App name or bundle id (macOS); default: the frontmost app."
|
|
1122
|
+
},
|
|
1123
|
+
"pid": {
|
|
1124
|
+
"type": "integer"
|
|
1125
|
+
},
|
|
1126
|
+
"window_id": {
|
|
1127
|
+
"type": "integer"
|
|
1128
|
+
},
|
|
1129
|
+
"max_steps": {
|
|
1130
|
+
"type": "integer",
|
|
1131
|
+
"description": "Most actions the subtask may take (default 12, at most 40)."
|
|
1132
|
+
},
|
|
1133
|
+
"budget_ms": {
|
|
1134
|
+
"type": "integer",
|
|
1135
|
+
"description": "Wall-clock budget in milliseconds (default 60000, at most 300000)."
|
|
1136
|
+
}
|
|
1137
|
+
},
|
|
1138
|
+
"required": [
|
|
1139
|
+
"name"
|
|
1140
|
+
],
|
|
1141
|
+
"additionalProperties": false
|
|
1142
|
+
},
|
|
1143
|
+
"skills": {
|
|
1144
|
+
"type": "object",
|
|
1145
|
+
"properties": {
|
|
1146
|
+
"forget": {
|
|
1147
|
+
"type": "string",
|
|
1148
|
+
"description": "Delete the saved skill with this name before listing."
|
|
1149
|
+
}
|
|
1150
|
+
},
|
|
1151
|
+
"required": [],
|
|
1152
|
+
"additionalProperties": false
|
|
1153
|
+
}
|
|
1154
|
+
};
|
|
1155
|
+
export const COMPUTER_V2_STRUCTURED_MEMBERS = [
|
|
1156
|
+
{
|
|
1157
|
+
"name": "read_ui",
|
|
1158
|
+
"description": "Read the target window or app's UI as a structured element tree (roles, names, values, bounds) with NO screenshot. Returns stable element ids for act, run_batch and verify. Pass `since` (a version from an earlier read_ui) to get only the diff; `query` filters by text; `max_elements` caps the list (default 200). When the window has no usable accessibility tree (canvas apps, games, remote desktops, custom-drawn UIs) the read falls back to vision: numbered set-of-marks regions come back as elements with `source: \"vision\"` and a `mark` number, and act on their ids works the same way. `source` and `source_reason` say which was used. Pass `target` (a description such as 'the Export button') to ground it with the local grounder: `grounded.id` is the element to act on.",
|
|
1159
|
+
"risk": "reversible",
|
|
1160
|
+
"default_enabled": true,
|
|
1161
|
+
"needs_confirm": false,
|
|
1162
|
+
"confirm": "never",
|
|
1163
|
+
"result_kind": "none",
|
|
1164
|
+
"ack_text": "Read the UI of {app}.",
|
|
1165
|
+
"scale_fields": {}
|
|
1166
|
+
},
|
|
1167
|
+
{
|
|
1168
|
+
"name": "act",
|
|
1169
|
+
"description": "Act on an element id from read_ui (tree ids and vision ids alike; a vision element is clicked or typed into at its centre after a pixel freshness check, and menu needs a window). op: click, set_value (clear and set the field's value), select (pick a child by name), press (a key), menu (choose a menu path such as 'File > Save'), focus. set_value/select/press on this Mac need the person's approval (the same rule as typing).",
|
|
1170
|
+
"risk": "risky",
|
|
1171
|
+
"default_enabled": true,
|
|
1172
|
+
"needs_confirm": false,
|
|
1173
|
+
"confirm": "never",
|
|
1174
|
+
"result_kind": "none",
|
|
1175
|
+
"ack_text": "{op} on {id}.",
|
|
1176
|
+
"scale_fields": {}
|
|
1177
|
+
},
|
|
1178
|
+
{
|
|
1179
|
+
"name": "run_batch",
|
|
1180
|
+
"description": "Run ordered steps in one window in a single call: act steps ({id, op, value?, key?}), menu steps (path), pixel steps ({tool, args}) and waits (ms). Each step may carry wait_for (checked before the step) and expect (checked after); a failed check stops the batch and one observe read returns the fresh map. This is the fast path: batch several steps instead of screenshot-per-action loops.",
|
|
1181
|
+
"risk": "risky",
|
|
1182
|
+
"default_enabled": true,
|
|
1183
|
+
"needs_confirm": true,
|
|
1184
|
+
"confirm": "non_sandbox",
|
|
1185
|
+
"result_kind": "none",
|
|
1186
|
+
"ack_text": "Ran {steps} batch steps.",
|
|
1187
|
+
"scale_fields": {}
|
|
1188
|
+
},
|
|
1189
|
+
{
|
|
1190
|
+
"name": "verify",
|
|
1191
|
+
"description": "Check bounded conditions on the current UI without acting: role, name, text, gone, value, enabled. Each check is {id?, role?, name?, text?, gone?, value?, enabled?}; all given parts must hold. Returns one ok per check. Use after act/run_batch, or to poll with timeout_ms.",
|
|
1192
|
+
"risk": "reversible",
|
|
1193
|
+
"default_enabled": true,
|
|
1194
|
+
"needs_confirm": false,
|
|
1195
|
+
"confirm": "never",
|
|
1196
|
+
"result_kind": "none",
|
|
1197
|
+
"ack_text": "Verified.",
|
|
1198
|
+
"scale_fields": {}
|
|
1199
|
+
},
|
|
1200
|
+
{
|
|
1201
|
+
"name": "request_human",
|
|
1202
|
+
"description": "Pause and hand the computer to a person: the live view opens with input enabled and the agent's hold on control is released. The same session resumes when the person signals done, or when timeout_ms elapses. Use for CAPTCHAs, push/SMS 2FA and judgment calls.",
|
|
1203
|
+
"risk": "reversible",
|
|
1204
|
+
"default_enabled": true,
|
|
1205
|
+
"needs_confirm": false,
|
|
1206
|
+
"confirm": "never",
|
|
1207
|
+
"result_kind": "none",
|
|
1208
|
+
"ack_text": "Human window closed ({reason}).",
|
|
1209
|
+
"scale_fields": {}
|
|
1210
|
+
},
|
|
1211
|
+
{
|
|
1212
|
+
"name": "use_credential",
|
|
1213
|
+
"description": "Type a vault credential into the FOCUSED field. The value never enters the model context, prompts or logs. kind: secret (the stored value) or totp (a fresh code from a stored TOTP seed). The credential is bound to an app or domain: pass `domain` (the app you are filling into); it must match the credential's binding. On this Mac a person approves each use, like typing.",
|
|
1214
|
+
"risk": "risky",
|
|
1215
|
+
"default_enabled": true,
|
|
1216
|
+
"needs_confirm": true,
|
|
1217
|
+
"confirm": "non_sandbox",
|
|
1218
|
+
"result_kind": "none",
|
|
1219
|
+
"ack_text": "Credential {name} typed.",
|
|
1220
|
+
"scale_fields": {}
|
|
1221
|
+
},
|
|
1222
|
+
{
|
|
1223
|
+
"name": "run_subtask",
|
|
1224
|
+
"description": "Hand a bounded UI subtask (fill a form, search and pick a result, toggle settings) to the fast decision loop instead of driving each step yourself. Give the goal, the literal inputs it may type (it never invents text), success checks in verify syntax, and limits. Each step reads the element map, picks one action from the visible elements x allowed ops with a ~100 ms typed decision (POST /v1/decisions), runs it through the same lease/approval/audit path (predictable steps batched with expect checks), and checks success with verify. Returns a step trace and a status: done, escalated (with the reason and the current screen: continue from there yourself), or failed. One approval covers the whole subtask on this computer. Repeat subtasks replay from the cache with zero decisions (the result's cache block says hit, healed, recorded or miss); only subtasks with at least one exact (non-ask) success check are cached. Input values are never stored, only their names. With api_options, an MCP/API call you hold is offered ahead of GUI steps (status use_api hands it back to you). With best_of N and computers, N rollouts run on N sandbox computers and a judge picks one by step narratives.",
|
|
1225
|
+
"risk": "risky",
|
|
1226
|
+
"default_enabled": true,
|
|
1227
|
+
"needs_confirm": true,
|
|
1228
|
+
"confirm": "non_sandbox",
|
|
1229
|
+
"result_kind": "none",
|
|
1230
|
+
"ack_text": "Ran the subtask.",
|
|
1231
|
+
"scale_fields": {}
|
|
1232
|
+
},
|
|
1233
|
+
{
|
|
1234
|
+
"name": "run_parallel",
|
|
1235
|
+
"description": "Run independent bounded subtasks on separate computers at the same time (one run_subtask per computer, each under its own computer's control lease), and get every result back in order. Each subtask names its computer (default: this one) and takes run_subtask's fields. One computer never runs two subtasks at once: a repeated computer, or one another subtask is already driving, is refused. Limits: max_parallel subtasks at once, a total budget_ms, and max_cost_usd on decision spend; subtasks that can't start inside them come back skipped. One approval covers the whole call when any subtask targets a non-sandbox computer.",
|
|
1236
|
+
"risk": "risky",
|
|
1237
|
+
"default_enabled": true,
|
|
1238
|
+
"needs_confirm": true,
|
|
1239
|
+
"confirm": "non_sandbox",
|
|
1240
|
+
"result_kind": "none",
|
|
1241
|
+
"ack_text": "Ran the subtasks.",
|
|
1242
|
+
"scale_fields": {}
|
|
1243
|
+
},
|
|
1244
|
+
{
|
|
1245
|
+
"name": "run_skill",
|
|
1246
|
+
"description": "Run a saved skill (a recording taught with run_subtask save_as) by name, with new values for its inputs. It replays the recorded steps with no decisions, checks each step and the skill's success checks with verify, re-infers a step whose element moved and rewrites the recording. Returns the same result as run_subtask. List skills with the skills member.",
|
|
1247
|
+
"risk": "risky",
|
|
1248
|
+
"default_enabled": true,
|
|
1249
|
+
"needs_confirm": true,
|
|
1250
|
+
"confirm": "non_sandbox",
|
|
1251
|
+
"result_kind": "none",
|
|
1252
|
+
"ack_text": "Ran the skill {name}.",
|
|
1253
|
+
"scale_fields": {}
|
|
1254
|
+
},
|
|
1255
|
+
{
|
|
1256
|
+
"name": "skills",
|
|
1257
|
+
"description": "List the saved skills (taught recordings) this person can run with run_skill: name, goal, app, input names, step count, last use. Pass forget to delete one first.",
|
|
1258
|
+
"risk": "reversible",
|
|
1259
|
+
"default_enabled": true,
|
|
1260
|
+
"needs_confirm": false,
|
|
1261
|
+
"confirm": "never",
|
|
1262
|
+
"result_kind": "none",
|
|
1263
|
+
"ack_text": "Listed the saved skills.",
|
|
1264
|
+
"scale_fields": {}
|
|
1265
|
+
}
|
|
1266
|
+
];
|