@allternit/computer-driver 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1445 @@
1
+ export declare const COMPUTER_V2_MEMBER_NAMES: readonly ["screenshot", "zoom", "left_click", "right_click", "middle_click", "double_click", "triple_click", "mouse_move", "left_click_drag", "left_mouse_down", "left_mouse_up", "scroll", "type", "key", "hold_key", "wait", "cursor_position", "read_ui", "act", "run_batch", "verify", "request_human", "use_credential", "run_subtask", "run_parallel", "run_skill", "skills"];
2
+ export type ComputerV2MemberName = (typeof COMPUTER_V2_MEMBER_NAMES)[number];
3
+ /** The ten driver-backed structured members (read_ui … skills). */
4
+ export declare const COMPUTER_V2_STRUCTURED_MEMBER_NAMES: readonly ["read_ui", "act", "run_batch", "verify", "request_human", "use_credential", "run_subtask", "run_parallel", "run_skill", "skills"];
5
+ export type ComputerV2StructuredMemberName = (typeof COMPUTER_V2_STRUCTURED_MEMBER_NAMES)[number];
6
+ /** Take a screenshot of the screen. */
7
+ export type ComputerV2ScreenshotInput = Record<string, unknown>;
8
+ /** Take a screenshot of a rectangular region. Region coordinates are in the full-screenshot space (not physical display pixels). The crop is scaled up to fill the image budget so fine details become legible. */
9
+ export type ComputerV2ZoomInput = {
10
+ region: Array<number>;
11
+ };
12
+ /** Click the left mouse button at the specified (x, y) pixel coordinate, or the current cursor position if `coordinate` is omitted. */
13
+ export type ComputerV2LeftClickInput = {
14
+ coordinate?: Array<number> | null;
15
+ text?: string | null;
16
+ };
17
+ /** Click the right mouse button at the specified (x, y) pixel coordinate, or the current cursor position if `coordinate` is omitted. */
18
+ export type ComputerV2RightClickInput = {
19
+ coordinate?: Array<number> | null;
20
+ text?: string | null;
21
+ };
22
+ /** Click the middle mouse button at the specified (x, y) pixel coordinate, or the current cursor position if `coordinate` is omitted. */
23
+ export type ComputerV2MiddleClickInput = {
24
+ coordinate?: Array<number> | null;
25
+ text?: string | null;
26
+ };
27
+ /** Double-click the left mouse button at the specified (x, y) pixel coordinate, or the current cursor position if `coordinate` is omitted. */
28
+ export type ComputerV2DoubleClickInput = {
29
+ coordinate?: Array<number> | null;
30
+ text?: string | null;
31
+ };
32
+ /** Triple-click the left mouse button at the specified (x, y) pixel coordinate, or the current cursor position if `coordinate` is omitted. */
33
+ export type ComputerV2TripleClickInput = {
34
+ coordinate?: Array<number> | null;
35
+ text?: string | null;
36
+ };
37
+ /** Move the cursor to a specified (x, y) pixel coordinate. Use this ONLY to hover without clicking; otherwise use a click action directly. */
38
+ export type ComputerV2MouseMoveInput = {
39
+ coordinate: Array<number>;
40
+ };
41
+ /** Click and drag the cursor from `start_coordinate` to `coordinate`. */
42
+ export type ComputerV2LeftClickDragInput = {
43
+ coordinate: Array<number>;
44
+ start_coordinate: Array<number>;
45
+ text?: string | null;
46
+ };
47
+ /** Press and hold the left mouse button at the current cursor position. */
48
+ export type ComputerV2LeftMouseDownInput = Record<string, unknown>;
49
+ /** Release the left mouse button. */
50
+ export type ComputerV2LeftMouseUpInput = Record<string, unknown>;
51
+ /** Scroll the screen at the specified (x, y) pixel coordinate, or the current cursor position if `coordinate` is omitted. Do NOT use PageUp/PageDown to scroll. */
52
+ export type ComputerV2ScrollInput = {
53
+ scroll_amount: number;
54
+ scroll_direction: "up" | "down" | "left" | "right";
55
+ coordinate?: Array<number> | null;
56
+ text?: string | null;
57
+ };
58
+ /** Type a string of text on the keyboard. */
59
+ export type ComputerV2TypeInput = {
60
+ text: string;
61
+ };
62
+ /** Press a key or key-combination on the keyboard. Use "+" to combine modifiers with a key (e.g. "ctrl+s", "alt+Tab", "ctrl+shift+Escape"). Key names are case-insensitive; common names like "Return", "Tab", "Escape", "Up", "Down", "Left", "Right", "Home", "End", "Page_Up", "Page_Down", "Delete", "BackSpace" are supported. */
63
+ export type ComputerV2KeyInput = {
64
+ text: string;
65
+ repeat?: number | null;
66
+ };
67
+ /** Hold down a key or key-combination for a specified duration. Uses the same key syntax as `key`. */
68
+ export type ComputerV2HoldKeyInput = {
69
+ duration: number;
70
+ text: string;
71
+ };
72
+ /** Wait for a specified duration. */
73
+ export type ComputerV2WaitInput = {
74
+ duration: number;
75
+ };
76
+ /** Get the current (x, y) pixel coordinate of the cursor. */
77
+ export type ComputerV2CursorPositionInput = Record<string, unknown>;
78
+ /** Read the target window or app's UI as a structured element tree (roles, names, values, bounds) with NO screenshot. Returns stable element ids for act, run_batch and verify. Pass `since` (a version from an earlier read_ui) to get only the diff; `query` filters by text; `max_elements` caps the list (default 200). When the window has no usable accessibility tree (canvas apps, games, remote desktops, custom-drawn UIs) the read falls back to vision: numbered set-of-marks regions come back as elements with `source: "vision"` and a `mark` number, and act on their ids works the same way. `source` and `source_reason` say which was used. Pass `target` (a description such as 'the Export button') to ground it with the local grounder: `grounded.id` is the element to act on. */
79
+ export type ComputerV2ReadUiInput = {
80
+ app?: string | null;
81
+ pid?: number | null;
82
+ window_id?: number | null;
83
+ query?: string | null;
84
+ max_elements?: number | null;
85
+ since?: number | null;
86
+ paths?: boolean | null;
87
+ crops?: boolean | null;
88
+ vision?: "auto" | "off" | "only" | null;
89
+ target?: string | null;
90
+ zoom?: "auto" | "always" | "never" | null;
91
+ };
92
+ /** Act on an element id from read_ui (tree ids and vision ids alike; a vision element is clicked or typed into at its centre after a pixel freshness check, and menu needs a window). op: click, set_value (clear and set the field's value), select (pick a child by name), press (a key), menu (choose a menu path such as 'File > Save'), focus. set_value/select/press on this Mac need the person's approval (the same rule as typing). */
93
+ export type ComputerV2ActInput = {
94
+ id: string;
95
+ op: "click" | "set_value" | "select" | "press" | "menu" | "focus";
96
+ value?: string | null;
97
+ key?: string | null;
98
+ path?: string | Array<string> | null;
99
+ version?: number | null;
100
+ app?: string | null;
101
+ pid?: number | null;
102
+ window_id?: number | null;
103
+ };
104
+ /** Run ordered steps in one window in a single call: act steps ({id, op, value?, key?}), menu steps (path), pixel steps ({tool, args}) and waits (ms). Each step may carry wait_for (checked before the step) and expect (checked after); a failed check stops the batch and one observe read returns the fresh map. This is the fast path: batch several steps instead of screenshot-per-action loops. */
105
+ export type ComputerV2RunBatchInput = {
106
+ app?: string | null;
107
+ pid?: number | null;
108
+ window_id?: number | null;
109
+ version?: number | null;
110
+ steps: Array<{
111
+ act?: {
112
+ id: string;
113
+ op: "click" | "set_value" | "select" | "press" | "menu" | "focus";
114
+ value?: string | null;
115
+ key?: string | null;
116
+ } | null;
117
+ menu?: string | Array<string> | null;
118
+ pixel?: {
119
+ tool: string;
120
+ args?: Record<string, unknown> | null;
121
+ } | null;
122
+ wait?: number | null;
123
+ wait_for?: {
124
+ id?: string | null;
125
+ role?: string | null;
126
+ name?: string | null;
127
+ text?: string | null;
128
+ gone?: boolean | null;
129
+ value?: string | null;
130
+ enabled?: boolean | null;
131
+ } | null;
132
+ expect?: {
133
+ id?: string | null;
134
+ role?: string | null;
135
+ name?: string | null;
136
+ text?: string | null;
137
+ gone?: boolean | null;
138
+ value?: string | null;
139
+ enabled?: boolean | null;
140
+ } | null;
141
+ timeout_ms?: number | null;
142
+ }>;
143
+ };
144
+ /** Check bounded conditions on the current UI without acting: role, name, text, gone, value, enabled. Each check is {id?, role?, name?, text?, gone?, value?, enabled?}; all given parts must hold. Returns one ok per check. Use after act/run_batch, or to poll with timeout_ms. */
145
+ export type ComputerV2VerifyInput = {
146
+ app?: string | null;
147
+ pid?: number | null;
148
+ window_id?: number | null;
149
+ checks: Array<{
150
+ id?: string | null;
151
+ role?: string | null;
152
+ name?: string | null;
153
+ text?: string | null;
154
+ gone?: boolean | null;
155
+ value?: string | null;
156
+ enabled?: boolean | null;
157
+ }>;
158
+ timeout_ms?: number | null;
159
+ };
160
+ /** Pause and hand the computer to a person: the live view opens with input enabled and the agent's hold on control is released. The same session resumes when the person signals done, or when timeout_ms elapses. Use for CAPTCHAs, push/SMS 2FA and judgment calls. */
161
+ export type ComputerV2RequestHumanInput = {
162
+ reason?: string | null;
163
+ timeout_ms?: number | null;
164
+ };
165
+ /** Type a vault credential into the FOCUSED field. The value never enters the model context, prompts or logs. kind: secret (the stored value) or totp (a fresh code from a stored TOTP seed). The credential is bound to an app or domain: pass `domain` (the app you are filling into); it must match the credential's binding. On this Mac a person approves each use, like typing. */
166
+ export type ComputerV2UseCredentialInput = {
167
+ name: string;
168
+ kind?: "secret" | "totp" | null;
169
+ source?: "vault" | "keychain" | "bitwarden" | "onepassword" | null;
170
+ domain?: string | null;
171
+ };
172
+ /** Hand a bounded UI subtask (fill a form, search and pick a result, toggle settings) to the fast decision loop instead of driving each step yourself. Give the goal, the literal inputs it may type (it never invents text), success checks in verify syntax, and limits. Each step reads the element map, picks one action from the visible elements x allowed ops with a ~100 ms typed decision (POST /v1/decisions), runs it through the same lease/approval/audit path (predictable steps batched with expect checks), and checks success with verify. Returns a step trace and a status: done, escalated (with the reason and the current screen: continue from there yourself), or failed. One approval covers the whole subtask on this computer. Repeat subtasks replay from the cache with zero decisions (the result's cache block says hit, healed, recorded or miss); only subtasks with at least one exact (non-ask) success check are cached. Input values are never stored, only their names. With api_options, an MCP/API call you hold is offered ahead of GUI steps (status use_api hands it back to you). With best_of N and computers, N rollouts run on N sandbox computers and a judge picks one by step narratives. */
173
+ export type ComputerV2RunSubtaskInput = {
174
+ goal: string;
175
+ inputs?: Array<{
176
+ name: string;
177
+ value: string;
178
+ }> | null;
179
+ success?: Array<{
180
+ id?: string | null;
181
+ role?: string | null;
182
+ name?: string | null;
183
+ text?: string | null;
184
+ gone?: boolean | null;
185
+ value?: string | null;
186
+ enabled?: boolean | null;
187
+ ask?: string | null;
188
+ }> | null;
189
+ constraints?: {
190
+ ops?: Array<"click" | "set_value" | "select" | "press" | "focus"> | null;
191
+ avoid?: Array<string> | null;
192
+ allow_irreversible?: boolean | null;
193
+ } | null;
194
+ max_steps?: number | null;
195
+ budget_ms?: number | null;
196
+ app?: string | null;
197
+ pid?: number | null;
198
+ window_id?: number | null;
199
+ query?: string | null;
200
+ max_elements?: number | null;
201
+ cache?: "auto" | "off" | "refresh" | null;
202
+ save_as?: string | null;
203
+ api_options?: Array<{
204
+ tool: string;
205
+ description?: string | null;
206
+ }> | null;
207
+ best_of?: number | null;
208
+ computers?: Array<string> | null;
209
+ max_cost_usd?: number | null;
210
+ };
211
+ /** Run independent bounded subtasks on separate computers at the same time (one run_subtask per computer, each under its own computer's control lease), and get every result back in order. Each subtask names its computer (default: this one) and takes run_subtask's fields. One computer never runs two subtasks at once: a repeated computer, or one another subtask is already driving, is refused. Limits: max_parallel subtasks at once, a total budget_ms, and max_cost_usd on decision spend; subtasks that can't start inside them come back skipped. One approval covers the whole call when any subtask targets a non-sandbox computer. */
212
+ export type ComputerV2RunParallelInput = {
213
+ subtasks: Array<{
214
+ computer?: string | null;
215
+ goal: string;
216
+ inputs?: Array<{
217
+ name: string;
218
+ value: string;
219
+ }> | null;
220
+ success?: Array<{
221
+ id?: string | null;
222
+ role?: string | null;
223
+ name?: string | null;
224
+ text?: string | null;
225
+ gone?: boolean | null;
226
+ value?: string | null;
227
+ enabled?: boolean | null;
228
+ ask?: string | null;
229
+ }> | null;
230
+ constraints?: {
231
+ ops?: Array<"click" | "set_value" | "select" | "press" | "focus"> | null;
232
+ avoid?: Array<string> | null;
233
+ allow_irreversible?: boolean | null;
234
+ } | null;
235
+ max_steps?: number | null;
236
+ budget_ms?: number | null;
237
+ app?: string | null;
238
+ pid?: number | null;
239
+ window_id?: number | null;
240
+ query?: string | null;
241
+ max_elements?: number | null;
242
+ api_options?: Array<{
243
+ tool: string;
244
+ description?: string | null;
245
+ }> | null;
246
+ }>;
247
+ max_parallel?: number | null;
248
+ budget_ms?: number | null;
249
+ max_cost_usd?: number | null;
250
+ };
251
+ /** Run a saved skill (a recording taught with run_subtask save_as) by name, with new values for its inputs. It replays the recorded steps with no decisions, checks each step and the skill's success checks with verify, re-infers a step whose element moved and rewrites the recording. Returns the same result as run_subtask. List skills with the skills member. */
252
+ export type ComputerV2RunSkillInput = {
253
+ name: string;
254
+ inputs?: Array<{
255
+ name: string;
256
+ value: string;
257
+ }> | null;
258
+ app?: string | null;
259
+ pid?: number | null;
260
+ window_id?: number | null;
261
+ max_steps?: number | null;
262
+ budget_ms?: number | null;
263
+ };
264
+ /** List the saved skills (taught recordings) this person can run with run_skill: name, goal, app, input names, step count, last use. Pass forget to delete one first. */
265
+ export type ComputerV2SkillsInput = {
266
+ forget?: string | null;
267
+ };
268
+ export interface ComputerV2MemberInputs {
269
+ screenshot: ComputerV2ScreenshotInput;
270
+ zoom: ComputerV2ZoomInput;
271
+ left_click: ComputerV2LeftClickInput;
272
+ right_click: ComputerV2RightClickInput;
273
+ middle_click: ComputerV2MiddleClickInput;
274
+ double_click: ComputerV2DoubleClickInput;
275
+ triple_click: ComputerV2TripleClickInput;
276
+ mouse_move: ComputerV2MouseMoveInput;
277
+ left_click_drag: ComputerV2LeftClickDragInput;
278
+ left_mouse_down: ComputerV2LeftMouseDownInput;
279
+ left_mouse_up: ComputerV2LeftMouseUpInput;
280
+ scroll: ComputerV2ScrollInput;
281
+ type: ComputerV2TypeInput;
282
+ key: ComputerV2KeyInput;
283
+ hold_key: ComputerV2HoldKeyInput;
284
+ wait: ComputerV2WaitInput;
285
+ cursor_position: ComputerV2CursorPositionInput;
286
+ read_ui: ComputerV2ReadUiInput;
287
+ act: ComputerV2ActInput;
288
+ run_batch: ComputerV2RunBatchInput;
289
+ verify: ComputerV2VerifyInput;
290
+ request_human: ComputerV2RequestHumanInput;
291
+ use_credential: ComputerV2UseCredentialInput;
292
+ run_subtask: ComputerV2RunSubtaskInput;
293
+ run_parallel: ComputerV2RunParallelInput;
294
+ run_skill: ComputerV2RunSkillInput;
295
+ skills: ComputerV2SkillsInput;
296
+ }
297
+ /** The per-member JSON Schemas, for building provider function tools. */
298
+ export declare const COMPUTER_V2_TOOL_SCHEMAS: {
299
+ readonly screenshot: {
300
+ readonly type: "object";
301
+ readonly properties: {};
302
+ readonly required: readonly [];
303
+ readonly additionalProperties: false;
304
+ };
305
+ readonly zoom: {
306
+ readonly type: "object";
307
+ readonly properties: {
308
+ readonly region: {
309
+ readonly type: "array";
310
+ readonly items: {
311
+ readonly type: "number";
312
+ };
313
+ readonly minItems: 4;
314
+ readonly maxItems: 4;
315
+ readonly description: "(x0, y0, x1, y1): The region to capture.";
316
+ };
317
+ };
318
+ readonly required: readonly ["region"];
319
+ readonly additionalProperties: false;
320
+ };
321
+ readonly left_click: {
322
+ readonly type: "object";
323
+ readonly properties: {
324
+ readonly coordinate: {
325
+ readonly type: "array";
326
+ readonly items: {
327
+ readonly type: "number";
328
+ };
329
+ readonly minItems: 2;
330
+ readonly maxItems: 2;
331
+ readonly description: "(x, y): x pixels from the left edge, y pixels from the top edge.";
332
+ };
333
+ readonly text: {
334
+ readonly type: "string";
335
+ readonly description: "Optional key combination to hold down during this action (e.g. \"ctrl\", \"shift\", \"ctrl+shift\").";
336
+ };
337
+ };
338
+ readonly required: readonly [];
339
+ readonly additionalProperties: false;
340
+ };
341
+ readonly right_click: {
342
+ readonly type: "object";
343
+ readonly properties: {
344
+ readonly coordinate: {
345
+ readonly type: "array";
346
+ readonly items: {
347
+ readonly type: "number";
348
+ };
349
+ readonly minItems: 2;
350
+ readonly maxItems: 2;
351
+ readonly description: "(x, y): x pixels from the left edge, y pixels from the top edge.";
352
+ };
353
+ readonly text: {
354
+ readonly type: "string";
355
+ readonly description: "Optional key combination to hold down during this action (e.g. \"ctrl\", \"shift\", \"ctrl+shift\").";
356
+ };
357
+ };
358
+ readonly required: readonly [];
359
+ readonly additionalProperties: false;
360
+ };
361
+ readonly middle_click: {
362
+ readonly type: "object";
363
+ readonly properties: {
364
+ readonly coordinate: {
365
+ readonly type: "array";
366
+ readonly items: {
367
+ readonly type: "number";
368
+ };
369
+ readonly minItems: 2;
370
+ readonly maxItems: 2;
371
+ readonly description: "(x, y): x pixels from the left edge, y pixels from the top edge.";
372
+ };
373
+ readonly text: {
374
+ readonly type: "string";
375
+ readonly description: "Optional key combination to hold down during this action (e.g. \"ctrl\", \"shift\", \"ctrl+shift\").";
376
+ };
377
+ };
378
+ readonly required: readonly [];
379
+ readonly additionalProperties: false;
380
+ };
381
+ readonly double_click: {
382
+ readonly type: "object";
383
+ readonly properties: {
384
+ readonly coordinate: {
385
+ readonly type: "array";
386
+ readonly items: {
387
+ readonly type: "number";
388
+ };
389
+ readonly minItems: 2;
390
+ readonly maxItems: 2;
391
+ readonly description: "(x, y): x pixels from the left edge, y pixels from the top edge.";
392
+ };
393
+ readonly text: {
394
+ readonly type: "string";
395
+ readonly description: "Optional key combination to hold down during this action (e.g. \"ctrl\", \"shift\", \"ctrl+shift\").";
396
+ };
397
+ };
398
+ readonly required: readonly [];
399
+ readonly additionalProperties: false;
400
+ };
401
+ readonly triple_click: {
402
+ readonly type: "object";
403
+ readonly properties: {
404
+ readonly coordinate: {
405
+ readonly type: "array";
406
+ readonly items: {
407
+ readonly type: "number";
408
+ };
409
+ readonly minItems: 2;
410
+ readonly maxItems: 2;
411
+ readonly description: "(x, y): x pixels from the left edge, y pixels from the top edge.";
412
+ };
413
+ readonly text: {
414
+ readonly type: "string";
415
+ readonly description: "Optional key combination to hold down during this action (e.g. \"ctrl\", \"shift\", \"ctrl+shift\").";
416
+ };
417
+ };
418
+ readonly required: readonly [];
419
+ readonly additionalProperties: false;
420
+ };
421
+ readonly mouse_move: {
422
+ readonly type: "object";
423
+ readonly properties: {
424
+ readonly coordinate: {
425
+ readonly type: "array";
426
+ readonly items: {
427
+ readonly type: "number";
428
+ };
429
+ readonly minItems: 2;
430
+ readonly maxItems: 2;
431
+ readonly description: "(x, y): x pixels from the left edge, y pixels from the top edge.";
432
+ };
433
+ };
434
+ readonly required: readonly ["coordinate"];
435
+ readonly additionalProperties: false;
436
+ };
437
+ readonly left_click_drag: {
438
+ readonly type: "object";
439
+ readonly properties: {
440
+ readonly coordinate: {
441
+ readonly type: "array";
442
+ readonly items: {
443
+ readonly type: "number";
444
+ };
445
+ readonly minItems: 2;
446
+ readonly maxItems: 2;
447
+ readonly description: "(x, y): x pixels from the left edge, y pixels from the top edge.";
448
+ };
449
+ readonly start_coordinate: {
450
+ readonly type: "array";
451
+ readonly items: {
452
+ readonly type: "number";
453
+ };
454
+ readonly minItems: 2;
455
+ readonly maxItems: 2;
456
+ readonly description: "(x, y): x pixels from the left edge, y pixels from the top edge.";
457
+ };
458
+ readonly text: {
459
+ readonly type: "string";
460
+ readonly description: "Optional key combination to hold down during this action (e.g. \"ctrl\", \"shift\", \"ctrl+shift\").";
461
+ };
462
+ };
463
+ readonly required: readonly ["coordinate", "start_coordinate"];
464
+ readonly additionalProperties: false;
465
+ };
466
+ readonly left_mouse_down: {
467
+ readonly type: "object";
468
+ readonly properties: {};
469
+ readonly required: readonly [];
470
+ readonly additionalProperties: false;
471
+ };
472
+ readonly left_mouse_up: {
473
+ readonly type: "object";
474
+ readonly properties: {};
475
+ readonly required: readonly [];
476
+ readonly additionalProperties: false;
477
+ };
478
+ readonly scroll: {
479
+ readonly type: "object";
480
+ readonly properties: {
481
+ readonly scroll_amount: {
482
+ readonly type: "number";
483
+ readonly description: "Number of 'clicks' of the scroll wheel.";
484
+ };
485
+ readonly scroll_direction: {
486
+ readonly type: "string";
487
+ readonly enum: readonly ["up", "down", "left", "right"];
488
+ };
489
+ readonly coordinate: {
490
+ readonly type: "array";
491
+ readonly items: {
492
+ readonly type: "number";
493
+ };
494
+ readonly minItems: 2;
495
+ readonly maxItems: 2;
496
+ readonly description: "(x, y): x pixels from the left edge, y pixels from the top edge.";
497
+ };
498
+ readonly text: {
499
+ readonly type: "string";
500
+ readonly description: "Optional key combination to hold down during this action (e.g. \"ctrl\", \"shift\", \"ctrl+shift\").";
501
+ };
502
+ };
503
+ readonly required: readonly ["scroll_amount", "scroll_direction"];
504
+ readonly additionalProperties: false;
505
+ };
506
+ readonly type: {
507
+ readonly type: "object";
508
+ readonly properties: {
509
+ readonly text: {
510
+ readonly type: "string";
511
+ readonly description: "The text to type.";
512
+ };
513
+ };
514
+ readonly required: readonly ["text"];
515
+ readonly additionalProperties: false;
516
+ };
517
+ readonly key: {
518
+ readonly type: "object";
519
+ readonly properties: {
520
+ readonly text: {
521
+ readonly type: "string";
522
+ readonly description: "The key or key-combination to press.";
523
+ };
524
+ readonly repeat: {
525
+ readonly type: "number";
526
+ readonly description: "Number of times to repeat the key press. Default is 1.";
527
+ };
528
+ };
529
+ readonly required: readonly ["text"];
530
+ readonly additionalProperties: false;
531
+ };
532
+ readonly hold_key: {
533
+ readonly type: "object";
534
+ readonly properties: {
535
+ readonly duration: {
536
+ readonly type: "number";
537
+ readonly description: "Duration to hold the key, in seconds.";
538
+ };
539
+ readonly text: {
540
+ readonly type: "string";
541
+ readonly description: "The key or key-combination to hold.";
542
+ };
543
+ };
544
+ readonly required: readonly ["duration", "text"];
545
+ readonly additionalProperties: false;
546
+ };
547
+ readonly wait: {
548
+ readonly type: "object";
549
+ readonly properties: {
550
+ readonly duration: {
551
+ readonly type: "number";
552
+ readonly description: "Duration to wait, in seconds.";
553
+ };
554
+ };
555
+ readonly required: readonly ["duration"];
556
+ readonly additionalProperties: false;
557
+ };
558
+ readonly cursor_position: {
559
+ readonly type: "object";
560
+ readonly properties: {};
561
+ readonly required: readonly [];
562
+ readonly additionalProperties: false;
563
+ };
564
+ readonly read_ui: {
565
+ readonly type: "object";
566
+ readonly properties: {
567
+ readonly app: {
568
+ readonly type: "string";
569
+ readonly description: "App name or bundle id (macOS); default: the frontmost app.";
570
+ };
571
+ readonly pid: {
572
+ readonly type: "integer";
573
+ };
574
+ readonly window_id: {
575
+ readonly type: "integer";
576
+ };
577
+ readonly query: {
578
+ readonly type: "string";
579
+ readonly description: "Only elements whose name or value contains this text.";
580
+ };
581
+ readonly max_elements: {
582
+ readonly type: "integer";
583
+ readonly description: "Cap the element list (default 200).";
584
+ };
585
+ readonly since: {
586
+ readonly type: "integer";
587
+ readonly description: "A map version from an earlier read_ui: answer only the changes since then.";
588
+ };
589
+ readonly paths: {
590
+ readonly type: "boolean";
591
+ readonly description: "Add each element's tree path (role:ordinal chain from the window). Replay locators use it; you rarely need it.";
592
+ };
593
+ readonly crops: {
594
+ readonly type: "boolean";
595
+ readonly description: "Add an 8x8 pixel hash per element (macOS). Slower: forces a fresh walk of the window. Replay's pixel fallback uses it.";
596
+ };
597
+ readonly vision: {
598
+ readonly type: "string";
599
+ readonly enum: readonly ["auto", "off", "only"];
600
+ readonly description: "The vision fallback: auto (default; used when the tree is empty or can't see into part of the window), off (tree only), only (vision only).";
601
+ };
602
+ readonly target: {
603
+ readonly type: "string";
604
+ readonly description: "Ground this description to one element with the local grounder; the answer's `grounded` has its id, point and confidence.";
605
+ };
606
+ readonly zoom: {
607
+ readonly type: "string";
608
+ readonly enum: readonly ["auto", "always", "never"];
609
+ readonly description: "target: zoom into a crop around a low-confidence answer and ground again (default auto).";
610
+ };
611
+ };
612
+ readonly required: readonly [];
613
+ readonly additionalProperties: false;
614
+ };
615
+ readonly act: {
616
+ readonly type: "object";
617
+ readonly properties: {
618
+ readonly id: {
619
+ readonly type: "string";
620
+ readonly description: "The element id from read_ui.";
621
+ };
622
+ readonly op: {
623
+ readonly type: "string";
624
+ readonly enum: readonly ["click", "set_value", "select", "press", "menu", "focus"];
625
+ readonly description: "What to do to the element.";
626
+ };
627
+ readonly value: {
628
+ readonly type: "string";
629
+ readonly description: "set_value: the text to type into the element; select: the child's name to pick.";
630
+ };
631
+ readonly key: {
632
+ readonly type: "string";
633
+ readonly description: "press: the key name (Return, Tab, Escape, ...).";
634
+ };
635
+ readonly path: {
636
+ readonly anyOf: readonly [{
637
+ readonly type: "string";
638
+ }, {
639
+ readonly type: "array";
640
+ readonly items: {
641
+ readonly type: "string";
642
+ };
643
+ }];
644
+ readonly description: "menu: the menu path, 'File > Save' or ['File', 'Save'].";
645
+ };
646
+ readonly version: {
647
+ readonly type: "integer";
648
+ readonly description: "The read_ui map version this action was planned on; a stale version is refused with the fresh map.";
649
+ };
650
+ readonly app: {
651
+ readonly type: "string";
652
+ readonly description: "App name or bundle id (macOS); default: the frontmost app.";
653
+ };
654
+ readonly pid: {
655
+ readonly type: "integer";
656
+ };
657
+ readonly window_id: {
658
+ readonly type: "integer";
659
+ };
660
+ };
661
+ readonly required: readonly ["id", "op"];
662
+ readonly additionalProperties: false;
663
+ };
664
+ readonly run_batch: {
665
+ readonly type: "object";
666
+ readonly properties: {
667
+ readonly app: {
668
+ readonly type: "string";
669
+ readonly description: "App name or bundle id (macOS); default: the frontmost app.";
670
+ };
671
+ readonly pid: {
672
+ readonly type: "integer";
673
+ };
674
+ readonly window_id: {
675
+ readonly type: "integer";
676
+ };
677
+ readonly version: {
678
+ readonly type: "integer";
679
+ readonly description: "Pin the read_ui map version the steps were planned on; stale is refused with the fresh map.";
680
+ };
681
+ readonly steps: {
682
+ readonly type: "array";
683
+ readonly minItems: 1;
684
+ readonly maxItems: 50;
685
+ readonly items: {
686
+ readonly type: "object";
687
+ readonly properties: {
688
+ readonly act: {
689
+ readonly type: "object";
690
+ readonly properties: {
691
+ readonly id: {
692
+ readonly type: "string";
693
+ };
694
+ readonly op: {
695
+ readonly type: "string";
696
+ readonly enum: readonly ["click", "set_value", "select", "press", "menu", "focus"];
697
+ };
698
+ readonly value: {
699
+ readonly type: "string";
700
+ };
701
+ readonly key: {
702
+ readonly type: "string";
703
+ };
704
+ };
705
+ readonly required: readonly ["id", "op"];
706
+ readonly additionalProperties: false;
707
+ };
708
+ readonly menu: {
709
+ readonly anyOf: readonly [{
710
+ readonly type: "string";
711
+ }, {
712
+ readonly type: "array";
713
+ readonly items: {
714
+ readonly type: "string";
715
+ };
716
+ }];
717
+ };
718
+ readonly pixel: {
719
+ readonly type: "object";
720
+ readonly properties: {
721
+ readonly tool: {
722
+ readonly type: "string";
723
+ };
724
+ readonly args: {
725
+ readonly type: "object";
726
+ };
727
+ };
728
+ readonly required: readonly ["tool"];
729
+ readonly additionalProperties: false;
730
+ };
731
+ readonly wait: {
732
+ readonly type: "number";
733
+ readonly description: "Wait this many milliseconds before the next step.";
734
+ };
735
+ readonly wait_for: {
736
+ readonly type: "object";
737
+ readonly properties: {
738
+ readonly id: {
739
+ readonly type: "string";
740
+ readonly description: "An element id from read_ui.";
741
+ };
742
+ readonly role: {
743
+ readonly type: "string";
744
+ readonly description: "An accessibility role, e.g. button, textfield, checkbox.";
745
+ };
746
+ readonly name: {
747
+ readonly type: "string";
748
+ readonly description: "Substring match against the element's name/label.";
749
+ };
750
+ readonly text: {
751
+ readonly type: "string";
752
+ readonly description: "Substring match against the element's displayed text or value.";
753
+ };
754
+ readonly gone: {
755
+ readonly type: "boolean";
756
+ readonly description: "True: no element may match; false: at least one must.";
757
+ };
758
+ readonly value: {
759
+ readonly type: "string";
760
+ readonly description: "The element's value must equal this exactly.";
761
+ };
762
+ readonly enabled: {
763
+ readonly type: "boolean";
764
+ readonly description: "The element's enabled state must equal this.";
765
+ };
766
+ };
767
+ readonly required: readonly [];
768
+ readonly additionalProperties: false;
769
+ readonly description: "Precondition: checked before the step; failure stops the batch.";
770
+ };
771
+ readonly expect: {
772
+ readonly type: "object";
773
+ readonly properties: {
774
+ readonly id: {
775
+ readonly type: "string";
776
+ readonly description: "An element id from read_ui.";
777
+ };
778
+ readonly role: {
779
+ readonly type: "string";
780
+ readonly description: "An accessibility role, e.g. button, textfield, checkbox.";
781
+ };
782
+ readonly name: {
783
+ readonly type: "string";
784
+ readonly description: "Substring match against the element's name/label.";
785
+ };
786
+ readonly text: {
787
+ readonly type: "string";
788
+ readonly description: "Substring match against the element's displayed text or value.";
789
+ };
790
+ readonly gone: {
791
+ readonly type: "boolean";
792
+ readonly description: "True: no element may match; false: at least one must.";
793
+ };
794
+ readonly value: {
795
+ readonly type: "string";
796
+ readonly description: "The element's value must equal this exactly.";
797
+ };
798
+ readonly enabled: {
799
+ readonly type: "boolean";
800
+ readonly description: "The element's enabled state must equal this.";
801
+ };
802
+ };
803
+ readonly required: readonly [];
804
+ readonly additionalProperties: false;
805
+ readonly description: "Postcondition: checked after the step; failure stops the batch.";
806
+ };
807
+ readonly timeout_ms: {
808
+ readonly type: "integer";
809
+ readonly description: "How long to wait for this step's checks (default 3000).";
810
+ };
811
+ };
812
+ readonly required: readonly [];
813
+ readonly additionalProperties: false;
814
+ };
815
+ };
816
+ };
817
+ readonly required: readonly ["steps"];
818
+ readonly additionalProperties: false;
819
+ };
820
+ readonly verify: {
821
+ readonly type: "object";
822
+ readonly properties: {
823
+ readonly app: {
824
+ readonly type: "string";
825
+ readonly description: "App name or bundle id (macOS); default: the frontmost app.";
826
+ };
827
+ readonly pid: {
828
+ readonly type: "integer";
829
+ };
830
+ readonly window_id: {
831
+ readonly type: "integer";
832
+ };
833
+ readonly checks: {
834
+ readonly type: "array";
835
+ readonly minItems: 1;
836
+ readonly maxItems: 20;
837
+ readonly items: {
838
+ readonly type: "object";
839
+ readonly properties: {
840
+ readonly id: {
841
+ readonly type: "string";
842
+ readonly description: "An element id from read_ui.";
843
+ };
844
+ readonly role: {
845
+ readonly type: "string";
846
+ readonly description: "An accessibility role, e.g. button, textfield, checkbox.";
847
+ };
848
+ readonly name: {
849
+ readonly type: "string";
850
+ readonly description: "Substring match against the element's name/label.";
851
+ };
852
+ readonly text: {
853
+ readonly type: "string";
854
+ readonly description: "Substring match against the element's displayed text or value.";
855
+ };
856
+ readonly gone: {
857
+ readonly type: "boolean";
858
+ readonly description: "True: no element may match; false: at least one must.";
859
+ };
860
+ readonly value: {
861
+ readonly type: "string";
862
+ readonly description: "The element's value must equal this exactly.";
863
+ };
864
+ readonly enabled: {
865
+ readonly type: "boolean";
866
+ readonly description: "The element's enabled state must equal this.";
867
+ };
868
+ };
869
+ readonly required: readonly [];
870
+ readonly additionalProperties: false;
871
+ };
872
+ };
873
+ readonly timeout_ms: {
874
+ readonly type: "integer";
875
+ readonly description: "Wait up to this long for every check to hold (default 0: one fresh read).";
876
+ };
877
+ };
878
+ readonly required: readonly ["checks"];
879
+ readonly additionalProperties: false;
880
+ };
881
+ readonly request_human: {
882
+ readonly type: "object";
883
+ readonly properties: {
884
+ readonly reason: {
885
+ readonly type: "string";
886
+ readonly description: "What the person should do, shown in the live view (e.g. 'Complete the CAPTCHA').";
887
+ };
888
+ readonly timeout_ms: {
889
+ readonly type: "integer";
890
+ readonly description: "How long to wait for the person (default 300000, max 1800000).";
891
+ };
892
+ };
893
+ readonly required: readonly [];
894
+ readonly additionalProperties: false;
895
+ };
896
+ readonly use_credential: {
897
+ readonly type: "object";
898
+ readonly properties: {
899
+ readonly name: {
900
+ readonly type: "string";
901
+ readonly description: "The credential's name in the vault.";
902
+ };
903
+ readonly kind: {
904
+ readonly type: "string";
905
+ readonly enum: readonly ["secret", "totp"];
906
+ readonly description: "secret: type the stored value; totp: generate and type a fresh TOTP code.";
907
+ };
908
+ readonly source: {
909
+ readonly type: "string";
910
+ readonly enum: readonly ["vault", "keychain", "bitwarden", "onepassword"];
911
+ readonly description: "Where to resolve the credential from (default: the sealed vault, then this Mac's Keychain).";
912
+ };
913
+ readonly domain: {
914
+ readonly type: "string";
915
+ readonly description: "The app or domain the credential is being typed into; checked against the credential's binding.";
916
+ };
917
+ };
918
+ readonly required: readonly ["name"];
919
+ readonly additionalProperties: false;
920
+ };
921
+ readonly run_subtask: {
922
+ readonly type: "object";
923
+ readonly properties: {
924
+ readonly goal: {
925
+ readonly type: "string";
926
+ readonly description: "What the subtask must achieve, in one or two sentences (e.g. 'Fill the sign-up form and submit it').";
927
+ };
928
+ readonly inputs: {
929
+ readonly type: "array";
930
+ readonly maxItems: 20;
931
+ readonly items: {
932
+ readonly type: "object";
933
+ readonly properties: {
934
+ readonly name: {
935
+ readonly type: "string";
936
+ readonly description: "What the value is (e.g. 'email', 'full name').";
937
+ };
938
+ readonly value: {
939
+ readonly type: "string";
940
+ readonly description: "The literal text to type.";
941
+ };
942
+ };
943
+ readonly required: readonly ["name", "value"];
944
+ readonly additionalProperties: false;
945
+ };
946
+ readonly description: "The only text the subtask may type. Not for secrets: use use_credential.";
947
+ };
948
+ readonly success: {
949
+ readonly type: "array";
950
+ readonly minItems: 1;
951
+ readonly maxItems: 10;
952
+ readonly items: {
953
+ readonly type: "object";
954
+ readonly properties: {
955
+ readonly id: {
956
+ readonly type: "string";
957
+ readonly description: "An element id from read_ui.";
958
+ };
959
+ readonly role: {
960
+ readonly type: "string";
961
+ readonly description: "An accessibility role, e.g. button, textfield, checkbox.";
962
+ };
963
+ readonly name: {
964
+ readonly type: "string";
965
+ readonly description: "Substring match against the element's name/label.";
966
+ };
967
+ readonly text: {
968
+ readonly type: "string";
969
+ readonly description: "Substring match against the element's displayed text or value.";
970
+ };
971
+ readonly gone: {
972
+ readonly type: "boolean";
973
+ readonly description: "True: no element may match; false: at least one must.";
974
+ };
975
+ readonly value: {
976
+ readonly type: "string";
977
+ readonly description: "The element's value must equal this exactly.";
978
+ };
979
+ readonly enabled: {
980
+ readonly type: "boolean";
981
+ readonly description: "The element's enabled state must equal this.";
982
+ };
983
+ readonly ask: {
984
+ readonly type: "string";
985
+ readonly description: "A fuzzy yes/no check in plain words (\"Is the confirmation message showing?\"), answered by a typed verify decision over the screen. Use only when role/name/text/value checks can't express it.";
986
+ };
987
+ };
988
+ readonly required: readonly [];
989
+ readonly additionalProperties: false;
990
+ };
991
+ readonly description: "Checks that all hold when the goal is met, in verify syntax ({role?, name?, text?, value?, gone?, enabled?}) or {ask: '...'} for a fuzzy yes/no. Without checks the loop stops when a decision picks done.";
992
+ };
993
+ readonly constraints: {
994
+ readonly type: "object";
995
+ readonly properties: {
996
+ readonly ops: {
997
+ readonly type: "array";
998
+ readonly items: {
999
+ readonly type: "string";
1000
+ readonly enum: readonly ["click", "set_value", "select", "press", "focus"];
1001
+ };
1002
+ readonly description: "Ops the subtask may use (default click, set_value, select, press).";
1003
+ };
1004
+ readonly avoid: {
1005
+ readonly type: "array";
1006
+ readonly maxItems: 20;
1007
+ readonly items: {
1008
+ readonly type: "string";
1009
+ };
1010
+ readonly description: "Never act on elements whose name contains any of these.";
1011
+ };
1012
+ readonly allow_irreversible: {
1013
+ readonly type: "boolean";
1014
+ readonly description: "Let the subtask click buttons named like send, pay, buy, delete, publish (default false: those hand back to you).";
1015
+ };
1016
+ };
1017
+ readonly required: readonly [];
1018
+ readonly additionalProperties: false;
1019
+ };
1020
+ readonly max_steps: {
1021
+ readonly type: "integer";
1022
+ readonly description: "Most actions the subtask may take (default 12, at most 40).";
1023
+ };
1024
+ readonly budget_ms: {
1025
+ readonly type: "integer";
1026
+ readonly description: "Wall-clock budget in milliseconds (default 60000, at most 300000).";
1027
+ };
1028
+ readonly app: {
1029
+ readonly type: "string";
1030
+ readonly description: "App name or bundle id (macOS); default: the frontmost app.";
1031
+ };
1032
+ readonly pid: {
1033
+ readonly type: "integer";
1034
+ };
1035
+ readonly window_id: {
1036
+ readonly type: "integer";
1037
+ };
1038
+ readonly query: {
1039
+ readonly type: "string";
1040
+ readonly description: "Only consider elements whose name or value contains this text.";
1041
+ };
1042
+ readonly max_elements: {
1043
+ readonly type: "integer";
1044
+ readonly description: "Cap the elements read per step (default 120).";
1045
+ };
1046
+ readonly cache: {
1047
+ readonly type: "string";
1048
+ readonly enum: readonly ["auto", "off", "refresh"];
1049
+ readonly description: "Replay cache (default auto). auto: a subtask seen before (same goal, app and window, same input names) replays its recorded steps with no decisions, checking each with verify, and re-infers only a step whose element moved; a new one runs the decision loop and is recorded once its success checks hold. refresh: ignore the cached steps and record anew. off: neither read nor write the cache.";
1050
+ };
1051
+ readonly save_as: {
1052
+ readonly type: "string";
1053
+ readonly description: "Save this subtask's recording as a named skill once it succeeds (overwrites a skill of that name). Run it later with run_skill and new input values.";
1054
+ };
1055
+ readonly api_options: {
1056
+ readonly type: "array";
1057
+ readonly maxItems: 10;
1058
+ readonly items: {
1059
+ readonly type: "object";
1060
+ readonly properties: {
1061
+ readonly tool: {
1062
+ readonly type: "string";
1063
+ readonly description: "The MCP tool or API action you can call yourself (e.g. 'github.create_issue').";
1064
+ };
1065
+ readonly description: {
1066
+ readonly type: "string";
1067
+ readonly description: "What it does, in one line.";
1068
+ };
1069
+ };
1070
+ readonly required: readonly ["tool"];
1071
+ readonly additionalProperties: false;
1072
+ };
1073
+ readonly description: "MCP tools or API actions you hold that could do this goal without the GUI. They are offered ahead of every GUI option on the first step; if the decision picks one, the subtask stops with status use_api and names it, and you make that call yourself. An API call beats GUI clicks when both exist.";
1074
+ };
1075
+ readonly best_of: {
1076
+ readonly type: "integer";
1077
+ readonly description: "Behavior Best-of-N for high-value subtasks: run N rollouts (2-5) at once on N sandbox computers listed in computers, then a judge decision picks one by the rollouts' step narratives. Sandbox/cloud computers only, never the person's own machine. Returns the chosen rollout, why, and every rollout's narrative.";
1078
+ };
1079
+ readonly computers: {
1080
+ readonly type: "array";
1081
+ readonly maxItems: 5;
1082
+ readonly items: {
1083
+ readonly type: "string";
1084
+ };
1085
+ readonly description: "With best_of: the sandbox computer ids the rollouts run on, one rollout per computer (at least best_of distinct ids).";
1086
+ };
1087
+ readonly max_cost_usd: {
1088
+ readonly type: "number";
1089
+ readonly description: "With best_of: the cap on decision spend for the whole call in USD (default 0.25, at most 5). The judge may use the planner-class tier only while under it.";
1090
+ };
1091
+ };
1092
+ readonly required: readonly ["goal"];
1093
+ readonly additionalProperties: false;
1094
+ };
1095
+ readonly run_parallel: {
1096
+ readonly type: "object";
1097
+ readonly properties: {
1098
+ readonly subtasks: {
1099
+ readonly type: "array";
1100
+ readonly minItems: 1;
1101
+ readonly maxItems: 8;
1102
+ readonly items: {
1103
+ readonly type: "object";
1104
+ readonly properties: {
1105
+ readonly computer: {
1106
+ readonly type: "string";
1107
+ readonly description: "The computer id this subtask runs on (default: the computer this call is made on).";
1108
+ };
1109
+ readonly goal: {
1110
+ readonly type: "string";
1111
+ readonly description: "What the subtask must achieve, in one or two sentences (e.g. 'Fill the sign-up form and submit it').";
1112
+ };
1113
+ readonly inputs: {
1114
+ readonly type: "array";
1115
+ readonly maxItems: 20;
1116
+ readonly items: {
1117
+ readonly type: "object";
1118
+ readonly properties: {
1119
+ readonly name: {
1120
+ readonly type: "string";
1121
+ readonly description: "What the value is (e.g. 'email', 'full name').";
1122
+ };
1123
+ readonly value: {
1124
+ readonly type: "string";
1125
+ readonly description: "The literal text to type.";
1126
+ };
1127
+ };
1128
+ readonly required: readonly ["name", "value"];
1129
+ readonly additionalProperties: false;
1130
+ };
1131
+ readonly description: "The only text the subtask may type. Not for secrets: use use_credential.";
1132
+ };
1133
+ readonly success: {
1134
+ readonly type: "array";
1135
+ readonly minItems: 1;
1136
+ readonly maxItems: 10;
1137
+ readonly items: {
1138
+ readonly type: "object";
1139
+ readonly properties: {
1140
+ readonly id: {
1141
+ readonly type: "string";
1142
+ readonly description: "An element id from read_ui.";
1143
+ };
1144
+ readonly role: {
1145
+ readonly type: "string";
1146
+ readonly description: "An accessibility role, e.g. button, textfield, checkbox.";
1147
+ };
1148
+ readonly name: {
1149
+ readonly type: "string";
1150
+ readonly description: "Substring match against the element's name/label.";
1151
+ };
1152
+ readonly text: {
1153
+ readonly type: "string";
1154
+ readonly description: "Substring match against the element's displayed text or value.";
1155
+ };
1156
+ readonly gone: {
1157
+ readonly type: "boolean";
1158
+ readonly description: "True: no element may match; false: at least one must.";
1159
+ };
1160
+ readonly value: {
1161
+ readonly type: "string";
1162
+ readonly description: "The element's value must equal this exactly.";
1163
+ };
1164
+ readonly enabled: {
1165
+ readonly type: "boolean";
1166
+ readonly description: "The element's enabled state must equal this.";
1167
+ };
1168
+ readonly ask: {
1169
+ readonly type: "string";
1170
+ readonly description: "A fuzzy yes/no check in plain words (\"Is the confirmation message showing?\"), answered by a typed verify decision over the screen. Use only when role/name/text/value checks can't express it.";
1171
+ };
1172
+ };
1173
+ readonly required: readonly [];
1174
+ readonly additionalProperties: false;
1175
+ };
1176
+ readonly description: "Checks that all hold when the goal is met, in verify syntax ({role?, name?, text?, value?, gone?, enabled?}) or {ask: '...'} for a fuzzy yes/no. Without checks the loop stops when a decision picks done.";
1177
+ };
1178
+ readonly constraints: {
1179
+ readonly type: "object";
1180
+ readonly properties: {
1181
+ readonly ops: {
1182
+ readonly type: "array";
1183
+ readonly items: {
1184
+ readonly type: "string";
1185
+ readonly enum: readonly ["click", "set_value", "select", "press", "focus"];
1186
+ };
1187
+ readonly description: "Ops the subtask may use (default click, set_value, select, press).";
1188
+ };
1189
+ readonly avoid: {
1190
+ readonly type: "array";
1191
+ readonly maxItems: 20;
1192
+ readonly items: {
1193
+ readonly type: "string";
1194
+ };
1195
+ readonly description: "Never act on elements whose name contains any of these.";
1196
+ };
1197
+ readonly allow_irreversible: {
1198
+ readonly type: "boolean";
1199
+ readonly description: "Let the subtask click buttons named like send, pay, buy, delete, publish (default false: those hand back to you).";
1200
+ };
1201
+ };
1202
+ readonly required: readonly [];
1203
+ readonly additionalProperties: false;
1204
+ };
1205
+ readonly max_steps: {
1206
+ readonly type: "integer";
1207
+ readonly description: "Most actions the subtask may take (default 12, at most 40).";
1208
+ };
1209
+ readonly budget_ms: {
1210
+ readonly type: "integer";
1211
+ readonly description: "Wall-clock budget in milliseconds (default 60000, at most 300000).";
1212
+ };
1213
+ readonly app: {
1214
+ readonly type: "string";
1215
+ readonly description: "App name or bundle id (macOS); default: the frontmost app.";
1216
+ };
1217
+ readonly pid: {
1218
+ readonly type: "integer";
1219
+ };
1220
+ readonly window_id: {
1221
+ readonly type: "integer";
1222
+ };
1223
+ readonly query: {
1224
+ readonly type: "string";
1225
+ readonly description: "Only consider elements whose name or value contains this text.";
1226
+ };
1227
+ readonly max_elements: {
1228
+ readonly type: "integer";
1229
+ readonly description: "Cap the elements read per step (default 120).";
1230
+ };
1231
+ readonly api_options: {
1232
+ readonly type: "array";
1233
+ readonly maxItems: 10;
1234
+ readonly items: {
1235
+ readonly type: "object";
1236
+ readonly properties: {
1237
+ readonly tool: {
1238
+ readonly type: "string";
1239
+ readonly description: "The MCP tool or API action you can call yourself (e.g. 'github.create_issue').";
1240
+ };
1241
+ readonly description: {
1242
+ readonly type: "string";
1243
+ readonly description: "What it does, in one line.";
1244
+ };
1245
+ };
1246
+ readonly required: readonly ["tool"];
1247
+ readonly additionalProperties: false;
1248
+ };
1249
+ readonly description: "MCP tools or API actions you hold that could do this goal without the GUI. They are offered ahead of every GUI option on the first step; if the decision picks one, the subtask stops with status use_api and names it, and you make that call yourself. An API call beats GUI clicks when both exist.";
1250
+ };
1251
+ };
1252
+ readonly required: readonly ["goal"];
1253
+ readonly additionalProperties: false;
1254
+ };
1255
+ readonly description: "Independent subtasks, one per computer. Same fields as run_subtask plus computer.";
1256
+ };
1257
+ readonly max_parallel: {
1258
+ readonly type: "integer";
1259
+ readonly description: "Most subtasks running at once (default 4, at most 8).";
1260
+ };
1261
+ readonly budget_ms: {
1262
+ readonly type: "integer";
1263
+ readonly description: "Wall-clock budget for the whole call in milliseconds (default 120000, at most 600000). Each subtask's own budget_ms is cut to what is left when it starts.";
1264
+ };
1265
+ readonly max_cost_usd: {
1266
+ readonly type: "number";
1267
+ readonly description: "Cap on decision spend for the whole call in USD (default 0.25, at most 5). Once reached, subtasks not yet started come back skipped.";
1268
+ };
1269
+ };
1270
+ readonly required: readonly ["subtasks"];
1271
+ readonly additionalProperties: false;
1272
+ };
1273
+ readonly run_skill: {
1274
+ readonly type: "object";
1275
+ readonly properties: {
1276
+ readonly name: {
1277
+ readonly type: "string";
1278
+ readonly description: "The skill's name (from skills).";
1279
+ };
1280
+ readonly inputs: {
1281
+ readonly type: "array";
1282
+ readonly maxItems: 20;
1283
+ readonly items: {
1284
+ readonly type: "object";
1285
+ readonly properties: {
1286
+ readonly name: {
1287
+ readonly type: "string";
1288
+ readonly description: "What the value is (e.g. 'email', 'full name').";
1289
+ };
1290
+ readonly value: {
1291
+ readonly type: "string";
1292
+ readonly description: "The literal text to type.";
1293
+ };
1294
+ };
1295
+ readonly required: readonly ["name", "value"];
1296
+ readonly additionalProperties: false;
1297
+ };
1298
+ readonly description: "A value for each of the skill's inputs (skills lists their names). Not for secrets: use use_credential.";
1299
+ };
1300
+ readonly app: {
1301
+ readonly type: "string";
1302
+ readonly description: "App name or bundle id (macOS); default: the frontmost app.";
1303
+ };
1304
+ readonly pid: {
1305
+ readonly type: "integer";
1306
+ };
1307
+ readonly window_id: {
1308
+ readonly type: "integer";
1309
+ };
1310
+ readonly max_steps: {
1311
+ readonly type: "integer";
1312
+ readonly description: "Most actions the subtask may take (default 12, at most 40).";
1313
+ };
1314
+ readonly budget_ms: {
1315
+ readonly type: "integer";
1316
+ readonly description: "Wall-clock budget in milliseconds (default 60000, at most 300000).";
1317
+ };
1318
+ };
1319
+ readonly required: readonly ["name"];
1320
+ readonly additionalProperties: false;
1321
+ };
1322
+ readonly skills: {
1323
+ readonly type: "object";
1324
+ readonly properties: {
1325
+ readonly forget: {
1326
+ readonly type: "string";
1327
+ readonly description: "Delete the saved skill with this name before listing.";
1328
+ };
1329
+ };
1330
+ readonly required: readonly [];
1331
+ readonly additionalProperties: false;
1332
+ };
1333
+ };
1334
+ export interface ComputerV2StructuredMemberSpec {
1335
+ name: string;
1336
+ description: string;
1337
+ risk: string;
1338
+ default_enabled: boolean;
1339
+ needs_confirm: boolean;
1340
+ confirm: string;
1341
+ result_kind: string;
1342
+ ack_text?: string;
1343
+ scale_fields: Record<string, string>;
1344
+ }
1345
+ export declare const COMPUTER_V2_STRUCTURED_MEMBERS: readonly [{
1346
+ readonly name: "read_ui";
1347
+ readonly description: "Read the target window or app's UI as a structured element tree (roles, names, values, bounds) with NO screenshot. Returns stable element ids for act, run_batch and verify. Pass `since` (a version from an earlier read_ui) to get only the diff; `query` filters by text; `max_elements` caps the list (default 200). When the window has no usable accessibility tree (canvas apps, games, remote desktops, custom-drawn UIs) the read falls back to vision: numbered set-of-marks regions come back as elements with `source: \"vision\"` and a `mark` number, and act on their ids works the same way. `source` and `source_reason` say which was used. Pass `target` (a description such as 'the Export button') to ground it with the local grounder: `grounded.id` is the element to act on.";
1348
+ readonly risk: "reversible";
1349
+ readonly default_enabled: true;
1350
+ readonly needs_confirm: false;
1351
+ readonly confirm: "never";
1352
+ readonly result_kind: "none";
1353
+ readonly ack_text: "Read the UI of {app}.";
1354
+ readonly scale_fields: {};
1355
+ }, {
1356
+ readonly name: "act";
1357
+ readonly description: "Act on an element id from read_ui (tree ids and vision ids alike; a vision element is clicked or typed into at its centre after a pixel freshness check, and menu needs a window). op: click, set_value (clear and set the field's value), select (pick a child by name), press (a key), menu (choose a menu path such as 'File > Save'), focus. set_value/select/press on this Mac need the person's approval (the same rule as typing).";
1358
+ readonly risk: "risky";
1359
+ readonly default_enabled: true;
1360
+ readonly needs_confirm: false;
1361
+ readonly confirm: "never";
1362
+ readonly result_kind: "none";
1363
+ readonly ack_text: "{op} on {id}.";
1364
+ readonly scale_fields: {};
1365
+ }, {
1366
+ readonly name: "run_batch";
1367
+ readonly description: "Run ordered steps in one window in a single call: act steps ({id, op, value?, key?}), menu steps (path), pixel steps ({tool, args}) and waits (ms). Each step may carry wait_for (checked before the step) and expect (checked after); a failed check stops the batch and one observe read returns the fresh map. This is the fast path: batch several steps instead of screenshot-per-action loops.";
1368
+ readonly risk: "risky";
1369
+ readonly default_enabled: true;
1370
+ readonly needs_confirm: true;
1371
+ readonly confirm: "non_sandbox";
1372
+ readonly result_kind: "none";
1373
+ readonly ack_text: "Ran {steps} batch steps.";
1374
+ readonly scale_fields: {};
1375
+ }, {
1376
+ readonly name: "verify";
1377
+ readonly description: "Check bounded conditions on the current UI without acting: role, name, text, gone, value, enabled. Each check is {id?, role?, name?, text?, gone?, value?, enabled?}; all given parts must hold. Returns one ok per check. Use after act/run_batch, or to poll with timeout_ms.";
1378
+ readonly risk: "reversible";
1379
+ readonly default_enabled: true;
1380
+ readonly needs_confirm: false;
1381
+ readonly confirm: "never";
1382
+ readonly result_kind: "none";
1383
+ readonly ack_text: "Verified.";
1384
+ readonly scale_fields: {};
1385
+ }, {
1386
+ readonly name: "request_human";
1387
+ readonly description: "Pause and hand the computer to a person: the live view opens with input enabled and the agent's hold on control is released. The same session resumes when the person signals done, or when timeout_ms elapses. Use for CAPTCHAs, push/SMS 2FA and judgment calls.";
1388
+ readonly risk: "reversible";
1389
+ readonly default_enabled: true;
1390
+ readonly needs_confirm: false;
1391
+ readonly confirm: "never";
1392
+ readonly result_kind: "none";
1393
+ readonly ack_text: "Human window closed ({reason}).";
1394
+ readonly scale_fields: {};
1395
+ }, {
1396
+ readonly name: "use_credential";
1397
+ readonly description: "Type a vault credential into the FOCUSED field. The value never enters the model context, prompts or logs. kind: secret (the stored value) or totp (a fresh code from a stored TOTP seed). The credential is bound to an app or domain: pass `domain` (the app you are filling into); it must match the credential's binding. On this Mac a person approves each use, like typing.";
1398
+ readonly risk: "risky";
1399
+ readonly default_enabled: true;
1400
+ readonly needs_confirm: true;
1401
+ readonly confirm: "non_sandbox";
1402
+ readonly result_kind: "none";
1403
+ readonly ack_text: "Credential {name} typed.";
1404
+ readonly scale_fields: {};
1405
+ }, {
1406
+ readonly name: "run_subtask";
1407
+ readonly description: "Hand a bounded UI subtask (fill a form, search and pick a result, toggle settings) to the fast decision loop instead of driving each step yourself. Give the goal, the literal inputs it may type (it never invents text), success checks in verify syntax, and limits. Each step reads the element map, picks one action from the visible elements x allowed ops with a ~100 ms typed decision (POST /v1/decisions), runs it through the same lease/approval/audit path (predictable steps batched with expect checks), and checks success with verify. Returns a step trace and a status: done, escalated (with the reason and the current screen: continue from there yourself), or failed. One approval covers the whole subtask on this computer. Repeat subtasks replay from the cache with zero decisions (the result's cache block says hit, healed, recorded or miss); only subtasks with at least one exact (non-ask) success check are cached. Input values are never stored, only their names. With api_options, an MCP/API call you hold is offered ahead of GUI steps (status use_api hands it back to you). With best_of N and computers, N rollouts run on N sandbox computers and a judge picks one by step narratives.";
1408
+ readonly risk: "risky";
1409
+ readonly default_enabled: true;
1410
+ readonly needs_confirm: true;
1411
+ readonly confirm: "non_sandbox";
1412
+ readonly result_kind: "none";
1413
+ readonly ack_text: "Ran the subtask.";
1414
+ readonly scale_fields: {};
1415
+ }, {
1416
+ readonly name: "run_parallel";
1417
+ readonly description: "Run independent bounded subtasks on separate computers at the same time (one run_subtask per computer, each under its own computer's control lease), and get every result back in order. Each subtask names its computer (default: this one) and takes run_subtask's fields. One computer never runs two subtasks at once: a repeated computer, or one another subtask is already driving, is refused. Limits: max_parallel subtasks at once, a total budget_ms, and max_cost_usd on decision spend; subtasks that can't start inside them come back skipped. One approval covers the whole call when any subtask targets a non-sandbox computer.";
1418
+ readonly risk: "risky";
1419
+ readonly default_enabled: true;
1420
+ readonly needs_confirm: true;
1421
+ readonly confirm: "non_sandbox";
1422
+ readonly result_kind: "none";
1423
+ readonly ack_text: "Ran the subtasks.";
1424
+ readonly scale_fields: {};
1425
+ }, {
1426
+ readonly name: "run_skill";
1427
+ readonly description: "Run a saved skill (a recording taught with run_subtask save_as) by name, with new values for its inputs. It replays the recorded steps with no decisions, checks each step and the skill's success checks with verify, re-infers a step whose element moved and rewrites the recording. Returns the same result as run_subtask. List skills with the skills member.";
1428
+ readonly risk: "risky";
1429
+ readonly default_enabled: true;
1430
+ readonly needs_confirm: true;
1431
+ readonly confirm: "non_sandbox";
1432
+ readonly result_kind: "none";
1433
+ readonly ack_text: "Ran the skill {name}.";
1434
+ readonly scale_fields: {};
1435
+ }, {
1436
+ readonly name: "skills";
1437
+ readonly description: "List the saved skills (taught recordings) this person can run with run_skill: name, goal, app, input names, step count, last use. Pass forget to delete one first.";
1438
+ readonly risk: "reversible";
1439
+ readonly default_enabled: true;
1440
+ readonly needs_confirm: false;
1441
+ readonly confirm: "never";
1442
+ readonly result_kind: "none";
1443
+ readonly ack_text: "Listed the saved skills.";
1444
+ readonly scale_fields: {};
1445
+ }];