@namzu/sdk 45.1.0 → 46.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +111 -0
- package/dist/authorization/gate.d.ts +5 -2
- package/dist/authorization/gate.d.ts.map +1 -1
- package/dist/authorization/gate.js +25 -4
- package/dist/authorization/gate.js.map +1 -1
- package/dist/authorization/rules.d.ts.map +1 -1
- package/dist/authorization/rules.js +19 -0
- package/dist/authorization/rules.js.map +1 -1
- package/dist/authorization/shell-lexer.d.ts +24 -0
- package/dist/authorization/shell-lexer.d.ts.map +1 -1
- package/dist/authorization/shell-lexer.js +69 -51
- package/dist/authorization/shell-lexer.js.map +1 -1
- package/dist/pricing/catalogue.generated.d.ts.map +1 -1
- package/dist/pricing/catalogue.generated.js +28 -4
- package/dist/pricing/catalogue.generated.js.map +1 -1
- package/dist/public-runtime.d.ts +3 -3
- package/dist/public-runtime.d.ts.map +1 -1
- package/dist/public-runtime.js +4 -2
- package/dist/public-runtime.js.map +1 -1
- package/dist/public-tools.d.ts +3 -2
- package/dist/public-tools.d.ts.map +1 -1
- package/dist/public-tools.js +5 -2
- package/dist/public-tools.js.map +1 -1
- package/dist/public-types.d.ts +3 -3
- package/dist/public-types.d.ts.map +1 -1
- package/dist/registry/tool/callable.d.ts +22 -0
- package/dist/registry/tool/callable.d.ts.map +1 -0
- package/dist/registry/tool/callable.js +29 -0
- package/dist/registry/tool/callable.js.map +1 -0
- package/dist/registry/tool/execute.d.ts.map +1 -1
- package/dist/registry/tool/execute.js +8 -1
- package/dist/registry/tool/execute.js.map +1 -1
- package/dist/runtime/query/executor/tool-call-admission.d.ts +37 -11
- package/dist/runtime/query/executor/tool-call-admission.d.ts.map +1 -1
- package/dist/runtime/query/executor/tool-call-admission.js +38 -12
- package/dist/runtime/query/executor/tool-call-admission.js.map +1 -1
- package/dist/runtime/query/executor.d.ts +4 -2
- package/dist/runtime/query/executor.d.ts.map +1 -1
- package/dist/runtime/query/executor.js +18 -5
- package/dist/runtime/query/executor.js.map +1 -1
- package/dist/runtime/query/review-policy.d.ts +43 -0
- package/dist/runtime/query/review-policy.d.ts.map +1 -1
- package/dist/runtime/query/review-policy.js +59 -16
- package/dist/runtime/query/review-policy.js.map +1 -1
- package/dist/skills/registry.d.ts +6 -0
- package/dist/skills/registry.d.ts.map +1 -1
- package/dist/skills/registry.js +1 -0
- package/dist/skills/registry.js.map +1 -1
- package/dist/tools/builtins/computer-use-coordinates.d.ts +65 -0
- package/dist/tools/builtins/computer-use-coordinates.d.ts.map +1 -0
- package/dist/tools/builtins/computer-use-coordinates.js +123 -0
- package/dist/tools/builtins/computer-use-coordinates.js.map +1 -0
- package/dist/tools/builtins/computer-use-image.d.ts +77 -0
- package/dist/tools/builtins/computer-use-image.d.ts.map +1 -0
- package/dist/tools/builtins/computer-use-image.js +223 -0
- package/dist/tools/builtins/computer-use-image.js.map +1 -0
- package/dist/tools/builtins/computer-use.d.ts +519 -14
- package/dist/tools/builtins/computer-use.d.ts.map +1 -1
- package/dist/tools/builtins/computer-use.js +1188 -183
- package/dist/tools/builtins/computer-use.js.map +1 -1
- package/dist/tools/builtins/index.d.ts +2 -1
- package/dist/tools/builtins/index.d.ts.map +1 -1
- package/dist/tools/builtins/index.js +1 -1
- package/dist/tools/builtins/index.js.map +1 -1
- package/dist/tools/builtins/skill.d.ts +74 -0
- package/dist/tools/builtins/skill.d.ts.map +1 -1
- package/dist/tools/builtins/skill.js +214 -174
- package/dist/tools/builtins/skill.js.map +1 -1
- package/dist/tools/defineTool.d.ts +2 -0
- package/dist/tools/defineTool.d.ts.map +1 -1
- package/dist/tools/defineTool.js +7 -0
- package/dist/tools/defineTool.js.map +1 -1
- package/dist/tools/schedules/present.d.ts.map +1 -1
- package/dist/tools/schedules/present.js +2 -0
- package/dist/tools/schedules/present.js.map +1 -1
- package/dist/tools/schedules/schedule-tool.d.ts +6 -5
- package/dist/tools/schedules/schedule-tool.d.ts.map +1 -1
- package/dist/tools/schedules/schedule-tool.js +121 -30
- package/dist/tools/schedules/schedule-tool.js.map +1 -1
- package/dist/tools/schedules/types.d.ts +69 -1
- package/dist/tools/schedules/types.d.ts.map +1 -1
- package/dist/types/authorization/index.d.ts +21 -0
- package/dist/types/authorization/index.d.ts.map +1 -1
- package/dist/types/authorization/index.js +5 -0
- package/dist/types/authorization/index.js.map +1 -1
- package/dist/types/computer-use/index.d.ts +176 -0
- package/dist/types/computer-use/index.d.ts.map +1 -1
- package/dist/types/computer-use/index.js.map +1 -1
- package/dist/types/tool/index.d.ts +19 -0
- package/dist/types/tool/index.d.ts.map +1 -1
- package/dist/types/tool/index.js.map +1 -1
- package/package.json +3 -1
- package/src/authorization/gate.ts +25 -4
- package/src/authorization/rules.ts +18 -0
- package/src/authorization/shell-lexer.ts +73 -45
- package/src/pricing/catalogue.generated.ts +28 -4
- package/src/pricing/rates.source.json +27 -6
- package/src/public-runtime.ts +16 -2
- package/src/public-tools.ts +19 -1
- package/src/public-types.ts +5 -0
- package/src/registry/tool/callable.ts +37 -0
- package/src/registry/tool/execute.ts +8 -1
- package/src/runtime/query/executor/tool-call-admission.ts +60 -16
- package/src/runtime/query/executor.ts +19 -4
- package/src/runtime/query/review-policy.ts +103 -15
- package/src/skills/registry.ts +8 -0
- package/src/tools/builtins/computer-use-coordinates.ts +144 -0
- package/src/tools/builtins/computer-use-image.ts +278 -0
- package/src/tools/builtins/computer-use.ts +1485 -191
- package/src/tools/builtins/index.ts +7 -1
- package/src/tools/builtins/skill.ts +304 -174
- package/src/tools/defineTool.ts +10 -0
- package/src/tools/schedules/present.ts +2 -0
- package/src/tools/schedules/schedule-tool.ts +139 -33
- package/src/tools/schedules/types.ts +69 -1
- package/src/types/authorization/index.ts +21 -0
- package/src/types/computer-use/index.ts +202 -0
- package/src/types/tool/index.ts +19 -0
|
@@ -1,19 +1,111 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
2
|
import type { ComputerUseHost } from '../../types/computer-use/index.js';
|
|
3
|
+
import type { LLMProvider } from '../../types/provider/index.js';
|
|
3
4
|
import type { ToolDefinition } from '../../types/tool/index.js';
|
|
5
|
+
import { type ScreenshotLimits } from './computer-use-image.js';
|
|
6
|
+
export { HIGH_RES_SCREENSHOT_LIMITS, STANDARD_SCREENSHOT_LIMITS, screenshotTargetSize, } from './computer-use-image.js';
|
|
7
|
+
export type { ImageSize, ScreenshotLimits } from './computer-use-image.js';
|
|
4
8
|
export declare const COMPUTER_USE_TOOL_NAME: "computer_use";
|
|
9
|
+
/**
|
|
10
|
+
* How `createComputerUseTool` sizes screenshots, paces actions and bounds a
|
|
11
|
+
* batch. Every field is optional.
|
|
12
|
+
*/
|
|
13
|
+
export interface ComputerUseToolOptions {
|
|
14
|
+
/**
|
|
15
|
+
* What every screenshot and zoom image is fitted to before the model sees
|
|
16
|
+
* it. Default {@link STANDARD_SCREENSHOT_LIMITS}, which every current vision
|
|
17
|
+
* model takes without a further resize; {@link HIGH_RES_SCREENSHOT_LIMITS}
|
|
18
|
+
* only when every model the session can reach is on that tier.
|
|
19
|
+
*/
|
|
20
|
+
readonly screenshotLimits?: ScreenshotLimits;
|
|
21
|
+
/**
|
|
22
|
+
* Milliseconds to wait after an action before the screenshot it returns,
|
|
23
|
+
* so a menu has opened or a page has started to paint. Default 500.
|
|
24
|
+
*/
|
|
25
|
+
readonly settleMs?: number;
|
|
26
|
+
/**
|
|
27
|
+
* Return a fresh screenshot after every call that changed something.
|
|
28
|
+
* Default true. Off, a model has to ask for one — an extra round trip per
|
|
29
|
+
* action, which is what this exists to remove.
|
|
30
|
+
*/
|
|
31
|
+
readonly screenshotAfterActions?: boolean;
|
|
32
|
+
/** Most actions one `batch` may carry. Default 20. */
|
|
33
|
+
readonly maxBatchActions?: number;
|
|
34
|
+
/** Longest single `wait`, in milliseconds. Default 10 000. */
|
|
35
|
+
readonly maxWaitMs?: number;
|
|
36
|
+
/**
|
|
37
|
+
* Why computer use cannot work in this session although the host could —
|
|
38
|
+
* typically that the provider cannot put an image in a tool result, so the
|
|
39
|
+
* model would never see a screenshot (see
|
|
40
|
+
* {@link computerUseUnavailableReason}). Set, the tool mounts as a
|
|
41
|
+
* diagnostic: its description says why and every call is refused without
|
|
42
|
+
* touching the host.
|
|
43
|
+
*/
|
|
44
|
+
readonly unavailableReason?: string;
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Why `provider` cannot drive `computer_use`, or undefined when it can.
|
|
48
|
+
*
|
|
49
|
+
* The tool's only way to show the model the screen is an image in a tool
|
|
50
|
+
* result. A driver that declares `supportsToolResultImages: false` replaces
|
|
51
|
+
* that image with a line of text, so the model acts on a screen it has never
|
|
52
|
+
* seen while every call reports success. Pass the answer to
|
|
53
|
+
* {@link ComputerUseToolOptions.unavailableReason}.
|
|
54
|
+
*/
|
|
55
|
+
export declare function computerUseUnavailableReason(provider: Pick<LLMProvider, 'id' | 'capabilities'>): string | undefined;
|
|
5
56
|
declare const actionSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
|
|
6
57
|
type: z.ZodLiteral<"screenshot">;
|
|
7
58
|
}, "strip", z.ZodTypeAny, {
|
|
8
59
|
type: "screenshot";
|
|
9
60
|
}, {
|
|
10
61
|
type: "screenshot";
|
|
62
|
+
}>, z.ZodObject<{
|
|
63
|
+
screenshot_id: z.ZodOptional<z.ZodString>;
|
|
64
|
+
type: z.ZodLiteral<"zoom">;
|
|
65
|
+
region: z.ZodObject<{
|
|
66
|
+
x: z.ZodNumber;
|
|
67
|
+
y: z.ZodNumber;
|
|
68
|
+
width: z.ZodNumber;
|
|
69
|
+
height: z.ZodNumber;
|
|
70
|
+
}, "strip", z.ZodTypeAny, {
|
|
71
|
+
y: number;
|
|
72
|
+
x: number;
|
|
73
|
+
width: number;
|
|
74
|
+
height: number;
|
|
75
|
+
}, {
|
|
76
|
+
y: number;
|
|
77
|
+
x: number;
|
|
78
|
+
width: number;
|
|
79
|
+
height: number;
|
|
80
|
+
}>;
|
|
81
|
+
}, "strip", z.ZodTypeAny, {
|
|
82
|
+
type: "zoom";
|
|
83
|
+
region: {
|
|
84
|
+
y: number;
|
|
85
|
+
x: number;
|
|
86
|
+
width: number;
|
|
87
|
+
height: number;
|
|
88
|
+
};
|
|
89
|
+
screenshot_id?: string | undefined;
|
|
90
|
+
}, {
|
|
91
|
+
type: "zoom";
|
|
92
|
+
region: {
|
|
93
|
+
y: number;
|
|
94
|
+
x: number;
|
|
95
|
+
width: number;
|
|
96
|
+
height: number;
|
|
97
|
+
};
|
|
98
|
+
screenshot_id?: string | undefined;
|
|
11
99
|
}>, z.ZodObject<{
|
|
12
100
|
type: z.ZodLiteral<"cursor_position">;
|
|
101
|
+
} & {
|
|
102
|
+
screenshot_id: z.ZodOptional<z.ZodString>;
|
|
13
103
|
}, "strip", z.ZodTypeAny, {
|
|
14
104
|
type: "cursor_position";
|
|
105
|
+
screenshot_id?: string | undefined;
|
|
15
106
|
}, {
|
|
16
107
|
type: "cursor_position";
|
|
108
|
+
screenshot_id?: string | undefined;
|
|
17
109
|
}>, z.ZodObject<{
|
|
18
110
|
type: z.ZodLiteral<"mouse_move">;
|
|
19
111
|
to: z.ZodObject<{
|
|
@@ -26,18 +118,22 @@ declare const actionSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
|
|
|
26
118
|
y: number;
|
|
27
119
|
x: number;
|
|
28
120
|
}>;
|
|
121
|
+
} & {
|
|
122
|
+
screenshot_id: z.ZodOptional<z.ZodString>;
|
|
29
123
|
}, "strip", z.ZodTypeAny, {
|
|
30
124
|
type: "mouse_move";
|
|
31
125
|
to: {
|
|
32
126
|
y: number;
|
|
33
127
|
x: number;
|
|
34
128
|
};
|
|
129
|
+
screenshot_id?: string | undefined;
|
|
35
130
|
}, {
|
|
36
131
|
type: "mouse_move";
|
|
37
132
|
to: {
|
|
38
133
|
y: number;
|
|
39
134
|
x: number;
|
|
40
135
|
};
|
|
136
|
+
screenshot_id?: string | undefined;
|
|
41
137
|
}>, z.ZodObject<{
|
|
42
138
|
type: z.ZodLiteral<"mouse_click">;
|
|
43
139
|
at: z.ZodObject<{
|
|
@@ -50,7 +146,9 @@ declare const actionSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
|
|
|
50
146
|
y: number;
|
|
51
147
|
x: number;
|
|
52
148
|
}>;
|
|
53
|
-
button: z.ZodEnum<["left", "right", "middle"]
|
|
149
|
+
button: z.ZodDefault<z.ZodEnum<["left", "right", "middle"]>>;
|
|
150
|
+
} & {
|
|
151
|
+
screenshot_id: z.ZodOptional<z.ZodString>;
|
|
54
152
|
}, "strip", z.ZodTypeAny, {
|
|
55
153
|
at: {
|
|
56
154
|
y: number;
|
|
@@ -58,13 +156,15 @@ declare const actionSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
|
|
|
58
156
|
};
|
|
59
157
|
type: "mouse_click";
|
|
60
158
|
button: "left" | "right" | "middle";
|
|
159
|
+
screenshot_id?: string | undefined;
|
|
61
160
|
}, {
|
|
62
161
|
at: {
|
|
63
162
|
y: number;
|
|
64
163
|
x: number;
|
|
65
164
|
};
|
|
66
165
|
type: "mouse_click";
|
|
67
|
-
button
|
|
166
|
+
button?: "left" | "right" | "middle" | undefined;
|
|
167
|
+
screenshot_id?: string | undefined;
|
|
68
168
|
}>, z.ZodObject<{
|
|
69
169
|
type: z.ZodLiteral<"mouse_drag">;
|
|
70
170
|
from: z.ZodObject<{
|
|
@@ -87,7 +187,9 @@ declare const actionSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
|
|
|
87
187
|
y: number;
|
|
88
188
|
x: number;
|
|
89
189
|
}>;
|
|
90
|
-
button: z.ZodEnum<["left", "right", "middle"]
|
|
190
|
+
button: z.ZodDefault<z.ZodEnum<["left", "right", "middle"]>>;
|
|
191
|
+
} & {
|
|
192
|
+
screenshot_id: z.ZodOptional<z.ZodString>;
|
|
91
193
|
}, "strip", z.ZodTypeAny, {
|
|
92
194
|
type: "mouse_drag";
|
|
93
195
|
from: {
|
|
@@ -99,6 +201,7 @@ declare const actionSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
|
|
|
99
201
|
x: number;
|
|
100
202
|
};
|
|
101
203
|
button: "left" | "right" | "middle";
|
|
204
|
+
screenshot_id?: string | undefined;
|
|
102
205
|
}, {
|
|
103
206
|
type: "mouse_drag";
|
|
104
207
|
from: {
|
|
@@ -109,7 +212,8 @@ declare const actionSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
|
|
|
109
212
|
y: number;
|
|
110
213
|
x: number;
|
|
111
214
|
};
|
|
112
|
-
button
|
|
215
|
+
button?: "left" | "right" | "middle" | undefined;
|
|
216
|
+
screenshot_id?: string | undefined;
|
|
113
217
|
}>, z.ZodObject<{
|
|
114
218
|
type: z.ZodLiteral<"scroll">;
|
|
115
219
|
at: z.ZodObject<{
|
|
@@ -124,6 +228,8 @@ declare const actionSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
|
|
|
124
228
|
}>;
|
|
125
229
|
direction: z.ZodEnum<["up", "down", "left", "right"]>;
|
|
126
230
|
amount: z.ZodNumber;
|
|
231
|
+
} & {
|
|
232
|
+
screenshot_id: z.ZodOptional<z.ZodString>;
|
|
127
233
|
}, "strip", z.ZodTypeAny, {
|
|
128
234
|
at: {
|
|
129
235
|
y: number;
|
|
@@ -132,6 +238,7 @@ declare const actionSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
|
|
|
132
238
|
type: "scroll";
|
|
133
239
|
amount: number;
|
|
134
240
|
direction: "left" | "right" | "up" | "down";
|
|
241
|
+
screenshot_id?: string | undefined;
|
|
135
242
|
}, {
|
|
136
243
|
at: {
|
|
137
244
|
y: number;
|
|
@@ -140,43 +247,442 @@ declare const actionSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
|
|
|
140
247
|
type: "scroll";
|
|
141
248
|
amount: number;
|
|
142
249
|
direction: "left" | "right" | "up" | "down";
|
|
250
|
+
screenshot_id?: string | undefined;
|
|
143
251
|
}>, z.ZodObject<{
|
|
144
252
|
type: z.ZodLiteral<"type_text">;
|
|
145
253
|
text: z.ZodString;
|
|
254
|
+
} & {
|
|
255
|
+
screenshot_id: z.ZodOptional<z.ZodString>;
|
|
146
256
|
}, "strip", z.ZodTypeAny, {
|
|
147
257
|
text: string;
|
|
148
258
|
type: "type_text";
|
|
259
|
+
screenshot_id?: string | undefined;
|
|
149
260
|
}, {
|
|
150
261
|
text: string;
|
|
151
262
|
type: "type_text";
|
|
263
|
+
screenshot_id?: string | undefined;
|
|
152
264
|
}>, z.ZodObject<{
|
|
153
265
|
type: z.ZodLiteral<"key">;
|
|
154
266
|
keys: z.ZodString;
|
|
267
|
+
} & {
|
|
268
|
+
screenshot_id: z.ZodOptional<z.ZodString>;
|
|
155
269
|
}, "strip", z.ZodTypeAny, {
|
|
156
270
|
keys: string;
|
|
157
271
|
type: "key";
|
|
272
|
+
screenshot_id?: string | undefined;
|
|
158
273
|
}, {
|
|
159
274
|
keys: string;
|
|
160
275
|
type: "key";
|
|
276
|
+
screenshot_id?: string | undefined;
|
|
277
|
+
}>, z.ZodObject<{
|
|
278
|
+
type: z.ZodLiteral<"wait">;
|
|
279
|
+
ms: z.ZodNumber;
|
|
280
|
+
} & {
|
|
281
|
+
screenshot_id: z.ZodOptional<z.ZodString>;
|
|
282
|
+
}, "strip", z.ZodTypeAny, {
|
|
283
|
+
type: "wait";
|
|
284
|
+
ms: number;
|
|
285
|
+
screenshot_id?: string | undefined;
|
|
286
|
+
}, {
|
|
287
|
+
type: "wait";
|
|
288
|
+
ms: number;
|
|
289
|
+
screenshot_id?: string | undefined;
|
|
290
|
+
}>, z.ZodObject<{
|
|
291
|
+
type: z.ZodLiteral<"list_windows">;
|
|
292
|
+
} & {
|
|
293
|
+
screenshot_id: z.ZodOptional<z.ZodString>;
|
|
294
|
+
}, "strip", z.ZodTypeAny, {
|
|
295
|
+
type: "list_windows";
|
|
296
|
+
screenshot_id?: string | undefined;
|
|
297
|
+
}, {
|
|
298
|
+
type: "list_windows";
|
|
299
|
+
screenshot_id?: string | undefined;
|
|
300
|
+
}>, z.ZodObject<{
|
|
301
|
+
type: z.ZodLiteral<"focus_window">;
|
|
302
|
+
window_id: z.ZodString;
|
|
303
|
+
} & {
|
|
304
|
+
screenshot_id: z.ZodOptional<z.ZodString>;
|
|
305
|
+
}, "strip", z.ZodTypeAny, {
|
|
306
|
+
type: "focus_window";
|
|
307
|
+
window_id: string;
|
|
308
|
+
screenshot_id?: string | undefined;
|
|
309
|
+
}, {
|
|
310
|
+
type: "focus_window";
|
|
311
|
+
window_id: string;
|
|
312
|
+
screenshot_id?: string | undefined;
|
|
313
|
+
}>, z.ZodObject<{
|
|
314
|
+
type: z.ZodLiteral<"ui_snapshot">;
|
|
315
|
+
window_id: z.ZodOptional<z.ZodString>;
|
|
316
|
+
}, "strip", z.ZodTypeAny, {
|
|
317
|
+
type: "ui_snapshot";
|
|
318
|
+
window_id?: string | undefined;
|
|
319
|
+
}, {
|
|
320
|
+
type: "ui_snapshot";
|
|
321
|
+
window_id?: string | undefined;
|
|
322
|
+
}>, z.ZodObject<{
|
|
323
|
+
type: z.ZodLiteral<"ui_act">;
|
|
324
|
+
ref: z.ZodString;
|
|
325
|
+
action: z.ZodEnum<["invoke", "set_value", "toggle", "select", "expand", "collapse", "focus", "scroll_into_view"]>;
|
|
326
|
+
value: z.ZodOptional<z.ZodString>;
|
|
327
|
+
} & {
|
|
328
|
+
screenshot_id: z.ZodOptional<z.ZodString>;
|
|
329
|
+
}, "strip", z.ZodTypeAny, {
|
|
330
|
+
type: "ui_act";
|
|
331
|
+
action: "select" | "invoke" | "focus" | "set_value" | "toggle" | "expand" | "collapse" | "scroll_into_view";
|
|
332
|
+
ref: string;
|
|
333
|
+
value?: string | undefined;
|
|
334
|
+
screenshot_id?: string | undefined;
|
|
335
|
+
}, {
|
|
336
|
+
type: "ui_act";
|
|
337
|
+
action: "select" | "invoke" | "focus" | "set_value" | "toggle" | "expand" | "collapse" | "scroll_into_view";
|
|
338
|
+
ref: string;
|
|
339
|
+
value?: string | undefined;
|
|
340
|
+
screenshot_id?: string | undefined;
|
|
341
|
+
}>, z.ZodObject<{
|
|
342
|
+
screenshot_id: z.ZodOptional<z.ZodString>;
|
|
343
|
+
type: z.ZodLiteral<"batch">;
|
|
344
|
+
actions: z.ZodArray<z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
|
|
345
|
+
type: z.ZodLiteral<"cursor_position">;
|
|
346
|
+
}, "strip", z.ZodTypeAny, {
|
|
347
|
+
type: "cursor_position";
|
|
348
|
+
}, {
|
|
349
|
+
type: "cursor_position";
|
|
350
|
+
}>, z.ZodObject<{
|
|
351
|
+
type: z.ZodLiteral<"mouse_move">;
|
|
352
|
+
to: z.ZodObject<{
|
|
353
|
+
x: z.ZodNumber;
|
|
354
|
+
y: z.ZodNumber;
|
|
355
|
+
}, "strip", z.ZodTypeAny, {
|
|
356
|
+
y: number;
|
|
357
|
+
x: number;
|
|
358
|
+
}, {
|
|
359
|
+
y: number;
|
|
360
|
+
x: number;
|
|
361
|
+
}>;
|
|
362
|
+
}, "strip", z.ZodTypeAny, {
|
|
363
|
+
type: "mouse_move";
|
|
364
|
+
to: {
|
|
365
|
+
y: number;
|
|
366
|
+
x: number;
|
|
367
|
+
};
|
|
368
|
+
}, {
|
|
369
|
+
type: "mouse_move";
|
|
370
|
+
to: {
|
|
371
|
+
y: number;
|
|
372
|
+
x: number;
|
|
373
|
+
};
|
|
374
|
+
}>, z.ZodObject<{
|
|
375
|
+
type: z.ZodLiteral<"mouse_click">;
|
|
376
|
+
at: z.ZodObject<{
|
|
377
|
+
x: z.ZodNumber;
|
|
378
|
+
y: z.ZodNumber;
|
|
379
|
+
}, "strip", z.ZodTypeAny, {
|
|
380
|
+
y: number;
|
|
381
|
+
x: number;
|
|
382
|
+
}, {
|
|
383
|
+
y: number;
|
|
384
|
+
x: number;
|
|
385
|
+
}>;
|
|
386
|
+
button: z.ZodDefault<z.ZodEnum<["left", "right", "middle"]>>;
|
|
387
|
+
}, "strip", z.ZodTypeAny, {
|
|
388
|
+
at: {
|
|
389
|
+
y: number;
|
|
390
|
+
x: number;
|
|
391
|
+
};
|
|
392
|
+
type: "mouse_click";
|
|
393
|
+
button: "left" | "right" | "middle";
|
|
394
|
+
}, {
|
|
395
|
+
at: {
|
|
396
|
+
y: number;
|
|
397
|
+
x: number;
|
|
398
|
+
};
|
|
399
|
+
type: "mouse_click";
|
|
400
|
+
button?: "left" | "right" | "middle" | undefined;
|
|
401
|
+
}>, z.ZodObject<{
|
|
402
|
+
type: z.ZodLiteral<"mouse_drag">;
|
|
403
|
+
from: z.ZodObject<{
|
|
404
|
+
x: z.ZodNumber;
|
|
405
|
+
y: z.ZodNumber;
|
|
406
|
+
}, "strip", z.ZodTypeAny, {
|
|
407
|
+
y: number;
|
|
408
|
+
x: number;
|
|
409
|
+
}, {
|
|
410
|
+
y: number;
|
|
411
|
+
x: number;
|
|
412
|
+
}>;
|
|
413
|
+
to: z.ZodObject<{
|
|
414
|
+
x: z.ZodNumber;
|
|
415
|
+
y: z.ZodNumber;
|
|
416
|
+
}, "strip", z.ZodTypeAny, {
|
|
417
|
+
y: number;
|
|
418
|
+
x: number;
|
|
419
|
+
}, {
|
|
420
|
+
y: number;
|
|
421
|
+
x: number;
|
|
422
|
+
}>;
|
|
423
|
+
button: z.ZodDefault<z.ZodEnum<["left", "right", "middle"]>>;
|
|
424
|
+
}, "strip", z.ZodTypeAny, {
|
|
425
|
+
type: "mouse_drag";
|
|
426
|
+
from: {
|
|
427
|
+
y: number;
|
|
428
|
+
x: number;
|
|
429
|
+
};
|
|
430
|
+
to: {
|
|
431
|
+
y: number;
|
|
432
|
+
x: number;
|
|
433
|
+
};
|
|
434
|
+
button: "left" | "right" | "middle";
|
|
435
|
+
}, {
|
|
436
|
+
type: "mouse_drag";
|
|
437
|
+
from: {
|
|
438
|
+
y: number;
|
|
439
|
+
x: number;
|
|
440
|
+
};
|
|
441
|
+
to: {
|
|
442
|
+
y: number;
|
|
443
|
+
x: number;
|
|
444
|
+
};
|
|
445
|
+
button?: "left" | "right" | "middle" | undefined;
|
|
446
|
+
}>, z.ZodObject<{
|
|
447
|
+
type: z.ZodLiteral<"scroll">;
|
|
448
|
+
at: z.ZodObject<{
|
|
449
|
+
x: z.ZodNumber;
|
|
450
|
+
y: z.ZodNumber;
|
|
451
|
+
}, "strip", z.ZodTypeAny, {
|
|
452
|
+
y: number;
|
|
453
|
+
x: number;
|
|
454
|
+
}, {
|
|
455
|
+
y: number;
|
|
456
|
+
x: number;
|
|
457
|
+
}>;
|
|
458
|
+
direction: z.ZodEnum<["up", "down", "left", "right"]>;
|
|
459
|
+
amount: z.ZodNumber;
|
|
460
|
+
}, "strip", z.ZodTypeAny, {
|
|
461
|
+
at: {
|
|
462
|
+
y: number;
|
|
463
|
+
x: number;
|
|
464
|
+
};
|
|
465
|
+
type: "scroll";
|
|
466
|
+
amount: number;
|
|
467
|
+
direction: "left" | "right" | "up" | "down";
|
|
468
|
+
}, {
|
|
469
|
+
at: {
|
|
470
|
+
y: number;
|
|
471
|
+
x: number;
|
|
472
|
+
};
|
|
473
|
+
type: "scroll";
|
|
474
|
+
amount: number;
|
|
475
|
+
direction: "left" | "right" | "up" | "down";
|
|
476
|
+
}>, z.ZodObject<{
|
|
477
|
+
type: z.ZodLiteral<"type_text">;
|
|
478
|
+
text: z.ZodString;
|
|
479
|
+
}, "strip", z.ZodTypeAny, {
|
|
480
|
+
text: string;
|
|
481
|
+
type: "type_text";
|
|
482
|
+
}, {
|
|
483
|
+
text: string;
|
|
484
|
+
type: "type_text";
|
|
485
|
+
}>, z.ZodObject<{
|
|
486
|
+
type: z.ZodLiteral<"key">;
|
|
487
|
+
keys: z.ZodString;
|
|
488
|
+
}, "strip", z.ZodTypeAny, {
|
|
489
|
+
keys: string;
|
|
490
|
+
type: "key";
|
|
491
|
+
}, {
|
|
492
|
+
keys: string;
|
|
493
|
+
type: "key";
|
|
494
|
+
}>, z.ZodObject<{
|
|
495
|
+
type: z.ZodLiteral<"wait">;
|
|
496
|
+
ms: z.ZodNumber;
|
|
497
|
+
}, "strip", z.ZodTypeAny, {
|
|
498
|
+
type: "wait";
|
|
499
|
+
ms: number;
|
|
500
|
+
}, {
|
|
501
|
+
type: "wait";
|
|
502
|
+
ms: number;
|
|
503
|
+
}>, z.ZodObject<{
|
|
504
|
+
type: z.ZodLiteral<"list_windows">;
|
|
505
|
+
}, "strip", z.ZodTypeAny, {
|
|
506
|
+
type: "list_windows";
|
|
507
|
+
}, {
|
|
508
|
+
type: "list_windows";
|
|
509
|
+
}>, z.ZodObject<{
|
|
510
|
+
type: z.ZodLiteral<"focus_window">;
|
|
511
|
+
window_id: z.ZodString;
|
|
512
|
+
}, "strip", z.ZodTypeAny, {
|
|
513
|
+
type: "focus_window";
|
|
514
|
+
window_id: string;
|
|
515
|
+
}, {
|
|
516
|
+
type: "focus_window";
|
|
517
|
+
window_id: string;
|
|
518
|
+
}>, z.ZodObject<{
|
|
519
|
+
type: z.ZodLiteral<"ui_act">;
|
|
520
|
+
ref: z.ZodString;
|
|
521
|
+
action: z.ZodEnum<["invoke", "set_value", "toggle", "select", "expand", "collapse", "focus", "scroll_into_view"]>;
|
|
522
|
+
value: z.ZodOptional<z.ZodString>;
|
|
523
|
+
}, "strip", z.ZodTypeAny, {
|
|
524
|
+
type: "ui_act";
|
|
525
|
+
action: "select" | "invoke" | "focus" | "set_value" | "toggle" | "expand" | "collapse" | "scroll_into_view";
|
|
526
|
+
ref: string;
|
|
527
|
+
value?: string | undefined;
|
|
528
|
+
}, {
|
|
529
|
+
type: "ui_act";
|
|
530
|
+
action: "select" | "invoke" | "focus" | "set_value" | "toggle" | "expand" | "collapse" | "scroll_into_view";
|
|
531
|
+
ref: string;
|
|
532
|
+
value?: string | undefined;
|
|
533
|
+
}>]>, "many">;
|
|
534
|
+
}, "strip", z.ZodTypeAny, {
|
|
535
|
+
type: "batch";
|
|
536
|
+
actions: ({
|
|
537
|
+
type: "cursor_position";
|
|
538
|
+
} | {
|
|
539
|
+
type: "mouse_move";
|
|
540
|
+
to: {
|
|
541
|
+
y: number;
|
|
542
|
+
x: number;
|
|
543
|
+
};
|
|
544
|
+
} | {
|
|
545
|
+
at: {
|
|
546
|
+
y: number;
|
|
547
|
+
x: number;
|
|
548
|
+
};
|
|
549
|
+
type: "mouse_click";
|
|
550
|
+
button: "left" | "right" | "middle";
|
|
551
|
+
} | {
|
|
552
|
+
type: "mouse_drag";
|
|
553
|
+
from: {
|
|
554
|
+
y: number;
|
|
555
|
+
x: number;
|
|
556
|
+
};
|
|
557
|
+
to: {
|
|
558
|
+
y: number;
|
|
559
|
+
x: number;
|
|
560
|
+
};
|
|
561
|
+
button: "left" | "right" | "middle";
|
|
562
|
+
} | {
|
|
563
|
+
at: {
|
|
564
|
+
y: number;
|
|
565
|
+
x: number;
|
|
566
|
+
};
|
|
567
|
+
type: "scroll";
|
|
568
|
+
amount: number;
|
|
569
|
+
direction: "left" | "right" | "up" | "down";
|
|
570
|
+
} | {
|
|
571
|
+
text: string;
|
|
572
|
+
type: "type_text";
|
|
573
|
+
} | {
|
|
574
|
+
keys: string;
|
|
575
|
+
type: "key";
|
|
576
|
+
} | {
|
|
577
|
+
type: "wait";
|
|
578
|
+
ms: number;
|
|
579
|
+
} | {
|
|
580
|
+
type: "list_windows";
|
|
581
|
+
} | {
|
|
582
|
+
type: "focus_window";
|
|
583
|
+
window_id: string;
|
|
584
|
+
} | {
|
|
585
|
+
type: "ui_act";
|
|
586
|
+
action: "select" | "invoke" | "focus" | "set_value" | "toggle" | "expand" | "collapse" | "scroll_into_view";
|
|
587
|
+
ref: string;
|
|
588
|
+
value?: string | undefined;
|
|
589
|
+
})[];
|
|
590
|
+
screenshot_id?: string | undefined;
|
|
591
|
+
}, {
|
|
592
|
+
type: "batch";
|
|
593
|
+
actions: ({
|
|
594
|
+
type: "cursor_position";
|
|
595
|
+
} | {
|
|
596
|
+
type: "mouse_move";
|
|
597
|
+
to: {
|
|
598
|
+
y: number;
|
|
599
|
+
x: number;
|
|
600
|
+
};
|
|
601
|
+
} | {
|
|
602
|
+
at: {
|
|
603
|
+
y: number;
|
|
604
|
+
x: number;
|
|
605
|
+
};
|
|
606
|
+
type: "mouse_click";
|
|
607
|
+
button?: "left" | "right" | "middle" | undefined;
|
|
608
|
+
} | {
|
|
609
|
+
type: "mouse_drag";
|
|
610
|
+
from: {
|
|
611
|
+
y: number;
|
|
612
|
+
x: number;
|
|
613
|
+
};
|
|
614
|
+
to: {
|
|
615
|
+
y: number;
|
|
616
|
+
x: number;
|
|
617
|
+
};
|
|
618
|
+
button?: "left" | "right" | "middle" | undefined;
|
|
619
|
+
} | {
|
|
620
|
+
at: {
|
|
621
|
+
y: number;
|
|
622
|
+
x: number;
|
|
623
|
+
};
|
|
624
|
+
type: "scroll";
|
|
625
|
+
amount: number;
|
|
626
|
+
direction: "left" | "right" | "up" | "down";
|
|
627
|
+
} | {
|
|
628
|
+
text: string;
|
|
629
|
+
type: "type_text";
|
|
630
|
+
} | {
|
|
631
|
+
keys: string;
|
|
632
|
+
type: "key";
|
|
633
|
+
} | {
|
|
634
|
+
type: "wait";
|
|
635
|
+
ms: number;
|
|
636
|
+
} | {
|
|
637
|
+
type: "list_windows";
|
|
638
|
+
} | {
|
|
639
|
+
type: "focus_window";
|
|
640
|
+
window_id: string;
|
|
641
|
+
} | {
|
|
642
|
+
type: "ui_act";
|
|
643
|
+
action: "select" | "invoke" | "focus" | "set_value" | "toggle" | "expand" | "collapse" | "scroll_into_view";
|
|
644
|
+
ref: string;
|
|
645
|
+
value?: string | undefined;
|
|
646
|
+
})[];
|
|
647
|
+
screenshot_id?: string | undefined;
|
|
161
648
|
}>]>;
|
|
162
649
|
/**
|
|
163
|
-
* The tool's input, inferred from its schema
|
|
650
|
+
* The tool's input, inferred from its schema: one action, or
|
|
651
|
+
* `{ type: 'batch', actions: [...] }`.
|
|
164
652
|
*
|
|
165
|
-
* Exported because `createComputerUseTool` returns a `ToolDefinition<ActionInput>`
|
|
653
|
+
* Exported because `createComputerUseTool` returns a `ToolDefinition<ActionInput>` (a `ComputerUseTool`)
|
|
166
654
|
* and a consumer typing that variable, or writing a wrapper around it, had no
|
|
167
655
|
* name for the parameter — the type was module-private while the function
|
|
168
656
|
* carrying it was public.
|
|
169
657
|
*/
|
|
170
658
|
export type ActionInput = z.infer<typeof actionSchema>;
|
|
659
|
+
/**
|
|
660
|
+
* The `computer_use` tool, with one question a host's review screen can ask
|
|
661
|
+
* of it.
|
|
662
|
+
*/
|
|
663
|
+
export interface ComputerUseTool extends ToolDefinition<ActionInput> {
|
|
664
|
+
/**
|
|
665
|
+
* What a `ui_act` ref names in the latest `ui_snapshot` — `Button "Beş"
|
|
666
|
+
* (e30)` — or undefined for a ref it does not hold. For a host that shows
|
|
667
|
+
* a person the call before it runs; the words are the application's, so a
|
|
668
|
+
* host shows them as text and nothing more.
|
|
669
|
+
*
|
|
670
|
+
* @experimental Follows the UI-tree surface; see `ComputerUseCapabilities.uiTree`.
|
|
671
|
+
*/
|
|
672
|
+
describeUiRef(ref: string): string | undefined;
|
|
673
|
+
}
|
|
171
674
|
/**
|
|
172
675
|
* Factory: given a ComputerUseHost (provided by the consumer — e.g.
|
|
173
|
-
* @namzu/computer-use's SubprocessComputerUseHost), returns
|
|
174
|
-
*
|
|
175
|
-
* the SDK's ToolResult shape.
|
|
676
|
+
* @namzu/computer-use's SubprocessComputerUseHost), returns the
|
|
677
|
+
* `computer_use` tool.
|
|
176
678
|
*
|
|
177
|
-
*
|
|
178
|
-
*
|
|
179
|
-
*
|
|
679
|
+
* Every screenshot is fitted to the model's image limits and numbered;
|
|
680
|
+
* every coordinate the model sends is a pixel of one of those screenshots and
|
|
681
|
+
* is mapped onto the host's display. Actions that change something return a
|
|
682
|
+
* fresh screenshot after a short settle delay, and `batch` runs several in
|
|
683
|
+
* one call. The description and schema reflect the host's frozen
|
|
684
|
+
* capabilities, and any action targeting an unavailable capability is
|
|
685
|
+
* rejected before the host is touched.
|
|
180
686
|
*
|
|
181
687
|
* @example
|
|
182
688
|
* ```ts
|
|
@@ -188,6 +694,5 @@ export type ActionInput = z.infer<typeof actionSchema>;
|
|
|
188
694
|
* registry.register(createComputerUseTool(host))
|
|
189
695
|
* ```
|
|
190
696
|
*/
|
|
191
|
-
export declare function createComputerUseTool(host: ComputerUseHost):
|
|
192
|
-
export {};
|
|
697
|
+
export declare function createComputerUseTool(host: ComputerUseHost, options?: ComputerUseToolOptions): ComputerUseTool;
|
|
193
698
|
//# sourceMappingURL=computer-use.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"computer-use.d.ts","sourceRoot":"","sources":["../../../src/tools/builtins/computer-use.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;
|
|
1
|
+
{"version":3,"file":"computer-use.d.ts","sourceRoot":"","sources":["../../../src/tools/builtins/computer-use.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AAEvB,OAAO,KAAK,EAGX,eAAe,EAUf,MAAM,mCAAmC,CAAA;AAE1C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAA;AAChE,OAAO,KAAK,EAAe,cAAc,EAAc,MAAM,2BAA2B,CAAA;AAcxF,OAAO,EAGN,KAAK,gBAAgB,EAIrB,MAAM,yBAAyB,CAAA;AAEhC,OAAO,EACN,0BAA0B,EAC1B,0BAA0B,EAC1B,oBAAoB,GACpB,MAAM,yBAAyB,CAAA;AAChC,YAAY,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAA;AAE1E,eAAO,MAAM,sBAAsB,EAAG,cAAuB,CAAA;AAS7D;;;GAGG;AACH,MAAM,WAAW,sBAAsB;IACtC;;;;;OAKG;IACH,QAAQ,CAAC,gBAAgB,CAAC,EAAE,gBAAgB,CAAA;IAC5C;;;OAGG;IACH,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;IAC1B;;;;OAIG;IACH,QAAQ,CAAC,sBAAsB,CAAC,EAAE,OAAO,CAAA;IACzC,sDAAsD;IACtD,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAA;IACjC,8DAA8D;IAC9D,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAA;IAC3B;;;;;;;OAOG;IACH,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,CAAA;CACnC;AAED;;;;;;;;GAQG;AACH,wBAAgB,4BAA4B,CAC3C,QAAQ,EAAE,IAAI,CAAC,WAAW,EAAE,IAAI,GAAG,cAAc,CAAC,GAChD,MAAM,GAAG,SAAS,CAGpB;AAwFD,QAAA,MAAM,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAoBhB,CAAA;AAEF;;;;;;;;GAQG;AACH,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAA;AAgmBtD;;;GAGG;AACH,MAAM,WAAW,eAAgB,SAAQ,cAAc,CAAC,WAAW,CAAC;IACnE;;;;;;;OAOG;IACH,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAAA;CAC9C;AA2DD;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAgB,qBAAqB,CACpC,IAAI,EAAE,eAAe,EACrB,OAAO,GAAE,sBAA2B,GAClC,eAAe,CAguBjB"}
|