@withgateway/sdk 0.2.8 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -8
- package/dist/cli.js +2695 -657
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +2675 -637
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.d.mts +1 -0
- package/dist/files/index.d.ts +1 -0
- package/dist/files/index.js +1443 -36
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +1441 -28
- package/dist/files/index.mjs.map +1 -1
- package/dist/{index-Cz8Q6Keo.d.mts → index-D6oi1TWE.d.mts} +93 -34
- package/dist/{index-BO2SNSgy.d.ts → index-DAb-FCMg.d.ts} +93 -34
- package/dist/index.d.mts +4 -3
- package/dist/index.d.ts +4 -3
- package/dist/index.js +1986 -235
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1941 -218
- package/dist/index.mjs.map +1 -1
- package/dist/media-CbcAfQvd.d.mts +83 -0
- package/dist/media-CbcAfQvd.d.ts +83 -0
- package/dist/traces/index.d.mts +1 -0
- package/dist/traces/index.d.ts +1 -0
- package/dist/traces/index.js +1440 -33
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +1438 -25
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +2 -1
- package/dist/tracing/index.d.ts +2 -1
- package/dist/tracing/index.js +512 -11
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +501 -10
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +614 -25
- package/dist/worlds/index.d.ts +614 -25
- package/dist/worlds/index.js +1923 -214
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +1910 -201
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_toml.py +3 -10
- package/runtime/gatewaysdk/environment/schema/api.py +38 -56
- package/runtime/gatewaysdk/environment/schema/grading.py +1 -5
- package/runtime/gatewaysdk/environment/schema/host.py +53 -90
- package/runtime/gatewaysdk/environment/schema/local.py +15 -7
- package/runtime/gatewaysdk/environment/schema/runtime.py +9 -9
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +3 -3
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +1 -1
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +18 -18
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +4 -2
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +5 -1
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/new-world-sandbox.md +5 -3
- package/runtime/gatewaysdk/environment/schema/store.py +15 -6
- package/runtime/gatewaysdk/environment/schema/tasks.py +25 -1
- package/skills/benchmarks-getting-started/SKILL.md +3 -3
- package/skills/connector-schema-authoring/SKILL.md +1 -1
- package/skills/connector-schema-authoring/world-schema.md +1 -1
- package/skills/worlds-getting-started/SKILL.md +18 -18
- package/skills/worlds-getting-started/references/cli-mcp-map.md +4 -2
- package/skills/worlds-getting-started/references/gotchas.md +5 -1
- package/skills/worlds-getting-started/references/new-world-sandbox.md +5 -3
package/dist/traces/index.js
CHANGED
|
@@ -35,6 +35,115 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
|
|
|
35
35
|
));
|
|
36
36
|
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
37
37
|
|
|
38
|
+
// src/worlds/transport.ts
|
|
39
|
+
function platformTimeoutMs(env = process.env) {
|
|
40
|
+
const raw = env[PLATFORM_TIMEOUT_ENV];
|
|
41
|
+
if (raw === void 0 || raw === "") return PLATFORM_TIMEOUT_DEFAULT_MS;
|
|
42
|
+
const value = Number(raw);
|
|
43
|
+
if (Number.isInteger(value) && value > 0) return value;
|
|
44
|
+
throw new Error(`${PLATFORM_TIMEOUT_ENV} must be a positive whole number of milliseconds, not \`${raw}\`.`);
|
|
45
|
+
}
|
|
46
|
+
var PLATFORM_TIMEOUT_ENV, PLATFORM_TIMEOUT_DEFAULT_MS, CHECK_HOST, RETRY, CAUSES, UNKNOWN_CAUSE, TransportCause, UnreachableError, PlatformTransport;
|
|
47
|
+
var init_transport = __esm({
|
|
48
|
+
"src/worlds/transport.ts"() {
|
|
49
|
+
"use strict";
|
|
50
|
+
PLATFORM_TIMEOUT_ENV = "GATEWAY_TIMEOUT_MS";
|
|
51
|
+
PLATFORM_TIMEOUT_DEFAULT_MS = 12e4;
|
|
52
|
+
CHECK_HOST = "Check GATEWAY_HOST (or --host) points at a running platform, then retry.";
|
|
53
|
+
RETRY = "Retry; if it repeats, check the network between here and the platform.";
|
|
54
|
+
CAUSES = {
|
|
55
|
+
ECONNREFUSED: { reason: "connection refused", next: CHECK_HOST },
|
|
56
|
+
ENOTFOUND: { reason: "DNS lookup failed", next: CHECK_HOST },
|
|
57
|
+
EAI_AGAIN: { reason: "DNS lookup failed", next: CHECK_HOST },
|
|
58
|
+
ETIMEDOUT: { reason: "connect timed out", next: CHECK_HOST },
|
|
59
|
+
UND_ERR_CONNECT_TIMEOUT: { reason: "connect timed out", next: CHECK_HOST },
|
|
60
|
+
EHOSTUNREACH: { reason: "host unreachable", next: CHECK_HOST },
|
|
61
|
+
ENETUNREACH: { reason: "network unreachable", next: RETRY },
|
|
62
|
+
ECONNRESET: { reason: "connection dropped", next: RETRY },
|
|
63
|
+
UND_ERR_SOCKET: { reason: "connection dropped", next: RETRY },
|
|
64
|
+
EPIPE: { reason: "connection dropped", next: RETRY }
|
|
65
|
+
};
|
|
66
|
+
UNKNOWN_CAUSE = { reason: "network error", next: CHECK_HOST };
|
|
67
|
+
TransportCause = class _TransportCause {
|
|
68
|
+
constructor(reason, detail, next) {
|
|
69
|
+
this.reason = reason;
|
|
70
|
+
this.detail = detail;
|
|
71
|
+
this.next = next;
|
|
72
|
+
}
|
|
73
|
+
reason;
|
|
74
|
+
detail;
|
|
75
|
+
next;
|
|
76
|
+
static timeout(timeoutMs) {
|
|
77
|
+
return new _TransportCause(
|
|
78
|
+
`timed out after ${timeoutMs / 1e3}s`,
|
|
79
|
+
null,
|
|
80
|
+
`Retry; if the platform is slow, raise ${PLATFORM_TIMEOUT_ENV} (milliseconds per request).`
|
|
81
|
+
);
|
|
82
|
+
}
|
|
83
|
+
static of(error) {
|
|
84
|
+
const cause = error instanceof Error && error.cause instanceof Error ? error.cause : void 0;
|
|
85
|
+
const code = cause && "code" in cause && typeof cause.code === "string" ? cause.code : null;
|
|
86
|
+
const message = cause?.message ?? (error instanceof Error ? error.message : String(error));
|
|
87
|
+
const known = code && CAUSES[code] || UNKNOWN_CAUSE;
|
|
88
|
+
return new _TransportCause(known.reason, code ? `${code}: ${message}` : message, known.next);
|
|
89
|
+
}
|
|
90
|
+
describe() {
|
|
91
|
+
return this.detail === null ? this.reason : `${this.reason} (${this.detail})`;
|
|
92
|
+
}
|
|
93
|
+
};
|
|
94
|
+
UnreachableError = class extends Error {
|
|
95
|
+
constructor(host, method, route, failure) {
|
|
96
|
+
super(`${method} ${route}: no answer from ${host}: ${failure.describe()}. ${failure.next}`);
|
|
97
|
+
this.host = host;
|
|
98
|
+
this.method = method;
|
|
99
|
+
this.route = route;
|
|
100
|
+
this.failure = failure;
|
|
101
|
+
this.name = "UnreachableError";
|
|
102
|
+
}
|
|
103
|
+
host;
|
|
104
|
+
method;
|
|
105
|
+
route;
|
|
106
|
+
failure;
|
|
107
|
+
};
|
|
108
|
+
PlatformTransport = class {
|
|
109
|
+
constructor(endpoint, timeoutMs = platformTimeoutMs()) {
|
|
110
|
+
this.endpoint = endpoint;
|
|
111
|
+
this.timeoutMs = timeoutMs;
|
|
112
|
+
}
|
|
113
|
+
endpoint;
|
|
114
|
+
timeoutMs;
|
|
115
|
+
/** A request that legitimately takes longer (a sandbox exec, a world create) waits its own budget; every other one waits the default. */
|
|
116
|
+
budget(ownMs) {
|
|
117
|
+
return Math.max(ownMs ?? 0, this.timeoutMs);
|
|
118
|
+
}
|
|
119
|
+
async send(method, route, body, opts = {}) {
|
|
120
|
+
const timeoutMs = this.budget(opts.timeoutMs);
|
|
121
|
+
const payload = body === void 0 ? {} : { body: JSON.stringify(body) };
|
|
122
|
+
const controller = new AbortController();
|
|
123
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
124
|
+
try {
|
|
125
|
+
const res = await fetch(`${this.endpoint.host}${route}`, {
|
|
126
|
+
method,
|
|
127
|
+
headers: {
|
|
128
|
+
"content-type": "application/json",
|
|
129
|
+
accept: "application/json",
|
|
130
|
+
authorization: "Basic " + Buffer.from(`${this.endpoint.publicKey}:${this.endpoint.secretKey}`).toString("base64")
|
|
131
|
+
},
|
|
132
|
+
...payload,
|
|
133
|
+
signal: controller.signal
|
|
134
|
+
});
|
|
135
|
+
return { status: res.status, ok: res.ok, text: await res.text() };
|
|
136
|
+
} catch (error) {
|
|
137
|
+
const cause = controller.signal.aborted ? TransportCause.timeout(timeoutMs) : TransportCause.of(error);
|
|
138
|
+
throw new UnreachableError(this.endpoint.host, method, route, cause);
|
|
139
|
+
} finally {
|
|
140
|
+
clearTimeout(timer);
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
};
|
|
144
|
+
}
|
|
145
|
+
});
|
|
146
|
+
|
|
38
147
|
// src/worlds/browserReconnect.ts
|
|
39
148
|
var init_browserReconnect = __esm({
|
|
40
149
|
"src/worlds/browserReconnect.ts"() {
|
|
@@ -43,11 +152,1294 @@ var init_browserReconnect = __esm({
|
|
|
43
152
|
});
|
|
44
153
|
|
|
45
154
|
// src/worlds/browser.ts
|
|
155
|
+
function redactActionArgs(action, args) {
|
|
156
|
+
const secretField = action === "browser_type" && SECRET_KEY.test(String(args.selector ?? ""));
|
|
157
|
+
const out = {};
|
|
158
|
+
for (const [key, value] of Object.entries(args)) {
|
|
159
|
+
if (value === null || value === void 0) out[key] = null;
|
|
160
|
+
else if (typeof value === "boolean" || typeof value === "number")
|
|
161
|
+
out[key] = value;
|
|
162
|
+
else if (typeof value === "string") {
|
|
163
|
+
if (SECRET_KEY.test(key) || SECRET_VALUE.test(value) || secretField && key === "text")
|
|
164
|
+
out[key] = "[redacted]";
|
|
165
|
+
else
|
|
166
|
+
out[key] = value.length <= EVENT_MAX_STRING ? value : value.slice(0, EVENT_MAX_STRING) + "\u2026";
|
|
167
|
+
} else out[key] = "[omitted]";
|
|
168
|
+
}
|
|
169
|
+
return out;
|
|
170
|
+
}
|
|
171
|
+
var EVENT_MAX_STRING, SECRET_KEY, SECRET_VALUE;
|
|
46
172
|
var init_browser = __esm({
|
|
47
173
|
"src/worlds/browser.ts"() {
|
|
48
174
|
"use strict";
|
|
49
|
-
|
|
175
|
+
init_session2();
|
|
50
176
|
init_browserReconnect();
|
|
177
|
+
EVENT_MAX_STRING = 500;
|
|
178
|
+
SECRET_KEY = /pass(word|wd)?|secret|token|api[-_]?key|authorization|cookie|(^|[-_ #.\["'=])(otp|pin|cvv|ssn)([-_ \]"']|$)/i;
|
|
179
|
+
SECRET_VALUE = /^((sk|pk|ghp|xox[abp])[-_][-_A-Za-z0-9]{8,}|AKIA[0-9A-Z]{12,}|Bearer [-_.A-Za-z0-9]{8,})/;
|
|
180
|
+
}
|
|
181
|
+
});
|
|
182
|
+
|
|
183
|
+
// src/worlds/nativeComputer/actions.ts
|
|
184
|
+
function toNumber(value) {
|
|
185
|
+
if (typeof value !== "number" || !Number.isFinite(value))
|
|
186
|
+
throw new Error(`not a number: ${JSON.stringify(value) ?? String(value)}`);
|
|
187
|
+
return value;
|
|
188
|
+
}
|
|
189
|
+
function toInt(value) {
|
|
190
|
+
return Math.trunc(toNumber(value));
|
|
191
|
+
}
|
|
192
|
+
var ACTION_NAMES, SECRET_TEXT, BOUNDS, ComputerAction;
|
|
193
|
+
var init_actions = __esm({
|
|
194
|
+
"src/worlds/nativeComputer/actions.ts"() {
|
|
195
|
+
"use strict";
|
|
196
|
+
ACTION_NAMES = [
|
|
197
|
+
"screenshot",
|
|
198
|
+
"zoom",
|
|
199
|
+
"click",
|
|
200
|
+
"move",
|
|
201
|
+
"mouse_down",
|
|
202
|
+
"mouse_up",
|
|
203
|
+
"drag",
|
|
204
|
+
"scroll",
|
|
205
|
+
"type",
|
|
206
|
+
"key",
|
|
207
|
+
"hold_key",
|
|
208
|
+
"wait",
|
|
209
|
+
"cursor_position",
|
|
210
|
+
"history"
|
|
211
|
+
];
|
|
212
|
+
SECRET_TEXT = "[redacted]";
|
|
213
|
+
BOUNDS = {
|
|
214
|
+
seconds: [0, 300],
|
|
215
|
+
repeat: [1, 100],
|
|
216
|
+
count: [1, 3],
|
|
217
|
+
dx: [-1e4, 1e4],
|
|
218
|
+
dy: [-1e4, 1e4]
|
|
219
|
+
};
|
|
220
|
+
ComputerAction = class {
|
|
221
|
+
constructor(name, args = {}) {
|
|
222
|
+
this.name = name;
|
|
223
|
+
this.args = args;
|
|
224
|
+
if (!ACTION_NAMES.includes(name))
|
|
225
|
+
throw new Error(`unknown computer action: ${name}`);
|
|
226
|
+
for (const [key, [low, high]] of Object.entries(BOUNDS)) {
|
|
227
|
+
const value = args[key];
|
|
228
|
+
if (value !== void 0 && !(Number(value) >= low && Number(value) <= high))
|
|
229
|
+
throw new Error(
|
|
230
|
+
`${name} ${key} must be between ${low} and ${high}, got ${String(value)}`
|
|
231
|
+
);
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
name;
|
|
235
|
+
args;
|
|
236
|
+
/** Flat arguments for the browser timeline and the trace: lists become
|
|
237
|
+
* `"x,y"` strings, and typed text is withheld when it went into a
|
|
238
|
+
* credential field. */
|
|
239
|
+
recordArgs(secretText) {
|
|
240
|
+
const out = {};
|
|
241
|
+
for (const [key, value] of Object.entries(this.args))
|
|
242
|
+
out[key] = Array.isArray(value) ? value.flat().join(",") : value;
|
|
243
|
+
if (secretText && "text" in out) out.text = SECRET_TEXT;
|
|
244
|
+
return out;
|
|
245
|
+
}
|
|
246
|
+
};
|
|
247
|
+
}
|
|
248
|
+
});
|
|
249
|
+
|
|
250
|
+
// src/worlds/nativeComputer/display.ts
|
|
251
|
+
var FIT_STEP, Display;
|
|
252
|
+
var init_display = __esm({
|
|
253
|
+
"src/worlds/nativeComputer/display.ts"() {
|
|
254
|
+
"use strict";
|
|
255
|
+
FIT_STEP = 0.98;
|
|
256
|
+
Display = class _Display {
|
|
257
|
+
constructor(width, height, viewportWidth, viewportHeight) {
|
|
258
|
+
this.width = width;
|
|
259
|
+
this.height = height;
|
|
260
|
+
this.viewportWidth = viewportWidth;
|
|
261
|
+
this.viewportHeight = viewportHeight;
|
|
262
|
+
}
|
|
263
|
+
width;
|
|
264
|
+
height;
|
|
265
|
+
viewportWidth;
|
|
266
|
+
viewportHeight;
|
|
267
|
+
/** `size` when the caller names one; otherwise the largest downscale of the
|
|
268
|
+
* viewport (at most 1:1) that `fits` the provider's limits. */
|
|
269
|
+
static forViewport(viewport, size, fits = () => true) {
|
|
270
|
+
const { width, height } = viewport;
|
|
271
|
+
if (size) return _Display.sized(size[0], size[1], width, height);
|
|
272
|
+
let scale = 1;
|
|
273
|
+
while (!fits(Math.round(width * scale), Math.round(height * scale)))
|
|
274
|
+
scale *= FIT_STEP;
|
|
275
|
+
return new _Display(
|
|
276
|
+
Math.round(width * scale),
|
|
277
|
+
Math.round(height * scale),
|
|
278
|
+
width,
|
|
279
|
+
height
|
|
280
|
+
);
|
|
281
|
+
}
|
|
282
|
+
/** A caller's display keeps the viewport's shape: screenshots are scaled
|
|
283
|
+
* uniformly, so another aspect ratio would map clicks off the image. */
|
|
284
|
+
static sized(width, height, viewportWidth, viewportHeight) {
|
|
285
|
+
if (width <= 0 || height <= 0)
|
|
286
|
+
throw new Error(`display ${width}x${height} must be positive`);
|
|
287
|
+
const sameShape = width * viewportHeight / viewportWidth;
|
|
288
|
+
if (Math.abs(sameShape - height) > 1)
|
|
289
|
+
throw new Error(
|
|
290
|
+
`display ${width}x${height} must keep the viewport's aspect ratio (${viewportWidth}x${viewportHeight}); try ${width}x${Math.round(sameShape)}`
|
|
291
|
+
);
|
|
292
|
+
return new _Display(width, height, viewportWidth, viewportHeight);
|
|
293
|
+
}
|
|
294
|
+
get scaled() {
|
|
295
|
+
return this.width !== this.viewportWidth || this.height !== this.viewportHeight;
|
|
296
|
+
}
|
|
297
|
+
get scale() {
|
|
298
|
+
return Math.min(
|
|
299
|
+
this.width / this.viewportWidth,
|
|
300
|
+
this.height / this.viewportHeight
|
|
301
|
+
);
|
|
302
|
+
}
|
|
303
|
+
toPage(x, y) {
|
|
304
|
+
return [
|
|
305
|
+
x * this.viewportWidth / this.width,
|
|
306
|
+
y * this.viewportHeight / this.height
|
|
307
|
+
];
|
|
308
|
+
}
|
|
309
|
+
};
|
|
310
|
+
}
|
|
311
|
+
});
|
|
312
|
+
|
|
313
|
+
// src/worlds/nativeComputer/screen.ts
|
|
314
|
+
function pngWidth(png) {
|
|
315
|
+
if (png.length < 24 || !png.subarray(0, 8).equals(PNG_SIGNATURE)) return null;
|
|
316
|
+
return png.readUInt32BE(16);
|
|
317
|
+
}
|
|
318
|
+
function isNativePage(page) {
|
|
319
|
+
const candidate = page;
|
|
320
|
+
const mouse = candidate?.mouse;
|
|
321
|
+
return typeof candidate?.context === "function" && NATIVE_METHODS.every((m) => typeof mouse?.[m] === "function");
|
|
322
|
+
}
|
|
323
|
+
var FOCUSED_FIELD_JS, SECRET_FOCUS, DRAG_STEPS, PNG_SIGNATURE, NATIVE_METHODS, Screen;
|
|
324
|
+
var init_screen = __esm({
|
|
325
|
+
"src/worlds/nativeComputer/screen.ts"() {
|
|
326
|
+
"use strict";
|
|
327
|
+
init_browser();
|
|
328
|
+
FOCUSED_FIELD_JS = "(() => { let e = document.activeElement; const inner = (el) => (el.shadowRoot && el.shadowRoot.activeElement) || (el.contentDocument && el.contentDocument.activeElement); while (e && inner(e)) e = inner(e); if (!e) return ''; if (e.tagName === 'IFRAME' || e.tagName === 'FRAME') return 'frame'; if (!(e.tagName === 'INPUT' || e.tagName === 'TEXTAREA' || e.isContentEditable)) return 'opaque'; return [e.type, e.name, e.id, e.getAttribute('autocomplete')].filter(Boolean).join(' '); })()";
|
|
329
|
+
SECRET_FOCUS = ["password", "frame", "opaque"];
|
|
330
|
+
DRAG_STEPS = 5;
|
|
331
|
+
PNG_SIGNATURE = Buffer.from([
|
|
332
|
+
137,
|
|
333
|
+
80,
|
|
334
|
+
78,
|
|
335
|
+
71,
|
|
336
|
+
13,
|
|
337
|
+
10,
|
|
338
|
+
26,
|
|
339
|
+
10
|
|
340
|
+
]);
|
|
341
|
+
NATIVE_METHODS = ["move", "click", "down", "up", "wheel"];
|
|
342
|
+
Screen = class {
|
|
343
|
+
constructor(browser, display) {
|
|
344
|
+
this.browser = browser;
|
|
345
|
+
this.display = display;
|
|
346
|
+
}
|
|
347
|
+
browser;
|
|
348
|
+
display;
|
|
349
|
+
cursor = [0, 0];
|
|
350
|
+
pixelRatio = 1;
|
|
351
|
+
handlers = {
|
|
352
|
+
screenshot: async () => ({ image: await this.screenshot() }),
|
|
353
|
+
zoom: (a) => this.zoom(a),
|
|
354
|
+
click: (a) => this.click(a),
|
|
355
|
+
move: (a) => this.held(a.args.modifiers ?? [], () => this.moveTo(this.point(a))),
|
|
356
|
+
mouse_down: async (a) => {
|
|
357
|
+
await this.page.mouse.down({ button: a.args.button ?? "left" });
|
|
358
|
+
return {};
|
|
359
|
+
},
|
|
360
|
+
mouse_up: async (a) => {
|
|
361
|
+
await this.page.mouse.up({ button: a.args.button ?? "left" });
|
|
362
|
+
return {};
|
|
363
|
+
},
|
|
364
|
+
drag: (a) => this.drag(a),
|
|
365
|
+
scroll: (a) => this.scroll(a),
|
|
366
|
+
type: async (a) => {
|
|
367
|
+
await this.page.keyboard.type(a.args.text ?? "");
|
|
368
|
+
return {};
|
|
369
|
+
},
|
|
370
|
+
key: (a) => this.key(a),
|
|
371
|
+
hold_key: (a) => this.held(
|
|
372
|
+
a.args.keys ?? [],
|
|
373
|
+
() => this.page.waitForTimeout((a.args.seconds ?? 0) * 1e3)
|
|
374
|
+
),
|
|
375
|
+
wait: async (a) => {
|
|
376
|
+
await this.page.waitForTimeout((a.args.seconds ?? 0) * 1e3);
|
|
377
|
+
return {};
|
|
378
|
+
},
|
|
379
|
+
cursor_position: async () => ({
|
|
380
|
+
text: `X=${this.cursor[0]}, Y=${this.cursor[1]}`
|
|
381
|
+
}),
|
|
382
|
+
history: async (a) => {
|
|
383
|
+
const navigate = {
|
|
384
|
+
back: () => this.page.goBack(),
|
|
385
|
+
forward: () => this.page.goForward()
|
|
386
|
+
};
|
|
387
|
+
await navigate[a.args.direction ?? "back"]();
|
|
388
|
+
return {};
|
|
389
|
+
}
|
|
390
|
+
};
|
|
391
|
+
/** The live page (a new object after a reconnect), as pixel-level Playwright. */
|
|
392
|
+
get page() {
|
|
393
|
+
const page = this.browser.page;
|
|
394
|
+
if (!isNativePage(page))
|
|
395
|
+
throw new Error(
|
|
396
|
+
"native computer use needs a Playwright page (mouse.move/click/down/up/wheel, context())"
|
|
397
|
+
);
|
|
398
|
+
return page;
|
|
399
|
+
}
|
|
400
|
+
run(action) {
|
|
401
|
+
return this.handlers[action.name](action);
|
|
402
|
+
}
|
|
403
|
+
/** A PNG of the viewport at the model's display size. */
|
|
404
|
+
async screenshot() {
|
|
405
|
+
if (!this.display.scaled) return this.page.screenshot({ type: "png" });
|
|
406
|
+
const { viewportWidth: width, viewportHeight: height } = this.display;
|
|
407
|
+
return this.capture({ x: 0, y: 0, width, height }, this.display.scale);
|
|
408
|
+
}
|
|
409
|
+
/** Whether keystrokes go to a credential field (a password input, or one
|
|
410
|
+
* named like a token, key or PIN), looking through shadow roots and
|
|
411
|
+
* same-origin frames. An unreadable page, a cross-origin frame or a focus
|
|
412
|
+
* that is not a text field (a closed shadow root's host) counts as one. */
|
|
413
|
+
async focusedFieldIsSecret() {
|
|
414
|
+
let described;
|
|
415
|
+
try {
|
|
416
|
+
described = String(await this.page.evaluate(FOCUSED_FIELD_JS) ?? "");
|
|
417
|
+
} catch {
|
|
418
|
+
return true;
|
|
419
|
+
}
|
|
420
|
+
return SECRET_FOCUS.some((prefix) => described.startsWith(prefix)) || SECRET_KEY.test(described);
|
|
421
|
+
}
|
|
422
|
+
/** CDP renders a clip at the page's device pixel ratio as well as its
|
|
423
|
+
* `scale` (a Retina host doubles it), so a capture of the wrong width
|
|
424
|
+
* measures that ratio and is taken once more, corrected. */
|
|
425
|
+
async capture(clip, scale) {
|
|
426
|
+
const wanted = Math.round(clip.width * scale);
|
|
427
|
+
const png = await this.cdpCapture(clip, scale / this.pixelRatio);
|
|
428
|
+
const width = pngWidth(png);
|
|
429
|
+
if (width === null || Math.abs(width - wanted) <= 1) return png;
|
|
430
|
+
this.pixelRatio *= width / wanted;
|
|
431
|
+
return this.cdpCapture(clip, scale / this.pixelRatio);
|
|
432
|
+
}
|
|
433
|
+
async cdpCapture(clip, scale) {
|
|
434
|
+
const page = this.page;
|
|
435
|
+
const session = await page.context().newCDPSession(page);
|
|
436
|
+
try {
|
|
437
|
+
const shot = await session.send("Page.captureScreenshot", {
|
|
438
|
+
format: "png",
|
|
439
|
+
clip: { ...clip, scale }
|
|
440
|
+
});
|
|
441
|
+
return Buffer.from(shot.data, "base64");
|
|
442
|
+
} finally {
|
|
443
|
+
await session.detach();
|
|
444
|
+
}
|
|
445
|
+
}
|
|
446
|
+
point(action) {
|
|
447
|
+
const { x, y } = action.args;
|
|
448
|
+
return x === void 0 || y === void 0 ? this.cursor : [x, y];
|
|
449
|
+
}
|
|
450
|
+
async moveTo(point2, steps = 1) {
|
|
451
|
+
const [x, y] = this.display.toPage(point2[0], point2[1]);
|
|
452
|
+
await this.page.mouse.move(x, y, { steps });
|
|
453
|
+
this.cursor = point2;
|
|
454
|
+
}
|
|
455
|
+
async held(modifiers, fn) {
|
|
456
|
+
for (const key of modifiers) await this.page.keyboard.down(key);
|
|
457
|
+
try {
|
|
458
|
+
await fn();
|
|
459
|
+
} finally {
|
|
460
|
+
for (const key of [...modifiers].reverse())
|
|
461
|
+
await this.page.keyboard.up(key);
|
|
462
|
+
}
|
|
463
|
+
return {};
|
|
464
|
+
}
|
|
465
|
+
async zoom(action) {
|
|
466
|
+
const [x0, y0, x1, y1] = action.args.region ?? [0, 0, 0, 0];
|
|
467
|
+
const [left, top] = this.display.toPage(x0, y0);
|
|
468
|
+
const [right, bottom] = this.display.toPage(x1, y1);
|
|
469
|
+
if (right <= left || bottom <= top)
|
|
470
|
+
throw new Error("zoom region must have x1 > x0 and y1 > y0");
|
|
471
|
+
const width = right - left;
|
|
472
|
+
const height = bottom - top;
|
|
473
|
+
const scale = Math.min(
|
|
474
|
+
this.display.width / width,
|
|
475
|
+
this.display.height / height
|
|
476
|
+
);
|
|
477
|
+
return {
|
|
478
|
+
image: await this.capture({ x: left, y: top, width, height }, scale)
|
|
479
|
+
};
|
|
480
|
+
}
|
|
481
|
+
async click(action) {
|
|
482
|
+
const point2 = this.point(action);
|
|
483
|
+
return this.held(action.args.modifiers ?? [], async () => {
|
|
484
|
+
await this.moveTo(point2);
|
|
485
|
+
const [x, y] = this.display.toPage(point2[0], point2[1]);
|
|
486
|
+
await this.page.mouse.click(x, y, {
|
|
487
|
+
button: action.args.button ?? "left",
|
|
488
|
+
clickCount: action.args.count ?? 1
|
|
489
|
+
});
|
|
490
|
+
});
|
|
491
|
+
}
|
|
492
|
+
async drag(action) {
|
|
493
|
+
const path7 = action.args.path ?? [];
|
|
494
|
+
if (path7.length < 2) throw new Error("a drag needs at least two points");
|
|
495
|
+
return this.held(action.args.modifiers ?? [], async () => {
|
|
496
|
+
await this.moveTo(path7[0]);
|
|
497
|
+
await this.page.mouse.down({ button: "left" });
|
|
498
|
+
try {
|
|
499
|
+
for (const point2 of path7.slice(1)) await this.moveTo(point2, DRAG_STEPS);
|
|
500
|
+
} finally {
|
|
501
|
+
await this.page.mouse.up({ button: "left" });
|
|
502
|
+
}
|
|
503
|
+
});
|
|
504
|
+
}
|
|
505
|
+
async scroll(action) {
|
|
506
|
+
await this.moveTo(this.point(action));
|
|
507
|
+
const scale = this.display.scale;
|
|
508
|
+
return this.held(
|
|
509
|
+
action.args.modifiers ?? [],
|
|
510
|
+
() => this.page.mouse.wheel(
|
|
511
|
+
(action.args.dx ?? 0) / scale,
|
|
512
|
+
(action.args.dy ?? 0) / scale
|
|
513
|
+
)
|
|
514
|
+
);
|
|
515
|
+
}
|
|
516
|
+
async key(action) {
|
|
517
|
+
const combo = (action.args.keys ?? []).join("+");
|
|
518
|
+
for (let i = 0; i < (action.args.repeat ?? 1); i += 1)
|
|
519
|
+
await this.page.keyboard.press(combo);
|
|
520
|
+
return {};
|
|
521
|
+
}
|
|
522
|
+
};
|
|
523
|
+
}
|
|
524
|
+
});
|
|
525
|
+
|
|
526
|
+
// src/tracing/media.ts
|
|
527
|
+
async function putBytes(url, data, headers) {
|
|
528
|
+
const controller = new AbortController();
|
|
529
|
+
const timer = setTimeout(() => controller.abort(), PUT_TIMEOUT_MS);
|
|
530
|
+
try {
|
|
531
|
+
const res = await fetch(url, {
|
|
532
|
+
method: "PUT",
|
|
533
|
+
body: new Uint8Array(data),
|
|
534
|
+
headers,
|
|
535
|
+
signal: controller.signal
|
|
536
|
+
});
|
|
537
|
+
return {
|
|
538
|
+
status: res.status,
|
|
539
|
+
error: res.ok ? null : (await res.text()).slice(0, MAX_UPLOAD_ERROR)
|
|
540
|
+
};
|
|
541
|
+
} catch (error) {
|
|
542
|
+
return {
|
|
543
|
+
status: NETWORK_FAILURE_STATUS,
|
|
544
|
+
error: error instanceof Error ? error.name : "Error"
|
|
545
|
+
};
|
|
546
|
+
} finally {
|
|
547
|
+
clearTimeout(timer);
|
|
548
|
+
}
|
|
549
|
+
}
|
|
550
|
+
var import_node_crypto, MEDIA_PATH, MAX_MEDIA_BYTES, PUT_TIMEOUT_MS, MAX_UPLOAD_ERROR, NETWORK_FAILURE_STATUS, MediaUploadError, MediaReference, MediaUploader;
|
|
551
|
+
var init_media = __esm({
|
|
552
|
+
"src/tracing/media.ts"() {
|
|
553
|
+
"use strict";
|
|
554
|
+
import_node_crypto = require("crypto");
|
|
555
|
+
init_transport();
|
|
556
|
+
MEDIA_PATH = "/api/public/media";
|
|
557
|
+
MAX_MEDIA_BYTES = 5 * 1024 * 1024;
|
|
558
|
+
PUT_TIMEOUT_MS = 3e4;
|
|
559
|
+
MAX_UPLOAD_ERROR = 500;
|
|
560
|
+
NETWORK_FAILURE_STATUS = 599;
|
|
561
|
+
MediaUploadError = class extends Error {
|
|
562
|
+
constructor(message, status) {
|
|
563
|
+
super(message);
|
|
564
|
+
this.status = status;
|
|
565
|
+
this.name = "MediaUploadError";
|
|
566
|
+
}
|
|
567
|
+
status;
|
|
568
|
+
};
|
|
569
|
+
MediaReference = class {
|
|
570
|
+
constructor(mediaId, contentType) {
|
|
571
|
+
this.mediaId = mediaId;
|
|
572
|
+
this.contentType = contentType;
|
|
573
|
+
}
|
|
574
|
+
mediaId;
|
|
575
|
+
contentType;
|
|
576
|
+
toString() {
|
|
577
|
+
return `@@@langfuseMedia:type=${this.contentType}|id=${this.mediaId}|source=bytes@@@`;
|
|
578
|
+
}
|
|
579
|
+
};
|
|
580
|
+
MediaUploader = class _MediaUploader {
|
|
581
|
+
constructor(transport, put = putBytes, maxBytes = MAX_MEDIA_BYTES) {
|
|
582
|
+
this.transport = transport;
|
|
583
|
+
this.put = put;
|
|
584
|
+
this.maxBytes = maxBytes;
|
|
585
|
+
}
|
|
586
|
+
transport;
|
|
587
|
+
put;
|
|
588
|
+
maxBytes;
|
|
589
|
+
/** The uploader for `GATEWAY_HOST` / `GATEWAY_PUBLIC_KEY` /
|
|
590
|
+
* `GATEWAY_SECRET_KEY` (the keys tracing exports with); null when any is unset. */
|
|
591
|
+
static fromEnv(env = process.env) {
|
|
592
|
+
const host = env.GATEWAY_HOST;
|
|
593
|
+
const publicKey = env.GATEWAY_PUBLIC_KEY;
|
|
594
|
+
const secretKey = env.GATEWAY_SECRET_KEY;
|
|
595
|
+
if (!host || !publicKey || !secretKey) return null;
|
|
596
|
+
return new _MediaUploader(
|
|
597
|
+
new PlatformTransport({
|
|
598
|
+
host: host.replace(/\/$/, ""),
|
|
599
|
+
publicKey,
|
|
600
|
+
secretKey
|
|
601
|
+
})
|
|
602
|
+
);
|
|
603
|
+
}
|
|
604
|
+
/** Store `data` linked to the trace (and observation) and return its reference. */
|
|
605
|
+
async upload(data, link) {
|
|
606
|
+
if (data.length > this.maxBytes)
|
|
607
|
+
throw new MediaUploadError(
|
|
608
|
+
`media is ${data.length} bytes, over the ${this.maxBytes}-byte cap`
|
|
609
|
+
);
|
|
610
|
+
const sha256Hash = (0, import_node_crypto.createHash)("sha256").update(data).digest("base64");
|
|
611
|
+
const answer = await this.send("POST", MEDIA_PATH, {
|
|
612
|
+
traceId: link.traceId,
|
|
613
|
+
observationId: link.observationId,
|
|
614
|
+
contentType: link.contentType,
|
|
615
|
+
contentLength: data.length,
|
|
616
|
+
sha256Hash,
|
|
617
|
+
field: link.field
|
|
618
|
+
});
|
|
619
|
+
if (typeof answer.mediaId !== "string" || !answer.mediaId)
|
|
620
|
+
throw new MediaUploadError(
|
|
621
|
+
"POST /api/public/media answered without a mediaId"
|
|
622
|
+
);
|
|
623
|
+
const reference = new MediaReference(answer.mediaId, link.contentType);
|
|
624
|
+
if (answer.uploadUrl)
|
|
625
|
+
await this.store(reference, answer.uploadUrl, data, sha256Hash);
|
|
626
|
+
return reference;
|
|
627
|
+
}
|
|
628
|
+
async store(reference, url, data, sha256Hash) {
|
|
629
|
+
const started = Date.now();
|
|
630
|
+
const { status, error } = await this.put(url, data, {
|
|
631
|
+
"Content-Type": reference.contentType,
|
|
632
|
+
"x-amz-checksum-sha256": sha256Hash,
|
|
633
|
+
"x-ms-blob-type": "BlockBlob"
|
|
634
|
+
});
|
|
635
|
+
await this.send(
|
|
636
|
+
"PATCH",
|
|
637
|
+
`${MEDIA_PATH}/${encodeURIComponent(reference.mediaId)}`,
|
|
638
|
+
{
|
|
639
|
+
uploadedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
640
|
+
uploadHttpStatus: status,
|
|
641
|
+
uploadHttpError: error,
|
|
642
|
+
uploadTimeMs: Date.now() - started
|
|
643
|
+
}
|
|
644
|
+
);
|
|
645
|
+
if (status < 200 || status >= 300)
|
|
646
|
+
throw new MediaUploadError(
|
|
647
|
+
`media storage refused the upload (HTTP ${status})`,
|
|
648
|
+
status
|
|
649
|
+
);
|
|
650
|
+
}
|
|
651
|
+
async send(method, route, body) {
|
|
652
|
+
const reply = await this.transport.send(method, route, body);
|
|
653
|
+
if (!reply.ok)
|
|
654
|
+
throw new MediaUploadError(
|
|
655
|
+
`${method} ${route} -> ${reply.status}`,
|
|
656
|
+
reply.status
|
|
657
|
+
);
|
|
658
|
+
return reply.text ? JSON.parse(reply.text) : {};
|
|
659
|
+
}
|
|
660
|
+
};
|
|
661
|
+
}
|
|
662
|
+
});
|
|
663
|
+
|
|
664
|
+
// src/worlds/nativeComputer/trace.ts
|
|
665
|
+
function loadOtelApi() {
|
|
666
|
+
if (otelApi !== void 0) return otelApi;
|
|
667
|
+
try {
|
|
668
|
+
otelApi = require("@opentelemetry/api");
|
|
669
|
+
} catch {
|
|
670
|
+
otelApi = null;
|
|
671
|
+
}
|
|
672
|
+
return otelApi;
|
|
673
|
+
}
|
|
674
|
+
function errorText(error) {
|
|
675
|
+
return error instanceof Error ? `${error.name}: ${error.message}` : String(error);
|
|
676
|
+
}
|
|
677
|
+
function errorName(error) {
|
|
678
|
+
return error instanceof Error ? error.name : "Error";
|
|
679
|
+
}
|
|
680
|
+
var SPAN_PREFIX, MAX_UPLOAD_FAILURES, TRACER_NAME, MAX_ERROR, otelApi, ActionTracer;
|
|
681
|
+
var init_trace = __esm({
|
|
682
|
+
"src/worlds/nativeComputer/trace.ts"() {
|
|
683
|
+
"use strict";
|
|
684
|
+
init_media();
|
|
685
|
+
SPAN_PREFIX = "computer.";
|
|
686
|
+
MAX_UPLOAD_FAILURES = 3;
|
|
687
|
+
TRACER_NAME = "@withgateway/sdk";
|
|
688
|
+
MAX_ERROR = 1e3;
|
|
689
|
+
ActionTracer = class _ActionTracer {
|
|
690
|
+
constructor(media, warn = (m) => console.warn(m)) {
|
|
691
|
+
this.media = media;
|
|
692
|
+
this.warn = warn;
|
|
693
|
+
}
|
|
694
|
+
media;
|
|
695
|
+
warn;
|
|
696
|
+
failures = 0;
|
|
697
|
+
warned = false;
|
|
698
|
+
static fromEnv() {
|
|
699
|
+
return new _ActionTracer(MediaUploader.fromEnv());
|
|
700
|
+
}
|
|
701
|
+
/** `args` are already redacted. A failure's status goes through `scrub`:
|
|
702
|
+
* browser errors quote the session's ws URL and UI token. */
|
|
703
|
+
async span(action, args, run, capture, scrub) {
|
|
704
|
+
const otel = loadOtelApi();
|
|
705
|
+
if (!otel) return run();
|
|
706
|
+
const { SpanStatusCode, trace } = otel;
|
|
707
|
+
const attributes = {
|
|
708
|
+
"gateway.observation.type": "tool",
|
|
709
|
+
"tool.name": SPAN_PREFIX + action.name,
|
|
710
|
+
"gateway.observation.input": JSON.stringify({
|
|
711
|
+
action: action.name,
|
|
712
|
+
...args
|
|
713
|
+
})
|
|
714
|
+
};
|
|
715
|
+
return trace.getTracer(TRACER_NAME).startActiveSpan(
|
|
716
|
+
SPAN_PREFIX + action.name,
|
|
717
|
+
{ attributes },
|
|
718
|
+
async (span) => {
|
|
719
|
+
try {
|
|
720
|
+
const result = await run();
|
|
721
|
+
await this.finish(span, result, capture);
|
|
722
|
+
span.setStatus({ code: SpanStatusCode.OK });
|
|
723
|
+
return result;
|
|
724
|
+
} catch (error) {
|
|
725
|
+
span.setStatus({
|
|
726
|
+
code: SpanStatusCode.ERROR,
|
|
727
|
+
message: scrub(errorText(error)).slice(0, MAX_ERROR)
|
|
728
|
+
});
|
|
729
|
+
throw error;
|
|
730
|
+
} finally {
|
|
731
|
+
span.end();
|
|
732
|
+
}
|
|
733
|
+
}
|
|
734
|
+
);
|
|
735
|
+
}
|
|
736
|
+
async finish(span, result, capture) {
|
|
737
|
+
if (!span.isRecording()) return;
|
|
738
|
+
const output = {
|
|
739
|
+
screenshot: await this.screenshot(span, result, capture)
|
|
740
|
+
};
|
|
741
|
+
if (result.text !== void 0) output.text = result.text;
|
|
742
|
+
span.setAttribute("gateway.observation.output", JSON.stringify(output));
|
|
743
|
+
}
|
|
744
|
+
async screenshot(span, result, capture) {
|
|
745
|
+
let png;
|
|
746
|
+
try {
|
|
747
|
+
png = result.image ?? await capture();
|
|
748
|
+
} catch (error) {
|
|
749
|
+
return `[screenshot unavailable: ${errorName(error)}]`;
|
|
750
|
+
}
|
|
751
|
+
return this.reference(png, span);
|
|
752
|
+
}
|
|
753
|
+
async reference(png, span) {
|
|
754
|
+
if (!this.media)
|
|
755
|
+
return "[screenshot not uploaded: GATEWAY_HOST / GATEWAY_PUBLIC_KEY / GATEWAY_SECRET_KEY unset]";
|
|
756
|
+
if (this.failures >= MAX_UPLOAD_FAILURES)
|
|
757
|
+
return "[screenshot not uploaded: uploads disabled after repeated failures]";
|
|
758
|
+
const { traceId, spanId } = span.spanContext();
|
|
759
|
+
try {
|
|
760
|
+
const reference = await this.media.upload(png, {
|
|
761
|
+
contentType: "image/png",
|
|
762
|
+
traceId,
|
|
763
|
+
observationId: spanId,
|
|
764
|
+
field: "output"
|
|
765
|
+
});
|
|
766
|
+
this.failures = 0;
|
|
767
|
+
return reference.toString();
|
|
768
|
+
} catch (error) {
|
|
769
|
+
return this.failed(error);
|
|
770
|
+
}
|
|
771
|
+
}
|
|
772
|
+
failed(error) {
|
|
773
|
+
this.failures += 1;
|
|
774
|
+
if (!this.warned) {
|
|
775
|
+
this.warned = true;
|
|
776
|
+
this.warn(
|
|
777
|
+
`native computer: screenshot upload failed (${errorName(error)}: ${error instanceof Error ? error.message : String(error)}); spans keep going without it`
|
|
778
|
+
);
|
|
779
|
+
}
|
|
780
|
+
return `[screenshot not uploaded: ${errorName(error)}]`;
|
|
781
|
+
}
|
|
782
|
+
};
|
|
783
|
+
}
|
|
784
|
+
});
|
|
785
|
+
|
|
786
|
+
// src/worlds/nativeComputer/base.ts
|
|
787
|
+
function readField(item, ...names) {
|
|
788
|
+
const record = item ?? {};
|
|
789
|
+
return names.map((name) => record[name]).find((value) => value !== void 0 && value !== null);
|
|
790
|
+
}
|
|
791
|
+
var REPLAYABLE, SETTLING, EVENT_PREFIX, NativeComputer;
|
|
792
|
+
var init_base = __esm({
|
|
793
|
+
"src/worlds/nativeComputer/base.ts"() {
|
|
794
|
+
"use strict";
|
|
795
|
+
init_browser();
|
|
796
|
+
init_actions();
|
|
797
|
+
init_display();
|
|
798
|
+
init_screen();
|
|
799
|
+
init_trace();
|
|
800
|
+
REPLAYABLE = /* @__PURE__ */ new Set(["screenshot", "zoom", "cursor_position", "wait"]);
|
|
801
|
+
SETTLING = /* @__PURE__ */ new Set(["click", "mouse_up", "drag", "key", "history"]);
|
|
802
|
+
EVENT_PREFIX = "computer_";
|
|
803
|
+
NativeComputer = class _NativeComputer {
|
|
804
|
+
constructor(browser, options = {}) {
|
|
805
|
+
this.browser = browser;
|
|
806
|
+
const kind = this.constructor;
|
|
807
|
+
this.display = Display.forViewport(
|
|
808
|
+
browser.viewport,
|
|
809
|
+
options.display,
|
|
810
|
+
(w, h) => kind.fits(w, h)
|
|
811
|
+
);
|
|
812
|
+
this.screen = new Screen(browser, this.display);
|
|
813
|
+
this.tracer = options.tracer ?? ActionTracer.fromEnv();
|
|
814
|
+
}
|
|
815
|
+
browser;
|
|
816
|
+
static kinds = /* @__PURE__ */ new Map();
|
|
817
|
+
display;
|
|
818
|
+
screen;
|
|
819
|
+
tracer;
|
|
820
|
+
queue = Promise.resolve();
|
|
821
|
+
static register(provider, kind) {
|
|
822
|
+
_NativeComputer.kinds.set(provider, kind);
|
|
823
|
+
}
|
|
824
|
+
static get providers() {
|
|
825
|
+
return [..._NativeComputer.kinds.keys()].sort();
|
|
826
|
+
}
|
|
827
|
+
static forProvider(provider, browser, options = {}) {
|
|
828
|
+
const Kind = _NativeComputer.kinds.get(provider);
|
|
829
|
+
if (!Kind)
|
|
830
|
+
throw new Error(
|
|
831
|
+
`unknown computer-use provider '${provider}'; choose from ${_NativeComputer.providers.join(", ")}`
|
|
832
|
+
);
|
|
833
|
+
return new Kind(browser, options);
|
|
834
|
+
}
|
|
835
|
+
/** Whether a screenshot of this size is within the provider's limits. */
|
|
836
|
+
static fits(_width, _height) {
|
|
837
|
+
return true;
|
|
838
|
+
}
|
|
839
|
+
/** Run one provider-shaped action (an OpenAI action object, or an Anthropic
|
|
840
|
+
* `{ action: "left_click", ... }` input) and return what it produced: the
|
|
841
|
+
* door for any agent framework that hands actions over one at a time. */
|
|
842
|
+
executeAction(raw) {
|
|
843
|
+
return this.perform(this.parse(raw));
|
|
844
|
+
}
|
|
845
|
+
/** Run one action on the page: reported on the session's browser timeline
|
|
846
|
+
* and, when tracing is on, as a span with the screen after. */
|
|
847
|
+
async perform(action) {
|
|
848
|
+
const secret = action.name === "type" && await this.screen.focusedFieldIsSecret();
|
|
849
|
+
const event = EVENT_PREFIX + action.name;
|
|
850
|
+
const args = redactActionArgs(event, action.recordArgs(secret));
|
|
851
|
+
return this.tracer.span(
|
|
852
|
+
action,
|
|
853
|
+
args,
|
|
854
|
+
() => this.browser.act(
|
|
855
|
+
event,
|
|
856
|
+
args,
|
|
857
|
+
() => this.runOnScreen(action),
|
|
858
|
+
this.onLoss(action)
|
|
859
|
+
),
|
|
860
|
+
() => this.screen.screenshot(),
|
|
861
|
+
(text) => this.browser.scrub(text)
|
|
862
|
+
);
|
|
863
|
+
}
|
|
864
|
+
async performAll(actions) {
|
|
865
|
+
const results = [];
|
|
866
|
+
for (const action of actions) results.push(await this.perform(action));
|
|
867
|
+
return results;
|
|
868
|
+
}
|
|
869
|
+
/** The screen as the model sees it, as a traced `screenshot` action. */
|
|
870
|
+
async screenshot() {
|
|
871
|
+
return (await this.perform(new ComputerAction("screenshot"))).image ?? Buffer.alloc(0);
|
|
872
|
+
}
|
|
873
|
+
/** Run `fn` after every earlier call made through here: the AI SDK starts a
|
|
874
|
+
* step's tool calls concurrently, but one screen takes one action at a time. */
|
|
875
|
+
serial(fn) {
|
|
876
|
+
const next = this.queue.then(fn, fn);
|
|
877
|
+
this.queue = next.catch(() => void 0);
|
|
878
|
+
return next;
|
|
879
|
+
}
|
|
880
|
+
async runOnScreen(action) {
|
|
881
|
+
const result = await this.screen.run(action);
|
|
882
|
+
if (SETTLING.has(action.name)) await this.browser.settle();
|
|
883
|
+
return result;
|
|
884
|
+
}
|
|
885
|
+
onLoss(action) {
|
|
886
|
+
return REPLAYABLE.has(action.name) ? "retry" : "report";
|
|
887
|
+
}
|
|
888
|
+
};
|
|
889
|
+
}
|
|
890
|
+
});
|
|
891
|
+
|
|
892
|
+
// src/worlds/nativeComputer/keys.ts
|
|
893
|
+
function playwrightKey(token) {
|
|
894
|
+
return KEY_NAMES[token.toLowerCase()] ?? token;
|
|
895
|
+
}
|
|
896
|
+
function letterOrName(key) {
|
|
897
|
+
return /^[a-zA-Z]$/.test(key) ? key.toLowerCase() : playwrightKey(key);
|
|
898
|
+
}
|
|
899
|
+
function modifierChord(text) {
|
|
900
|
+
return typeof text === "string" && text ? [...KeyChord.fromXdotool(text).keys] : [];
|
|
901
|
+
}
|
|
902
|
+
function modifierKeys(keys) {
|
|
903
|
+
return Array.isArray(keys) && keys.length > 0 ? [...KeyChord.fromKeys(keys.map(String)).keys] : [];
|
|
904
|
+
}
|
|
905
|
+
var F_KEYS, KEY_NAMES, KeyChord;
|
|
906
|
+
var init_keys = __esm({
|
|
907
|
+
"src/worlds/nativeComputer/keys.ts"() {
|
|
908
|
+
"use strict";
|
|
909
|
+
F_KEYS = Object.fromEntries(
|
|
910
|
+
Array.from({ length: 12 }, (_, i) => [`f${i + 1}`, `F${i + 1}`])
|
|
911
|
+
);
|
|
912
|
+
KEY_NAMES = {
|
|
913
|
+
return: "Enter",
|
|
914
|
+
enter: "Enter",
|
|
915
|
+
kp_enter: "Enter",
|
|
916
|
+
esc: "Escape",
|
|
917
|
+
escape: "Escape",
|
|
918
|
+
tab: "Tab",
|
|
919
|
+
space: "Space",
|
|
920
|
+
backspace: "Backspace",
|
|
921
|
+
delete: "Delete",
|
|
922
|
+
del: "Delete",
|
|
923
|
+
insert: "Insert",
|
|
924
|
+
home: "Home",
|
|
925
|
+
end: "End",
|
|
926
|
+
pageup: "PageUp",
|
|
927
|
+
page_up: "PageUp",
|
|
928
|
+
prior: "PageUp",
|
|
929
|
+
pagedown: "PageDown",
|
|
930
|
+
page_down: "PageDown",
|
|
931
|
+
next: "PageDown",
|
|
932
|
+
up: "ArrowUp",
|
|
933
|
+
arrowup: "ArrowUp",
|
|
934
|
+
down: "ArrowDown",
|
|
935
|
+
arrowdown: "ArrowDown",
|
|
936
|
+
left: "ArrowLeft",
|
|
937
|
+
arrowleft: "ArrowLeft",
|
|
938
|
+
right: "ArrowRight",
|
|
939
|
+
arrowright: "ArrowRight",
|
|
940
|
+
ctrl: "Control",
|
|
941
|
+
control: "Control",
|
|
942
|
+
shift: "Shift",
|
|
943
|
+
alt: "Alt",
|
|
944
|
+
option: "Alt",
|
|
945
|
+
super: "Meta",
|
|
946
|
+
meta: "Meta",
|
|
947
|
+
cmd: "Meta",
|
|
948
|
+
command: "Meta",
|
|
949
|
+
win: "Meta",
|
|
950
|
+
capslock: "CapsLock",
|
|
951
|
+
caps_lock: "CapsLock",
|
|
952
|
+
plus: "+",
|
|
953
|
+
minus: "-",
|
|
954
|
+
...F_KEYS
|
|
955
|
+
};
|
|
956
|
+
KeyChord = class _KeyChord {
|
|
957
|
+
constructor(keys) {
|
|
958
|
+
this.keys = keys;
|
|
959
|
+
if (keys.length === 0)
|
|
960
|
+
throw new Error("a key action needs at least one key");
|
|
961
|
+
}
|
|
962
|
+
keys;
|
|
963
|
+
/** `ctrl+shift+t` / `Return`: Anthropic's vocabulary, case kept. */
|
|
964
|
+
static fromXdotool(text) {
|
|
965
|
+
return new _KeyChord(
|
|
966
|
+
text.trim().split("+").filter(Boolean).map(playwrightKey)
|
|
967
|
+
);
|
|
968
|
+
}
|
|
969
|
+
/** `["CTRL", "A"]`: OpenAI's vocabulary. A lone letter is the unshifted key. */
|
|
970
|
+
static fromKeys(keys) {
|
|
971
|
+
return new _KeyChord(keys.map(letterOrName));
|
|
972
|
+
}
|
|
973
|
+
/** What `page.keyboard.press` takes: `Control+a`. */
|
|
974
|
+
get combo() {
|
|
975
|
+
return this.keys.join("+");
|
|
976
|
+
}
|
|
977
|
+
};
|
|
978
|
+
}
|
|
979
|
+
});
|
|
980
|
+
|
|
981
|
+
// src/worlds/nativeComputer/anthropic.ts
|
|
982
|
+
function required(raw, key) {
|
|
983
|
+
const value = raw[key];
|
|
984
|
+
if (value === void 0 || value === null)
|
|
985
|
+
throw new Error(`${String(raw.action)} needs ${key}`);
|
|
986
|
+
return value;
|
|
987
|
+
}
|
|
988
|
+
function pair(raw, key) {
|
|
989
|
+
const value = required(raw, key);
|
|
990
|
+
return [toInt(value[0]), toInt(value[1])];
|
|
991
|
+
}
|
|
992
|
+
function point(raw) {
|
|
993
|
+
if (raw.coordinate === void 0 || raw.coordinate === null) return {};
|
|
994
|
+
const [x, y] = pair(raw, "coordinate");
|
|
995
|
+
return { x, y };
|
|
996
|
+
}
|
|
997
|
+
function chord(raw) {
|
|
998
|
+
return [...KeyChord.fromXdotool(String(required(raw, "text"))).keys];
|
|
999
|
+
}
|
|
1000
|
+
function click(button, count) {
|
|
1001
|
+
return (raw) => new ComputerAction("click", {
|
|
1002
|
+
...point(raw),
|
|
1003
|
+
button,
|
|
1004
|
+
count,
|
|
1005
|
+
modifiers: modifierChord(raw.text)
|
|
1006
|
+
});
|
|
1007
|
+
}
|
|
1008
|
+
var SCROLL_STEP_PX, NOT_EXECUTED, MAX_ERROR2, SCROLL_DIRECTIONS, MEMBERS, AnthropicComputer, AnthropicComputer20251124;
|
|
1009
|
+
var init_anthropic = __esm({
|
|
1010
|
+
"src/worlds/nativeComputer/anthropic.ts"() {
|
|
1011
|
+
"use strict";
|
|
1012
|
+
init_actions();
|
|
1013
|
+
init_base();
|
|
1014
|
+
init_keys();
|
|
1015
|
+
SCROLL_STEP_PX = 100;
|
|
1016
|
+
NOT_EXECUTED = "Not executed: an earlier computer action in this turn failed.";
|
|
1017
|
+
MAX_ERROR2 = 1e3;
|
|
1018
|
+
SCROLL_DIRECTIONS = {
|
|
1019
|
+
up: [0, -1],
|
|
1020
|
+
down: [0, 1],
|
|
1021
|
+
left: [-1, 0],
|
|
1022
|
+
right: [1, 0]
|
|
1023
|
+
};
|
|
1024
|
+
MEMBERS = {
|
|
1025
|
+
screenshot: () => new ComputerAction("screenshot"),
|
|
1026
|
+
zoom: (raw) => new ComputerAction("zoom", {
|
|
1027
|
+
region: required(raw, "region").map(toInt)
|
|
1028
|
+
}),
|
|
1029
|
+
left_click: click("left", 1),
|
|
1030
|
+
right_click: click("right", 1),
|
|
1031
|
+
middle_click: click("middle", 1),
|
|
1032
|
+
double_click: click("left", 2),
|
|
1033
|
+
triple_click: click("left", 3),
|
|
1034
|
+
left_click_drag: (raw) => new ComputerAction("drag", {
|
|
1035
|
+
path: [pair(raw, "start_coordinate"), pair(raw, "coordinate")],
|
|
1036
|
+
modifiers: modifierChord(raw.text)
|
|
1037
|
+
}),
|
|
1038
|
+
mouse_move: (raw) => {
|
|
1039
|
+
const [x, y] = pair(raw, "coordinate");
|
|
1040
|
+
return new ComputerAction("move", { x, y });
|
|
1041
|
+
},
|
|
1042
|
+
left_mouse_down: () => new ComputerAction("mouse_down", { button: "left" }),
|
|
1043
|
+
left_mouse_up: () => new ComputerAction("mouse_up", { button: "left" }),
|
|
1044
|
+
cursor_position: () => new ComputerAction("cursor_position"),
|
|
1045
|
+
scroll: (raw) => {
|
|
1046
|
+
const direction = String(raw.scroll_direction ?? "down");
|
|
1047
|
+
if (!Object.hasOwn(SCROLL_DIRECTIONS, direction))
|
|
1048
|
+
throw new Error(
|
|
1049
|
+
`scroll_direction must be one of ${Object.keys(SCROLL_DIRECTIONS).join(", ")}`
|
|
1050
|
+
);
|
|
1051
|
+
const [dx, dy] = SCROLL_DIRECTIONS[direction];
|
|
1052
|
+
const amount = toInt(raw.scroll_amount ?? 1) * SCROLL_STEP_PX;
|
|
1053
|
+
return new ComputerAction("scroll", {
|
|
1054
|
+
...point(raw),
|
|
1055
|
+
dx: dx * amount,
|
|
1056
|
+
dy: dy * amount,
|
|
1057
|
+
modifiers: modifierChord(raw.text)
|
|
1058
|
+
});
|
|
1059
|
+
},
|
|
1060
|
+
type: (raw) => new ComputerAction("type", { text: String(required(raw, "text")) }),
|
|
1061
|
+
key: (raw) => new ComputerAction("key", {
|
|
1062
|
+
keys: chord(raw),
|
|
1063
|
+
repeat: toInt(raw.repeat ?? 1)
|
|
1064
|
+
}),
|
|
1065
|
+
hold_key: (raw) => new ComputerAction("hold_key", {
|
|
1066
|
+
keys: chord(raw),
|
|
1067
|
+
seconds: toNumber(required(raw, "duration"))
|
|
1068
|
+
}),
|
|
1069
|
+
wait: (raw) => new ComputerAction("wait", { seconds: toNumber(raw.duration ?? 1) })
|
|
1070
|
+
};
|
|
1071
|
+
AnthropicComputer = class _AnthropicComputer extends NativeComputer {
|
|
1072
|
+
static TOOL_TYPE = "computer_toolset_20260801";
|
|
1073
|
+
static TOOLSET = "computer";
|
|
1074
|
+
static BETA = null;
|
|
1075
|
+
static MAX_LONG_EDGE = 2576;
|
|
1076
|
+
static MAX_VISUAL_TOKENS = 4784;
|
|
1077
|
+
static PATCH_PX = 28;
|
|
1078
|
+
static fits(width, height) {
|
|
1079
|
+
const tokens = Math.ceil(width / this.PATCH_PX) * Math.ceil(height / this.PATCH_PX);
|
|
1080
|
+
return Math.max(width, height) <= this.MAX_LONG_EDGE && tokens <= this.MAX_VISUAL_TOKENS;
|
|
1081
|
+
}
|
|
1082
|
+
tool() {
|
|
1083
|
+
return { type: _AnthropicComputer.TOOL_TYPE };
|
|
1084
|
+
}
|
|
1085
|
+
/** Whether a content block is a call to this computer. */
|
|
1086
|
+
owns(block) {
|
|
1087
|
+
return readField(block, "type") === "tool_use" && readField(block, "toolset_name") === _AnthropicComputer.TOOLSET;
|
|
1088
|
+
}
|
|
1089
|
+
member(block) {
|
|
1090
|
+
return [
|
|
1091
|
+
String(readField(block, "name")),
|
|
1092
|
+
{ ...readField(block, "input") ?? {} }
|
|
1093
|
+
];
|
|
1094
|
+
}
|
|
1095
|
+
parse(raw) {
|
|
1096
|
+
const name = String(readField(raw, "action"));
|
|
1097
|
+
const parser = Object.hasOwn(MEMBERS, name) ? MEMBERS[name] : void 0;
|
|
1098
|
+
if (!parser) throw new Error(`unknown computer action: ${name}`);
|
|
1099
|
+
return parser(raw);
|
|
1100
|
+
}
|
|
1101
|
+
/** Run every computer `tool_use` in a response's content, in order, and
|
|
1102
|
+
* return one `tool_result` per block. Other blocks are skipped. */
|
|
1103
|
+
async execute(content) {
|
|
1104
|
+
const results = [];
|
|
1105
|
+
let failed = false;
|
|
1106
|
+
for (const block of content) {
|
|
1107
|
+
if (!this.owns(block)) continue;
|
|
1108
|
+
if (failed) {
|
|
1109
|
+
results.push(this.envelope(block, NOT_EXECUTED, true));
|
|
1110
|
+
continue;
|
|
1111
|
+
}
|
|
1112
|
+
const [result, blockFailed] = await this.runBlock(block);
|
|
1113
|
+
results.push(result);
|
|
1114
|
+
failed = blockFailed;
|
|
1115
|
+
}
|
|
1116
|
+
return results;
|
|
1117
|
+
}
|
|
1118
|
+
/** The content an `ActionResult` goes back to the model as. */
|
|
1119
|
+
static content(result) {
|
|
1120
|
+
if (result.image)
|
|
1121
|
+
return [
|
|
1122
|
+
{
|
|
1123
|
+
type: "image",
|
|
1124
|
+
source: {
|
|
1125
|
+
type: "base64",
|
|
1126
|
+
media_type: "image/png",
|
|
1127
|
+
data: result.image.toString("base64")
|
|
1128
|
+
}
|
|
1129
|
+
}
|
|
1130
|
+
];
|
|
1131
|
+
return [{ type: "text", text: result.text ?? "OK" }];
|
|
1132
|
+
}
|
|
1133
|
+
/** For `anthropic.tools.computerToolset_20260801({ ...computer.aiSdk() })`
|
|
1134
|
+
* (or `computer_20251124` with the legacy kind): `execute` runs one action
|
|
1135
|
+
* and returns `{ type: "image", data }` or text; `toModelOutput` turns that
|
|
1136
|
+
* into the tool result. Calls run one at a time, and once a call fails the
|
|
1137
|
+
* rest of that step's calls answer as not executed. */
|
|
1138
|
+
aiSdk() {
|
|
1139
|
+
let failedStep;
|
|
1140
|
+
const execute = (input, options) => this.serial(async () => {
|
|
1141
|
+
const step = options?.messages?.length;
|
|
1142
|
+
if (step !== void 0 && step === failedStep)
|
|
1143
|
+
throw new Error(NOT_EXECUTED);
|
|
1144
|
+
try {
|
|
1145
|
+
return _AnthropicComputer.aiSdkOutput(await this.executeAction(input));
|
|
1146
|
+
} catch (error) {
|
|
1147
|
+
failedStep = step;
|
|
1148
|
+
throw error instanceof Error ? new Error(this.browser.scrub(error.message)) : error;
|
|
1149
|
+
}
|
|
1150
|
+
});
|
|
1151
|
+
return {
|
|
1152
|
+
execute,
|
|
1153
|
+
toModelOutput: ({ output }) => _AnthropicComputer.aiSdkModelOutput(output)
|
|
1154
|
+
};
|
|
1155
|
+
}
|
|
1156
|
+
static aiSdkOutput(result) {
|
|
1157
|
+
if (result.image)
|
|
1158
|
+
return { type: "image", data: result.image.toString("base64") };
|
|
1159
|
+
return result.text ?? "OK";
|
|
1160
|
+
}
|
|
1161
|
+
/** `image-data` is what AI SDK 6's Anthropic provider turns into an image
|
|
1162
|
+
* block; AI SDK 7 still accepts it (and converts it to a `file` part). */
|
|
1163
|
+
static aiSdkModelOutput(output) {
|
|
1164
|
+
if (typeof output === "string") return { type: "text", value: output };
|
|
1165
|
+
return {
|
|
1166
|
+
type: "content",
|
|
1167
|
+
value: [
|
|
1168
|
+
{ type: "image-data", data: output.data, mediaType: "image/png" }
|
|
1169
|
+
]
|
|
1170
|
+
};
|
|
1171
|
+
}
|
|
1172
|
+
envelope(block, content, isError = false) {
|
|
1173
|
+
return {
|
|
1174
|
+
type: "tool_result",
|
|
1175
|
+
tool_use_id: String(readField(block, "id")),
|
|
1176
|
+
toolset_name: _AnthropicComputer.TOOLSET,
|
|
1177
|
+
content,
|
|
1178
|
+
...isError ? { is_error: true } : {}
|
|
1179
|
+
};
|
|
1180
|
+
}
|
|
1181
|
+
async runBlock(block) {
|
|
1182
|
+
const [name, payload] = this.member(block);
|
|
1183
|
+
try {
|
|
1184
|
+
const result = await this.executeAction({ ...payload, action: name });
|
|
1185
|
+
return [this.envelope(block, _AnthropicComputer.content(result)), false];
|
|
1186
|
+
} catch (error) {
|
|
1187
|
+
const detail = error instanceof Error ? `${error.name}: ${error.message}` : String(error);
|
|
1188
|
+
return [
|
|
1189
|
+
this.envelope(
|
|
1190
|
+
block,
|
|
1191
|
+
this.browser.scrub(`Error: ${detail}`).slice(0, MAX_ERROR2),
|
|
1192
|
+
true
|
|
1193
|
+
),
|
|
1194
|
+
true
|
|
1195
|
+
];
|
|
1196
|
+
}
|
|
1197
|
+
}
|
|
1198
|
+
};
|
|
1199
|
+
AnthropicComputer20251124 = class _AnthropicComputer20251124 extends AnthropicComputer {
|
|
1200
|
+
static TOOL_TYPE = "computer_20251124";
|
|
1201
|
+
static BETA = "computer-use-2025-11-24";
|
|
1202
|
+
static MAX_LONG_EDGE = 1568;
|
|
1203
|
+
static MAX_PIXELS = 115e4;
|
|
1204
|
+
static fits(width, height) {
|
|
1205
|
+
return Math.max(width, height) <= this.MAX_LONG_EDGE && width * height <= this.MAX_PIXELS;
|
|
1206
|
+
}
|
|
1207
|
+
tool() {
|
|
1208
|
+
return {
|
|
1209
|
+
type: _AnthropicComputer20251124.TOOL_TYPE,
|
|
1210
|
+
name: AnthropicComputer.TOOLSET,
|
|
1211
|
+
display_width_px: this.display.width,
|
|
1212
|
+
display_height_px: this.display.height,
|
|
1213
|
+
enable_zoom: true
|
|
1214
|
+
};
|
|
1215
|
+
}
|
|
1216
|
+
owns(block) {
|
|
1217
|
+
return readField(block, "type") === "tool_use" && readField(block, "name") === AnthropicComputer.TOOLSET;
|
|
1218
|
+
}
|
|
1219
|
+
member(block) {
|
|
1220
|
+
const { action, ...payload } = readField(block, "input") ?? {};
|
|
1221
|
+
return [String(action ?? ""), payload];
|
|
1222
|
+
}
|
|
1223
|
+
envelope(block, content, isError = false) {
|
|
1224
|
+
const { toolset_name: _dropped, ...rest } = super.envelope(
|
|
1225
|
+
block,
|
|
1226
|
+
content,
|
|
1227
|
+
isError
|
|
1228
|
+
);
|
|
1229
|
+
return rest;
|
|
1230
|
+
}
|
|
1231
|
+
};
|
|
1232
|
+
NativeComputer.register("anthropic", AnthropicComputer);
|
|
1233
|
+
NativeComputer.register(
|
|
1234
|
+
"anthropic-computer-20251124",
|
|
1235
|
+
AnthropicComputer20251124
|
|
1236
|
+
);
|
|
1237
|
+
}
|
|
1238
|
+
});
|
|
1239
|
+
|
|
1240
|
+
// src/worlds/nativeComputer/openai.ts
|
|
1241
|
+
function safetyCheck(raw) {
|
|
1242
|
+
const check = { id: String(readField(raw, "id") ?? "") };
|
|
1243
|
+
const code = readField(raw, "code");
|
|
1244
|
+
const message = readField(raw, "message");
|
|
1245
|
+
if (code !== void 0) check.code = String(code);
|
|
1246
|
+
if (message !== void 0) check.message = String(message);
|
|
1247
|
+
return check;
|
|
1248
|
+
}
|
|
1249
|
+
function dataUrl(png) {
|
|
1250
|
+
return `data:image/png;base64,${png.toString("base64")}`;
|
|
1251
|
+
}
|
|
1252
|
+
function xy(raw) {
|
|
1253
|
+
return {
|
|
1254
|
+
x: toInt(readField(raw, "x") ?? 0),
|
|
1255
|
+
y: toInt(readField(raw, "y") ?? 0)
|
|
1256
|
+
};
|
|
1257
|
+
}
|
|
1258
|
+
function held(raw) {
|
|
1259
|
+
return modifierKeys(readField(raw, "keys"));
|
|
1260
|
+
}
|
|
1261
|
+
var WAIT_SECONDS, BUTTONS, HISTORY_BUTTONS, PendingSafetyChecks, ACTIONS, OpenAIComputer, OpenAIPreviewComputer;
|
|
1262
|
+
var init_openai = __esm({
|
|
1263
|
+
"src/worlds/nativeComputer/openai.ts"() {
|
|
1264
|
+
"use strict";
|
|
1265
|
+
init_actions();
|
|
1266
|
+
init_base();
|
|
1267
|
+
init_keys();
|
|
1268
|
+
WAIT_SECONDS = 1;
|
|
1269
|
+
BUTTONS = {
|
|
1270
|
+
left: "left",
|
|
1271
|
+
right: "right",
|
|
1272
|
+
wheel: "middle",
|
|
1273
|
+
middle: "middle"
|
|
1274
|
+
};
|
|
1275
|
+
HISTORY_BUTTONS = /* @__PURE__ */ new Set(["back", "forward"]);
|
|
1276
|
+
PendingSafetyChecks = class extends Error {
|
|
1277
|
+
constructor(callId, checks) {
|
|
1278
|
+
const listed = checks.map((c) => `${c.code ?? "check"}: ${c.message ?? ""}`.trim()).join("; ");
|
|
1279
|
+
super(
|
|
1280
|
+
`computer_call ${callId} needs ${checks.length} safety check(s) acknowledged: ${listed}`
|
|
1281
|
+
);
|
|
1282
|
+
this.callId = callId;
|
|
1283
|
+
this.checks = checks;
|
|
1284
|
+
this.name = "PendingSafetyChecks";
|
|
1285
|
+
}
|
|
1286
|
+
callId;
|
|
1287
|
+
checks;
|
|
1288
|
+
};
|
|
1289
|
+
ACTIONS = {
|
|
1290
|
+
click: (raw) => {
|
|
1291
|
+
const button = String(readField(raw, "button") ?? "left");
|
|
1292
|
+
if (HISTORY_BUTTONS.has(button))
|
|
1293
|
+
return new ComputerAction("history", {
|
|
1294
|
+
direction: button
|
|
1295
|
+
});
|
|
1296
|
+
if (!Object.hasOwn(BUTTONS, button))
|
|
1297
|
+
throw new Error(`unknown mouse button: ${button}`);
|
|
1298
|
+
return new ComputerAction("click", {
|
|
1299
|
+
...xy(raw),
|
|
1300
|
+
button: BUTTONS[button],
|
|
1301
|
+
count: 1,
|
|
1302
|
+
modifiers: held(raw)
|
|
1303
|
+
});
|
|
1304
|
+
},
|
|
1305
|
+
double_click: (raw) => new ComputerAction("click", {
|
|
1306
|
+
...xy(raw),
|
|
1307
|
+
button: "left",
|
|
1308
|
+
count: 2,
|
|
1309
|
+
modifiers: held(raw)
|
|
1310
|
+
}),
|
|
1311
|
+
drag: (raw) => new ComputerAction("drag", {
|
|
1312
|
+
path: (readField(raw, "path") ?? []).map(
|
|
1313
|
+
(p) => [toInt(readField(p, "x")), toInt(readField(p, "y"))]
|
|
1314
|
+
),
|
|
1315
|
+
modifiers: held(raw)
|
|
1316
|
+
}),
|
|
1317
|
+
keypress: (raw) => new ComputerAction("key", {
|
|
1318
|
+
keys: [
|
|
1319
|
+
...KeyChord.fromKeys(readField(raw, "keys") ?? []).keys
|
|
1320
|
+
],
|
|
1321
|
+
repeat: 1
|
|
1322
|
+
}),
|
|
1323
|
+
move: (raw) => new ComputerAction("move", { ...xy(raw), modifiers: held(raw) }),
|
|
1324
|
+
scroll: (raw) => new ComputerAction("scroll", {
|
|
1325
|
+
...xy(raw),
|
|
1326
|
+
dx: toInt(readField(raw, "scroll_x", "scrollX") ?? 0),
|
|
1327
|
+
dy: toInt(readField(raw, "scroll_y", "scrollY") ?? 0),
|
|
1328
|
+
modifiers: held(raw)
|
|
1329
|
+
}),
|
|
1330
|
+
type: (raw) => new ComputerAction("type", { text: String(readField(raw, "text") ?? "") }),
|
|
1331
|
+
wait: () => new ComputerAction("wait", { seconds: WAIT_SECONDS }),
|
|
1332
|
+
screenshot: () => new ComputerAction("screenshot")
|
|
1333
|
+
};
|
|
1334
|
+
OpenAIComputer = class _OpenAIComputer extends NativeComputer {
|
|
1335
|
+
tool() {
|
|
1336
|
+
return { type: "computer" };
|
|
1337
|
+
}
|
|
1338
|
+
owns(item) {
|
|
1339
|
+
return readField(item, "type") === "computer_call";
|
|
1340
|
+
}
|
|
1341
|
+
parse(raw) {
|
|
1342
|
+
const kind = String(readField(raw, "type"));
|
|
1343
|
+
const parser = Object.hasOwn(ACTIONS, kind) ? ACTIONS[kind] : void 0;
|
|
1344
|
+
if (!parser) throw new Error(`unknown computer action: ${kind}`);
|
|
1345
|
+
return parser(raw);
|
|
1346
|
+
}
|
|
1347
|
+
static actionsOf(call) {
|
|
1348
|
+
const batch = readField(call, "actions");
|
|
1349
|
+
if (Array.isArray(batch)) return batch;
|
|
1350
|
+
const single = readField(call, "action");
|
|
1351
|
+
return single === void 0 ? [] : [single];
|
|
1352
|
+
}
|
|
1353
|
+
/** Run a `computer_call`'s actions and return its `computer_call_output`.
|
|
1354
|
+
* Throws `PendingSafetyChecks` (before anything runs) while a check the
|
|
1355
|
+
* model raised is not in `acknowledged`. */
|
|
1356
|
+
async execute(call, acknowledged = []) {
|
|
1357
|
+
const callId = String(readField(call, "call_id", "callId"));
|
|
1358
|
+
const checks = (readField(
|
|
1359
|
+
call,
|
|
1360
|
+
"pending_safety_checks",
|
|
1361
|
+
"pendingSafetyChecks"
|
|
1362
|
+
) ?? []).map(safetyCheck);
|
|
1363
|
+
const png = await this.runCall(
|
|
1364
|
+
callId,
|
|
1365
|
+
_OpenAIComputer.actionsOf(call),
|
|
1366
|
+
checks,
|
|
1367
|
+
acknowledged
|
|
1368
|
+
);
|
|
1369
|
+
return {
|
|
1370
|
+
type: "computer_call_output",
|
|
1371
|
+
call_id: callId,
|
|
1372
|
+
output: {
|
|
1373
|
+
type: "computer_screenshot",
|
|
1374
|
+
image_url: dataUrl(png),
|
|
1375
|
+
detail: "original"
|
|
1376
|
+
},
|
|
1377
|
+
...checks.length > 0 ? { acknowledged_safety_checks: checks } : {}
|
|
1378
|
+
};
|
|
1379
|
+
}
|
|
1380
|
+
/** For `openai.tools.computer({ ...computer.aiSdk() })`: `execute` runs a
|
|
1381
|
+
* call's `actions` and returns the screenshot output. A call with pending
|
|
1382
|
+
* safety checks asks for approval first (`needsApproval`), so the AI SDK's
|
|
1383
|
+
* approval flow puts them in front of a person; once approved, `execute`
|
|
1384
|
+
* acknowledges them. */
|
|
1385
|
+
aiSdk() {
|
|
1386
|
+
return {
|
|
1387
|
+
needsApproval: (input) => (input.pendingSafetyChecks ?? []).length > 0,
|
|
1388
|
+
execute: (input) => this.serial(async () => {
|
|
1389
|
+
const checks = (input.pendingSafetyChecks ?? []).map(safetyCheck);
|
|
1390
|
+
const png = await this.runCall(
|
|
1391
|
+
"ai-sdk",
|
|
1392
|
+
input.actions ?? [],
|
|
1393
|
+
checks,
|
|
1394
|
+
checks
|
|
1395
|
+
);
|
|
1396
|
+
return {
|
|
1397
|
+
output: { type: "computer_screenshot", imageUrl: dataUrl(png) },
|
|
1398
|
+
...checks.length > 0 ? { acknowledgedSafetyChecks: checks } : {}
|
|
1399
|
+
};
|
|
1400
|
+
})
|
|
1401
|
+
};
|
|
1402
|
+
}
|
|
1403
|
+
/** The batch's actions, run after the safety checks clear; the screen after them. */
|
|
1404
|
+
async runCall(callId, rawActions, checks, acknowledged) {
|
|
1405
|
+
const cleared = new Set(acknowledged.map((c) => safetyCheck(c).id));
|
|
1406
|
+
const waiting = checks.filter((c) => !c.id || !cleared.has(c.id));
|
|
1407
|
+
if (waiting.length > 0) throw new PendingSafetyChecks(callId, waiting);
|
|
1408
|
+
const actions = rawActions.map((raw) => this.parse(raw));
|
|
1409
|
+
const results = await this.performAll(actions);
|
|
1410
|
+
const last = actions.at(-1);
|
|
1411
|
+
if (last?.name === "screenshot")
|
|
1412
|
+
return results.at(-1)?.image ?? Buffer.alloc(0);
|
|
1413
|
+
return this.screenshot();
|
|
1414
|
+
}
|
|
1415
|
+
};
|
|
1416
|
+
OpenAIPreviewComputer = class extends OpenAIComputer {
|
|
1417
|
+
tool() {
|
|
1418
|
+
return {
|
|
1419
|
+
type: "computer_use_preview",
|
|
1420
|
+
display_width: this.display.width,
|
|
1421
|
+
display_height: this.display.height,
|
|
1422
|
+
environment: "browser"
|
|
1423
|
+
};
|
|
1424
|
+
}
|
|
1425
|
+
};
|
|
1426
|
+
NativeComputer.register("openai", OpenAIComputer);
|
|
1427
|
+
NativeComputer.register("openai-preview", OpenAIPreviewComputer);
|
|
1428
|
+
}
|
|
1429
|
+
});
|
|
1430
|
+
|
|
1431
|
+
// src/worlds/nativeComputer/index.ts
|
|
1432
|
+
var init_nativeComputer = __esm({
|
|
1433
|
+
"src/worlds/nativeComputer/index.ts"() {
|
|
1434
|
+
"use strict";
|
|
1435
|
+
init_actions();
|
|
1436
|
+
init_base();
|
|
1437
|
+
init_display();
|
|
1438
|
+
init_keys();
|
|
1439
|
+
init_screen();
|
|
1440
|
+
init_trace();
|
|
1441
|
+
init_anthropic();
|
|
1442
|
+
init_openai();
|
|
51
1443
|
}
|
|
52
1444
|
});
|
|
53
1445
|
|
|
@@ -63,6 +1455,23 @@ var init_sessionContract = __esm({
|
|
|
63
1455
|
}
|
|
64
1456
|
});
|
|
65
1457
|
|
|
1458
|
+
// src/tracing/session.ts
|
|
1459
|
+
var import_async_hooks, processStorage, currentSessionStorage, currentWorldSessionStorage;
|
|
1460
|
+
var init_session = __esm({
|
|
1461
|
+
"src/tracing/session.ts"() {
|
|
1462
|
+
"use strict";
|
|
1463
|
+
import_async_hooks = require("async_hooks");
|
|
1464
|
+
processStorage = (name) => {
|
|
1465
|
+
const key = /* @__PURE__ */ Symbol.for(`@withgateway/sdk/${name}`);
|
|
1466
|
+
const slots = globalThis;
|
|
1467
|
+
slots[key] ??= new import_async_hooks.AsyncLocalStorage();
|
|
1468
|
+
return slots[key];
|
|
1469
|
+
};
|
|
1470
|
+
currentSessionStorage = processStorage("session-storage");
|
|
1471
|
+
currentWorldSessionStorage = processStorage("world-session-storage");
|
|
1472
|
+
}
|
|
1473
|
+
});
|
|
1474
|
+
|
|
66
1475
|
// src/worlds/taskRef.ts
|
|
67
1476
|
var init_taskRef = __esm({
|
|
68
1477
|
"src/worlds/taskRef.ts"() {
|
|
@@ -71,14 +1480,16 @@ var init_taskRef = __esm({
|
|
|
71
1480
|
});
|
|
72
1481
|
|
|
73
1482
|
// src/worlds/session.ts
|
|
74
|
-
var
|
|
75
|
-
var
|
|
1483
|
+
var import_node_crypto2, READY_TIMEOUT_MS, CALL_TIMEOUT_MS, JUDGED_GRADE_TIMEOUT_MS;
|
|
1484
|
+
var init_session2 = __esm({
|
|
76
1485
|
"src/worlds/session.ts"() {
|
|
77
1486
|
"use strict";
|
|
78
|
-
|
|
1487
|
+
import_node_crypto2 = require("crypto");
|
|
79
1488
|
init_worlds();
|
|
80
1489
|
init_browser();
|
|
1490
|
+
init_nativeComputer();
|
|
81
1491
|
init_sessionContract();
|
|
1492
|
+
init_session();
|
|
82
1493
|
init_taskRef();
|
|
83
1494
|
init_taskRef();
|
|
84
1495
|
READY_TIMEOUT_MS = 10 * 6e4;
|
|
@@ -109,7 +1520,7 @@ var init_taskSessions = __esm({
|
|
|
109
1520
|
"use strict";
|
|
110
1521
|
init_worlds();
|
|
111
1522
|
init_browser();
|
|
112
|
-
|
|
1523
|
+
init_session2();
|
|
113
1524
|
}
|
|
114
1525
|
});
|
|
115
1526
|
|
|
@@ -122,16 +1533,16 @@ var init_evals = __esm({
|
|
|
122
1533
|
});
|
|
123
1534
|
|
|
124
1535
|
// src/worlds/taskRun.ts
|
|
125
|
-
var
|
|
1536
|
+
var import_node_crypto3, fs, path;
|
|
126
1537
|
var init_taskRun = __esm({
|
|
127
1538
|
"src/worlds/taskRun.ts"() {
|
|
128
1539
|
"use strict";
|
|
129
|
-
|
|
1540
|
+
import_node_crypto3 = require("crypto");
|
|
130
1541
|
fs = __toESM(require("fs"));
|
|
131
1542
|
path = __toESM(require("path"));
|
|
132
1543
|
init_browser();
|
|
133
1544
|
init_evals();
|
|
134
|
-
|
|
1545
|
+
init_session2();
|
|
135
1546
|
init_taskSessions();
|
|
136
1547
|
}
|
|
137
1548
|
});
|
|
@@ -166,11 +1577,11 @@ var init_checkoutTarget = __esm({
|
|
|
166
1577
|
});
|
|
167
1578
|
|
|
168
1579
|
// src/worlds/worldDir.ts
|
|
169
|
-
var
|
|
1580
|
+
var import_node_crypto4, fs3, path3, CONNECTOR_SPEC, WORLD_SCHEMA_MARKERS, WORLD_ROOT_MARKERS;
|
|
170
1581
|
var init_worldDir = __esm({
|
|
171
1582
|
"src/worlds/worldDir.ts"() {
|
|
172
1583
|
"use strict";
|
|
173
|
-
|
|
1584
|
+
import_node_crypto4 = require("crypto");
|
|
174
1585
|
fs3 = __toESM(require("fs"));
|
|
175
1586
|
path3 = __toESM(require("path"));
|
|
176
1587
|
CONNECTOR_SPEC = "connector.toml";
|
|
@@ -191,11 +1602,11 @@ var init_tar = __esm({
|
|
|
191
1602
|
});
|
|
192
1603
|
|
|
193
1604
|
// src/worlds/hub.ts
|
|
194
|
-
var
|
|
1605
|
+
var import_node_crypto5, import_node_zlib2, fs5, path5, META_RELPATH;
|
|
195
1606
|
var init_hub = __esm({
|
|
196
1607
|
"src/worlds/hub.ts"() {
|
|
197
1608
|
"use strict";
|
|
198
|
-
|
|
1609
|
+
import_node_crypto5 = require("crypto");
|
|
199
1610
|
import_node_zlib2 = require("zlib");
|
|
200
1611
|
fs5 = __toESM(require("fs"));
|
|
201
1612
|
path5 = __toESM(require("path"));
|
|
@@ -241,11 +1652,11 @@ var init_json = __esm({
|
|
|
241
1652
|
});
|
|
242
1653
|
|
|
243
1654
|
// src/worlds/transforms.ts
|
|
244
|
-
var
|
|
1655
|
+
var import_node_crypto6, WHITESPACE, WHITESPACE_EDGES, WHITESPACE_RUN, NOT_DIGIT, STRIP_PREFIX, trim, emailLocalPart, emailDomain, hex, SIMPLE, TRANSFORMS;
|
|
245
1656
|
var init_transforms = __esm({
|
|
246
1657
|
"src/worlds/transforms.ts"() {
|
|
247
1658
|
"use strict";
|
|
248
|
-
|
|
1659
|
+
import_node_crypto6 = require("crypto");
|
|
249
1660
|
init_json();
|
|
250
1661
|
WHITESPACE = " \n\r\f\v";
|
|
251
1662
|
WHITESPACE_EDGES = new RegExp(`^[${WHITESPACE}]+|[${WHITESPACE}]+$`, "g");
|
|
@@ -261,7 +1672,7 @@ var init_transforms = __esm({
|
|
|
261
1672
|
const at = value.lastIndexOf("@");
|
|
262
1673
|
return at === -1 ? "" : value.slice(at + 1).toLowerCase();
|
|
263
1674
|
};
|
|
264
|
-
hex = (algorithm, value) => (0,
|
|
1675
|
+
hex = (algorithm, value) => (0, import_node_crypto6.createHash)(algorithm).update(value, "utf8").digest("hex");
|
|
265
1676
|
SIMPLE = {
|
|
266
1677
|
lower: (value) => value.toLowerCase(),
|
|
267
1678
|
upper: (value) => value.toUpperCase(),
|
|
@@ -281,11 +1692,11 @@ var init_transforms = __esm({
|
|
|
281
1692
|
});
|
|
282
1693
|
|
|
283
1694
|
// src/worlds/redaction.ts
|
|
284
|
-
var
|
|
1695
|
+
var import_node_crypto7;
|
|
285
1696
|
var init_redaction = __esm({
|
|
286
1697
|
"src/worlds/redaction.ts"() {
|
|
287
1698
|
"use strict";
|
|
288
|
-
|
|
1699
|
+
import_node_crypto7 = require("crypto");
|
|
289
1700
|
init_json();
|
|
290
1701
|
init_transforms();
|
|
291
1702
|
}
|
|
@@ -366,25 +1777,18 @@ function parseErrorBody(text, limit = 600) {
|
|
|
366
1777
|
code: stableCode
|
|
367
1778
|
};
|
|
368
1779
|
}
|
|
369
|
-
async function apiRequest(options, method, path7, body) {
|
|
370
|
-
const
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
headers: {
|
|
374
|
-
"content-type": "application/json",
|
|
375
|
-
authorization: "Basic " + Buffer.from(`${publicKey}:${secretKey}`).toString("base64")
|
|
376
|
-
},
|
|
377
|
-
...body === void 0 ? {} : { body: JSON.stringify(body) }
|
|
378
|
-
});
|
|
379
|
-
const text = await res.text();
|
|
380
|
-
if (!res.ok) throw new ApiError(res.status, method, path7, text);
|
|
381
|
-
return JSON.parse(text);
|
|
1780
|
+
async function apiRequest(options, method, path7, body, init = {}) {
|
|
1781
|
+
const reply = await new PlatformTransport(resolveEnv(options)).send(method, path7, body, init);
|
|
1782
|
+
if (!reply.ok) throw new ApiError(reply.status, method, path7, reply.text);
|
|
1783
|
+
return JSON.parse(reply.text);
|
|
382
1784
|
}
|
|
383
1785
|
var ApiError;
|
|
384
1786
|
var init_worlds = __esm({
|
|
385
1787
|
"src/worlds/index.ts"() {
|
|
386
1788
|
"use strict";
|
|
387
|
-
|
|
1789
|
+
init_transport();
|
|
1790
|
+
init_transport();
|
|
1791
|
+
init_session2();
|
|
388
1792
|
init_sessionList();
|
|
389
1793
|
init_sessionContract();
|
|
390
1794
|
init_taskRef();
|
|
@@ -396,6 +1800,7 @@ var init_worlds = __esm({
|
|
|
396
1800
|
init_evals();
|
|
397
1801
|
init_browser();
|
|
398
1802
|
init_browserReconnect();
|
|
1803
|
+
init_nativeComputer();
|
|
399
1804
|
init_hub();
|
|
400
1805
|
init_validate();
|
|
401
1806
|
init_describe();
|
|
@@ -408,14 +1813,16 @@ var init_worlds = __esm({
|
|
|
408
1813
|
init_editSessions();
|
|
409
1814
|
init_dryRun();
|
|
410
1815
|
ApiError = class extends Error {
|
|
411
|
-
constructor(status, method, path7,
|
|
412
|
-
const parsed = parseErrorBody(
|
|
1816
|
+
constructor(status, method, path7, text) {
|
|
1817
|
+
const parsed = parseErrorBody(text);
|
|
413
1818
|
super(`${method} ${path7} -> ${status}: ${parsed.detail}`);
|
|
414
1819
|
this.status = status;
|
|
1820
|
+
this.text = text;
|
|
415
1821
|
this.name = "ApiError";
|
|
416
1822
|
this.code = parsed.code;
|
|
417
1823
|
}
|
|
418
1824
|
status;
|
|
1825
|
+
text;
|
|
419
1826
|
code;
|
|
420
1827
|
};
|
|
421
1828
|
}
|