@withgateway/sdk 0.2.9 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/README.md +5 -1
  2. package/dist/cli.js +3812 -1158
  3. package/dist/cli.js.map +1 -1
  4. package/dist/cli.mjs +3796 -1142
  5. package/dist/cli.mjs.map +1 -1
  6. package/dist/files/index.d.mts +1 -0
  7. package/dist/files/index.d.ts +1 -0
  8. package/dist/files/index.js +1358 -21
  9. package/dist/files/index.js.map +1 -1
  10. package/dist/files/index.mjs +1356 -13
  11. package/dist/files/index.mjs.map +1 -1
  12. package/dist/{index-BO2SNSgy.d.ts → index-CBqbB_0J.d.ts} +95 -46
  13. package/dist/{index-Cz8Q6Keo.d.mts → index-D-AMnAMa.d.mts} +95 -46
  14. package/dist/index.d.mts +4 -3
  15. package/dist/index.d.ts +4 -3
  16. package/dist/index.js +1942 -177
  17. package/dist/index.js.map +1 -1
  18. package/dist/index.mjs +1890 -146
  19. package/dist/index.mjs.map +1 -1
  20. package/dist/media-CbcAfQvd.d.mts +83 -0
  21. package/dist/media-CbcAfQvd.d.ts +83 -0
  22. package/dist/traces/index.d.mts +1 -0
  23. package/dist/traces/index.d.ts +1 -0
  24. package/dist/traces/index.js +1355 -18
  25. package/dist/traces/index.js.map +1 -1
  26. package/dist/traces/index.mjs +1353 -10
  27. package/dist/traces/index.mjs.map +1 -1
  28. package/dist/tracing/index.d.mts +2 -1
  29. package/dist/tracing/index.d.ts +2 -1
  30. package/dist/tracing/index.js +550 -30
  31. package/dist/tracing/index.js.map +1 -1
  32. package/dist/tracing/index.mjs +539 -29
  33. package/dist/tracing/index.mjs.map +1 -1
  34. package/dist/worlds/index.d.mts +648 -80
  35. package/dist/worlds/index.d.ts +648 -80
  36. package/dist/worlds/index.js +1875 -145
  37. package/dist/worlds/index.js.map +1 -1
  38. package/dist/worlds/index.mjs +1861 -131
  39. package/dist/worlds/index.mjs.map +1 -1
  40. package/package.json +1 -1
  41. package/runtime/gatewaysdk/environment/schema/api.py +43 -62
  42. package/runtime/gatewaysdk/environment/schema/cast.py +31 -4
  43. package/runtime/gatewaysdk/environment/schema/grading.py +1 -5
  44. package/runtime/gatewaysdk/environment/schema/host.py +53 -90
  45. package/runtime/gatewaysdk/environment/schema/local.py +15 -7
  46. package/runtime/gatewaysdk/environment/schema/platform.py +1 -7
  47. package/runtime/gatewaysdk/environment/schema/runtime.py +9 -9
  48. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +3 -3
  49. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +2 -2
  50. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +13 -13
  51. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +2 -1
  52. package/runtime/gatewaysdk/environment/schema/tasks.py +25 -1
  53. package/skills/connector-schema-authoring/SKILL.md +3 -3
  54. package/skills/connector-schema-authoring/connector-worlds.md +12 -5
  55. package/skills/connector-schema-authoring/world-schema.md +8 -0
  56. package/skills/world-data-ingestion/SKILL.md +2 -2
  57. package/skills/worlds-getting-started/SKILL.md +13 -13
  58. package/skills/worlds-getting-started/references/cli-mcp-map.md +2 -1
@@ -1,6 +1,12 @@
1
1
  import { createRequire as __gatewayCreateRequire } from "module";
2
2
  const require = __gatewayCreateRequire(import.meta.url);
3
3
  var __getOwnPropNames = Object.getOwnPropertyNames;
4
+ var __require = /* @__PURE__ */ ((x) => typeof require !== "undefined" ? require : typeof Proxy !== "undefined" ? new Proxy(x, {
5
+ get: (a, b) => (typeof require !== "undefined" ? require : a)[b]
6
+ }) : x)(function(x) {
7
+ if (typeof require !== "undefined") return require.apply(this, arguments);
8
+ throw Error('Dynamic require of "' + x + '" is not supported');
9
+ });
4
10
  var __esm = (fn, res, err) => function __init() {
5
11
  if (err) throw err[0];
6
12
  try {
@@ -127,11 +133,1294 @@ var init_browserReconnect = __esm({
127
133
  });
128
134
 
129
135
  // src/worlds/browser.ts
136
+ function redactActionArgs(action, args) {
137
+ const secretField = action === "browser_type" && SECRET_KEY.test(String(args.selector ?? ""));
138
+ const out = {};
139
+ for (const [key, value] of Object.entries(args)) {
140
+ if (value === null || value === void 0) out[key] = null;
141
+ else if (typeof value === "boolean" || typeof value === "number")
142
+ out[key] = value;
143
+ else if (typeof value === "string") {
144
+ if (SECRET_KEY.test(key) || SECRET_VALUE.test(value) || secretField && key === "text")
145
+ out[key] = "[redacted]";
146
+ else
147
+ out[key] = value.length <= EVENT_MAX_STRING ? value : value.slice(0, EVENT_MAX_STRING) + "\u2026";
148
+ } else out[key] = "[omitted]";
149
+ }
150
+ return out;
151
+ }
152
+ var EVENT_MAX_STRING, SECRET_KEY, SECRET_VALUE;
130
153
  var init_browser = __esm({
131
154
  "src/worlds/browser.ts"() {
132
155
  "use strict";
133
- init_session();
156
+ init_session2();
134
157
  init_browserReconnect();
158
+ EVENT_MAX_STRING = 500;
159
+ SECRET_KEY = /pass(word|wd)?|secret|token|api[-_]?key|authorization|cookie|(^|[-_ #.\["'=])(otp|pin|cvv|ssn)([-_ \]"']|$)/i;
160
+ SECRET_VALUE = /^((sk|pk|ghp|xox[abp])[-_][-_A-Za-z0-9]{8,}|AKIA[0-9A-Z]{12,}|Bearer [-_.A-Za-z0-9]{8,})/;
161
+ }
162
+ });
163
+
164
+ // src/worlds/nativeComputer/actions.ts
165
+ function toNumber(value) {
166
+ if (typeof value !== "number" || !Number.isFinite(value))
167
+ throw new Error(`not a number: ${JSON.stringify(value) ?? String(value)}`);
168
+ return value;
169
+ }
170
+ function toInt(value) {
171
+ return Math.trunc(toNumber(value));
172
+ }
173
+ var ACTION_NAMES, SECRET_TEXT, BOUNDS, ComputerAction;
174
+ var init_actions = __esm({
175
+ "src/worlds/nativeComputer/actions.ts"() {
176
+ "use strict";
177
+ ACTION_NAMES = [
178
+ "screenshot",
179
+ "zoom",
180
+ "click",
181
+ "move",
182
+ "mouse_down",
183
+ "mouse_up",
184
+ "drag",
185
+ "scroll",
186
+ "type",
187
+ "key",
188
+ "hold_key",
189
+ "wait",
190
+ "cursor_position",
191
+ "history"
192
+ ];
193
+ SECRET_TEXT = "[redacted]";
194
+ BOUNDS = {
195
+ seconds: [0, 300],
196
+ repeat: [1, 100],
197
+ count: [1, 3],
198
+ dx: [-1e4, 1e4],
199
+ dy: [-1e4, 1e4]
200
+ };
201
+ ComputerAction = class {
202
+ constructor(name, args = {}) {
203
+ this.name = name;
204
+ this.args = args;
205
+ if (!ACTION_NAMES.includes(name))
206
+ throw new Error(`unknown computer action: ${name}`);
207
+ for (const [key, [low, high]] of Object.entries(BOUNDS)) {
208
+ const value = args[key];
209
+ if (value !== void 0 && !(Number(value) >= low && Number(value) <= high))
210
+ throw new Error(
211
+ `${name} ${key} must be between ${low} and ${high}, got ${String(value)}`
212
+ );
213
+ }
214
+ }
215
+ name;
216
+ args;
217
+ /** Flat arguments for the browser timeline and the trace: lists become
218
+ * `"x,y"` strings, and typed text is withheld when it went into a
219
+ * credential field. */
220
+ recordArgs(secretText) {
221
+ const out = {};
222
+ for (const [key, value] of Object.entries(this.args))
223
+ out[key] = Array.isArray(value) ? value.flat().join(",") : value;
224
+ if (secretText && "text" in out) out.text = SECRET_TEXT;
225
+ return out;
226
+ }
227
+ };
228
+ }
229
+ });
230
+
231
+ // src/worlds/nativeComputer/display.ts
232
+ var FIT_STEP, Display;
233
+ var init_display = __esm({
234
+ "src/worlds/nativeComputer/display.ts"() {
235
+ "use strict";
236
+ FIT_STEP = 0.98;
237
+ Display = class _Display {
238
+ constructor(width, height, viewportWidth, viewportHeight) {
239
+ this.width = width;
240
+ this.height = height;
241
+ this.viewportWidth = viewportWidth;
242
+ this.viewportHeight = viewportHeight;
243
+ }
244
+ width;
245
+ height;
246
+ viewportWidth;
247
+ viewportHeight;
248
+ /** `size` when the caller names one; otherwise the largest downscale of the
249
+ * viewport (at most 1:1) that `fits` the provider's limits. */
250
+ static forViewport(viewport, size, fits = () => true) {
251
+ const { width, height } = viewport;
252
+ if (size) return _Display.sized(size[0], size[1], width, height);
253
+ let scale = 1;
254
+ while (!fits(Math.round(width * scale), Math.round(height * scale)))
255
+ scale *= FIT_STEP;
256
+ return new _Display(
257
+ Math.round(width * scale),
258
+ Math.round(height * scale),
259
+ width,
260
+ height
261
+ );
262
+ }
263
+ /** A caller's display keeps the viewport's shape: screenshots are scaled
264
+ * uniformly, so another aspect ratio would map clicks off the image. */
265
+ static sized(width, height, viewportWidth, viewportHeight) {
266
+ if (width <= 0 || height <= 0)
267
+ throw new Error(`display ${width}x${height} must be positive`);
268
+ const sameShape = width * viewportHeight / viewportWidth;
269
+ if (Math.abs(sameShape - height) > 1)
270
+ throw new Error(
271
+ `display ${width}x${height} must keep the viewport's aspect ratio (${viewportWidth}x${viewportHeight}); try ${width}x${Math.round(sameShape)}`
272
+ );
273
+ return new _Display(width, height, viewportWidth, viewportHeight);
274
+ }
275
+ get scaled() {
276
+ return this.width !== this.viewportWidth || this.height !== this.viewportHeight;
277
+ }
278
+ get scale() {
279
+ return Math.min(
280
+ this.width / this.viewportWidth,
281
+ this.height / this.viewportHeight
282
+ );
283
+ }
284
+ toPage(x, y) {
285
+ return [
286
+ x * this.viewportWidth / this.width,
287
+ y * this.viewportHeight / this.height
288
+ ];
289
+ }
290
+ };
291
+ }
292
+ });
293
+
294
+ // src/worlds/nativeComputer/screen.ts
295
+ function pngWidth(png) {
296
+ if (png.length < 24 || !png.subarray(0, 8).equals(PNG_SIGNATURE)) return null;
297
+ return png.readUInt32BE(16);
298
+ }
299
+ function isNativePage(page) {
300
+ const candidate = page;
301
+ const mouse = candidate?.mouse;
302
+ return typeof candidate?.context === "function" && NATIVE_METHODS.every((m) => typeof mouse?.[m] === "function");
303
+ }
304
+ var FOCUSED_FIELD_JS, SECRET_FOCUS, DRAG_STEPS, PNG_SIGNATURE, NATIVE_METHODS, Screen;
305
+ var init_screen = __esm({
306
+ "src/worlds/nativeComputer/screen.ts"() {
307
+ "use strict";
308
+ init_browser();
309
+ FOCUSED_FIELD_JS = "(() => { let e = document.activeElement; const inner = (el) => (el.shadowRoot && el.shadowRoot.activeElement) || (el.contentDocument && el.contentDocument.activeElement); while (e && inner(e)) e = inner(e); if (!e) return ''; if (e.tagName === 'IFRAME' || e.tagName === 'FRAME') return 'frame'; if (!(e.tagName === 'INPUT' || e.tagName === 'TEXTAREA' || e.isContentEditable)) return 'opaque'; return [e.type, e.name, e.id, e.getAttribute('autocomplete')].filter(Boolean).join(' '); })()";
310
+ SECRET_FOCUS = ["password", "frame", "opaque"];
311
+ DRAG_STEPS = 5;
312
+ PNG_SIGNATURE = Buffer.from([
313
+ 137,
314
+ 80,
315
+ 78,
316
+ 71,
317
+ 13,
318
+ 10,
319
+ 26,
320
+ 10
321
+ ]);
322
+ NATIVE_METHODS = ["move", "click", "down", "up", "wheel"];
323
+ Screen = class {
324
+ constructor(browser, display) {
325
+ this.browser = browser;
326
+ this.display = display;
327
+ }
328
+ browser;
329
+ display;
330
+ cursor = [0, 0];
331
+ pixelRatio = 1;
332
+ handlers = {
333
+ screenshot: async () => ({ image: await this.screenshot() }),
334
+ zoom: (a) => this.zoom(a),
335
+ click: (a) => this.click(a),
336
+ move: (a) => this.held(a.args.modifiers ?? [], () => this.moveTo(this.point(a))),
337
+ mouse_down: async (a) => {
338
+ await this.page.mouse.down({ button: a.args.button ?? "left" });
339
+ return {};
340
+ },
341
+ mouse_up: async (a) => {
342
+ await this.page.mouse.up({ button: a.args.button ?? "left" });
343
+ return {};
344
+ },
345
+ drag: (a) => this.drag(a),
346
+ scroll: (a) => this.scroll(a),
347
+ type: async (a) => {
348
+ await this.page.keyboard.type(a.args.text ?? "");
349
+ return {};
350
+ },
351
+ key: (a) => this.key(a),
352
+ hold_key: (a) => this.held(
353
+ a.args.keys ?? [],
354
+ () => this.page.waitForTimeout((a.args.seconds ?? 0) * 1e3)
355
+ ),
356
+ wait: async (a) => {
357
+ await this.page.waitForTimeout((a.args.seconds ?? 0) * 1e3);
358
+ return {};
359
+ },
360
+ cursor_position: async () => ({
361
+ text: `X=${this.cursor[0]}, Y=${this.cursor[1]}`
362
+ }),
363
+ history: async (a) => {
364
+ const navigate = {
365
+ back: () => this.page.goBack(),
366
+ forward: () => this.page.goForward()
367
+ };
368
+ await navigate[a.args.direction ?? "back"]();
369
+ return {};
370
+ }
371
+ };
372
+ /** The live page (a new object after a reconnect), as pixel-level Playwright. */
373
+ get page() {
374
+ const page = this.browser.page;
375
+ if (!isNativePage(page))
376
+ throw new Error(
377
+ "native computer use needs a Playwright page (mouse.move/click/down/up/wheel, context())"
378
+ );
379
+ return page;
380
+ }
381
+ run(action) {
382
+ return this.handlers[action.name](action);
383
+ }
384
+ /** A PNG of the viewport at the model's display size. */
385
+ async screenshot() {
386
+ if (!this.display.scaled) return this.page.screenshot({ type: "png" });
387
+ const { viewportWidth: width, viewportHeight: height } = this.display;
388
+ return this.capture({ x: 0, y: 0, width, height }, this.display.scale);
389
+ }
390
+ /** Whether keystrokes go to a credential field (a password input, or one
391
+ * named like a token, key or PIN), looking through shadow roots and
392
+ * same-origin frames. An unreadable page, a cross-origin frame or a focus
393
+ * that is not a text field (a closed shadow root's host) counts as one. */
394
+ async focusedFieldIsSecret() {
395
+ let described;
396
+ try {
397
+ described = String(await this.page.evaluate(FOCUSED_FIELD_JS) ?? "");
398
+ } catch {
399
+ return true;
400
+ }
401
+ return SECRET_FOCUS.some((prefix) => described.startsWith(prefix)) || SECRET_KEY.test(described);
402
+ }
403
+ /** CDP renders a clip at the page's device pixel ratio as well as its
404
+ * `scale` (a Retina host doubles it), so a capture of the wrong width
405
+ * measures that ratio and is taken once more, corrected. */
406
+ async capture(clip, scale) {
407
+ const wanted = Math.round(clip.width * scale);
408
+ const png = await this.cdpCapture(clip, scale / this.pixelRatio);
409
+ const width = pngWidth(png);
410
+ if (width === null || Math.abs(width - wanted) <= 1) return png;
411
+ this.pixelRatio *= width / wanted;
412
+ return this.cdpCapture(clip, scale / this.pixelRatio);
413
+ }
414
+ async cdpCapture(clip, scale) {
415
+ const page = this.page;
416
+ const session = await page.context().newCDPSession(page);
417
+ try {
418
+ const shot = await session.send("Page.captureScreenshot", {
419
+ format: "png",
420
+ clip: { ...clip, scale }
421
+ });
422
+ return Buffer.from(shot.data, "base64");
423
+ } finally {
424
+ await session.detach();
425
+ }
426
+ }
427
+ point(action) {
428
+ const { x, y } = action.args;
429
+ return x === void 0 || y === void 0 ? this.cursor : [x, y];
430
+ }
431
+ async moveTo(point2, steps = 1) {
432
+ const [x, y] = this.display.toPage(point2[0], point2[1]);
433
+ await this.page.mouse.move(x, y, { steps });
434
+ this.cursor = point2;
435
+ }
436
+ async held(modifiers, fn) {
437
+ for (const key of modifiers) await this.page.keyboard.down(key);
438
+ try {
439
+ await fn();
440
+ } finally {
441
+ for (const key of [...modifiers].reverse())
442
+ await this.page.keyboard.up(key);
443
+ }
444
+ return {};
445
+ }
446
+ async zoom(action) {
447
+ const [x0, y0, x1, y1] = action.args.region ?? [0, 0, 0, 0];
448
+ const [left, top] = this.display.toPage(x0, y0);
449
+ const [right, bottom] = this.display.toPage(x1, y1);
450
+ if (right <= left || bottom <= top)
451
+ throw new Error("zoom region must have x1 > x0 and y1 > y0");
452
+ const width = right - left;
453
+ const height = bottom - top;
454
+ const scale = Math.min(
455
+ this.display.width / width,
456
+ this.display.height / height
457
+ );
458
+ return {
459
+ image: await this.capture({ x: left, y: top, width, height }, scale)
460
+ };
461
+ }
462
+ async click(action) {
463
+ const point2 = this.point(action);
464
+ return this.held(action.args.modifiers ?? [], async () => {
465
+ await this.moveTo(point2);
466
+ const [x, y] = this.display.toPage(point2[0], point2[1]);
467
+ await this.page.mouse.click(x, y, {
468
+ button: action.args.button ?? "left",
469
+ clickCount: action.args.count ?? 1
470
+ });
471
+ });
472
+ }
473
+ async drag(action) {
474
+ const path8 = action.args.path ?? [];
475
+ if (path8.length < 2) throw new Error("a drag needs at least two points");
476
+ return this.held(action.args.modifiers ?? [], async () => {
477
+ await this.moveTo(path8[0]);
478
+ await this.page.mouse.down({ button: "left" });
479
+ try {
480
+ for (const point2 of path8.slice(1)) await this.moveTo(point2, DRAG_STEPS);
481
+ } finally {
482
+ await this.page.mouse.up({ button: "left" });
483
+ }
484
+ });
485
+ }
486
+ async scroll(action) {
487
+ await this.moveTo(this.point(action));
488
+ const scale = this.display.scale;
489
+ return this.held(
490
+ action.args.modifiers ?? [],
491
+ () => this.page.mouse.wheel(
492
+ (action.args.dx ?? 0) / scale,
493
+ (action.args.dy ?? 0) / scale
494
+ )
495
+ );
496
+ }
497
+ async key(action) {
498
+ const combo = (action.args.keys ?? []).join("+");
499
+ for (let i = 0; i < (action.args.repeat ?? 1); i += 1)
500
+ await this.page.keyboard.press(combo);
501
+ return {};
502
+ }
503
+ };
504
+ }
505
+ });
506
+
507
+ // src/tracing/media.ts
508
+ import { createHash } from "crypto";
509
+ async function putBytes(url, data, headers) {
510
+ const controller = new AbortController();
511
+ const timer = setTimeout(() => controller.abort(), PUT_TIMEOUT_MS);
512
+ try {
513
+ const res = await fetch(url, {
514
+ method: "PUT",
515
+ body: new Uint8Array(data),
516
+ headers,
517
+ signal: controller.signal
518
+ });
519
+ return {
520
+ status: res.status,
521
+ error: res.ok ? null : (await res.text()).slice(0, MAX_UPLOAD_ERROR)
522
+ };
523
+ } catch (error) {
524
+ return {
525
+ status: NETWORK_FAILURE_STATUS,
526
+ error: error instanceof Error ? error.name : "Error"
527
+ };
528
+ } finally {
529
+ clearTimeout(timer);
530
+ }
531
+ }
532
+ var MEDIA_PATH, MAX_MEDIA_BYTES, PUT_TIMEOUT_MS, MAX_UPLOAD_ERROR, NETWORK_FAILURE_STATUS, MediaUploadError, MediaReference, MediaUploader;
533
+ var init_media = __esm({
534
+ "src/tracing/media.ts"() {
535
+ "use strict";
536
+ init_transport();
537
+ MEDIA_PATH = "/api/public/media";
538
+ MAX_MEDIA_BYTES = 5 * 1024 * 1024;
539
+ PUT_TIMEOUT_MS = 3e4;
540
+ MAX_UPLOAD_ERROR = 500;
541
+ NETWORK_FAILURE_STATUS = 599;
542
+ MediaUploadError = class extends Error {
543
+ constructor(message, status) {
544
+ super(message);
545
+ this.status = status;
546
+ this.name = "MediaUploadError";
547
+ }
548
+ status;
549
+ };
550
+ MediaReference = class {
551
+ constructor(mediaId, contentType) {
552
+ this.mediaId = mediaId;
553
+ this.contentType = contentType;
554
+ }
555
+ mediaId;
556
+ contentType;
557
+ toString() {
558
+ return `@@@langfuseMedia:type=${this.contentType}|id=${this.mediaId}|source=bytes@@@`;
559
+ }
560
+ };
561
+ MediaUploader = class _MediaUploader {
562
+ constructor(transport, put = putBytes, maxBytes = MAX_MEDIA_BYTES) {
563
+ this.transport = transport;
564
+ this.put = put;
565
+ this.maxBytes = maxBytes;
566
+ }
567
+ transport;
568
+ put;
569
+ maxBytes;
570
+ /** The uploader for `GATEWAY_HOST` / `GATEWAY_PUBLIC_KEY` /
571
+ * `GATEWAY_SECRET_KEY` (the keys tracing exports with); null when any is unset. */
572
+ static fromEnv(env = process.env) {
573
+ const host = env.GATEWAY_HOST;
574
+ const publicKey = env.GATEWAY_PUBLIC_KEY;
575
+ const secretKey = env.GATEWAY_SECRET_KEY;
576
+ if (!host || !publicKey || !secretKey) return null;
577
+ return new _MediaUploader(
578
+ new PlatformTransport({
579
+ host: host.replace(/\/$/, ""),
580
+ publicKey,
581
+ secretKey
582
+ })
583
+ );
584
+ }
585
+ /** Store `data` linked to the trace (and observation) and return its reference. */
586
+ async upload(data, link) {
587
+ if (data.length > this.maxBytes)
588
+ throw new MediaUploadError(
589
+ `media is ${data.length} bytes, over the ${this.maxBytes}-byte cap`
590
+ );
591
+ const sha256Hash = createHash("sha256").update(data).digest("base64");
592
+ const answer = await this.send("POST", MEDIA_PATH, {
593
+ traceId: link.traceId,
594
+ observationId: link.observationId,
595
+ contentType: link.contentType,
596
+ contentLength: data.length,
597
+ sha256Hash,
598
+ field: link.field
599
+ });
600
+ if (typeof answer.mediaId !== "string" || !answer.mediaId)
601
+ throw new MediaUploadError(
602
+ "POST /api/public/media answered without a mediaId"
603
+ );
604
+ const reference = new MediaReference(answer.mediaId, link.contentType);
605
+ if (answer.uploadUrl)
606
+ await this.store(reference, answer.uploadUrl, data, sha256Hash);
607
+ return reference;
608
+ }
609
+ async store(reference, url, data, sha256Hash) {
610
+ const started = Date.now();
611
+ const { status, error } = await this.put(url, data, {
612
+ "Content-Type": reference.contentType,
613
+ "x-amz-checksum-sha256": sha256Hash,
614
+ "x-ms-blob-type": "BlockBlob"
615
+ });
616
+ await this.send(
617
+ "PATCH",
618
+ `${MEDIA_PATH}/${encodeURIComponent(reference.mediaId)}`,
619
+ {
620
+ uploadedAt: (/* @__PURE__ */ new Date()).toISOString(),
621
+ uploadHttpStatus: status,
622
+ uploadHttpError: error,
623
+ uploadTimeMs: Date.now() - started
624
+ }
625
+ );
626
+ if (status < 200 || status >= 300)
627
+ throw new MediaUploadError(
628
+ `media storage refused the upload (HTTP ${status})`,
629
+ status
630
+ );
631
+ }
632
+ async send(method, route, body) {
633
+ const reply = await this.transport.send(method, route, body);
634
+ if (!reply.ok)
635
+ throw new MediaUploadError(
636
+ `${method} ${route} -> ${reply.status}`,
637
+ reply.status
638
+ );
639
+ return reply.text ? JSON.parse(reply.text) : {};
640
+ }
641
+ };
642
+ }
643
+ });
644
+
645
+ // src/worlds/nativeComputer/trace.ts
646
+ function loadOtelApi() {
647
+ if (otelApi !== void 0) return otelApi;
648
+ try {
649
+ otelApi = __require("@opentelemetry/api");
650
+ } catch {
651
+ otelApi = null;
652
+ }
653
+ return otelApi;
654
+ }
655
+ function errorText(error) {
656
+ return error instanceof Error ? `${error.name}: ${error.message}` : String(error);
657
+ }
658
+ function errorName(error) {
659
+ return error instanceof Error ? error.name : "Error";
660
+ }
661
+ var SPAN_PREFIX, MAX_UPLOAD_FAILURES, TRACER_NAME, MAX_ERROR, otelApi, ActionTracer;
662
+ var init_trace = __esm({
663
+ "src/worlds/nativeComputer/trace.ts"() {
664
+ "use strict";
665
+ init_media();
666
+ SPAN_PREFIX = "computer.";
667
+ MAX_UPLOAD_FAILURES = 3;
668
+ TRACER_NAME = "@withgateway/sdk";
669
+ MAX_ERROR = 1e3;
670
+ ActionTracer = class _ActionTracer {
671
+ constructor(media, warn = (m) => console.warn(m)) {
672
+ this.media = media;
673
+ this.warn = warn;
674
+ }
675
+ media;
676
+ warn;
677
+ failures = 0;
678
+ warned = false;
679
+ static fromEnv() {
680
+ return new _ActionTracer(MediaUploader.fromEnv());
681
+ }
682
+ /** `args` are already redacted. A failure's status goes through `scrub`:
683
+ * browser errors quote the session's ws URL and UI token. */
684
+ async span(action, args, run, capture, scrub) {
685
+ const otel = loadOtelApi();
686
+ if (!otel) return run();
687
+ const { SpanStatusCode, trace } = otel;
688
+ const attributes = {
689
+ "gateway.observation.type": "tool",
690
+ "tool.name": SPAN_PREFIX + action.name,
691
+ "gateway.observation.input": JSON.stringify({
692
+ action: action.name,
693
+ ...args
694
+ })
695
+ };
696
+ return trace.getTracer(TRACER_NAME).startActiveSpan(
697
+ SPAN_PREFIX + action.name,
698
+ { attributes },
699
+ async (span) => {
700
+ try {
701
+ const result = await run();
702
+ await this.finish(span, result, capture);
703
+ span.setStatus({ code: SpanStatusCode.OK });
704
+ return result;
705
+ } catch (error) {
706
+ span.setStatus({
707
+ code: SpanStatusCode.ERROR,
708
+ message: scrub(errorText(error)).slice(0, MAX_ERROR)
709
+ });
710
+ throw error;
711
+ } finally {
712
+ span.end();
713
+ }
714
+ }
715
+ );
716
+ }
717
+ async finish(span, result, capture) {
718
+ if (!span.isRecording()) return;
719
+ const output = {
720
+ screenshot: await this.screenshot(span, result, capture)
721
+ };
722
+ if (result.text !== void 0) output.text = result.text;
723
+ span.setAttribute("gateway.observation.output", JSON.stringify(output));
724
+ }
725
+ async screenshot(span, result, capture) {
726
+ let png;
727
+ try {
728
+ png = result.image ?? await capture();
729
+ } catch (error) {
730
+ return `[screenshot unavailable: ${errorName(error)}]`;
731
+ }
732
+ return this.reference(png, span);
733
+ }
734
+ async reference(png, span) {
735
+ if (!this.media)
736
+ return "[screenshot not uploaded: GATEWAY_HOST / GATEWAY_PUBLIC_KEY / GATEWAY_SECRET_KEY unset]";
737
+ if (this.failures >= MAX_UPLOAD_FAILURES)
738
+ return "[screenshot not uploaded: uploads disabled after repeated failures]";
739
+ const { traceId, spanId } = span.spanContext();
740
+ try {
741
+ const reference = await this.media.upload(png, {
742
+ contentType: "image/png",
743
+ traceId,
744
+ observationId: spanId,
745
+ field: "output"
746
+ });
747
+ this.failures = 0;
748
+ return reference.toString();
749
+ } catch (error) {
750
+ return this.failed(error);
751
+ }
752
+ }
753
+ failed(error) {
754
+ this.failures += 1;
755
+ if (!this.warned) {
756
+ this.warned = true;
757
+ this.warn(
758
+ `native computer: screenshot upload failed (${errorName(error)}: ${error instanceof Error ? error.message : String(error)}); spans keep going without it`
759
+ );
760
+ }
761
+ return `[screenshot not uploaded: ${errorName(error)}]`;
762
+ }
763
+ };
764
+ }
765
+ });
766
+
767
+ // src/worlds/nativeComputer/base.ts
768
+ function readField(item, ...names) {
769
+ const record = item ?? {};
770
+ return names.map((name) => record[name]).find((value) => value !== void 0 && value !== null);
771
+ }
772
+ var REPLAYABLE, SETTLING, EVENT_PREFIX, NativeComputer;
773
+ var init_base = __esm({
774
+ "src/worlds/nativeComputer/base.ts"() {
775
+ "use strict";
776
+ init_browser();
777
+ init_actions();
778
+ init_display();
779
+ init_screen();
780
+ init_trace();
781
+ REPLAYABLE = /* @__PURE__ */ new Set(["screenshot", "zoom", "cursor_position", "wait"]);
782
+ SETTLING = /* @__PURE__ */ new Set(["click", "mouse_up", "drag", "key", "history"]);
783
+ EVENT_PREFIX = "computer_";
784
+ NativeComputer = class _NativeComputer {
785
+ constructor(browser, options = {}) {
786
+ this.browser = browser;
787
+ const kind = this.constructor;
788
+ this.display = Display.forViewport(
789
+ browser.viewport,
790
+ options.display,
791
+ (w, h) => kind.fits(w, h)
792
+ );
793
+ this.screen = new Screen(browser, this.display);
794
+ this.tracer = options.tracer ?? ActionTracer.fromEnv();
795
+ }
796
+ browser;
797
+ static kinds = /* @__PURE__ */ new Map();
798
+ display;
799
+ screen;
800
+ tracer;
801
+ queue = Promise.resolve();
802
+ static register(provider, kind) {
803
+ _NativeComputer.kinds.set(provider, kind);
804
+ }
805
+ static get providers() {
806
+ return [..._NativeComputer.kinds.keys()].sort();
807
+ }
808
+ static forProvider(provider, browser, options = {}) {
809
+ const Kind = _NativeComputer.kinds.get(provider);
810
+ if (!Kind)
811
+ throw new Error(
812
+ `unknown computer-use provider '${provider}'; choose from ${_NativeComputer.providers.join(", ")}`
813
+ );
814
+ return new Kind(browser, options);
815
+ }
816
+ /** Whether a screenshot of this size is within the provider's limits. */
817
+ static fits(_width, _height) {
818
+ return true;
819
+ }
820
+ /** Run one provider-shaped action (an OpenAI action object, or an Anthropic
821
+ * `{ action: "left_click", ... }` input) and return what it produced: the
822
+ * door for any agent framework that hands actions over one at a time. */
823
+ executeAction(raw) {
824
+ return this.perform(this.parse(raw));
825
+ }
826
+ /** Run one action on the page: reported on the session's browser timeline
827
+ * and, when tracing is on, as a span with the screen after. */
828
+ async perform(action) {
829
+ const secret = action.name === "type" && await this.screen.focusedFieldIsSecret();
830
+ const event = EVENT_PREFIX + action.name;
831
+ const args = redactActionArgs(event, action.recordArgs(secret));
832
+ return this.tracer.span(
833
+ action,
834
+ args,
835
+ () => this.browser.act(
836
+ event,
837
+ args,
838
+ () => this.runOnScreen(action),
839
+ this.onLoss(action)
840
+ ),
841
+ () => this.screen.screenshot(),
842
+ (text) => this.browser.scrub(text)
843
+ );
844
+ }
845
+ async performAll(actions) {
846
+ const results = [];
847
+ for (const action of actions) results.push(await this.perform(action));
848
+ return results;
849
+ }
850
+ /** The screen as the model sees it, as a traced `screenshot` action. */
851
+ async screenshot() {
852
+ return (await this.perform(new ComputerAction("screenshot"))).image ?? Buffer.alloc(0);
853
+ }
854
+ /** Run `fn` after every earlier call made through here: the AI SDK starts a
855
+ * step's tool calls concurrently, but one screen takes one action at a time. */
856
+ serial(fn) {
857
+ const next = this.queue.then(fn, fn);
858
+ this.queue = next.catch(() => void 0);
859
+ return next;
860
+ }
861
+ async runOnScreen(action) {
862
+ const result = await this.screen.run(action);
863
+ if (SETTLING.has(action.name)) await this.browser.settle();
864
+ return result;
865
+ }
866
+ onLoss(action) {
867
+ return REPLAYABLE.has(action.name) ? "retry" : "report";
868
+ }
869
+ };
870
+ }
871
+ });
872
+
873
+ // src/worlds/nativeComputer/keys.ts
874
+ function playwrightKey(token) {
875
+ return KEY_NAMES[token.toLowerCase()] ?? token;
876
+ }
877
+ function letterOrName(key) {
878
+ return /^[a-zA-Z]$/.test(key) ? key.toLowerCase() : playwrightKey(key);
879
+ }
880
+ function modifierChord(text) {
881
+ return typeof text === "string" && text ? [...KeyChord.fromXdotool(text).keys] : [];
882
+ }
883
+ function modifierKeys(keys) {
884
+ return Array.isArray(keys) && keys.length > 0 ? [...KeyChord.fromKeys(keys.map(String)).keys] : [];
885
+ }
886
+ var F_KEYS, KEY_NAMES, KeyChord;
887
+ var init_keys = __esm({
888
+ "src/worlds/nativeComputer/keys.ts"() {
889
+ "use strict";
890
+ F_KEYS = Object.fromEntries(
891
+ Array.from({ length: 12 }, (_, i) => [`f${i + 1}`, `F${i + 1}`])
892
+ );
893
+ KEY_NAMES = {
894
+ return: "Enter",
895
+ enter: "Enter",
896
+ kp_enter: "Enter",
897
+ esc: "Escape",
898
+ escape: "Escape",
899
+ tab: "Tab",
900
+ space: "Space",
901
+ backspace: "Backspace",
902
+ delete: "Delete",
903
+ del: "Delete",
904
+ insert: "Insert",
905
+ home: "Home",
906
+ end: "End",
907
+ pageup: "PageUp",
908
+ page_up: "PageUp",
909
+ prior: "PageUp",
910
+ pagedown: "PageDown",
911
+ page_down: "PageDown",
912
+ next: "PageDown",
913
+ up: "ArrowUp",
914
+ arrowup: "ArrowUp",
915
+ down: "ArrowDown",
916
+ arrowdown: "ArrowDown",
917
+ left: "ArrowLeft",
918
+ arrowleft: "ArrowLeft",
919
+ right: "ArrowRight",
920
+ arrowright: "ArrowRight",
921
+ ctrl: "Control",
922
+ control: "Control",
923
+ shift: "Shift",
924
+ alt: "Alt",
925
+ option: "Alt",
926
+ super: "Meta",
927
+ meta: "Meta",
928
+ cmd: "Meta",
929
+ command: "Meta",
930
+ win: "Meta",
931
+ capslock: "CapsLock",
932
+ caps_lock: "CapsLock",
933
+ plus: "+",
934
+ minus: "-",
935
+ ...F_KEYS
936
+ };
937
+ KeyChord = class _KeyChord {
938
+ constructor(keys) {
939
+ this.keys = keys;
940
+ if (keys.length === 0)
941
+ throw new Error("a key action needs at least one key");
942
+ }
943
+ keys;
944
+ /** `ctrl+shift+t` / `Return`: Anthropic's vocabulary, case kept. */
945
+ static fromXdotool(text) {
946
+ return new _KeyChord(
947
+ text.trim().split("+").filter(Boolean).map(playwrightKey)
948
+ );
949
+ }
950
+ /** `["CTRL", "A"]`: OpenAI's vocabulary. A lone letter is the unshifted key. */
951
+ static fromKeys(keys) {
952
+ return new _KeyChord(keys.map(letterOrName));
953
+ }
954
+ /** What `page.keyboard.press` takes: `Control+a`. */
955
+ get combo() {
956
+ return this.keys.join("+");
957
+ }
958
+ };
959
+ }
960
+ });
961
+
962
+ // src/worlds/nativeComputer/anthropic.ts
963
+ function required(raw, key) {
964
+ const value = raw[key];
965
+ if (value === void 0 || value === null)
966
+ throw new Error(`${String(raw.action)} needs ${key}`);
967
+ return value;
968
+ }
969
+ function pair(raw, key) {
970
+ const value = required(raw, key);
971
+ return [toInt(value[0]), toInt(value[1])];
972
+ }
973
+ function point(raw) {
974
+ if (raw.coordinate === void 0 || raw.coordinate === null) return {};
975
+ const [x, y] = pair(raw, "coordinate");
976
+ return { x, y };
977
+ }
978
+ function chord(raw) {
979
+ return [...KeyChord.fromXdotool(String(required(raw, "text"))).keys];
980
+ }
981
+ function click(button, count) {
982
+ return (raw) => new ComputerAction("click", {
983
+ ...point(raw),
984
+ button,
985
+ count,
986
+ modifiers: modifierChord(raw.text)
987
+ });
988
+ }
989
+ var SCROLL_STEP_PX, NOT_EXECUTED, MAX_ERROR2, SCROLL_DIRECTIONS, MEMBERS, AnthropicComputer, AnthropicComputer20251124;
990
+ var init_anthropic = __esm({
991
+ "src/worlds/nativeComputer/anthropic.ts"() {
992
+ "use strict";
993
+ init_actions();
994
+ init_base();
995
+ init_keys();
996
+ SCROLL_STEP_PX = 100;
997
+ NOT_EXECUTED = "Not executed: an earlier computer action in this turn failed.";
998
+ MAX_ERROR2 = 1e3;
999
+ SCROLL_DIRECTIONS = {
1000
+ up: [0, -1],
1001
+ down: [0, 1],
1002
+ left: [-1, 0],
1003
+ right: [1, 0]
1004
+ };
1005
+ MEMBERS = {
1006
+ screenshot: () => new ComputerAction("screenshot"),
1007
+ zoom: (raw) => new ComputerAction("zoom", {
1008
+ region: required(raw, "region").map(toInt)
1009
+ }),
1010
+ left_click: click("left", 1),
1011
+ right_click: click("right", 1),
1012
+ middle_click: click("middle", 1),
1013
+ double_click: click("left", 2),
1014
+ triple_click: click("left", 3),
1015
+ left_click_drag: (raw) => new ComputerAction("drag", {
1016
+ path: [pair(raw, "start_coordinate"), pair(raw, "coordinate")],
1017
+ modifiers: modifierChord(raw.text)
1018
+ }),
1019
+ mouse_move: (raw) => {
1020
+ const [x, y] = pair(raw, "coordinate");
1021
+ return new ComputerAction("move", { x, y });
1022
+ },
1023
+ left_mouse_down: () => new ComputerAction("mouse_down", { button: "left" }),
1024
+ left_mouse_up: () => new ComputerAction("mouse_up", { button: "left" }),
1025
+ cursor_position: () => new ComputerAction("cursor_position"),
1026
+ scroll: (raw) => {
1027
+ const direction = String(raw.scroll_direction ?? "down");
1028
+ if (!Object.hasOwn(SCROLL_DIRECTIONS, direction))
1029
+ throw new Error(
1030
+ `scroll_direction must be one of ${Object.keys(SCROLL_DIRECTIONS).join(", ")}`
1031
+ );
1032
+ const [dx, dy] = SCROLL_DIRECTIONS[direction];
1033
+ const amount = toInt(raw.scroll_amount ?? 1) * SCROLL_STEP_PX;
1034
+ return new ComputerAction("scroll", {
1035
+ ...point(raw),
1036
+ dx: dx * amount,
1037
+ dy: dy * amount,
1038
+ modifiers: modifierChord(raw.text)
1039
+ });
1040
+ },
1041
+ type: (raw) => new ComputerAction("type", { text: String(required(raw, "text")) }),
1042
+ key: (raw) => new ComputerAction("key", {
1043
+ keys: chord(raw),
1044
+ repeat: toInt(raw.repeat ?? 1)
1045
+ }),
1046
+ hold_key: (raw) => new ComputerAction("hold_key", {
1047
+ keys: chord(raw),
1048
+ seconds: toNumber(required(raw, "duration"))
1049
+ }),
1050
+ wait: (raw) => new ComputerAction("wait", { seconds: toNumber(raw.duration ?? 1) })
1051
+ };
1052
+ AnthropicComputer = class _AnthropicComputer extends NativeComputer {
1053
+ static TOOL_TYPE = "computer_toolset_20260801";
1054
+ static TOOLSET = "computer";
1055
+ static BETA = null;
1056
+ static MAX_LONG_EDGE = 2576;
1057
+ static MAX_VISUAL_TOKENS = 4784;
1058
+ static PATCH_PX = 28;
1059
+ static fits(width, height) {
1060
+ const tokens = Math.ceil(width / this.PATCH_PX) * Math.ceil(height / this.PATCH_PX);
1061
+ return Math.max(width, height) <= this.MAX_LONG_EDGE && tokens <= this.MAX_VISUAL_TOKENS;
1062
+ }
1063
+ tool() {
1064
+ return { type: _AnthropicComputer.TOOL_TYPE };
1065
+ }
1066
+ /** Whether a content block is a call to this computer. */
1067
+ owns(block) {
1068
+ return readField(block, "type") === "tool_use" && readField(block, "toolset_name") === _AnthropicComputer.TOOLSET;
1069
+ }
1070
+ member(block) {
1071
+ return [
1072
+ String(readField(block, "name")),
1073
+ { ...readField(block, "input") ?? {} }
1074
+ ];
1075
+ }
1076
+ parse(raw) {
1077
+ const name = String(readField(raw, "action"));
1078
+ const parser = Object.hasOwn(MEMBERS, name) ? MEMBERS[name] : void 0;
1079
+ if (!parser) throw new Error(`unknown computer action: ${name}`);
1080
+ return parser(raw);
1081
+ }
1082
+ /** Run every computer `tool_use` in a response's content, in order, and
1083
+ * return one `tool_result` per block. Other blocks are skipped. */
1084
+ async execute(content) {
1085
+ const results = [];
1086
+ let failed = false;
1087
+ for (const block of content) {
1088
+ if (!this.owns(block)) continue;
1089
+ if (failed) {
1090
+ results.push(this.envelope(block, NOT_EXECUTED, true));
1091
+ continue;
1092
+ }
1093
+ const [result, blockFailed] = await this.runBlock(block);
1094
+ results.push(result);
1095
+ failed = blockFailed;
1096
+ }
1097
+ return results;
1098
+ }
1099
+ /** The content an `ActionResult` goes back to the model as. */
1100
+ static content(result) {
1101
+ if (result.image)
1102
+ return [
1103
+ {
1104
+ type: "image",
1105
+ source: {
1106
+ type: "base64",
1107
+ media_type: "image/png",
1108
+ data: result.image.toString("base64")
1109
+ }
1110
+ }
1111
+ ];
1112
+ return [{ type: "text", text: result.text ?? "OK" }];
1113
+ }
1114
+ /** For `anthropic.tools.computerToolset_20260801({ ...computer.aiSdk() })`
1115
+ * (or `computer_20251124` with the legacy kind): `execute` runs one action
1116
+ * and returns `{ type: "image", data }` or text; `toModelOutput` turns that
1117
+ * into the tool result. Calls run one at a time, and once a call fails the
1118
+ * rest of that step's calls answer as not executed. */
1119
+ aiSdk() {
1120
+ let failedStep;
1121
+ const execute = (input, options) => this.serial(async () => {
1122
+ const step = options?.messages?.length;
1123
+ if (step !== void 0 && step === failedStep)
1124
+ throw new Error(NOT_EXECUTED);
1125
+ try {
1126
+ return _AnthropicComputer.aiSdkOutput(await this.executeAction(input));
1127
+ } catch (error) {
1128
+ failedStep = step;
1129
+ throw error instanceof Error ? new Error(this.browser.scrub(error.message)) : error;
1130
+ }
1131
+ });
1132
+ return {
1133
+ execute,
1134
+ toModelOutput: ({ output }) => _AnthropicComputer.aiSdkModelOutput(output)
1135
+ };
1136
+ }
1137
+ static aiSdkOutput(result) {
1138
+ if (result.image)
1139
+ return { type: "image", data: result.image.toString("base64") };
1140
+ return result.text ?? "OK";
1141
+ }
1142
+ /** `image-data` is what AI SDK 6's Anthropic provider turns into an image
1143
+ * block; AI SDK 7 still accepts it (and converts it to a `file` part). */
1144
+ static aiSdkModelOutput(output) {
1145
+ if (typeof output === "string") return { type: "text", value: output };
1146
+ return {
1147
+ type: "content",
1148
+ value: [
1149
+ { type: "image-data", data: output.data, mediaType: "image/png" }
1150
+ ]
1151
+ };
1152
+ }
1153
+ envelope(block, content, isError = false) {
1154
+ return {
1155
+ type: "tool_result",
1156
+ tool_use_id: String(readField(block, "id")),
1157
+ toolset_name: _AnthropicComputer.TOOLSET,
1158
+ content,
1159
+ ...isError ? { is_error: true } : {}
1160
+ };
1161
+ }
1162
+ async runBlock(block) {
1163
+ const [name, payload] = this.member(block);
1164
+ try {
1165
+ const result = await this.executeAction({ ...payload, action: name });
1166
+ return [this.envelope(block, _AnthropicComputer.content(result)), false];
1167
+ } catch (error) {
1168
+ const detail = error instanceof Error ? `${error.name}: ${error.message}` : String(error);
1169
+ return [
1170
+ this.envelope(
1171
+ block,
1172
+ this.browser.scrub(`Error: ${detail}`).slice(0, MAX_ERROR2),
1173
+ true
1174
+ ),
1175
+ true
1176
+ ];
1177
+ }
1178
+ }
1179
+ };
1180
+ AnthropicComputer20251124 = class _AnthropicComputer20251124 extends AnthropicComputer {
1181
+ static TOOL_TYPE = "computer_20251124";
1182
+ static BETA = "computer-use-2025-11-24";
1183
+ static MAX_LONG_EDGE = 1568;
1184
+ static MAX_PIXELS = 115e4;
1185
+ static fits(width, height) {
1186
+ return Math.max(width, height) <= this.MAX_LONG_EDGE && width * height <= this.MAX_PIXELS;
1187
+ }
1188
+ tool() {
1189
+ return {
1190
+ type: _AnthropicComputer20251124.TOOL_TYPE,
1191
+ name: AnthropicComputer.TOOLSET,
1192
+ display_width_px: this.display.width,
1193
+ display_height_px: this.display.height,
1194
+ enable_zoom: true
1195
+ };
1196
+ }
1197
+ owns(block) {
1198
+ return readField(block, "type") === "tool_use" && readField(block, "name") === AnthropicComputer.TOOLSET;
1199
+ }
1200
+ member(block) {
1201
+ const { action, ...payload } = readField(block, "input") ?? {};
1202
+ return [String(action ?? ""), payload];
1203
+ }
1204
+ envelope(block, content, isError = false) {
1205
+ const { toolset_name: _dropped, ...rest } = super.envelope(
1206
+ block,
1207
+ content,
1208
+ isError
1209
+ );
1210
+ return rest;
1211
+ }
1212
+ };
1213
+ NativeComputer.register("anthropic", AnthropicComputer);
1214
+ NativeComputer.register(
1215
+ "anthropic-computer-20251124",
1216
+ AnthropicComputer20251124
1217
+ );
1218
+ }
1219
+ });
1220
+
1221
+ // src/worlds/nativeComputer/openai.ts
1222
+ function safetyCheck(raw) {
1223
+ const check = { id: String(readField(raw, "id") ?? "") };
1224
+ const code = readField(raw, "code");
1225
+ const message = readField(raw, "message");
1226
+ if (code !== void 0) check.code = String(code);
1227
+ if (message !== void 0) check.message = String(message);
1228
+ return check;
1229
+ }
1230
+ function dataUrl(png) {
1231
+ return `data:image/png;base64,${png.toString("base64")}`;
1232
+ }
1233
+ function xy(raw) {
1234
+ return {
1235
+ x: toInt(readField(raw, "x") ?? 0),
1236
+ y: toInt(readField(raw, "y") ?? 0)
1237
+ };
1238
+ }
1239
+ function held(raw) {
1240
+ return modifierKeys(readField(raw, "keys"));
1241
+ }
1242
+ var WAIT_SECONDS, BUTTONS, HISTORY_BUTTONS, PendingSafetyChecks, ACTIONS, OpenAIComputer, OpenAIPreviewComputer;
1243
+ var init_openai = __esm({
1244
+ "src/worlds/nativeComputer/openai.ts"() {
1245
+ "use strict";
1246
+ init_actions();
1247
+ init_base();
1248
+ init_keys();
1249
+ WAIT_SECONDS = 1;
1250
+ BUTTONS = {
1251
+ left: "left",
1252
+ right: "right",
1253
+ wheel: "middle",
1254
+ middle: "middle"
1255
+ };
1256
+ HISTORY_BUTTONS = /* @__PURE__ */ new Set(["back", "forward"]);
1257
+ PendingSafetyChecks = class extends Error {
1258
+ constructor(callId, checks) {
1259
+ const listed = checks.map((c) => `${c.code ?? "check"}: ${c.message ?? ""}`.trim()).join("; ");
1260
+ super(
1261
+ `computer_call ${callId} needs ${checks.length} safety check(s) acknowledged: ${listed}`
1262
+ );
1263
+ this.callId = callId;
1264
+ this.checks = checks;
1265
+ this.name = "PendingSafetyChecks";
1266
+ }
1267
+ callId;
1268
+ checks;
1269
+ };
1270
+ ACTIONS = {
1271
+ click: (raw) => {
1272
+ const button = String(readField(raw, "button") ?? "left");
1273
+ if (HISTORY_BUTTONS.has(button))
1274
+ return new ComputerAction("history", {
1275
+ direction: button
1276
+ });
1277
+ if (!Object.hasOwn(BUTTONS, button))
1278
+ throw new Error(`unknown mouse button: ${button}`);
1279
+ return new ComputerAction("click", {
1280
+ ...xy(raw),
1281
+ button: BUTTONS[button],
1282
+ count: 1,
1283
+ modifiers: held(raw)
1284
+ });
1285
+ },
1286
+ double_click: (raw) => new ComputerAction("click", {
1287
+ ...xy(raw),
1288
+ button: "left",
1289
+ count: 2,
1290
+ modifiers: held(raw)
1291
+ }),
1292
+ drag: (raw) => new ComputerAction("drag", {
1293
+ path: (readField(raw, "path") ?? []).map(
1294
+ (p) => [toInt(readField(p, "x")), toInt(readField(p, "y"))]
1295
+ ),
1296
+ modifiers: held(raw)
1297
+ }),
1298
+ keypress: (raw) => new ComputerAction("key", {
1299
+ keys: [
1300
+ ...KeyChord.fromKeys(readField(raw, "keys") ?? []).keys
1301
+ ],
1302
+ repeat: 1
1303
+ }),
1304
+ move: (raw) => new ComputerAction("move", { ...xy(raw), modifiers: held(raw) }),
1305
+ scroll: (raw) => new ComputerAction("scroll", {
1306
+ ...xy(raw),
1307
+ dx: toInt(readField(raw, "scroll_x", "scrollX") ?? 0),
1308
+ dy: toInt(readField(raw, "scroll_y", "scrollY") ?? 0),
1309
+ modifiers: held(raw)
1310
+ }),
1311
+ type: (raw) => new ComputerAction("type", { text: String(readField(raw, "text") ?? "") }),
1312
+ wait: () => new ComputerAction("wait", { seconds: WAIT_SECONDS }),
1313
+ screenshot: () => new ComputerAction("screenshot")
1314
+ };
1315
+ OpenAIComputer = class _OpenAIComputer extends NativeComputer {
1316
+ tool() {
1317
+ return { type: "computer" };
1318
+ }
1319
+ owns(item) {
1320
+ return readField(item, "type") === "computer_call";
1321
+ }
1322
+ parse(raw) {
1323
+ const kind = String(readField(raw, "type"));
1324
+ const parser = Object.hasOwn(ACTIONS, kind) ? ACTIONS[kind] : void 0;
1325
+ if (!parser) throw new Error(`unknown computer action: ${kind}`);
1326
+ return parser(raw);
1327
+ }
1328
+ static actionsOf(call) {
1329
+ const batch = readField(call, "actions");
1330
+ if (Array.isArray(batch)) return batch;
1331
+ const single = readField(call, "action");
1332
+ return single === void 0 ? [] : [single];
1333
+ }
1334
+ /** Run a `computer_call`'s actions and return its `computer_call_output`.
1335
+ * Throws `PendingSafetyChecks` (before anything runs) while a check the
1336
+ * model raised is not in `acknowledged`. */
1337
+ async execute(call, acknowledged = []) {
1338
+ const callId = String(readField(call, "call_id", "callId"));
1339
+ const checks = (readField(
1340
+ call,
1341
+ "pending_safety_checks",
1342
+ "pendingSafetyChecks"
1343
+ ) ?? []).map(safetyCheck);
1344
+ const png = await this.runCall(
1345
+ callId,
1346
+ _OpenAIComputer.actionsOf(call),
1347
+ checks,
1348
+ acknowledged
1349
+ );
1350
+ return {
1351
+ type: "computer_call_output",
1352
+ call_id: callId,
1353
+ output: {
1354
+ type: "computer_screenshot",
1355
+ image_url: dataUrl(png),
1356
+ detail: "original"
1357
+ },
1358
+ ...checks.length > 0 ? { acknowledged_safety_checks: checks } : {}
1359
+ };
1360
+ }
1361
+ /** For `openai.tools.computer({ ...computer.aiSdk() })`: `execute` runs a
1362
+ * call's `actions` and returns the screenshot output. A call with pending
1363
+ * safety checks asks for approval first (`needsApproval`), so the AI SDK's
1364
+ * approval flow puts them in front of a person; once approved, `execute`
1365
+ * acknowledges them. */
1366
+ aiSdk() {
1367
+ return {
1368
+ needsApproval: (input) => (input.pendingSafetyChecks ?? []).length > 0,
1369
+ execute: (input) => this.serial(async () => {
1370
+ const checks = (input.pendingSafetyChecks ?? []).map(safetyCheck);
1371
+ const png = await this.runCall(
1372
+ "ai-sdk",
1373
+ input.actions ?? [],
1374
+ checks,
1375
+ checks
1376
+ );
1377
+ return {
1378
+ output: { type: "computer_screenshot", imageUrl: dataUrl(png) },
1379
+ ...checks.length > 0 ? { acknowledgedSafetyChecks: checks } : {}
1380
+ };
1381
+ })
1382
+ };
1383
+ }
1384
+ /** The batch's actions, run after the safety checks clear; the screen after them. */
1385
+ async runCall(callId, rawActions, checks, acknowledged) {
1386
+ const cleared = new Set(acknowledged.map((c) => safetyCheck(c).id));
1387
+ const waiting = checks.filter((c) => !c.id || !cleared.has(c.id));
1388
+ if (waiting.length > 0) throw new PendingSafetyChecks(callId, waiting);
1389
+ const actions = rawActions.map((raw) => this.parse(raw));
1390
+ const results = await this.performAll(actions);
1391
+ const last = actions.at(-1);
1392
+ if (last?.name === "screenshot")
1393
+ return results.at(-1)?.image ?? Buffer.alloc(0);
1394
+ return this.screenshot();
1395
+ }
1396
+ };
1397
+ OpenAIPreviewComputer = class extends OpenAIComputer {
1398
+ tool() {
1399
+ return {
1400
+ type: "computer_use_preview",
1401
+ display_width: this.display.width,
1402
+ display_height: this.display.height,
1403
+ environment: "browser"
1404
+ };
1405
+ }
1406
+ };
1407
+ NativeComputer.register("openai", OpenAIComputer);
1408
+ NativeComputer.register("openai-preview", OpenAIPreviewComputer);
1409
+ }
1410
+ });
1411
+
1412
+ // src/worlds/nativeComputer/index.ts
1413
+ var init_nativeComputer = __esm({
1414
+ "src/worlds/nativeComputer/index.ts"() {
1415
+ "use strict";
1416
+ init_actions();
1417
+ init_base();
1418
+ init_display();
1419
+ init_keys();
1420
+ init_screen();
1421
+ init_trace();
1422
+ init_anthropic();
1423
+ init_openai();
135
1424
  }
136
1425
  });
137
1426
 
@@ -147,6 +1436,23 @@ var init_sessionContract = __esm({
147
1436
  }
148
1437
  });
149
1438
 
1439
+ // src/tracing/session.ts
1440
+ import { AsyncLocalStorage } from "async_hooks";
1441
+ var processStorage, currentSessionStorage, currentWorldSessionStorage;
1442
+ var init_session = __esm({
1443
+ "src/tracing/session.ts"() {
1444
+ "use strict";
1445
+ processStorage = (name) => {
1446
+ const key = /* @__PURE__ */ Symbol.for(`@withgateway/sdk/${name}`);
1447
+ const slots = globalThis;
1448
+ slots[key] ??= new AsyncLocalStorage();
1449
+ return slots[key];
1450
+ };
1451
+ currentSessionStorage = processStorage("session-storage");
1452
+ currentWorldSessionStorage = processStorage("world-session-storage");
1453
+ }
1454
+ });
1455
+
150
1456
  // src/worlds/taskRef.ts
151
1457
  var init_taskRef = __esm({
152
1458
  "src/worlds/taskRef.ts"() {
@@ -157,12 +1463,14 @@ var init_taskRef = __esm({
157
1463
  // src/worlds/session.ts
158
1464
  import { randomUUID } from "crypto";
159
1465
  var READY_TIMEOUT_MS, CALL_TIMEOUT_MS, JUDGED_GRADE_TIMEOUT_MS;
160
- var init_session = __esm({
1466
+ var init_session2 = __esm({
161
1467
  "src/worlds/session.ts"() {
162
1468
  "use strict";
163
1469
  init_worlds();
164
1470
  init_browser();
1471
+ init_nativeComputer();
165
1472
  init_sessionContract();
1473
+ init_session();
166
1474
  init_taskRef();
167
1475
  init_taskRef();
168
1476
  READY_TIMEOUT_MS = 10 * 6e4;
@@ -187,13 +1495,22 @@ var init_tasks = __esm({
187
1495
  }
188
1496
  });
189
1497
 
1498
+ // src/worlds/surfaceDescriptor.ts
1499
+ var init_surfaceDescriptor = __esm({
1500
+ "src/worlds/surfaceDescriptor.ts"() {
1501
+ "use strict";
1502
+ init_browser();
1503
+ }
1504
+ });
1505
+
190
1506
  // src/worlds/taskSessions.ts
191
1507
  var init_taskSessions = __esm({
192
1508
  "src/worlds/taskSessions.ts"() {
193
1509
  "use strict";
194
1510
  init_worlds();
195
1511
  init_browser();
196
- init_session();
1512
+ init_surfaceDescriptor();
1513
+ init_session2();
197
1514
  }
198
1515
  });
199
1516
 
@@ -205,6 +1522,21 @@ var init_evals = __esm({
205
1522
  }
206
1523
  });
207
1524
 
1525
+ // src/tracing/prerequisites.ts
1526
+ var init_prerequisites = __esm({
1527
+ "src/tracing/prerequisites.ts"() {
1528
+ "use strict";
1529
+ }
1530
+ });
1531
+
1532
+ // src/worlds/taskTracing.ts
1533
+ var init_taskTracing = __esm({
1534
+ "src/worlds/taskTracing.ts"() {
1535
+ "use strict";
1536
+ init_prerequisites();
1537
+ }
1538
+ });
1539
+
208
1540
  // src/worlds/taskRun.ts
209
1541
  import { randomBytes } from "crypto";
210
1542
  import * as fs from "fs";
@@ -214,8 +1546,9 @@ var init_taskRun = __esm({
214
1546
  "use strict";
215
1547
  init_browser();
216
1548
  init_evals();
217
- init_session();
1549
+ init_session2();
218
1550
  init_taskSessions();
1551
+ init_taskTracing();
219
1552
  }
220
1553
  });
221
1554
 
@@ -237,6 +1570,13 @@ var init_setRun = __esm({
237
1570
  }
238
1571
  });
239
1572
 
1573
+ // src/worlds/requestLog.ts
1574
+ var init_requestLog = __esm({
1575
+ "src/worlds/requestLog.ts"() {
1576
+ "use strict";
1577
+ }
1578
+ });
1579
+
240
1580
  // src/worlds/checkoutTarget.ts
241
1581
  import * as fs2 from "fs";
242
1582
  import * as path2 from "path";
@@ -247,7 +1587,7 @@ var init_checkoutTarget = __esm({
247
1587
  });
248
1588
 
249
1589
  // src/worlds/worldDir.ts
250
- import { createHash } from "crypto";
1590
+ import { createHash as createHash2 } from "crypto";
251
1591
  import * as fs3 from "fs";
252
1592
  import * as path3 from "path";
253
1593
  var CONNECTOR_SPEC, WORLD_SCHEMA_MARKERS, WORLD_ROOT_MARKERS;
@@ -271,7 +1611,7 @@ var init_tar = __esm({
271
1611
  });
272
1612
 
273
1613
  // src/worlds/hub.ts
274
- import { createHash as createHash2 } from "crypto";
1614
+ import { createHash as createHash3 } from "crypto";
275
1615
  import { gzipSync } from "zlib";
276
1616
  import * as fs5 from "fs";
277
1617
  import * as path5 from "path";
@@ -321,7 +1661,7 @@ var init_json = __esm({
321
1661
  });
322
1662
 
323
1663
  // src/worlds/transforms.ts
324
- import { createHash as createHash3 } from "crypto";
1664
+ import { createHash as createHash4 } from "crypto";
325
1665
  var WHITESPACE, WHITESPACE_EDGES, WHITESPACE_RUN, NOT_DIGIT, STRIP_PREFIX, trim, emailLocalPart, emailDomain, hex, SIMPLE, TRANSFORMS;
326
1666
  var init_transforms = __esm({
327
1667
  "src/worlds/transforms.ts"() {
@@ -341,7 +1681,7 @@ var init_transforms = __esm({
341
1681
  const at = value.lastIndexOf("@");
342
1682
  return at === -1 ? "" : value.slice(at + 1).toLowerCase();
343
1683
  };
344
- hex = (algorithm, value) => createHash3(algorithm).update(value, "utf8").digest("hex");
1684
+ hex = (algorithm, value) => createHash4(algorithm).update(value, "utf8").digest("hex");
345
1685
  SIMPLE = {
346
1686
  lower: (value) => value.toLowerCase(),
347
1687
  upper: (value) => value.toUpperCase(),
@@ -361,7 +1701,7 @@ var init_transforms = __esm({
361
1701
  });
362
1702
 
363
1703
  // src/worlds/redaction.ts
364
- import { createHash as createHash4 } from "crypto";
1704
+ import { createHash as createHash5 } from "crypto";
365
1705
  var init_redaction = __esm({
366
1706
  "src/worlds/redaction.ts"() {
367
1707
  "use strict";
@@ -455,7 +1795,7 @@ var init_worlds = __esm({
455
1795
  "use strict";
456
1796
  init_transport();
457
1797
  init_transport();
458
- init_session();
1798
+ init_session2();
459
1799
  init_sessionList();
460
1800
  init_sessionContract();
461
1801
  init_taskRef();
@@ -466,7 +1806,10 @@ var init_worlds = __esm({
466
1806
  init_checkoutProvenance();
467
1807
  init_evals();
468
1808
  init_browser();
1809
+ init_surfaceDescriptor();
1810
+ init_requestLog();
469
1811
  init_browserReconnect();
1812
+ init_nativeComputer();
470
1813
  init_hub();
471
1814
  init_validate();
472
1815
  init_describe();
@@ -496,7 +1839,7 @@ var init_worlds = __esm({
496
1839
 
497
1840
  // src/files/index.ts
498
1841
  init_worlds();
499
- import { createHash as createHash5 } from "crypto";
1842
+ import { createHash as createHash6 } from "crypto";
500
1843
  import * as fs7 from "fs";
501
1844
  import * as path7 from "path";
502
1845
  import { Readable, Transform } from "stream";
@@ -561,7 +1904,7 @@ async function confirmUpload(options = {}, fileId, ok = true) {
561
1904
  return apiRequest(options, "POST", `${BASE}/confirm`, { fileId, ok });
562
1905
  }
563
1906
  async function sha256Base64(localPath) {
564
- const hash = createHash5("sha256");
1907
+ const hash = createHash6("sha256");
565
1908
  for await (const chunk of fs7.createReadStream(localPath)) hash.update(chunk);
566
1909
  return hash.digest("base64");
567
1910
  }
@@ -646,7 +1989,7 @@ async function uploadBytes(options = {}, bytes, targetPath, upload = {}) {
646
1989
  if (!bytes.length) throw new Error("nothing to upload: the content is empty");
647
1990
  const target = drivePath(targetPath);
648
1991
  const contentType = upload.contentType ?? contentTypeFor(target);
649
- const sha256 = createHash5("sha256").update(bytes).digest("base64");
1992
+ const sha256 = createHash6("sha256").update(bytes).digest("base64");
650
1993
  const granted = await requestUpload(options, { path: target, contentLength: bytes.length, contentType, sha256, ...upload.overwrite ? { overwrite: true } : {} });
651
1994
  try {
652
1995
  await putPresigned(granted.uploadUrl, new Uint8Array(bytes), { contentType, contentLength: bytes.length, sha256 });