@withgateway/sdk 0.2.9 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/README.md +5 -1
  2. package/dist/cli.js +3812 -1158
  3. package/dist/cli.js.map +1 -1
  4. package/dist/cli.mjs +3796 -1142
  5. package/dist/cli.mjs.map +1 -1
  6. package/dist/files/index.d.mts +1 -0
  7. package/dist/files/index.d.ts +1 -0
  8. package/dist/files/index.js +1358 -21
  9. package/dist/files/index.js.map +1 -1
  10. package/dist/files/index.mjs +1356 -13
  11. package/dist/files/index.mjs.map +1 -1
  12. package/dist/{index-BO2SNSgy.d.ts → index-CBqbB_0J.d.ts} +95 -46
  13. package/dist/{index-Cz8Q6Keo.d.mts → index-D-AMnAMa.d.mts} +95 -46
  14. package/dist/index.d.mts +4 -3
  15. package/dist/index.d.ts +4 -3
  16. package/dist/index.js +1942 -177
  17. package/dist/index.js.map +1 -1
  18. package/dist/index.mjs +1890 -146
  19. package/dist/index.mjs.map +1 -1
  20. package/dist/media-CbcAfQvd.d.mts +83 -0
  21. package/dist/media-CbcAfQvd.d.ts +83 -0
  22. package/dist/traces/index.d.mts +1 -0
  23. package/dist/traces/index.d.ts +1 -0
  24. package/dist/traces/index.js +1355 -18
  25. package/dist/traces/index.js.map +1 -1
  26. package/dist/traces/index.mjs +1353 -10
  27. package/dist/traces/index.mjs.map +1 -1
  28. package/dist/tracing/index.d.mts +2 -1
  29. package/dist/tracing/index.d.ts +2 -1
  30. package/dist/tracing/index.js +550 -30
  31. package/dist/tracing/index.js.map +1 -1
  32. package/dist/tracing/index.mjs +539 -29
  33. package/dist/tracing/index.mjs.map +1 -1
  34. package/dist/worlds/index.d.mts +648 -80
  35. package/dist/worlds/index.d.ts +648 -80
  36. package/dist/worlds/index.js +1875 -145
  37. package/dist/worlds/index.js.map +1 -1
  38. package/dist/worlds/index.mjs +1861 -131
  39. package/dist/worlds/index.mjs.map +1 -1
  40. package/package.json +1 -1
  41. package/runtime/gatewaysdk/environment/schema/api.py +43 -62
  42. package/runtime/gatewaysdk/environment/schema/cast.py +31 -4
  43. package/runtime/gatewaysdk/environment/schema/grading.py +1 -5
  44. package/runtime/gatewaysdk/environment/schema/host.py +53 -90
  45. package/runtime/gatewaysdk/environment/schema/local.py +15 -7
  46. package/runtime/gatewaysdk/environment/schema/platform.py +1 -7
  47. package/runtime/gatewaysdk/environment/schema/runtime.py +9 -9
  48. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +3 -3
  49. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +2 -2
  50. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +13 -13
  51. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +2 -1
  52. package/runtime/gatewaysdk/environment/schema/tasks.py +25 -1
  53. package/skills/connector-schema-authoring/SKILL.md +3 -3
  54. package/skills/connector-schema-authoring/connector-worlds.md +12 -5
  55. package/skills/connector-schema-authoring/world-schema.md +8 -0
  56. package/skills/world-data-ingestion/SKILL.md +2 -2
  57. package/skills/worlds-getting-started/SKILL.md +13 -13
  58. package/skills/worlds-getting-started/references/cli-mcp-map.md +2 -1
@@ -152,11 +152,1294 @@ var init_browserReconnect = __esm({
152
152
  });
153
153
 
154
154
  // src/worlds/browser.ts
155
+ function redactActionArgs(action, args) {
156
+ const secretField = action === "browser_type" && SECRET_KEY.test(String(args.selector ?? ""));
157
+ const out = {};
158
+ for (const [key, value] of Object.entries(args)) {
159
+ if (value === null || value === void 0) out[key] = null;
160
+ else if (typeof value === "boolean" || typeof value === "number")
161
+ out[key] = value;
162
+ else if (typeof value === "string") {
163
+ if (SECRET_KEY.test(key) || SECRET_VALUE.test(value) || secretField && key === "text")
164
+ out[key] = "[redacted]";
165
+ else
166
+ out[key] = value.length <= EVENT_MAX_STRING ? value : value.slice(0, EVENT_MAX_STRING) + "\u2026";
167
+ } else out[key] = "[omitted]";
168
+ }
169
+ return out;
170
+ }
171
+ var EVENT_MAX_STRING, SECRET_KEY, SECRET_VALUE;
155
172
  var init_browser = __esm({
156
173
  "src/worlds/browser.ts"() {
157
174
  "use strict";
158
- init_session();
175
+ init_session2();
159
176
  init_browserReconnect();
177
+ EVENT_MAX_STRING = 500;
178
+ SECRET_KEY = /pass(word|wd)?|secret|token|api[-_]?key|authorization|cookie|(^|[-_ #.\["'=])(otp|pin|cvv|ssn)([-_ \]"']|$)/i;
179
+ SECRET_VALUE = /^((sk|pk|ghp|xox[abp])[-_][-_A-Za-z0-9]{8,}|AKIA[0-9A-Z]{12,}|Bearer [-_.A-Za-z0-9]{8,})/;
180
+ }
181
+ });
182
+
183
+ // src/worlds/nativeComputer/actions.ts
184
+ function toNumber(value) {
185
+ if (typeof value !== "number" || !Number.isFinite(value))
186
+ throw new Error(`not a number: ${JSON.stringify(value) ?? String(value)}`);
187
+ return value;
188
+ }
189
+ function toInt(value) {
190
+ return Math.trunc(toNumber(value));
191
+ }
192
+ var ACTION_NAMES, SECRET_TEXT, BOUNDS, ComputerAction;
193
+ var init_actions = __esm({
194
+ "src/worlds/nativeComputer/actions.ts"() {
195
+ "use strict";
196
+ ACTION_NAMES = [
197
+ "screenshot",
198
+ "zoom",
199
+ "click",
200
+ "move",
201
+ "mouse_down",
202
+ "mouse_up",
203
+ "drag",
204
+ "scroll",
205
+ "type",
206
+ "key",
207
+ "hold_key",
208
+ "wait",
209
+ "cursor_position",
210
+ "history"
211
+ ];
212
+ SECRET_TEXT = "[redacted]";
213
+ BOUNDS = {
214
+ seconds: [0, 300],
215
+ repeat: [1, 100],
216
+ count: [1, 3],
217
+ dx: [-1e4, 1e4],
218
+ dy: [-1e4, 1e4]
219
+ };
220
+ ComputerAction = class {
221
+ constructor(name, args = {}) {
222
+ this.name = name;
223
+ this.args = args;
224
+ if (!ACTION_NAMES.includes(name))
225
+ throw new Error(`unknown computer action: ${name}`);
226
+ for (const [key, [low, high]] of Object.entries(BOUNDS)) {
227
+ const value = args[key];
228
+ if (value !== void 0 && !(Number(value) >= low && Number(value) <= high))
229
+ throw new Error(
230
+ `${name} ${key} must be between ${low} and ${high}, got ${String(value)}`
231
+ );
232
+ }
233
+ }
234
+ name;
235
+ args;
236
+ /** Flat arguments for the browser timeline and the trace: lists become
237
+ * `"x,y"` strings, and typed text is withheld when it went into a
238
+ * credential field. */
239
+ recordArgs(secretText) {
240
+ const out = {};
241
+ for (const [key, value] of Object.entries(this.args))
242
+ out[key] = Array.isArray(value) ? value.flat().join(",") : value;
243
+ if (secretText && "text" in out) out.text = SECRET_TEXT;
244
+ return out;
245
+ }
246
+ };
247
+ }
248
+ });
249
+
250
+ // src/worlds/nativeComputer/display.ts
251
+ var FIT_STEP, Display;
252
+ var init_display = __esm({
253
+ "src/worlds/nativeComputer/display.ts"() {
254
+ "use strict";
255
+ FIT_STEP = 0.98;
256
+ Display = class _Display {
257
+ constructor(width, height, viewportWidth, viewportHeight) {
258
+ this.width = width;
259
+ this.height = height;
260
+ this.viewportWidth = viewportWidth;
261
+ this.viewportHeight = viewportHeight;
262
+ }
263
+ width;
264
+ height;
265
+ viewportWidth;
266
+ viewportHeight;
267
+ /** `size` when the caller names one; otherwise the largest downscale of the
268
+ * viewport (at most 1:1) that `fits` the provider's limits. */
269
+ static forViewport(viewport, size, fits = () => true) {
270
+ const { width, height } = viewport;
271
+ if (size) return _Display.sized(size[0], size[1], width, height);
272
+ let scale = 1;
273
+ while (!fits(Math.round(width * scale), Math.round(height * scale)))
274
+ scale *= FIT_STEP;
275
+ return new _Display(
276
+ Math.round(width * scale),
277
+ Math.round(height * scale),
278
+ width,
279
+ height
280
+ );
281
+ }
282
+ /** A caller's display keeps the viewport's shape: screenshots are scaled
283
+ * uniformly, so another aspect ratio would map clicks off the image. */
284
+ static sized(width, height, viewportWidth, viewportHeight) {
285
+ if (width <= 0 || height <= 0)
286
+ throw new Error(`display ${width}x${height} must be positive`);
287
+ const sameShape = width * viewportHeight / viewportWidth;
288
+ if (Math.abs(sameShape - height) > 1)
289
+ throw new Error(
290
+ `display ${width}x${height} must keep the viewport's aspect ratio (${viewportWidth}x${viewportHeight}); try ${width}x${Math.round(sameShape)}`
291
+ );
292
+ return new _Display(width, height, viewportWidth, viewportHeight);
293
+ }
294
+ get scaled() {
295
+ return this.width !== this.viewportWidth || this.height !== this.viewportHeight;
296
+ }
297
+ get scale() {
298
+ return Math.min(
299
+ this.width / this.viewportWidth,
300
+ this.height / this.viewportHeight
301
+ );
302
+ }
303
+ toPage(x, y) {
304
+ return [
305
+ x * this.viewportWidth / this.width,
306
+ y * this.viewportHeight / this.height
307
+ ];
308
+ }
309
+ };
310
+ }
311
+ });
312
+
313
+ // src/worlds/nativeComputer/screen.ts
314
+ function pngWidth(png) {
315
+ if (png.length < 24 || !png.subarray(0, 8).equals(PNG_SIGNATURE)) return null;
316
+ return png.readUInt32BE(16);
317
+ }
318
+ function isNativePage(page) {
319
+ const candidate = page;
320
+ const mouse = candidate?.mouse;
321
+ return typeof candidate?.context === "function" && NATIVE_METHODS.every((m) => typeof mouse?.[m] === "function");
322
+ }
323
+ var FOCUSED_FIELD_JS, SECRET_FOCUS, DRAG_STEPS, PNG_SIGNATURE, NATIVE_METHODS, Screen;
324
+ var init_screen = __esm({
325
+ "src/worlds/nativeComputer/screen.ts"() {
326
+ "use strict";
327
+ init_browser();
328
+ FOCUSED_FIELD_JS = "(() => { let e = document.activeElement; const inner = (el) => (el.shadowRoot && el.shadowRoot.activeElement) || (el.contentDocument && el.contentDocument.activeElement); while (e && inner(e)) e = inner(e); if (!e) return ''; if (e.tagName === 'IFRAME' || e.tagName === 'FRAME') return 'frame'; if (!(e.tagName === 'INPUT' || e.tagName === 'TEXTAREA' || e.isContentEditable)) return 'opaque'; return [e.type, e.name, e.id, e.getAttribute('autocomplete')].filter(Boolean).join(' '); })()";
329
+ SECRET_FOCUS = ["password", "frame", "opaque"];
330
+ DRAG_STEPS = 5;
331
+ PNG_SIGNATURE = Buffer.from([
332
+ 137,
333
+ 80,
334
+ 78,
335
+ 71,
336
+ 13,
337
+ 10,
338
+ 26,
339
+ 10
340
+ ]);
341
+ NATIVE_METHODS = ["move", "click", "down", "up", "wheel"];
342
+ Screen = class {
343
+ constructor(browser, display) {
344
+ this.browser = browser;
345
+ this.display = display;
346
+ }
347
+ browser;
348
+ display;
349
+ cursor = [0, 0];
350
+ pixelRatio = 1;
351
+ handlers = {
352
+ screenshot: async () => ({ image: await this.screenshot() }),
353
+ zoom: (a) => this.zoom(a),
354
+ click: (a) => this.click(a),
355
+ move: (a) => this.held(a.args.modifiers ?? [], () => this.moveTo(this.point(a))),
356
+ mouse_down: async (a) => {
357
+ await this.page.mouse.down({ button: a.args.button ?? "left" });
358
+ return {};
359
+ },
360
+ mouse_up: async (a) => {
361
+ await this.page.mouse.up({ button: a.args.button ?? "left" });
362
+ return {};
363
+ },
364
+ drag: (a) => this.drag(a),
365
+ scroll: (a) => this.scroll(a),
366
+ type: async (a) => {
367
+ await this.page.keyboard.type(a.args.text ?? "");
368
+ return {};
369
+ },
370
+ key: (a) => this.key(a),
371
+ hold_key: (a) => this.held(
372
+ a.args.keys ?? [],
373
+ () => this.page.waitForTimeout((a.args.seconds ?? 0) * 1e3)
374
+ ),
375
+ wait: async (a) => {
376
+ await this.page.waitForTimeout((a.args.seconds ?? 0) * 1e3);
377
+ return {};
378
+ },
379
+ cursor_position: async () => ({
380
+ text: `X=${this.cursor[0]}, Y=${this.cursor[1]}`
381
+ }),
382
+ history: async (a) => {
383
+ const navigate = {
384
+ back: () => this.page.goBack(),
385
+ forward: () => this.page.goForward()
386
+ };
387
+ await navigate[a.args.direction ?? "back"]();
388
+ return {};
389
+ }
390
+ };
391
+ /** The live page (a new object after a reconnect), as pixel-level Playwright. */
392
+ get page() {
393
+ const page = this.browser.page;
394
+ if (!isNativePage(page))
395
+ throw new Error(
396
+ "native computer use needs a Playwright page (mouse.move/click/down/up/wheel, context())"
397
+ );
398
+ return page;
399
+ }
400
+ run(action) {
401
+ return this.handlers[action.name](action);
402
+ }
403
+ /** A PNG of the viewport at the model's display size. */
404
+ async screenshot() {
405
+ if (!this.display.scaled) return this.page.screenshot({ type: "png" });
406
+ const { viewportWidth: width, viewportHeight: height } = this.display;
407
+ return this.capture({ x: 0, y: 0, width, height }, this.display.scale);
408
+ }
409
+ /** Whether keystrokes go to a credential field (a password input, or one
410
+ * named like a token, key or PIN), looking through shadow roots and
411
+ * same-origin frames. An unreadable page, a cross-origin frame or a focus
412
+ * that is not a text field (a closed shadow root's host) counts as one. */
413
+ async focusedFieldIsSecret() {
414
+ let described;
415
+ try {
416
+ described = String(await this.page.evaluate(FOCUSED_FIELD_JS) ?? "");
417
+ } catch {
418
+ return true;
419
+ }
420
+ return SECRET_FOCUS.some((prefix) => described.startsWith(prefix)) || SECRET_KEY.test(described);
421
+ }
422
+ /** CDP renders a clip at the page's device pixel ratio as well as its
423
+ * `scale` (a Retina host doubles it), so a capture of the wrong width
424
+ * measures that ratio and is taken once more, corrected. */
425
+ async capture(clip, scale) {
426
+ const wanted = Math.round(clip.width * scale);
427
+ const png = await this.cdpCapture(clip, scale / this.pixelRatio);
428
+ const width = pngWidth(png);
429
+ if (width === null || Math.abs(width - wanted) <= 1) return png;
430
+ this.pixelRatio *= width / wanted;
431
+ return this.cdpCapture(clip, scale / this.pixelRatio);
432
+ }
433
+ async cdpCapture(clip, scale) {
434
+ const page = this.page;
435
+ const session = await page.context().newCDPSession(page);
436
+ try {
437
+ const shot = await session.send("Page.captureScreenshot", {
438
+ format: "png",
439
+ clip: { ...clip, scale }
440
+ });
441
+ return Buffer.from(shot.data, "base64");
442
+ } finally {
443
+ await session.detach();
444
+ }
445
+ }
446
+ point(action) {
447
+ const { x, y } = action.args;
448
+ return x === void 0 || y === void 0 ? this.cursor : [x, y];
449
+ }
450
+ async moveTo(point2, steps = 1) {
451
+ const [x, y] = this.display.toPage(point2[0], point2[1]);
452
+ await this.page.mouse.move(x, y, { steps });
453
+ this.cursor = point2;
454
+ }
455
+ async held(modifiers, fn) {
456
+ for (const key of modifiers) await this.page.keyboard.down(key);
457
+ try {
458
+ await fn();
459
+ } finally {
460
+ for (const key of [...modifiers].reverse())
461
+ await this.page.keyboard.up(key);
462
+ }
463
+ return {};
464
+ }
465
+ async zoom(action) {
466
+ const [x0, y0, x1, y1] = action.args.region ?? [0, 0, 0, 0];
467
+ const [left, top] = this.display.toPage(x0, y0);
468
+ const [right, bottom] = this.display.toPage(x1, y1);
469
+ if (right <= left || bottom <= top)
470
+ throw new Error("zoom region must have x1 > x0 and y1 > y0");
471
+ const width = right - left;
472
+ const height = bottom - top;
473
+ const scale = Math.min(
474
+ this.display.width / width,
475
+ this.display.height / height
476
+ );
477
+ return {
478
+ image: await this.capture({ x: left, y: top, width, height }, scale)
479
+ };
480
+ }
481
+ async click(action) {
482
+ const point2 = this.point(action);
483
+ return this.held(action.args.modifiers ?? [], async () => {
484
+ await this.moveTo(point2);
485
+ const [x, y] = this.display.toPage(point2[0], point2[1]);
486
+ await this.page.mouse.click(x, y, {
487
+ button: action.args.button ?? "left",
488
+ clickCount: action.args.count ?? 1
489
+ });
490
+ });
491
+ }
492
+ async drag(action) {
493
+ const path7 = action.args.path ?? [];
494
+ if (path7.length < 2) throw new Error("a drag needs at least two points");
495
+ return this.held(action.args.modifiers ?? [], async () => {
496
+ await this.moveTo(path7[0]);
497
+ await this.page.mouse.down({ button: "left" });
498
+ try {
499
+ for (const point2 of path7.slice(1)) await this.moveTo(point2, DRAG_STEPS);
500
+ } finally {
501
+ await this.page.mouse.up({ button: "left" });
502
+ }
503
+ });
504
+ }
505
+ async scroll(action) {
506
+ await this.moveTo(this.point(action));
507
+ const scale = this.display.scale;
508
+ return this.held(
509
+ action.args.modifiers ?? [],
510
+ () => this.page.mouse.wheel(
511
+ (action.args.dx ?? 0) / scale,
512
+ (action.args.dy ?? 0) / scale
513
+ )
514
+ );
515
+ }
516
+ async key(action) {
517
+ const combo = (action.args.keys ?? []).join("+");
518
+ for (let i = 0; i < (action.args.repeat ?? 1); i += 1)
519
+ await this.page.keyboard.press(combo);
520
+ return {};
521
+ }
522
+ };
523
+ }
524
+ });
525
+
526
+ // src/tracing/media.ts
527
+ async function putBytes(url, data, headers) {
528
+ const controller = new AbortController();
529
+ const timer = setTimeout(() => controller.abort(), PUT_TIMEOUT_MS);
530
+ try {
531
+ const res = await fetch(url, {
532
+ method: "PUT",
533
+ body: new Uint8Array(data),
534
+ headers,
535
+ signal: controller.signal
536
+ });
537
+ return {
538
+ status: res.status,
539
+ error: res.ok ? null : (await res.text()).slice(0, MAX_UPLOAD_ERROR)
540
+ };
541
+ } catch (error) {
542
+ return {
543
+ status: NETWORK_FAILURE_STATUS,
544
+ error: error instanceof Error ? error.name : "Error"
545
+ };
546
+ } finally {
547
+ clearTimeout(timer);
548
+ }
549
+ }
550
+ var import_node_crypto, MEDIA_PATH, MAX_MEDIA_BYTES, PUT_TIMEOUT_MS, MAX_UPLOAD_ERROR, NETWORK_FAILURE_STATUS, MediaUploadError, MediaReference, MediaUploader;
551
+ var init_media = __esm({
552
+ "src/tracing/media.ts"() {
553
+ "use strict";
554
+ import_node_crypto = require("crypto");
555
+ init_transport();
556
+ MEDIA_PATH = "/api/public/media";
557
+ MAX_MEDIA_BYTES = 5 * 1024 * 1024;
558
+ PUT_TIMEOUT_MS = 3e4;
559
+ MAX_UPLOAD_ERROR = 500;
560
+ NETWORK_FAILURE_STATUS = 599;
561
+ MediaUploadError = class extends Error {
562
+ constructor(message, status) {
563
+ super(message);
564
+ this.status = status;
565
+ this.name = "MediaUploadError";
566
+ }
567
+ status;
568
+ };
569
+ MediaReference = class {
570
+ constructor(mediaId, contentType) {
571
+ this.mediaId = mediaId;
572
+ this.contentType = contentType;
573
+ }
574
+ mediaId;
575
+ contentType;
576
+ toString() {
577
+ return `@@@langfuseMedia:type=${this.contentType}|id=${this.mediaId}|source=bytes@@@`;
578
+ }
579
+ };
580
+ MediaUploader = class _MediaUploader {
581
+ constructor(transport, put = putBytes, maxBytes = MAX_MEDIA_BYTES) {
582
+ this.transport = transport;
583
+ this.put = put;
584
+ this.maxBytes = maxBytes;
585
+ }
586
+ transport;
587
+ put;
588
+ maxBytes;
589
+ /** The uploader for `GATEWAY_HOST` / `GATEWAY_PUBLIC_KEY` /
590
+ * `GATEWAY_SECRET_KEY` (the keys tracing exports with); null when any is unset. */
591
+ static fromEnv(env = process.env) {
592
+ const host = env.GATEWAY_HOST;
593
+ const publicKey = env.GATEWAY_PUBLIC_KEY;
594
+ const secretKey = env.GATEWAY_SECRET_KEY;
595
+ if (!host || !publicKey || !secretKey) return null;
596
+ return new _MediaUploader(
597
+ new PlatformTransport({
598
+ host: host.replace(/\/$/, ""),
599
+ publicKey,
600
+ secretKey
601
+ })
602
+ );
603
+ }
604
+ /** Store `data` linked to the trace (and observation) and return its reference. */
605
+ async upload(data, link) {
606
+ if (data.length > this.maxBytes)
607
+ throw new MediaUploadError(
608
+ `media is ${data.length} bytes, over the ${this.maxBytes}-byte cap`
609
+ );
610
+ const sha256Hash = (0, import_node_crypto.createHash)("sha256").update(data).digest("base64");
611
+ const answer = await this.send("POST", MEDIA_PATH, {
612
+ traceId: link.traceId,
613
+ observationId: link.observationId,
614
+ contentType: link.contentType,
615
+ contentLength: data.length,
616
+ sha256Hash,
617
+ field: link.field
618
+ });
619
+ if (typeof answer.mediaId !== "string" || !answer.mediaId)
620
+ throw new MediaUploadError(
621
+ "POST /api/public/media answered without a mediaId"
622
+ );
623
+ const reference = new MediaReference(answer.mediaId, link.contentType);
624
+ if (answer.uploadUrl)
625
+ await this.store(reference, answer.uploadUrl, data, sha256Hash);
626
+ return reference;
627
+ }
628
+ async store(reference, url, data, sha256Hash) {
629
+ const started = Date.now();
630
+ const { status, error } = await this.put(url, data, {
631
+ "Content-Type": reference.contentType,
632
+ "x-amz-checksum-sha256": sha256Hash,
633
+ "x-ms-blob-type": "BlockBlob"
634
+ });
635
+ await this.send(
636
+ "PATCH",
637
+ `${MEDIA_PATH}/${encodeURIComponent(reference.mediaId)}`,
638
+ {
639
+ uploadedAt: (/* @__PURE__ */ new Date()).toISOString(),
640
+ uploadHttpStatus: status,
641
+ uploadHttpError: error,
642
+ uploadTimeMs: Date.now() - started
643
+ }
644
+ );
645
+ if (status < 200 || status >= 300)
646
+ throw new MediaUploadError(
647
+ `media storage refused the upload (HTTP ${status})`,
648
+ status
649
+ );
650
+ }
651
+ async send(method, route, body) {
652
+ const reply = await this.transport.send(method, route, body);
653
+ if (!reply.ok)
654
+ throw new MediaUploadError(
655
+ `${method} ${route} -> ${reply.status}`,
656
+ reply.status
657
+ );
658
+ return reply.text ? JSON.parse(reply.text) : {};
659
+ }
660
+ };
661
+ }
662
+ });
663
+
664
+ // src/worlds/nativeComputer/trace.ts
665
+ function loadOtelApi() {
666
+ if (otelApi !== void 0) return otelApi;
667
+ try {
668
+ otelApi = require("@opentelemetry/api");
669
+ } catch {
670
+ otelApi = null;
671
+ }
672
+ return otelApi;
673
+ }
674
+ function errorText(error) {
675
+ return error instanceof Error ? `${error.name}: ${error.message}` : String(error);
676
+ }
677
+ function errorName(error) {
678
+ return error instanceof Error ? error.name : "Error";
679
+ }
680
+ var SPAN_PREFIX, MAX_UPLOAD_FAILURES, TRACER_NAME, MAX_ERROR, otelApi, ActionTracer;
681
+ var init_trace = __esm({
682
+ "src/worlds/nativeComputer/trace.ts"() {
683
+ "use strict";
684
+ init_media();
685
+ SPAN_PREFIX = "computer.";
686
+ MAX_UPLOAD_FAILURES = 3;
687
+ TRACER_NAME = "@withgateway/sdk";
688
+ MAX_ERROR = 1e3;
689
+ ActionTracer = class _ActionTracer {
690
+ constructor(media, warn = (m) => console.warn(m)) {
691
+ this.media = media;
692
+ this.warn = warn;
693
+ }
694
+ media;
695
+ warn;
696
+ failures = 0;
697
+ warned = false;
698
+ static fromEnv() {
699
+ return new _ActionTracer(MediaUploader.fromEnv());
700
+ }
701
+ /** `args` are already redacted. A failure's status goes through `scrub`:
702
+ * browser errors quote the session's ws URL and UI token. */
703
+ async span(action, args, run, capture, scrub) {
704
+ const otel = loadOtelApi();
705
+ if (!otel) return run();
706
+ const { SpanStatusCode, trace } = otel;
707
+ const attributes = {
708
+ "gateway.observation.type": "tool",
709
+ "tool.name": SPAN_PREFIX + action.name,
710
+ "gateway.observation.input": JSON.stringify({
711
+ action: action.name,
712
+ ...args
713
+ })
714
+ };
715
+ return trace.getTracer(TRACER_NAME).startActiveSpan(
716
+ SPAN_PREFIX + action.name,
717
+ { attributes },
718
+ async (span) => {
719
+ try {
720
+ const result = await run();
721
+ await this.finish(span, result, capture);
722
+ span.setStatus({ code: SpanStatusCode.OK });
723
+ return result;
724
+ } catch (error) {
725
+ span.setStatus({
726
+ code: SpanStatusCode.ERROR,
727
+ message: scrub(errorText(error)).slice(0, MAX_ERROR)
728
+ });
729
+ throw error;
730
+ } finally {
731
+ span.end();
732
+ }
733
+ }
734
+ );
735
+ }
736
+ async finish(span, result, capture) {
737
+ if (!span.isRecording()) return;
738
+ const output = {
739
+ screenshot: await this.screenshot(span, result, capture)
740
+ };
741
+ if (result.text !== void 0) output.text = result.text;
742
+ span.setAttribute("gateway.observation.output", JSON.stringify(output));
743
+ }
744
+ async screenshot(span, result, capture) {
745
+ let png;
746
+ try {
747
+ png = result.image ?? await capture();
748
+ } catch (error) {
749
+ return `[screenshot unavailable: ${errorName(error)}]`;
750
+ }
751
+ return this.reference(png, span);
752
+ }
753
+ async reference(png, span) {
754
+ if (!this.media)
755
+ return "[screenshot not uploaded: GATEWAY_HOST / GATEWAY_PUBLIC_KEY / GATEWAY_SECRET_KEY unset]";
756
+ if (this.failures >= MAX_UPLOAD_FAILURES)
757
+ return "[screenshot not uploaded: uploads disabled after repeated failures]";
758
+ const { traceId, spanId } = span.spanContext();
759
+ try {
760
+ const reference = await this.media.upload(png, {
761
+ contentType: "image/png",
762
+ traceId,
763
+ observationId: spanId,
764
+ field: "output"
765
+ });
766
+ this.failures = 0;
767
+ return reference.toString();
768
+ } catch (error) {
769
+ return this.failed(error);
770
+ }
771
+ }
772
+ failed(error) {
773
+ this.failures += 1;
774
+ if (!this.warned) {
775
+ this.warned = true;
776
+ this.warn(
777
+ `native computer: screenshot upload failed (${errorName(error)}: ${error instanceof Error ? error.message : String(error)}); spans keep going without it`
778
+ );
779
+ }
780
+ return `[screenshot not uploaded: ${errorName(error)}]`;
781
+ }
782
+ };
783
+ }
784
+ });
785
+
786
+ // src/worlds/nativeComputer/base.ts
787
+ function readField(item, ...names) {
788
+ const record = item ?? {};
789
+ return names.map((name) => record[name]).find((value) => value !== void 0 && value !== null);
790
+ }
791
+ var REPLAYABLE, SETTLING, EVENT_PREFIX, NativeComputer;
792
+ var init_base = __esm({
793
+ "src/worlds/nativeComputer/base.ts"() {
794
+ "use strict";
795
+ init_browser();
796
+ init_actions();
797
+ init_display();
798
+ init_screen();
799
+ init_trace();
800
+ REPLAYABLE = /* @__PURE__ */ new Set(["screenshot", "zoom", "cursor_position", "wait"]);
801
+ SETTLING = /* @__PURE__ */ new Set(["click", "mouse_up", "drag", "key", "history"]);
802
+ EVENT_PREFIX = "computer_";
803
+ NativeComputer = class _NativeComputer {
804
+ constructor(browser, options = {}) {
805
+ this.browser = browser;
806
+ const kind = this.constructor;
807
+ this.display = Display.forViewport(
808
+ browser.viewport,
809
+ options.display,
810
+ (w, h) => kind.fits(w, h)
811
+ );
812
+ this.screen = new Screen(browser, this.display);
813
+ this.tracer = options.tracer ?? ActionTracer.fromEnv();
814
+ }
815
+ browser;
816
+ static kinds = /* @__PURE__ */ new Map();
817
+ display;
818
+ screen;
819
+ tracer;
820
+ queue = Promise.resolve();
821
+ static register(provider, kind) {
822
+ _NativeComputer.kinds.set(provider, kind);
823
+ }
824
+ static get providers() {
825
+ return [..._NativeComputer.kinds.keys()].sort();
826
+ }
827
+ static forProvider(provider, browser, options = {}) {
828
+ const Kind = _NativeComputer.kinds.get(provider);
829
+ if (!Kind)
830
+ throw new Error(
831
+ `unknown computer-use provider '${provider}'; choose from ${_NativeComputer.providers.join(", ")}`
832
+ );
833
+ return new Kind(browser, options);
834
+ }
835
+ /** Whether a screenshot of this size is within the provider's limits. */
836
+ static fits(_width, _height) {
837
+ return true;
838
+ }
839
+ /** Run one provider-shaped action (an OpenAI action object, or an Anthropic
840
+ * `{ action: "left_click", ... }` input) and return what it produced: the
841
+ * door for any agent framework that hands actions over one at a time. */
842
+ executeAction(raw) {
843
+ return this.perform(this.parse(raw));
844
+ }
845
+ /** Run one action on the page: reported on the session's browser timeline
846
+ * and, when tracing is on, as a span with the screen after. */
847
+ async perform(action) {
848
+ const secret = action.name === "type" && await this.screen.focusedFieldIsSecret();
849
+ const event = EVENT_PREFIX + action.name;
850
+ const args = redactActionArgs(event, action.recordArgs(secret));
851
+ return this.tracer.span(
852
+ action,
853
+ args,
854
+ () => this.browser.act(
855
+ event,
856
+ args,
857
+ () => this.runOnScreen(action),
858
+ this.onLoss(action)
859
+ ),
860
+ () => this.screen.screenshot(),
861
+ (text) => this.browser.scrub(text)
862
+ );
863
+ }
864
+ async performAll(actions) {
865
+ const results = [];
866
+ for (const action of actions) results.push(await this.perform(action));
867
+ return results;
868
+ }
869
+ /** The screen as the model sees it, as a traced `screenshot` action. */
870
+ async screenshot() {
871
+ return (await this.perform(new ComputerAction("screenshot"))).image ?? Buffer.alloc(0);
872
+ }
873
+ /** Run `fn` after every earlier call made through here: the AI SDK starts a
874
+ * step's tool calls concurrently, but one screen takes one action at a time. */
875
+ serial(fn) {
876
+ const next = this.queue.then(fn, fn);
877
+ this.queue = next.catch(() => void 0);
878
+ return next;
879
+ }
880
+ async runOnScreen(action) {
881
+ const result = await this.screen.run(action);
882
+ if (SETTLING.has(action.name)) await this.browser.settle();
883
+ return result;
884
+ }
885
+ onLoss(action) {
886
+ return REPLAYABLE.has(action.name) ? "retry" : "report";
887
+ }
888
+ };
889
+ }
890
+ });
891
+
892
+ // src/worlds/nativeComputer/keys.ts
893
+ function playwrightKey(token) {
894
+ return KEY_NAMES[token.toLowerCase()] ?? token;
895
+ }
896
+ function letterOrName(key) {
897
+ return /^[a-zA-Z]$/.test(key) ? key.toLowerCase() : playwrightKey(key);
898
+ }
899
+ function modifierChord(text) {
900
+ return typeof text === "string" && text ? [...KeyChord.fromXdotool(text).keys] : [];
901
+ }
902
+ function modifierKeys(keys) {
903
+ return Array.isArray(keys) && keys.length > 0 ? [...KeyChord.fromKeys(keys.map(String)).keys] : [];
904
+ }
905
+ var F_KEYS, KEY_NAMES, KeyChord;
906
+ var init_keys = __esm({
907
+ "src/worlds/nativeComputer/keys.ts"() {
908
+ "use strict";
909
+ F_KEYS = Object.fromEntries(
910
+ Array.from({ length: 12 }, (_, i) => [`f${i + 1}`, `F${i + 1}`])
911
+ );
912
+ KEY_NAMES = {
913
+ return: "Enter",
914
+ enter: "Enter",
915
+ kp_enter: "Enter",
916
+ esc: "Escape",
917
+ escape: "Escape",
918
+ tab: "Tab",
919
+ space: "Space",
920
+ backspace: "Backspace",
921
+ delete: "Delete",
922
+ del: "Delete",
923
+ insert: "Insert",
924
+ home: "Home",
925
+ end: "End",
926
+ pageup: "PageUp",
927
+ page_up: "PageUp",
928
+ prior: "PageUp",
929
+ pagedown: "PageDown",
930
+ page_down: "PageDown",
931
+ next: "PageDown",
932
+ up: "ArrowUp",
933
+ arrowup: "ArrowUp",
934
+ down: "ArrowDown",
935
+ arrowdown: "ArrowDown",
936
+ left: "ArrowLeft",
937
+ arrowleft: "ArrowLeft",
938
+ right: "ArrowRight",
939
+ arrowright: "ArrowRight",
940
+ ctrl: "Control",
941
+ control: "Control",
942
+ shift: "Shift",
943
+ alt: "Alt",
944
+ option: "Alt",
945
+ super: "Meta",
946
+ meta: "Meta",
947
+ cmd: "Meta",
948
+ command: "Meta",
949
+ win: "Meta",
950
+ capslock: "CapsLock",
951
+ caps_lock: "CapsLock",
952
+ plus: "+",
953
+ minus: "-",
954
+ ...F_KEYS
955
+ };
956
+ KeyChord = class _KeyChord {
957
+ constructor(keys) {
958
+ this.keys = keys;
959
+ if (keys.length === 0)
960
+ throw new Error("a key action needs at least one key");
961
+ }
962
+ keys;
963
+ /** `ctrl+shift+t` / `Return`: Anthropic's vocabulary, case kept. */
964
+ static fromXdotool(text) {
965
+ return new _KeyChord(
966
+ text.trim().split("+").filter(Boolean).map(playwrightKey)
967
+ );
968
+ }
969
+ /** `["CTRL", "A"]`: OpenAI's vocabulary. A lone letter is the unshifted key. */
970
+ static fromKeys(keys) {
971
+ return new _KeyChord(keys.map(letterOrName));
972
+ }
973
+ /** What `page.keyboard.press` takes: `Control+a`. */
974
+ get combo() {
975
+ return this.keys.join("+");
976
+ }
977
+ };
978
+ }
979
+ });
980
+
981
+ // src/worlds/nativeComputer/anthropic.ts
982
+ function required(raw, key) {
983
+ const value = raw[key];
984
+ if (value === void 0 || value === null)
985
+ throw new Error(`${String(raw.action)} needs ${key}`);
986
+ return value;
987
+ }
988
+ function pair(raw, key) {
989
+ const value = required(raw, key);
990
+ return [toInt(value[0]), toInt(value[1])];
991
+ }
992
+ function point(raw) {
993
+ if (raw.coordinate === void 0 || raw.coordinate === null) return {};
994
+ const [x, y] = pair(raw, "coordinate");
995
+ return { x, y };
996
+ }
997
+ function chord(raw) {
998
+ return [...KeyChord.fromXdotool(String(required(raw, "text"))).keys];
999
+ }
1000
+ function click(button, count) {
1001
+ return (raw) => new ComputerAction("click", {
1002
+ ...point(raw),
1003
+ button,
1004
+ count,
1005
+ modifiers: modifierChord(raw.text)
1006
+ });
1007
+ }
1008
+ var SCROLL_STEP_PX, NOT_EXECUTED, MAX_ERROR2, SCROLL_DIRECTIONS, MEMBERS, AnthropicComputer, AnthropicComputer20251124;
1009
+ var init_anthropic = __esm({
1010
+ "src/worlds/nativeComputer/anthropic.ts"() {
1011
+ "use strict";
1012
+ init_actions();
1013
+ init_base();
1014
+ init_keys();
1015
+ SCROLL_STEP_PX = 100;
1016
+ NOT_EXECUTED = "Not executed: an earlier computer action in this turn failed.";
1017
+ MAX_ERROR2 = 1e3;
1018
+ SCROLL_DIRECTIONS = {
1019
+ up: [0, -1],
1020
+ down: [0, 1],
1021
+ left: [-1, 0],
1022
+ right: [1, 0]
1023
+ };
1024
+ MEMBERS = {
1025
+ screenshot: () => new ComputerAction("screenshot"),
1026
+ zoom: (raw) => new ComputerAction("zoom", {
1027
+ region: required(raw, "region").map(toInt)
1028
+ }),
1029
+ left_click: click("left", 1),
1030
+ right_click: click("right", 1),
1031
+ middle_click: click("middle", 1),
1032
+ double_click: click("left", 2),
1033
+ triple_click: click("left", 3),
1034
+ left_click_drag: (raw) => new ComputerAction("drag", {
1035
+ path: [pair(raw, "start_coordinate"), pair(raw, "coordinate")],
1036
+ modifiers: modifierChord(raw.text)
1037
+ }),
1038
+ mouse_move: (raw) => {
1039
+ const [x, y] = pair(raw, "coordinate");
1040
+ return new ComputerAction("move", { x, y });
1041
+ },
1042
+ left_mouse_down: () => new ComputerAction("mouse_down", { button: "left" }),
1043
+ left_mouse_up: () => new ComputerAction("mouse_up", { button: "left" }),
1044
+ cursor_position: () => new ComputerAction("cursor_position"),
1045
+ scroll: (raw) => {
1046
+ const direction = String(raw.scroll_direction ?? "down");
1047
+ if (!Object.hasOwn(SCROLL_DIRECTIONS, direction))
1048
+ throw new Error(
1049
+ `scroll_direction must be one of ${Object.keys(SCROLL_DIRECTIONS).join(", ")}`
1050
+ );
1051
+ const [dx, dy] = SCROLL_DIRECTIONS[direction];
1052
+ const amount = toInt(raw.scroll_amount ?? 1) * SCROLL_STEP_PX;
1053
+ return new ComputerAction("scroll", {
1054
+ ...point(raw),
1055
+ dx: dx * amount,
1056
+ dy: dy * amount,
1057
+ modifiers: modifierChord(raw.text)
1058
+ });
1059
+ },
1060
+ type: (raw) => new ComputerAction("type", { text: String(required(raw, "text")) }),
1061
+ key: (raw) => new ComputerAction("key", {
1062
+ keys: chord(raw),
1063
+ repeat: toInt(raw.repeat ?? 1)
1064
+ }),
1065
+ hold_key: (raw) => new ComputerAction("hold_key", {
1066
+ keys: chord(raw),
1067
+ seconds: toNumber(required(raw, "duration"))
1068
+ }),
1069
+ wait: (raw) => new ComputerAction("wait", { seconds: toNumber(raw.duration ?? 1) })
1070
+ };
1071
+ AnthropicComputer = class _AnthropicComputer extends NativeComputer {
1072
+ static TOOL_TYPE = "computer_toolset_20260801";
1073
+ static TOOLSET = "computer";
1074
+ static BETA = null;
1075
+ static MAX_LONG_EDGE = 2576;
1076
+ static MAX_VISUAL_TOKENS = 4784;
1077
+ static PATCH_PX = 28;
1078
+ static fits(width, height) {
1079
+ const tokens = Math.ceil(width / this.PATCH_PX) * Math.ceil(height / this.PATCH_PX);
1080
+ return Math.max(width, height) <= this.MAX_LONG_EDGE && tokens <= this.MAX_VISUAL_TOKENS;
1081
+ }
1082
+ tool() {
1083
+ return { type: _AnthropicComputer.TOOL_TYPE };
1084
+ }
1085
+ /** Whether a content block is a call to this computer. */
1086
+ owns(block) {
1087
+ return readField(block, "type") === "tool_use" && readField(block, "toolset_name") === _AnthropicComputer.TOOLSET;
1088
+ }
1089
+ member(block) {
1090
+ return [
1091
+ String(readField(block, "name")),
1092
+ { ...readField(block, "input") ?? {} }
1093
+ ];
1094
+ }
1095
+ parse(raw) {
1096
+ const name = String(readField(raw, "action"));
1097
+ const parser = Object.hasOwn(MEMBERS, name) ? MEMBERS[name] : void 0;
1098
+ if (!parser) throw new Error(`unknown computer action: ${name}`);
1099
+ return parser(raw);
1100
+ }
1101
+ /** Run every computer `tool_use` in a response's content, in order, and
1102
+ * return one `tool_result` per block. Other blocks are skipped. */
1103
+ async execute(content) {
1104
+ const results = [];
1105
+ let failed = false;
1106
+ for (const block of content) {
1107
+ if (!this.owns(block)) continue;
1108
+ if (failed) {
1109
+ results.push(this.envelope(block, NOT_EXECUTED, true));
1110
+ continue;
1111
+ }
1112
+ const [result, blockFailed] = await this.runBlock(block);
1113
+ results.push(result);
1114
+ failed = blockFailed;
1115
+ }
1116
+ return results;
1117
+ }
1118
+ /** The content an `ActionResult` goes back to the model as. */
1119
+ static content(result) {
1120
+ if (result.image)
1121
+ return [
1122
+ {
1123
+ type: "image",
1124
+ source: {
1125
+ type: "base64",
1126
+ media_type: "image/png",
1127
+ data: result.image.toString("base64")
1128
+ }
1129
+ }
1130
+ ];
1131
+ return [{ type: "text", text: result.text ?? "OK" }];
1132
+ }
1133
+ /** For `anthropic.tools.computerToolset_20260801({ ...computer.aiSdk() })`
1134
+ * (or `computer_20251124` with the legacy kind): `execute` runs one action
1135
+ * and returns `{ type: "image", data }` or text; `toModelOutput` turns that
1136
+ * into the tool result. Calls run one at a time, and once a call fails the
1137
+ * rest of that step's calls answer as not executed. */
1138
+ aiSdk() {
1139
+ let failedStep;
1140
+ const execute = (input, options) => this.serial(async () => {
1141
+ const step = options?.messages?.length;
1142
+ if (step !== void 0 && step === failedStep)
1143
+ throw new Error(NOT_EXECUTED);
1144
+ try {
1145
+ return _AnthropicComputer.aiSdkOutput(await this.executeAction(input));
1146
+ } catch (error) {
1147
+ failedStep = step;
1148
+ throw error instanceof Error ? new Error(this.browser.scrub(error.message)) : error;
1149
+ }
1150
+ });
1151
+ return {
1152
+ execute,
1153
+ toModelOutput: ({ output }) => _AnthropicComputer.aiSdkModelOutput(output)
1154
+ };
1155
+ }
1156
+ static aiSdkOutput(result) {
1157
+ if (result.image)
1158
+ return { type: "image", data: result.image.toString("base64") };
1159
+ return result.text ?? "OK";
1160
+ }
1161
+ /** `image-data` is what AI SDK 6's Anthropic provider turns into an image
1162
+ * block; AI SDK 7 still accepts it (and converts it to a `file` part). */
1163
+ static aiSdkModelOutput(output) {
1164
+ if (typeof output === "string") return { type: "text", value: output };
1165
+ return {
1166
+ type: "content",
1167
+ value: [
1168
+ { type: "image-data", data: output.data, mediaType: "image/png" }
1169
+ ]
1170
+ };
1171
+ }
1172
+ envelope(block, content, isError = false) {
1173
+ return {
1174
+ type: "tool_result",
1175
+ tool_use_id: String(readField(block, "id")),
1176
+ toolset_name: _AnthropicComputer.TOOLSET,
1177
+ content,
1178
+ ...isError ? { is_error: true } : {}
1179
+ };
1180
+ }
1181
+ async runBlock(block) {
1182
+ const [name, payload] = this.member(block);
1183
+ try {
1184
+ const result = await this.executeAction({ ...payload, action: name });
1185
+ return [this.envelope(block, _AnthropicComputer.content(result)), false];
1186
+ } catch (error) {
1187
+ const detail = error instanceof Error ? `${error.name}: ${error.message}` : String(error);
1188
+ return [
1189
+ this.envelope(
1190
+ block,
1191
+ this.browser.scrub(`Error: ${detail}`).slice(0, MAX_ERROR2),
1192
+ true
1193
+ ),
1194
+ true
1195
+ ];
1196
+ }
1197
+ }
1198
+ };
1199
+ AnthropicComputer20251124 = class _AnthropicComputer20251124 extends AnthropicComputer {
1200
+ static TOOL_TYPE = "computer_20251124";
1201
+ static BETA = "computer-use-2025-11-24";
1202
+ static MAX_LONG_EDGE = 1568;
1203
+ static MAX_PIXELS = 115e4;
1204
+ static fits(width, height) {
1205
+ return Math.max(width, height) <= this.MAX_LONG_EDGE && width * height <= this.MAX_PIXELS;
1206
+ }
1207
+ tool() {
1208
+ return {
1209
+ type: _AnthropicComputer20251124.TOOL_TYPE,
1210
+ name: AnthropicComputer.TOOLSET,
1211
+ display_width_px: this.display.width,
1212
+ display_height_px: this.display.height,
1213
+ enable_zoom: true
1214
+ };
1215
+ }
1216
+ owns(block) {
1217
+ return readField(block, "type") === "tool_use" && readField(block, "name") === AnthropicComputer.TOOLSET;
1218
+ }
1219
+ member(block) {
1220
+ const { action, ...payload } = readField(block, "input") ?? {};
1221
+ return [String(action ?? ""), payload];
1222
+ }
1223
+ envelope(block, content, isError = false) {
1224
+ const { toolset_name: _dropped, ...rest } = super.envelope(
1225
+ block,
1226
+ content,
1227
+ isError
1228
+ );
1229
+ return rest;
1230
+ }
1231
+ };
1232
+ NativeComputer.register("anthropic", AnthropicComputer);
1233
+ NativeComputer.register(
1234
+ "anthropic-computer-20251124",
1235
+ AnthropicComputer20251124
1236
+ );
1237
+ }
1238
+ });
1239
+
1240
+ // src/worlds/nativeComputer/openai.ts
1241
+ function safetyCheck(raw) {
1242
+ const check = { id: String(readField(raw, "id") ?? "") };
1243
+ const code = readField(raw, "code");
1244
+ const message = readField(raw, "message");
1245
+ if (code !== void 0) check.code = String(code);
1246
+ if (message !== void 0) check.message = String(message);
1247
+ return check;
1248
+ }
1249
+ function dataUrl(png) {
1250
+ return `data:image/png;base64,${png.toString("base64")}`;
1251
+ }
1252
+ function xy(raw) {
1253
+ return {
1254
+ x: toInt(readField(raw, "x") ?? 0),
1255
+ y: toInt(readField(raw, "y") ?? 0)
1256
+ };
1257
+ }
1258
+ function held(raw) {
1259
+ return modifierKeys(readField(raw, "keys"));
1260
+ }
1261
+ var WAIT_SECONDS, BUTTONS, HISTORY_BUTTONS, PendingSafetyChecks, ACTIONS, OpenAIComputer, OpenAIPreviewComputer;
1262
+ var init_openai = __esm({
1263
+ "src/worlds/nativeComputer/openai.ts"() {
1264
+ "use strict";
1265
+ init_actions();
1266
+ init_base();
1267
+ init_keys();
1268
+ WAIT_SECONDS = 1;
1269
+ BUTTONS = {
1270
+ left: "left",
1271
+ right: "right",
1272
+ wheel: "middle",
1273
+ middle: "middle"
1274
+ };
1275
+ HISTORY_BUTTONS = /* @__PURE__ */ new Set(["back", "forward"]);
1276
+ PendingSafetyChecks = class extends Error {
1277
+ constructor(callId, checks) {
1278
+ const listed = checks.map((c) => `${c.code ?? "check"}: ${c.message ?? ""}`.trim()).join("; ");
1279
+ super(
1280
+ `computer_call ${callId} needs ${checks.length} safety check(s) acknowledged: ${listed}`
1281
+ );
1282
+ this.callId = callId;
1283
+ this.checks = checks;
1284
+ this.name = "PendingSafetyChecks";
1285
+ }
1286
+ callId;
1287
+ checks;
1288
+ };
1289
+ ACTIONS = {
1290
+ click: (raw) => {
1291
+ const button = String(readField(raw, "button") ?? "left");
1292
+ if (HISTORY_BUTTONS.has(button))
1293
+ return new ComputerAction("history", {
1294
+ direction: button
1295
+ });
1296
+ if (!Object.hasOwn(BUTTONS, button))
1297
+ throw new Error(`unknown mouse button: ${button}`);
1298
+ return new ComputerAction("click", {
1299
+ ...xy(raw),
1300
+ button: BUTTONS[button],
1301
+ count: 1,
1302
+ modifiers: held(raw)
1303
+ });
1304
+ },
1305
+ double_click: (raw) => new ComputerAction("click", {
1306
+ ...xy(raw),
1307
+ button: "left",
1308
+ count: 2,
1309
+ modifiers: held(raw)
1310
+ }),
1311
+ drag: (raw) => new ComputerAction("drag", {
1312
+ path: (readField(raw, "path") ?? []).map(
1313
+ (p) => [toInt(readField(p, "x")), toInt(readField(p, "y"))]
1314
+ ),
1315
+ modifiers: held(raw)
1316
+ }),
1317
+ keypress: (raw) => new ComputerAction("key", {
1318
+ keys: [
1319
+ ...KeyChord.fromKeys(readField(raw, "keys") ?? []).keys
1320
+ ],
1321
+ repeat: 1
1322
+ }),
1323
+ move: (raw) => new ComputerAction("move", { ...xy(raw), modifiers: held(raw) }),
1324
+ scroll: (raw) => new ComputerAction("scroll", {
1325
+ ...xy(raw),
1326
+ dx: toInt(readField(raw, "scroll_x", "scrollX") ?? 0),
1327
+ dy: toInt(readField(raw, "scroll_y", "scrollY") ?? 0),
1328
+ modifiers: held(raw)
1329
+ }),
1330
+ type: (raw) => new ComputerAction("type", { text: String(readField(raw, "text") ?? "") }),
1331
+ wait: () => new ComputerAction("wait", { seconds: WAIT_SECONDS }),
1332
+ screenshot: () => new ComputerAction("screenshot")
1333
+ };
1334
+ OpenAIComputer = class _OpenAIComputer extends NativeComputer {
1335
+ tool() {
1336
+ return { type: "computer" };
1337
+ }
1338
+ owns(item) {
1339
+ return readField(item, "type") === "computer_call";
1340
+ }
1341
+ parse(raw) {
1342
+ const kind = String(readField(raw, "type"));
1343
+ const parser = Object.hasOwn(ACTIONS, kind) ? ACTIONS[kind] : void 0;
1344
+ if (!parser) throw new Error(`unknown computer action: ${kind}`);
1345
+ return parser(raw);
1346
+ }
1347
+ static actionsOf(call) {
1348
+ const batch = readField(call, "actions");
1349
+ if (Array.isArray(batch)) return batch;
1350
+ const single = readField(call, "action");
1351
+ return single === void 0 ? [] : [single];
1352
+ }
1353
+ /** Run a `computer_call`'s actions and return its `computer_call_output`.
1354
+ * Throws `PendingSafetyChecks` (before anything runs) while a check the
1355
+ * model raised is not in `acknowledged`. */
1356
+ async execute(call, acknowledged = []) {
1357
+ const callId = String(readField(call, "call_id", "callId"));
1358
+ const checks = (readField(
1359
+ call,
1360
+ "pending_safety_checks",
1361
+ "pendingSafetyChecks"
1362
+ ) ?? []).map(safetyCheck);
1363
+ const png = await this.runCall(
1364
+ callId,
1365
+ _OpenAIComputer.actionsOf(call),
1366
+ checks,
1367
+ acknowledged
1368
+ );
1369
+ return {
1370
+ type: "computer_call_output",
1371
+ call_id: callId,
1372
+ output: {
1373
+ type: "computer_screenshot",
1374
+ image_url: dataUrl(png),
1375
+ detail: "original"
1376
+ },
1377
+ ...checks.length > 0 ? { acknowledged_safety_checks: checks } : {}
1378
+ };
1379
+ }
1380
+ /** For `openai.tools.computer({ ...computer.aiSdk() })`: `execute` runs a
1381
+ * call's `actions` and returns the screenshot output. A call with pending
1382
+ * safety checks asks for approval first (`needsApproval`), so the AI SDK's
1383
+ * approval flow puts them in front of a person; once approved, `execute`
1384
+ * acknowledges them. */
1385
+ aiSdk() {
1386
+ return {
1387
+ needsApproval: (input) => (input.pendingSafetyChecks ?? []).length > 0,
1388
+ execute: (input) => this.serial(async () => {
1389
+ const checks = (input.pendingSafetyChecks ?? []).map(safetyCheck);
1390
+ const png = await this.runCall(
1391
+ "ai-sdk",
1392
+ input.actions ?? [],
1393
+ checks,
1394
+ checks
1395
+ );
1396
+ return {
1397
+ output: { type: "computer_screenshot", imageUrl: dataUrl(png) },
1398
+ ...checks.length > 0 ? { acknowledgedSafetyChecks: checks } : {}
1399
+ };
1400
+ })
1401
+ };
1402
+ }
1403
+ /** The batch's actions, run after the safety checks clear; the screen after them. */
1404
+ async runCall(callId, rawActions, checks, acknowledged) {
1405
+ const cleared = new Set(acknowledged.map((c) => safetyCheck(c).id));
1406
+ const waiting = checks.filter((c) => !c.id || !cleared.has(c.id));
1407
+ if (waiting.length > 0) throw new PendingSafetyChecks(callId, waiting);
1408
+ const actions = rawActions.map((raw) => this.parse(raw));
1409
+ const results = await this.performAll(actions);
1410
+ const last = actions.at(-1);
1411
+ if (last?.name === "screenshot")
1412
+ return results.at(-1)?.image ?? Buffer.alloc(0);
1413
+ return this.screenshot();
1414
+ }
1415
+ };
1416
+ OpenAIPreviewComputer = class extends OpenAIComputer {
1417
+ tool() {
1418
+ return {
1419
+ type: "computer_use_preview",
1420
+ display_width: this.display.width,
1421
+ display_height: this.display.height,
1422
+ environment: "browser"
1423
+ };
1424
+ }
1425
+ };
1426
+ NativeComputer.register("openai", OpenAIComputer);
1427
+ NativeComputer.register("openai-preview", OpenAIPreviewComputer);
1428
+ }
1429
+ });
1430
+
1431
+ // src/worlds/nativeComputer/index.ts
1432
+ var init_nativeComputer = __esm({
1433
+ "src/worlds/nativeComputer/index.ts"() {
1434
+ "use strict";
1435
+ init_actions();
1436
+ init_base();
1437
+ init_display();
1438
+ init_keys();
1439
+ init_screen();
1440
+ init_trace();
1441
+ init_anthropic();
1442
+ init_openai();
160
1443
  }
161
1444
  });
162
1445
 
@@ -172,6 +1455,23 @@ var init_sessionContract = __esm({
172
1455
  }
173
1456
  });
174
1457
 
1458
+ // src/tracing/session.ts
1459
+ var import_async_hooks, processStorage, currentSessionStorage, currentWorldSessionStorage;
1460
+ var init_session = __esm({
1461
+ "src/tracing/session.ts"() {
1462
+ "use strict";
1463
+ import_async_hooks = require("async_hooks");
1464
+ processStorage = (name) => {
1465
+ const key = /* @__PURE__ */ Symbol.for(`@withgateway/sdk/${name}`);
1466
+ const slots = globalThis;
1467
+ slots[key] ??= new import_async_hooks.AsyncLocalStorage();
1468
+ return slots[key];
1469
+ };
1470
+ currentSessionStorage = processStorage("session-storage");
1471
+ currentWorldSessionStorage = processStorage("world-session-storage");
1472
+ }
1473
+ });
1474
+
175
1475
  // src/worlds/taskRef.ts
176
1476
  var init_taskRef = __esm({
177
1477
  "src/worlds/taskRef.ts"() {
@@ -180,14 +1480,16 @@ var init_taskRef = __esm({
180
1480
  });
181
1481
 
182
1482
  // src/worlds/session.ts
183
- var import_node_crypto, READY_TIMEOUT_MS, CALL_TIMEOUT_MS, JUDGED_GRADE_TIMEOUT_MS;
184
- var init_session = __esm({
1483
+ var import_node_crypto2, READY_TIMEOUT_MS, CALL_TIMEOUT_MS, JUDGED_GRADE_TIMEOUT_MS;
1484
+ var init_session2 = __esm({
185
1485
  "src/worlds/session.ts"() {
186
1486
  "use strict";
187
- import_node_crypto = require("crypto");
1487
+ import_node_crypto2 = require("crypto");
188
1488
  init_worlds();
189
1489
  init_browser();
1490
+ init_nativeComputer();
190
1491
  init_sessionContract();
1492
+ init_session();
191
1493
  init_taskRef();
192
1494
  init_taskRef();
193
1495
  READY_TIMEOUT_MS = 10 * 6e4;
@@ -212,13 +1514,22 @@ var init_tasks = __esm({
212
1514
  }
213
1515
  });
214
1516
 
1517
+ // src/worlds/surfaceDescriptor.ts
1518
+ var init_surfaceDescriptor = __esm({
1519
+ "src/worlds/surfaceDescriptor.ts"() {
1520
+ "use strict";
1521
+ init_browser();
1522
+ }
1523
+ });
1524
+
215
1525
  // src/worlds/taskSessions.ts
216
1526
  var init_taskSessions = __esm({
217
1527
  "src/worlds/taskSessions.ts"() {
218
1528
  "use strict";
219
1529
  init_worlds();
220
1530
  init_browser();
221
- init_session();
1531
+ init_surfaceDescriptor();
1532
+ init_session2();
222
1533
  }
223
1534
  });
224
1535
 
@@ -230,18 +1541,34 @@ var init_evals = __esm({
230
1541
  }
231
1542
  });
232
1543
 
1544
+ // src/tracing/prerequisites.ts
1545
+ var init_prerequisites = __esm({
1546
+ "src/tracing/prerequisites.ts"() {
1547
+ "use strict";
1548
+ }
1549
+ });
1550
+
1551
+ // src/worlds/taskTracing.ts
1552
+ var init_taskTracing = __esm({
1553
+ "src/worlds/taskTracing.ts"() {
1554
+ "use strict";
1555
+ init_prerequisites();
1556
+ }
1557
+ });
1558
+
233
1559
  // src/worlds/taskRun.ts
234
- var import_node_crypto2, fs, path;
1560
+ var import_node_crypto3, fs, path;
235
1561
  var init_taskRun = __esm({
236
1562
  "src/worlds/taskRun.ts"() {
237
1563
  "use strict";
238
- import_node_crypto2 = require("crypto");
1564
+ import_node_crypto3 = require("crypto");
239
1565
  fs = __toESM(require("fs"));
240
1566
  path = __toESM(require("path"));
241
1567
  init_browser();
242
1568
  init_evals();
243
- init_session();
1569
+ init_session2();
244
1570
  init_taskSessions();
1571
+ init_taskTracing();
245
1572
  }
246
1573
  });
247
1574
 
@@ -264,6 +1591,13 @@ var init_setRun = __esm({
264
1591
  }
265
1592
  });
266
1593
 
1594
+ // src/worlds/requestLog.ts
1595
+ var init_requestLog = __esm({
1596
+ "src/worlds/requestLog.ts"() {
1597
+ "use strict";
1598
+ }
1599
+ });
1600
+
267
1601
  // src/worlds/checkoutTarget.ts
268
1602
  var fs2, path2;
269
1603
  var init_checkoutTarget = __esm({
@@ -275,11 +1609,11 @@ var init_checkoutTarget = __esm({
275
1609
  });
276
1610
 
277
1611
  // src/worlds/worldDir.ts
278
- var import_node_crypto3, fs3, path3, CONNECTOR_SPEC, WORLD_SCHEMA_MARKERS, WORLD_ROOT_MARKERS;
1612
+ var import_node_crypto4, fs3, path3, CONNECTOR_SPEC, WORLD_SCHEMA_MARKERS, WORLD_ROOT_MARKERS;
279
1613
  var init_worldDir = __esm({
280
1614
  "src/worlds/worldDir.ts"() {
281
1615
  "use strict";
282
- import_node_crypto3 = require("crypto");
1616
+ import_node_crypto4 = require("crypto");
283
1617
  fs3 = __toESM(require("fs"));
284
1618
  path3 = __toESM(require("path"));
285
1619
  CONNECTOR_SPEC = "connector.toml";
@@ -300,11 +1634,11 @@ var init_tar = __esm({
300
1634
  });
301
1635
 
302
1636
  // src/worlds/hub.ts
303
- var import_node_crypto4, import_node_zlib2, fs5, path5, META_RELPATH;
1637
+ var import_node_crypto5, import_node_zlib2, fs5, path5, META_RELPATH;
304
1638
  var init_hub = __esm({
305
1639
  "src/worlds/hub.ts"() {
306
1640
  "use strict";
307
- import_node_crypto4 = require("crypto");
1641
+ import_node_crypto5 = require("crypto");
308
1642
  import_node_zlib2 = require("zlib");
309
1643
  fs5 = __toESM(require("fs"));
310
1644
  path5 = __toESM(require("path"));
@@ -350,11 +1684,11 @@ var init_json = __esm({
350
1684
  });
351
1685
 
352
1686
  // src/worlds/transforms.ts
353
- var import_node_crypto5, WHITESPACE, WHITESPACE_EDGES, WHITESPACE_RUN, NOT_DIGIT, STRIP_PREFIX, trim, emailLocalPart, emailDomain, hex, SIMPLE, TRANSFORMS;
1687
+ var import_node_crypto6, WHITESPACE, WHITESPACE_EDGES, WHITESPACE_RUN, NOT_DIGIT, STRIP_PREFIX, trim, emailLocalPart, emailDomain, hex, SIMPLE, TRANSFORMS;
354
1688
  var init_transforms = __esm({
355
1689
  "src/worlds/transforms.ts"() {
356
1690
  "use strict";
357
- import_node_crypto5 = require("crypto");
1691
+ import_node_crypto6 = require("crypto");
358
1692
  init_json();
359
1693
  WHITESPACE = " \n\r\f\v";
360
1694
  WHITESPACE_EDGES = new RegExp(`^[${WHITESPACE}]+|[${WHITESPACE}]+$`, "g");
@@ -370,7 +1704,7 @@ var init_transforms = __esm({
370
1704
  const at = value.lastIndexOf("@");
371
1705
  return at === -1 ? "" : value.slice(at + 1).toLowerCase();
372
1706
  };
373
- hex = (algorithm, value) => (0, import_node_crypto5.createHash)(algorithm).update(value, "utf8").digest("hex");
1707
+ hex = (algorithm, value) => (0, import_node_crypto6.createHash)(algorithm).update(value, "utf8").digest("hex");
374
1708
  SIMPLE = {
375
1709
  lower: (value) => value.toLowerCase(),
376
1710
  upper: (value) => value.toUpperCase(),
@@ -390,11 +1724,11 @@ var init_transforms = __esm({
390
1724
  });
391
1725
 
392
1726
  // src/worlds/redaction.ts
393
- var import_node_crypto6;
1727
+ var import_node_crypto7;
394
1728
  var init_redaction = __esm({
395
1729
  "src/worlds/redaction.ts"() {
396
1730
  "use strict";
397
- import_node_crypto6 = require("crypto");
1731
+ import_node_crypto7 = require("crypto");
398
1732
  init_json();
399
1733
  init_transforms();
400
1734
  }
@@ -486,7 +1820,7 @@ var init_worlds = __esm({
486
1820
  "use strict";
487
1821
  init_transport();
488
1822
  init_transport();
489
- init_session();
1823
+ init_session2();
490
1824
  init_sessionList();
491
1825
  init_sessionContract();
492
1826
  init_taskRef();
@@ -497,7 +1831,10 @@ var init_worlds = __esm({
497
1831
  init_checkoutProvenance();
498
1832
  init_evals();
499
1833
  init_browser();
1834
+ init_surfaceDescriptor();
1835
+ init_requestLog();
500
1836
  init_browserReconnect();
1837
+ init_nativeComputer();
501
1838
  init_hub();
502
1839
  init_validate();
503
1840
  init_describe();