@namzu/sdk 48.2.0 → 48.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (16) hide show
  1. package/CHANGELOG.md +6 -0
  2. package/dist/tools/builtins/computer-use-coordinates.d.ts +13 -1
  3. package/dist/tools/builtins/computer-use-coordinates.d.ts.map +1 -1
  4. package/dist/tools/builtins/computer-use-coordinates.js +46 -0
  5. package/dist/tools/builtins/computer-use-coordinates.js.map +1 -1
  6. package/dist/tools/builtins/computer-use.d.ts +135 -0
  7. package/dist/tools/builtins/computer-use.d.ts.map +1 -1
  8. package/dist/tools/builtins/computer-use.js +255 -54
  9. package/dist/tools/builtins/computer-use.js.map +1 -1
  10. package/dist/types/computer-use/index.d.ts +56 -1
  11. package/dist/types/computer-use/index.d.ts.map +1 -1
  12. package/dist/types/computer-use/index.js.map +1 -1
  13. package/package.json +1 -1
  14. package/src/tools/builtins/computer-use-coordinates.ts +58 -1
  15. package/src/tools/builtins/computer-use.ts +336 -54
  16. package/src/types/computer-use/index.ts +65 -7
package/CHANGELOG.md CHANGED
@@ -1,5 +1,11 @@
1
1
  # Changelog
2
2
 
3
+ ## 48.3.0
4
+
5
+ ### Minor Changes
6
+
7
+ - 072bb74: Windows computer use can capture pixels for one named window with `screenshot {window_id}` and keep background clicks and drags, text and keys bound to that window. Window `type_text` and `key` can explicitly request `delivery_mode: 'foreground'` for browser text or modifier shortcuts that cannot run in the background; this may briefly bring the verified window forward. The pinned driver sets `windowScroll: false` and refuses scoped pixel scrolling because its wheel input cannot prove which window receives it; a host that supports safe window scrolling can offer it. Capture identity and window geometry are checked before input; moved windows, old captures and restarted driver sessions require a new window screenshot. Hosts can offer window capture without display capture, in which case `window_id` is required and plain display screenshots remain unavailable. A failed window capture never broadens to a display capture. When both browser and computer-use tools are ready, the CLI routes a named existing browser window to computer use, leaving browser tools for the separate Namzu-managed profile.
8
+
3
9
  ## 48.2.0
4
10
 
5
11
  ### Minor Changes
package/dist/tools/builtins/computer-use-coordinates.d.ts CHANGED
@@ -1,4 +1,4 @@
1
- import type { DisplayInfo, Point, Rect } from '../../types/computer-use/index.js';
1
+ import type { DisplayInfo, Point, Rect, WindowScreenshotResult } from '../../types/computer-use/index.js';
2
2
  /**
3
3
  * One screenshot the model was shown: its id, the size of the image it saw,
4
4
  * and the display that image was made from. Every coordinate the model sends
@@ -11,6 +11,10 @@ export interface ScreenshotFrame {
11
11
  readonly imageHeight: number;
12
12
  /** Physical size and virtual-desktop origin of what the image shows. */
13
13
  readonly display: DisplayInfo;
14
+ /** Present only for an explicit capture of one window. */
15
+ readonly window?: Omit<WindowScreenshotResult, 'data'> & {
16
+ readonly data?: Buffer;
17
+ };
14
18
  }
15
19
  /**
16
20
  * The frames this tool instance has returned, newest last. A model reads a
@@ -25,10 +29,16 @@ export declare class ScreenshotFrames {
25
29
  private counter;
26
30
  private newest;
27
31
  constructor(capacity?: number);
32
+ /** Only the newest window PNG is needed for zoom; older tokens cannot act. */
33
+ private discardOldWindowPixels;
28
34
  record(image: {
29
35
  width: number;
30
36
  height: number;
31
37
  }, display: DisplayInfo): ScreenshotFrame;
38
+ recordWindow(image: {
39
+ width: number;
40
+ height: number;
41
+ }, window: WindowScreenshotResult): ScreenshotFrame;
32
42
  latest(): ScreenshotFrame | undefined;
33
43
  get(id: string): ScreenshotFrame | undefined;
34
44
  }
@@ -49,6 +59,8 @@ export declare function pointOnImage(frame: ScreenshotFrame, point: Point): bool
49
59
  * display.
50
60
  */
51
61
  export declare function toDisplayPoint(frame: ScreenshotFrame, point: Point): Point;
62
+ /** Model image pixels to the exact PNG pixel space accepted by the window driver. */
63
+ export declare function toWindowPoint(frame: ScreenshotFrame, point: Point): Point;
52
64
  /** A display-relative physical pixel as the image pixel of `frame` that shows it. */
53
65
  export declare function toImagePoint(frame: ScreenshotFrame, point: Point): Point;
54
66
  /**
package/dist/tools/builtins/computer-use-coordinates.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"computer-use-coordinates.d.ts","sourceRoot":"","sources":["../../../src/tools/builtins/computer-use-coordinates.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,mCAAmC,CAAA;AAEjF;;;;GAIG;AACH,MAAM,WAAW,eAAe;IAC/B,yDAAyD;IACzD,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;IACnB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;IAC5B,wEAAwE;IACxE,QAAQ,CAAC,OAAO,EAAE,WAAW,CAAA;CAC7B;AAED;;;;;;GAMG;AACH,qBAAa,gBAAgB;IAKhB,OAAO,CAAC,QAAQ,CAAC,QAAQ;IAJrC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAqC;IAC5D,OAAO,CAAC,OAAO,CAAI;IACnB,OAAO,CAAC,MAAM,CAA6B;gBAEd,QAAQ,SAAK;IAE1C,MAAM,CAAC,KAAK,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,EAAE,OAAO,EAAE,WAAW,GAAG,eAAe;IAkBvF,MAAM,IAAI,eAAe,GAAG,SAAS;IAIrC,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,eAAe,GAAG,SAAS;CAG5C;AAED,oEAAoE;AACpE,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,WAAW,CAEzE;AAED;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,eAAe,EAAE,KAAK,EAAE,KAAK,GAAG,OAAO,CAE1E;AAcD;;;;;;GAMG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,eAAe,EAAE,KAAK,EAAE,KAAK,GAAG,KAAK,CAK1E;AAED,qFAAqF;AACrF,wBAAgB,YAAY,CAAC,KAAK,EAAE,eAAe,EAAE,KAAK,EAAE,KAAK,GAAG,KAAK,CAKxE;AAED;;;;GAIG;AACH,wBAAgB,aAAa,CAAC,KAAK,EAAE,eAAe,EAAE,IAAI,EAAE,IAAI,GAAG,IAAI,GAAG,IAAI,CAS7E;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,eAAe,EAAE,MAAM,EAAE,IAAI,GAAG,IAAI,GAAG,IAAI,CAiBpF"}
1
+ {"version":3,"file":"computer-use-coordinates.d.ts","sourceRoot":"","sources":["../../../src/tools/builtins/computer-use-coordinates.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACX,WAAW,EACX,KAAK,EACL,IAAI,EACJ,sBAAsB,EACtB,MAAM,mCAAmC,CAAA;AAE1C;;;;GAIG;AACH,MAAM,WAAW,eAAe;IAC/B,yDAAyD;IACzD,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;IACnB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;IAC5B,wEAAwE;IACxE,QAAQ,CAAC,OAAO,EAAE,WAAW,CAAA;IAC7B,0DAA0D;IAC1D,QAAQ,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,sBAAsB,EAAE,MAAM,CAAC,GAAG;QAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,CAAA;CACnF;AAED;;;;;;GAMG;AACH,qBAAa,gBAAgB;IAKhB,OAAO,CAAC,QAAQ,CAAC,QAAQ;IAJrC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAqC;IAC5D,OAAO,CAAC,OAAO,CAAI;IACnB,OAAO,CAAC,MAAM,CAA6B;gBAEd,QAAQ,SAAK;IAE1C,8EAA8E;IAC9E,OAAO,CAAC,sBAAsB;IAO9B,MAAM,CAAC,KAAK,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,EAAE,OAAO,EAAE,WAAW,GAAG,eAAe;IAmBvF,YAAY,CACX,KAAK,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,EACxC,MAAM,EAAE,sBAAsB,GAC5B,eAAe;IA6BlB,MAAM,IAAI,eAAe,GAAG,SAAS;IAIrC,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,eAAe,GAAG,SAAS;CAG5C;AAED,oEAAoE;AACpE,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,WAAW,CAEzE;AAED;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,eAAe,EAAE,KAAK,EAAE,KAAK,GAAG,OAAO,CAE1E;AAcD;;;;;;GAMG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,eAAe,EAAE,KAAK,EAAE,KAAK,GAAG,KAAK,CAK1E;AAED,qFAAqF;AACrF,wBAAgB,aAAa,CAAC,KAAK,EAAE,eAAe,EAAE,KAAK,EAAE,KAAK,GAAG,KAAK,CAMzE;AAED,qFAAqF;AACrF,wBAAgB,YAAY,CAAC,KAAK,EAAE,eAAe,EAAE,KAAK,EAAE,KAAK,GAAG,KAAK,CAKxE;AAED;;;;GAIG;AACH,wBAAgB,aAAa,CAAC,KAAK,EAAE,eAAe,EAAE,IAAI,EAAE,IAAI,GAAG,IAAI,GAAG,IAAI,CAS7E;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,eAAe,EAAE,MAAM,EAAE,IAAI,GAAG,IAAI,GAAG,IAAI,CAiBpF"}
package/dist/tools/builtins/computer-use-coordinates.js CHANGED
@@ -13,7 +13,15 @@ export class ScreenshotFrames {
13
13
  constructor(capacity = 32) {
14
14
  this.capacity = capacity;
15
15
  }
16
+ /** Only the newest window PNG is needed for zoom; older tokens cannot act. */
17
+ discardOldWindowPixels() {
18
+ for (const [id, frame] of this.frames) {
19
+ if (frame.window?.data)
20
+ this.frames.set(id, { ...frame, window: { ...frame.window, data: undefined } });
21
+ }
22
+ }
16
23
  record(image, display) {
24
+ this.discardOldWindowPixels();
17
25
  this.counter += 1;
18
26
  const frame = {
19
27
  id: `s${this.counter}`,
@@ -31,6 +39,35 @@ export class ScreenshotFrames {
31
39
  }
32
40
  return frame;
33
41
  }
42
+ recordWindow(image, window) {
43
+ this.discardOldWindowPixels();
44
+ this.counter += 1;
45
+ const frame = {
46
+ id: `s${this.counter}`,
47
+ imageWidth: image.width,
48
+ imageHeight: image.height,
49
+ // Kept only for the common image-size helpers; a window frame never
50
+ // enters a display-coordinate action or virtual-desktop projection.
51
+ display: {
52
+ id: window.window.id,
53
+ x: window.window.bounds.x,
54
+ y: window.window.bounds.y,
55
+ width: window.width,
56
+ height: window.height,
57
+ scaleFactor: 1,
58
+ },
59
+ window,
60
+ };
61
+ this.frames.set(frame.id, frame);
62
+ this.newest = frame;
63
+ while (this.frames.size > this.capacity) {
64
+ const oldest = this.frames.keys().next().value;
65
+ if (oldest === undefined)
66
+ break;
67
+ this.frames.delete(oldest);
68
+ }
69
+ return frame;
70
+ }
34
71
  latest() {
35
72
  return this.newest;
36
73
  }
@@ -74,6 +111,15 @@ export function toDisplayPoint(frame, point) {
74
111
  y: axisToDisplay(point.y, frame.imageHeight, frame.display.height),
75
112
  };
76
113
  }
114
+ /** Model image pixels to the exact PNG pixel space accepted by the window driver. */
115
+ export function toWindowPoint(frame, point) {
116
+ if (!frame.window)
117
+ throw new Error('computer_use: expected a window screenshot');
118
+ return {
119
+ x: axisToDisplay(point.x, frame.imageWidth, frame.window.width),
120
+ y: axisToDisplay(point.y, frame.imageHeight, frame.window.height),
121
+ };
122
+ }
77
123
  /** A display-relative physical pixel as the image pixel of `frame` that shows it. */
78
124
  export function toImagePoint(frame, point) {
79
125
  return {
package/dist/tools/builtins/computer-use-coordinates.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"computer-use-coordinates.js","sourceRoot":"","sources":["../../../src/tools/builtins/computer-use-coordinates.ts"],"names":[],"mappings":"AAgBA;;;;;;GAMG;AACH,MAAM,OAAO,gBAAgB;IAKC;IAJZ,MAAM,GAAG,IAAI,GAAG,EAA2B,CAAA;IACpD,OAAO,GAAG,CAAC,CAAA;IACX,MAAM,CAA6B;IAE3C,YAA6B,WAAW,EAAE;QAAb,aAAQ,GAAR,QAAQ,CAAK;IAAG,CAAC;IAE9C,MAAM,CAAC,KAAwC,EAAE,OAAoB;QACpE,IAAI,CAAC,OAAO,IAAI,CAAC,CAAA;QACjB,MAAM,KAAK,GAAoB;YAC9B,EAAE,EAAE,IAAI,IAAI,CAAC,OAAO,EAAE;YACtB,UAAU,EAAE,KAAK,CAAC,KAAK;YACvB,WAAW,EAAE,KAAK,CAAC,MAAM;YACzB,OAAO;SACP,CAAA;QACD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,KAAK,CAAC,CAAA;QAChC,IAAI,CAAC,MAAM,GAAG,KAAK,CAAA;QACnB,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;YACzC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAA;YAC9C,IAAI,MAAM,KAAK,SAAS;gBAAE,MAAK;YAC/B,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;QAC3B,CAAC;QACD,OAAO,KAAK,CAAA;IACb,CAAC;IAED,MAAM;QACL,OAAO,IAAI,CAAC,MAAM,CAAA;IACnB,CAAC;IAED,GAAG,CAAC,EAAU;QACb,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;IAC3B,CAAC;CACD;AAED,oEAAoE;AACpE,MAAM,UAAU,cAAc,CAAC,KAAa,EAAE,MAAc;IAC3D,OAAO,EAAE,EAAE,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAA;AACnF,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,YAAY,CAAC,KAAsB,EAAE,KAAY;IAChE,OAAO,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,KAAK,CAAC,UAAU,IAAI,KAAK,CAAC,CAAC,IAAI,KAAK,CAAC,WAAW,CAAA;AACnG,CAAC;AAED,SAAS,aAAa,CAAC,KAAa,EAAE,KAAa,EAAE,OAAe;IACnE,wEAAwE;IACxE,0EAA0E;IAC1E,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,GAAG,CAAC,GAAG,OAAO,CAAC,GAAG,KAAK,CAAC,CAAA;IAC5D,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,CAAA;AAClD,CAAC;AAED,SAAS,WAAW,CAAC,KAAa,EAAE,OAAe,EAAE,KAAa;IACjE,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,GAAG,CAAC,GAAG,KAAK,CAAC,GAAG,OAAO,CAAC,CAAA;IAC5D,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,KAAK,GAAG,CAAC,CAAC,CAAA;AAChD,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,cAAc,CAAC,KAAsB,EAAE,KAAY;IAClE,OAAO;QACN,CAAC,EAAE,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,UAAU,EAAE,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAChE,CAAC,EAAE,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,WAAW,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;KAClE,CAAA;AACF,CAAC;AAED,qFAAqF;AACrF,MAAM,UAAU,YAAY,CAAC,KAAsB,EAAE,KAAY;IAChE,OAAO;QACN,CAAC,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,UAAU,CAAC;QAC9D,CAAC,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,KAAK,CAAC,WAAW,CAAC;KAChE,CAAA;AACF,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,aAAa,CAAC,KAAsB,EAAE,IAAU;IAC/D,MAAM,EAAE,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,GAAG,KAAK,CAAC,UAAU,CAAA;IACjD,MAAM,EAAE,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,KAAK,CAAC,WAAW,CAAA;IACnD,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;IACjD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;IAChD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;IAClF,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;IACrF,IAAI,KAAK,IAAI,IAAI,IAAI,MAAM,IAAI,GAAG;QAAE,OAAO,IAAI,CAAA;IAC/C,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,KAAK,GAAG,IAAI,EAAE,MAAM,EAAE,MAAM,GAAG,GAAG,EAAE,CAAA;AACtE,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,KAAsB,EAAE,MAAY;IACtE,MAAM,EAAE,OAAO,EAAE,GAAG,KAAK,CAAA;IACzB,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;IAC9C,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;IAC7C,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK,GAAG,OAAO,CAAC,CAAC,EAAE,OAAO,CAAC,KAAK,CAAC,CAAA;IAC1E,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,GAAG,OAAO,CAAC,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,CAAA;IAC7E,IAAI,KAAK,IAAI,IAAI,IAAI,MAAM,IAAI,GAAG;QAAE,OAAO,IAAI,CAAA;IAC/C,MAAM,EAAE,GAAG,KAAK,CAAC,UAAU,GAAG,OAAO,CAAC,KAAK,CAAA;IAC3C,MAAM,EAAE,GAAG,KAAK,CAAC,WAAW,GAAG,OAAO,CAAC,MAAM,CAAA;IAC7C,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,EAAE,CAAC,CAAA;IAC/B,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,EAAE,CAAC,CAAA;IAC9B,OAAO;QACN,CAAC;QACD,CAAC;QACD,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC;QAC7C,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC;KAC/C,CAAA;AACF,CAAC"}
1
+ {"version":3,"file":"computer-use-coordinates.js","sourceRoot":"","sources":["../../../src/tools/builtins/computer-use-coordinates.ts"],"names":[],"mappings":"AAuBA;;;;;;GAMG;AACH,MAAM,OAAO,gBAAgB;IAKC;IAJZ,MAAM,GAAG,IAAI,GAAG,EAA2B,CAAA;IACpD,OAAO,GAAG,CAAC,CAAA;IACX,MAAM,CAA6B;IAE3C,YAA6B,WAAW,EAAE;QAAb,aAAQ,GAAR,QAAQ,CAAK;IAAG,CAAC;IAE9C,8EAA8E;IACtE,sBAAsB;QAC7B,KAAK,MAAM,CAAC,EAAE,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACvC,IAAI,KAAK,CAAC,MAAM,EAAE,IAAI;gBACrB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,GAAG,KAAK,EAAE,MAAM,EAAE,EAAE,GAAG,KAAK,CAAC,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE,CAAC,CAAA;QACjF,CAAC;IACF,CAAC;IAED,MAAM,CAAC,KAAwC,EAAE,OAAoB;QACpE,IAAI,CAAC,sBAAsB,EAAE,CAAA;QAC7B,IAAI,CAAC,OAAO,IAAI,CAAC,CAAA;QACjB,MAAM,KAAK,GAAoB;YAC9B,EAAE,EAAE,IAAI,IAAI,CAAC,OAAO,EAAE;YACtB,UAAU,EAAE,KAAK,CAAC,KAAK;YACvB,WAAW,EAAE,KAAK,CAAC,MAAM;YACzB,OAAO;SACP,CAAA;QACD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,KAAK,CAAC,CAAA;QAChC,IAAI,CAAC,MAAM,GAAG,KAAK,CAAA;QACnB,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;YACzC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAA;YAC9C,IAAI,MAAM,KAAK,SAAS;gBAAE,MAAK;YAC/B,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;QAC3B,CAAC;QACD,OAAO,KAAK,CAAA;IACb,CAAC;IAED,YAAY,CACX,KAAwC,EACxC,MAA8B;QAE9B,IAAI,CAAC,sBAAsB,EAAE,CAAA;QAC7B,IAAI,CAAC,OAAO,IAAI,CAAC,CAAA;QACjB,MAAM,KAAK,GAAoB;YAC9B,EAAE,EAAE,IAAI,IAAI,CAAC,OAAO,EAAE;YACtB,UAAU,EAAE,KAAK,CAAC,KAAK;YACvB,WAAW,EAAE,KAAK,CAAC,MAAM;YACzB,oEAAoE;YACpE,oEAAoE;YACpE,OAAO,EAAE;gBACR,EAAE,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE;gBACpB,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;gBACzB,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;gBACzB,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,WAAW,EAAE,CAAC;aACd;YACD,MAAM;SACN,CAAA;QACD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,KAAK,CAAC,CAAA;QAChC,IAAI,CAAC,MAAM,GAAG,KAAK,CAAA;QACnB,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;YACzC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAA;YAC9C,IAAI,MAAM,KAAK,SAAS;gBAAE,MAAK;YAC/B,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;QAC3B,CAAC;QACD,OAAO,KAAK,CAAA;IACb,CAAC;IAED,MAAM;QACL,OAAO,IAAI,CAAC,MAAM,CAAA;IACnB,CAAC;IAED,GAAG,CAAC,EAAU;QACb,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;IAC3B,CAAC;CACD;AAED,oEAAoE;AACpE,MAAM,UAAU,cAAc,CAAC,KAAa,EAAE,MAAc;IAC3D,OAAO,EAAE,EAAE,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAA;AACnF,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,YAAY,CAAC,KAAsB,EAAE,KAAY;IAChE,OAAO,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,KAAK,CAAC,UAAU,IAAI,KAAK,CAAC,CAAC,IAAI,KAAK,CAAC,WAAW,CAAA;AACnG,CAAC;AAED,SAAS,aAAa,CAAC,KAAa,EAAE,KAAa,EAAE,OAAe;IACnE,wEAAwE;IACxE,0EAA0E;IAC1E,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,GAAG,CAAC,GAAG,OAAO,CAAC,GAAG,KAAK,CAAC,CAAA;IAC5D,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,CAAA;AAClD,CAAC;AAED,SAAS,WAAW,CAAC,KAAa,EAAE,OAAe,EAAE,KAAa;IACjE,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,GAAG,CAAC,GAAG,KAAK,CAAC,GAAG,OAAO,CAAC,CAAA;IAC5D,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,KAAK,GAAG,CAAC,CAAC,CAAA;AAChD,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,cAAc,CAAC,KAAsB,EAAE,KAAY;IAClE,OAAO;QACN,CAAC,EAAE,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,UAAU,EAAE,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAChE,CAAC,EAAE,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,WAAW,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;KAClE,CAAA;AACF,CAAC;AAED,qFAAqF;AACrF,MAAM,UAAU,aAAa,CAAC,KAAsB,EAAE,KAAY;IACjE,IAAI,CAAC,KAAK,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAA;IAChF,OAAO;QACN,CAAC,EAAE,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC;QAC/D,CAAC,EAAE,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;KACjE,CAAA;AACF,CAAC;AAED,qFAAqF;AACrF,MAAM,UAAU,YAAY,CAAC,KAAsB,EAAE,KAAY;IAChE,OAAO;QACN,CAAC,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,UAAU,CAAC;QAC9D,CAAC,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,KAAK,CAAC,WAAW,CAAC;KAChE,CAAA;AACF,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,aAAa,CAAC,KAAsB,EAAE,IAAU;IAC/D,MAAM,EAAE,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,GAAG,KAAK,CAAC,UAAU,CAAA;IACjD,MAAM,EAAE,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,KAAK,CAAC,WAAW,CAAA;IACnD,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;IACjD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;IAChD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;IAClF,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;IACrF,IAAI,KAAK,IAAI,IAAI,IAAI,MAAM,IAAI,GAAG;QAAE,OAAO,IAAI,CAAA;IAC/C,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,KAAK,GAAG,IAAI,EAAE,MAAM,EAAE,MAAM,GAAG,GAAG,EAAE,CAAA;AACtE,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,KAAsB,EAAE,MAAY;IACtE,MAAM,EAAE,OAAO,EAAE,GAAG,KAAK,CAAA;IACzB,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;IAC9C,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;IAC7C,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK,GAAG,OAAO,CAAC,CAAC,EAAE,OAAO,CAAC,KAAK,CAAC,CAAA;IAC1E,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,GAAG,OAAO,CAAC,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,CAAA;IAC7E,IAAI,KAAK,IAAI,IAAI,IAAI,MAAM,IAAI,GAAG;QAAE,OAAO,IAAI,CAAA;IAC/C,MAAM,EAAE,GAAG,KAAK,CAAC,UAAU,GAAG,OAAO,CAAC,KAAK,CAAA;IAC3C,MAAM,EAAE,GAAG,KAAK,CAAC,WAAW,GAAG,OAAO,CAAC,MAAM,CAAA;IAC7C,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,EAAE,CAAC,CAAA;IAC/B,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,EAAE,CAAC,CAAA;IAC9B,OAAO;QACN,CAAC;QACD,CAAC;QACD,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC;QAC7C,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC;KAC/C,CAAA;AACF,CAAC"}
package/dist/tools/builtins/computer-use.d.ts CHANGED
@@ -55,10 +55,13 @@ export interface ComputerUseToolOptions {
55
55
  export declare function computerUseUnavailableReason(provider: Pick<LLMProvider, 'id' | 'capabilities'>): string | undefined;
56
56
  declare const actionSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
57
57
  type: z.ZodLiteral<"screenshot">;
58
+ window_id: z.ZodOptional<z.ZodString>;
58
59
  }, "strip", z.ZodTypeAny, {
59
60
  type: "screenshot";
61
+ window_id?: string | undefined;
60
62
  }, {
61
63
  type: "screenshot";
64
+ window_id?: string | undefined;
62
65
  }>, z.ZodObject<{
63
66
  screenshot_id: z.ZodOptional<z.ZodString>;
64
67
  type: z.ZodLiteral<"zoom">;
@@ -147,6 +150,8 @@ declare const actionSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
147
150
  x: number;
148
151
  }>;
149
152
  button: z.ZodDefault<z.ZodEnum<["left", "right", "middle"]>>;
153
+ /** Accepted only to return an explicit refusal; pointer foreground is unsafe. */
154
+ delivery_mode: z.ZodOptional<z.ZodEnum<["background", "foreground"]>>;
150
155
  } & {
151
156
  screenshot_id: z.ZodOptional<z.ZodString>;
152
157
  }, "strip", z.ZodTypeAny, {
@@ -156,6 +161,7 @@ declare const actionSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
156
161
  };
157
162
  type: "mouse_click";
158
163
  button: "left" | "right" | "middle";
164
+ delivery_mode?: "background" | "foreground" | undefined;
159
165
  screenshot_id?: string | undefined;
160
166
  }, {
161
167
  at: {
@@ -164,6 +170,7 @@ declare const actionSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
164
170
  };
165
171
  type: "mouse_click";
166
172
  button?: "left" | "right" | "middle" | undefined;
173
+ delivery_mode?: "background" | "foreground" | undefined;
167
174
  screenshot_id?: string | undefined;
168
175
  }>, z.ZodObject<{
169
176
  type: z.ZodLiteral<"mouse_drag">;
@@ -188,6 +195,7 @@ declare const actionSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
188
195
  x: number;
189
196
  }>;
190
197
  button: z.ZodDefault<z.ZodEnum<["left", "right", "middle"]>>;
198
+ delivery_mode: z.ZodOptional<z.ZodEnum<["background", "foreground"]>>;
191
199
  } & {
192
200
  screenshot_id: z.ZodOptional<z.ZodString>;
193
201
  }, "strip", z.ZodTypeAny, {
@@ -201,6 +209,7 @@ declare const actionSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
201
209
  x: number;
202
210
  };
203
211
  button: "left" | "right" | "middle";
212
+ delivery_mode?: "background" | "foreground" | undefined;
204
213
  screenshot_id?: string | undefined;
205
214
  }, {
206
215
  type: "mouse_drag";
@@ -213,6 +222,7 @@ declare const actionSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
213
222
  x: number;
214
223
  };
215
224
  button?: "left" | "right" | "middle" | undefined;
225
+ delivery_mode?: "background" | "foreground" | undefined;
216
226
  screenshot_id?: string | undefined;
217
227
  }>, z.ZodObject<{
218
228
  type: z.ZodLiteral<"scroll">;
@@ -228,6 +238,7 @@ declare const actionSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
228
238
  }>;
229
239
  direction: z.ZodEnum<["up", "down", "left", "right"]>;
230
240
  amount: z.ZodNumber;
241
+ delivery_mode: z.ZodOptional<z.ZodEnum<["background", "foreground"]>>;
231
242
  } & {
232
243
  screenshot_id: z.ZodOptional<z.ZodString>;
233
244
  }, "strip", z.ZodTypeAny, {
@@ -238,6 +249,7 @@ declare const actionSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
238
249
  type: "scroll";
239
250
  amount: number;
240
251
  direction: "left" | "right" | "up" | "down";
252
+ delivery_mode?: "background" | "foreground" | undefined;
241
253
  screenshot_id?: string | undefined;
242
254
  }, {
243
255
  at: {
@@ -247,32 +259,76 @@ declare const actionSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
247
259
  type: "scroll";
248
260
  amount: number;
249
261
  direction: "left" | "right" | "up" | "down";
262
+ delivery_mode?: "background" | "foreground" | undefined;
250
263
  screenshot_id?: string | undefined;
251
264
  }>, z.ZodObject<{
252
265
  type: z.ZodLiteral<"type_text">;
253
266
  text: z.ZodString;
267
+ /** Accepted only to refuse an unsafe atomic foreground focus click. */
268
+ at: z.ZodOptional<z.ZodObject<{
269
+ x: z.ZodNumber;
270
+ y: z.ZodNumber;
271
+ }, "strip", z.ZodTypeAny, {
272
+ y: number;
273
+ x: number;
274
+ }, {
275
+ y: number;
276
+ x: number;
277
+ }>>;
278
+ delivery_mode: z.ZodOptional<z.ZodEnum<["background", "foreground"]>>;
254
279
  } & {
255
280
  screenshot_id: z.ZodOptional<z.ZodString>;
256
281
  }, "strip", z.ZodTypeAny, {
257
282
  text: string;
258
283
  type: "type_text";
284
+ at?: {
285
+ y: number;
286
+ x: number;
287
+ } | undefined;
288
+ delivery_mode?: "background" | "foreground" | undefined;
259
289
  screenshot_id?: string | undefined;
260
290
  }, {
261
291
  text: string;
262
292
  type: "type_text";
293
+ at?: {
294
+ y: number;
295
+ x: number;
296
+ } | undefined;
297
+ delivery_mode?: "background" | "foreground" | undefined;
263
298
  screenshot_id?: string | undefined;
264
299
  }>, z.ZodObject<{
265
300
  type: z.ZodLiteral<"key">;
266
301
  keys: z.ZodString;
302
+ at: z.ZodOptional<z.ZodObject<{
303
+ x: z.ZodNumber;
304
+ y: z.ZodNumber;
305
+ }, "strip", z.ZodTypeAny, {
306
+ y: number;
307
+ x: number;
308
+ }, {
309
+ y: number;
310
+ x: number;
311
+ }>>;
312
+ delivery_mode: z.ZodOptional<z.ZodEnum<["background", "foreground"]>>;
267
313
  } & {
268
314
  screenshot_id: z.ZodOptional<z.ZodString>;
269
315
  }, "strip", z.ZodTypeAny, {
270
316
  keys: string;
271
317
  type: "key";
318
+ at?: {
319
+ y: number;
320
+ x: number;
321
+ } | undefined;
322
+ delivery_mode?: "background" | "foreground" | undefined;
272
323
  screenshot_id?: string | undefined;
273
324
  }, {
274
325
  keys: string;
275
326
  type: "key";
327
+ at?: {
328
+ y: number;
329
+ x: number;
330
+ } | undefined;
331
+ delivery_mode?: "background" | "foreground" | undefined;
276
332
  screenshot_id?: string | undefined;
277
333
  }>, z.ZodObject<{
278
334
  type: z.ZodLiteral<"wait">;
@@ -384,6 +440,8 @@ declare const actionSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
384
440
  x: number;
385
441
  }>;
386
442
  button: z.ZodDefault<z.ZodEnum<["left", "right", "middle"]>>;
443
+ /** Accepted only to return an explicit refusal; pointer foreground is unsafe. */
444
+ delivery_mode: z.ZodOptional<z.ZodEnum<["background", "foreground"]>>;
387
445
  }, "strip", z.ZodTypeAny, {
388
446
  at: {
389
447
  y: number;
@@ -391,6 +449,7 @@ declare const actionSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
391
449
  };
392
450
  type: "mouse_click";
393
451
  button: "left" | "right" | "middle";
452
+ delivery_mode?: "background" | "foreground" | undefined;
394
453
  }, {
395
454
  at: {
396
455
  y: number;
@@ -398,6 +457,7 @@ declare const actionSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
398
457
  };
399
458
  type: "mouse_click";
400
459
  button?: "left" | "right" | "middle" | undefined;
460
+ delivery_mode?: "background" | "foreground" | undefined;
401
461
  }>, z.ZodObject<{
402
462
  type: z.ZodLiteral<"mouse_drag">;
403
463
  from: z.ZodObject<{
@@ -421,6 +481,7 @@ declare const actionSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
421
481
  x: number;
422
482
  }>;
423
483
  button: z.ZodDefault<z.ZodEnum<["left", "right", "middle"]>>;
484
+ delivery_mode: z.ZodOptional<z.ZodEnum<["background", "foreground"]>>;
424
485
  }, "strip", z.ZodTypeAny, {
425
486
  type: "mouse_drag";
426
487
  from: {
@@ -432,6 +493,7 @@ declare const actionSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
432
493
  x: number;
433
494
  };
434
495
  button: "left" | "right" | "middle";
496
+ delivery_mode?: "background" | "foreground" | undefined;
435
497
  }, {
436
498
  type: "mouse_drag";
437
499
  from: {
@@ -443,6 +505,7 @@ declare const actionSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
443
505
  x: number;
444
506
  };
445
507
  button?: "left" | "right" | "middle" | undefined;
508
+ delivery_mode?: "background" | "foreground" | undefined;
446
509
  }>, z.ZodObject<{
447
510
  type: z.ZodLiteral<"scroll">;
448
511
  at: z.ZodObject<{
@@ -457,6 +520,7 @@ declare const actionSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
457
520
  }>;
458
521
  direction: z.ZodEnum<["up", "down", "left", "right"]>;
459
522
  amount: z.ZodNumber;
523
+ delivery_mode: z.ZodOptional<z.ZodEnum<["background", "foreground"]>>;
460
524
  }, "strip", z.ZodTypeAny, {
461
525
  at: {
462
526
  y: number;
@@ -465,6 +529,7 @@ declare const actionSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
465
529
  type: "scroll";
466
530
  amount: number;
467
531
  direction: "left" | "right" | "up" | "down";
532
+ delivery_mode?: "background" | "foreground" | undefined;
468
533
  }, {
469
534
  at: {
470
535
  y: number;
@@ -473,24 +538,68 @@ declare const actionSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
473
538
  type: "scroll";
474
539
  amount: number;
475
540
  direction: "left" | "right" | "up" | "down";
541
+ delivery_mode?: "background" | "foreground" | undefined;
476
542
  }>, z.ZodObject<{
477
543
  type: z.ZodLiteral<"type_text">;
478
544
  text: z.ZodString;
545
+ /** Accepted only to refuse an unsafe atomic foreground focus click. */
546
+ at: z.ZodOptional<z.ZodObject<{
547
+ x: z.ZodNumber;
548
+ y: z.ZodNumber;
549
+ }, "strip", z.ZodTypeAny, {
550
+ y: number;
551
+ x: number;
552
+ }, {
553
+ y: number;
554
+ x: number;
555
+ }>>;
556
+ delivery_mode: z.ZodOptional<z.ZodEnum<["background", "foreground"]>>;
479
557
  }, "strip", z.ZodTypeAny, {
480
558
  text: string;
481
559
  type: "type_text";
560
+ at?: {
561
+ y: number;
562
+ x: number;
563
+ } | undefined;
564
+ delivery_mode?: "background" | "foreground" | undefined;
482
565
  }, {
483
566
  text: string;
484
567
  type: "type_text";
568
+ at?: {
569
+ y: number;
570
+ x: number;
571
+ } | undefined;
572
+ delivery_mode?: "background" | "foreground" | undefined;
485
573
  }>, z.ZodObject<{
486
574
  type: z.ZodLiteral<"key">;
487
575
  keys: z.ZodString;
576
+ at: z.ZodOptional<z.ZodObject<{
577
+ x: z.ZodNumber;
578
+ y: z.ZodNumber;
579
+ }, "strip", z.ZodTypeAny, {
580
+ y: number;
581
+ x: number;
582
+ }, {
583
+ y: number;
584
+ x: number;
585
+ }>>;
586
+ delivery_mode: z.ZodOptional<z.ZodEnum<["background", "foreground"]>>;
488
587
  }, "strip", z.ZodTypeAny, {
489
588
  keys: string;
490
589
  type: "key";
590
+ at?: {
591
+ y: number;
592
+ x: number;
593
+ } | undefined;
594
+ delivery_mode?: "background" | "foreground" | undefined;
491
595
  }, {
492
596
  keys: string;
493
597
  type: "key";
598
+ at?: {
599
+ y: number;
600
+ x: number;
601
+ } | undefined;
602
+ delivery_mode?: "background" | "foreground" | undefined;
494
603
  }>, z.ZodObject<{
495
604
  type: z.ZodLiteral<"wait">;
496
605
  ms: z.ZodNumber;
@@ -548,6 +657,7 @@ declare const actionSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
548
657
  };
549
658
  type: "mouse_click";
550
659
  button: "left" | "right" | "middle";
660
+ delivery_mode?: "background" | "foreground" | undefined;
551
661
  } | {
552
662
  type: "mouse_drag";
553
663
  from: {
@@ -559,6 +669,7 @@ declare const actionSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
559
669
  x: number;
560
670
  };
561
671
  button: "left" | "right" | "middle";
672
+ delivery_mode?: "background" | "foreground" | undefined;
562
673
  } | {
563
674
  at: {
564
675
  y: number;
@@ -567,12 +678,23 @@ declare const actionSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
567
678
  type: "scroll";
568
679
  amount: number;
569
680
  direction: "left" | "right" | "up" | "down";
681
+ delivery_mode?: "background" | "foreground" | undefined;
570
682
  } | {
571
683
  text: string;
572
684
  type: "type_text";
685
+ at?: {
686
+ y: number;
687
+ x: number;
688
+ } | undefined;
689
+ delivery_mode?: "background" | "foreground" | undefined;
573
690
  } | {
574
691
  keys: string;
575
692
  type: "key";
693
+ at?: {
694
+ y: number;
695
+ x: number;
696
+ } | undefined;
697
+ delivery_mode?: "background" | "foreground" | undefined;
576
698
  } | {
577
699
  type: "wait";
578
700
  ms: number;
@@ -605,6 +727,7 @@ declare const actionSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
605
727
  };
606
728
  type: "mouse_click";
607
729
  button?: "left" | "right" | "middle" | undefined;
730
+ delivery_mode?: "background" | "foreground" | undefined;
608
731
  } | {
609
732
  type: "mouse_drag";
610
733
  from: {
@@ -616,6 +739,7 @@ declare const actionSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
616
739
  x: number;
617
740
  };
618
741
  button?: "left" | "right" | "middle" | undefined;
742
+ delivery_mode?: "background" | "foreground" | undefined;
619
743
  } | {
620
744
  at: {
621
745
  y: number;
@@ -624,12 +748,23 @@ declare const actionSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
624
748
  type: "scroll";
625
749
  amount: number;
626
750
  direction: "left" | "right" | "up" | "down";
751
+ delivery_mode?: "background" | "foreground" | undefined;
627
752
  } | {
628
753
  text: string;
629
754
  type: "type_text";
755
+ at?: {
756
+ y: number;
757
+ x: number;
758
+ } | undefined;
759
+ delivery_mode?: "background" | "foreground" | undefined;
630
760
  } | {
631
761
  keys: string;
632
762
  type: "key";
763
+ at?: {
764
+ y: number;
765
+ x: number;
766
+ } | undefined;
767
+ delivery_mode?: "background" | "foreground" | undefined;
633
768
  } | {
634
769
  type: "wait";
635
770
  ms: number;
package/dist/tools/builtins/computer-use.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"computer-use.d.ts","sourceRoot":"","sources":["../../../src/tools/builtins/computer-use.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AAEvB,OAAO,KAAK,EAGX,eAAe,EAUf,MAAM,mCAAmC,CAAA;AAE1C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAA;AAChE,OAAO,KAAK,EAAe,cAAc,EAAc,MAAM,2BAA2B,CAAA;AAcxF,OAAO,EAGN,KAAK,gBAAgB,EAIrB,MAAM,yBAAyB,CAAA;AAEhC,OAAO,EACN,0BAA0B,EAC1B,0BAA0B,EAC1B,oBAAoB,GACpB,MAAM,yBAAyB,CAAA;AAChC,YAAY,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAA;AAE1E,eAAO,MAAM,sBAAsB,EAAG,cAAuB,CAAA;AAS7D;;;GAGG;AACH,MAAM,WAAW,sBAAsB;IACtC;;;;;OAKG;IACH,QAAQ,CAAC,gBAAgB,CAAC,EAAE,gBAAgB,CAAA;IAC5C;;;OAGG;IACH,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;IAC1B;;;;OAIG;IACH,QAAQ,CAAC,sBAAsB,CAAC,EAAE,OAAO,CAAA;IACzC,sDAAsD;IACtD,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAA;IACjC,8DAA8D;IAC9D,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAA;IAC3B;;;;;;;OAOG;IACH,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,CAAA;CACnC;AAED;;;;;;;;GAQG;AACH,wBAAgB,4BAA4B,CAC3C,QAAQ,EAAE,IAAI,CAAC,WAAW,EAAE,IAAI,GAAG,cAAc,CAAC,GAChD,MAAM,GAAG,SAAS,CAGpB;AAwFD,QAAA,MAAM,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAoBhB,CAAA;AAEF;;;;;;;;GAQG;AACH,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAA;AA0mBtD;;;GAGG;AACH,MAAM,WAAW,eAAgB,SAAQ,cAAc,CAAC,WAAW,CAAC;IACnE;;;;;;;OAOG;IACH,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAAA;CAC9C;AA2DD;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAgB,qBAAqB,CACpC,IAAI,EAAE,eAAe,EACrB,OAAO,GAAE,sBAA2B,GAClC,eAAe,CAguBjB"}
1
+ {"version":3,"file":"computer-use.d.ts","sourceRoot":"","sources":["../../../src/tools/builtins/computer-use.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AAEvB,OAAO,KAAK,EAGX,eAAe,EAWf,MAAM,mCAAmC,CAAA;AAE1C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAA;AAChE,OAAO,KAAK,EAAe,cAAc,EAAc,MAAM,2BAA2B,CAAA;AAexF,OAAO,EAGN,KAAK,gBAAgB,EAIrB,MAAM,yBAAyB,CAAA;AAEhC,OAAO,EACN,0BAA0B,EAC1B,0BAA0B,EAC1B,oBAAoB,GACpB,MAAM,yBAAyB,CAAA;AAChC,YAAY,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAA;AAE1E,eAAO,MAAM,sBAAsB,EAAG,cAAuB,CAAA;AAS7D;;;GAGG;AACH,MAAM,WAAW,sBAAsB;IACtC;;;;;OAKG;IACH,QAAQ,CAAC,gBAAgB,CAAC,EAAE,gBAAgB,CAAA;IAC5C;;;OAGG;IACH,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;IAC1B;;;;OAIG;IACH,QAAQ,CAAC,sBAAsB,CAAC,EAAE,OAAO,CAAA;IACzC,sDAAsD;IACtD,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAA;IACjC,8DAA8D;IAC9D,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAA;IAC3B;;;;;;;OAOG;IACH,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,CAAA;CACnC;AAED;;;;;;;;GAQG;AACH,wBAAgB,4BAA4B,CAC3C,QAAQ,EAAE,IAAI,CAAC,WAAW,EAAE,IAAI,GAAG,cAAc,CAAC,GAChD,MAAM,GAAG,SAAS,CAGpB;AAuGD,QAAA,MAAM,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IA1EjB,iFAAiF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAoBjF,uEAAuE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;QApBvE,iFAAiF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;QAoBjF,uEAAuE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IA0EtE,CAAA;AAEF;;;;;;;;GAQG;AACH,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAA;AA+qBtD;;;GAGG;AACH,MAAM,WAAW,eAAgB,SAAQ,cAAc,CAAC,WAAW,CAAC;IACnE;;;;;;;OAOG;IACH,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAAA;CAC9C;AA4DD;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAgB,qBAAqB,CACpC,IAAI,EAAE,eAAe,EACrB,OAAO,GAAE,sBAA2B,GAClC,eAAe,CAm6BjB"}