@forsion/tangu-computer-use 0.5.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. package/CHANGELOG.md +223 -0
  2. package/LICENSE +26 -0
  3. package/LICENSE.upstream +21 -0
  4. package/README.md +169 -0
  5. package/UPSTREAM.md +277 -0
  6. package/check.mjs +369 -0
  7. package/icon.png +0 -0
  8. package/install.sh +112 -0
  9. package/main.js +541 -0
  10. package/manifest.json +47 -0
  11. package/native/linux/bridge-rs/Cargo.lock +1204 -0
  12. package/native/linux/bridge-rs/Cargo.toml +18 -0
  13. package/native/linux/bridge-rs/src/atspi.rs +640 -0
  14. package/native/linux/bridge-rs/src/error.rs +65 -0
  15. package/native/linux/bridge-rs/src/lib.rs +11 -0
  16. package/native/linux/bridge-rs/src/main.rs +1074 -0
  17. package/native/linux/bridge-rs/src/protocol.rs +49 -0
  18. package/native/linux/bridge-rs/src/state.rs +293 -0
  19. package/native/linux/bridge-rs/src/wayland.rs +89 -0
  20. package/native/linux/bridge-rs/src/x11.rs +909 -0
  21. package/native/linux/bridge-rs/tests/protocol_tests.rs +73 -0
  22. package/native/macos/agent_cursor.swift +184 -0
  23. package/native/macos/agent_cursor_motion.swift +252 -0
  24. package/native/macos/agent_cursor_tests.swift +128 -0
  25. package/native/macos/agent_highlight.swift +229 -0
  26. package/native/macos/agent_highlight_tests.swift +99 -0
  27. package/native/macos/bridge.swift +3852 -0
  28. package/native/macos/foreground_activity.swift +57 -0
  29. package/native/macos/foreground_activity_tests.swift +31 -0
  30. package/native/macos/live_stream.swift +289 -0
  31. package/native/windows/bridge-rs/Cargo.lock +396 -0
  32. package/native/windows/bridge-rs/Cargo.toml +30 -0
  33. package/native/windows/bridge-rs/src/capture.rs +518 -0
  34. package/native/windows/bridge-rs/src/error.rs +81 -0
  35. package/native/windows/bridge-rs/src/input.rs +501 -0
  36. package/native/windows/bridge-rs/src/lib.rs +16 -0
  37. package/native/windows/bridge-rs/src/main.rs +1683 -0
  38. package/native/windows/bridge-rs/src/protocol.rs +53 -0
  39. package/native/windows/bridge-rs/src/refs.rs +237 -0
  40. package/native/windows/bridge-rs/src/state.rs +55 -0
  41. package/native/windows/bridge-rs/src/uia.rs +1252 -0
  42. package/native/windows/bridge-rs/src/window.rs +718 -0
  43. package/native/windows/bridge-rs/tests/protocol_tests.rs +252 -0
  44. package/native/windows/bridge-rs/tests/refs_tests.rs +86 -0
  45. package/native/windows/bridge-rs/tests/state_tests.rs +33 -0
  46. package/package.json +74 -0
  47. package/prebuilt/linux/arm64/linux-bridge +0 -0
  48. package/prebuilt/linux/x64/linux-bridge +0 -0
  49. package/prebuilt/macos/arm64/bridge +0 -0
  50. package/prebuilt/macos/arm64/tangu-computer-use.app.json +10 -0
  51. package/prebuilt/macos/arm64/tangu-computer-use.app.zip +0 -0
  52. package/prebuilt/macos/x64/bridge +0 -0
  53. package/prebuilt/macos/x64/tangu-computer-use.app.json +10 -0
  54. package/prebuilt/macos/x64/tangu-computer-use.app.zip +0 -0
  55. package/prebuilt/windows/windows-bridge.exe +0 -0
  56. package/scripts/blind-click.check.mjs +128 -0
  57. package/scripts/build-native.mjs +302 -0
  58. package/scripts/build.mjs +84 -0
  59. package/scripts/calc-fixture.mjs +78 -0
  60. package/scripts/cli-exit.check.mjs +49 -0
  61. package/scripts/helper-path.check.mjs +82 -0
  62. package/scripts/helper-refresh.check.mjs +62 -0
  63. package/scripts/helper-signal.check.mjs +59 -0
  64. package/scripts/highlight.check.mjs +32 -0
  65. package/scripts/keychain-free-install.check.mjs +140 -0
  66. package/scripts/live-view.check.mjs +124 -0
  67. package/scripts/macos-bundle.d.mts +1 -0
  68. package/scripts/macos-bundle.mjs +119 -0
  69. package/scripts/make-signing-cert.sh +57 -0
  70. package/scripts/mini-foreground.check.mjs +11 -0
  71. package/scripts/no-foreground.check.mjs +81 -0
  72. package/scripts/overlay-visible.check.mjs +221 -0
  73. package/scripts/package-macos-app.mjs +39 -0
  74. package/scripts/permissions.check.mjs +199 -0
  75. package/scripts/platform-contract.check.mjs +30 -0
  76. package/scripts/setup-helper.mjs +154 -0
  77. package/scripts/tangu-computer-use.entitlements +10 -0
  78. package/scripts/verify-macos-bundles.mjs +25 -0
  79. package/scripts/verify-package.mjs +74 -0
  80. package/skills/computer-use/SKILL.md +110 -0
  81. package/src/foregroundNote.ts +43 -0
  82. package/src/helperState.ts +31 -0
  83. package/src/index.ts +63 -0
  84. package/src/onboarding.ts +198 -0
  85. package/src/pi-compat.ts +49 -0
  86. package/src/settings.ts +11 -0
  87. package/src/setup.ts +97 -0
  88. package/src/tools.ts +289 -0
  89. package/src/vendor/actions.ts +130 -0
  90. package/src/vendor/bridge.ts +2405 -0
  91. package/src/vendor/cdp.ts +658 -0
  92. package/src/vendor/config.ts +113 -0
  93. package/src/vendor/contract.ts +104 -0
  94. package/src/vendor/note.ts +195 -0
  95. package/src/vendor/outline.ts +651 -0
  96. package/src/vendor/output.ts +134 -0
  97. package/src/vendor/permissions.ts +111 -0
  98. package/src/vendor/platform/architecture.ts +23 -0
  99. package/src/vendor/platform/coerce.ts +16 -0
  100. package/src/vendor/platform/index.ts +59 -0
  101. package/src/vendor/platform/linux/backend.ts +186 -0
  102. package/src/vendor/platform/linux/helper.ts +238 -0
  103. package/src/vendor/platform/macos/backend.ts +131 -0
  104. package/src/vendor/platform/macos/browser.ts +110 -0
  105. package/src/vendor/platform/macos/helper-path.d.mts +9 -0
  106. package/src/vendor/platform/macos/helper-path.mjs +34 -0
  107. package/src/vendor/platform/macos/helper.ts +291 -0
  108. package/src/vendor/platform/macos/permissions.ts +146 -0
  109. package/src/vendor/platform/types.ts +222 -0
  110. package/src/vendor/platform/windows/backend.ts +142 -0
  111. package/src/vendor/platform/windows/helper.ts +140 -0
  112. package/src/vendor/root-selection.ts +25 -0
  113. package/src/vendor/runtime.ts +129 -0
  114. package/src/vendor/state.ts +146 -0
  115. package/src/vendor/view.ts +147 -0
  116. package/tangu-plugins/computer-use/dist/foregroundNote.js +19 -0
  117. package/tangu-plugins/computer-use/dist/index.js +5672 -0
  118. package/tangu-plugins/computer-use/tangu-plugin.json +9 -0
  119. package/tsconfig.json +22 -0
  120. package/types/tangu-agent.d.ts +142 -0
@@ -0,0 +1,909 @@
1
+ //! X11/EWMH enrichment, capture, focus, and guarded XTEST physical input.
2
+ use crate::atspi::{Rect, RootSnapshot};
3
+ use crate::{ErrorCode, ProtocolError};
4
+ use base64::Engine as _;
5
+ use serde_json::{json, Value};
6
+ use x11rb::connection::Connection;
7
+ use x11rb::protocol::xproto::{
8
+ Atom, AtomEnum, ButtonIndex, ClientMessageData, ClientMessageEvent, ConnectionExt as _,
9
+ Drawable, EventMask, ImageFormat, ImageOrder, Keycode, MapState, Window, BUTTON_PRESS_EVENT,
10
+ BUTTON_RELEASE_EVENT, CLIENT_MESSAGE_EVENT, KEY_PRESS_EVENT, KEY_RELEASE_EVENT,
11
+ MOTION_NOTIFY_EVENT,
12
+ };
13
+ use x11rb::protocol::{composite, xtest};
14
+ use x11rb::rust_connection::RustConnection;
15
+
16
+ const MAX_CAPTURE_PIXELS: u64 = 64 * 1024 * 1024;
17
+
18
+ #[derive(Debug, Clone, Copy, PartialEq, Eq)]
19
+ pub enum SessionKind {
20
+ X11,
21
+ Wayland,
22
+ Headless,
23
+ }
24
+ impl SessionKind {
25
+ pub fn detect() -> Self {
26
+ let kind = std::env::var("XDG_SESSION_TYPE")
27
+ .unwrap_or_default()
28
+ .to_ascii_lowercase();
29
+ if kind == "wayland" || (kind.is_empty() && std::env::var_os("WAYLAND_DISPLAY").is_some()) {
30
+ Self::Wayland
31
+ } else if kind == "x11" || std::env::var_os("DISPLAY").is_some() {
32
+ Self::X11
33
+ } else {
34
+ Self::Headless
35
+ }
36
+ }
37
+ pub fn as_str(self) -> &'static str {
38
+ match self {
39
+ Self::X11 => "x11",
40
+ Self::Wayland => "wayland",
41
+ Self::Headless => "headless",
42
+ }
43
+ }
44
+ }
45
+
46
+ #[derive(Debug, Clone, Copy, PartialEq, Eq)]
47
+ pub enum PhysicalPolicy {
48
+ AxOnly,
49
+ Background,
50
+ Default,
51
+ Foreground,
52
+ }
53
+ impl PhysicalPolicy {
54
+ pub fn parse(value: Option<&str>) -> Result<Self, ProtocolError> {
55
+ match value.unwrap_or("default") {
56
+ "ax_only" => Ok(Self::AxOnly),
57
+ "background" => Ok(Self::Background),
58
+ "default" => Ok(Self::Default),
59
+ "foreground" => Ok(Self::Foreground),
60
+ value => Err(invalid(format!("Unknown delivery policy '{value}'"))),
61
+ }
62
+ }
63
+ pub fn require_physical(self, session: SessionKind) -> Result<(), ProtocolError> {
64
+ if session == SessionKind::Headless {
65
+ return Err(err(
66
+ "Physical input is unavailable in a headless Linux session",
67
+ ErrorCode::CoordinateBlocked,
68
+ ));
69
+ }
70
+ if session != SessionKind::X11 {
71
+ return Err(err(
72
+ "XTEST is unavailable in native Wayland sessions; use AT-SPI",
73
+ ErrorCode::CapabilityDeferred,
74
+ ));
75
+ }
76
+ match self {
77
+ Self::AxOnly => Err(err(
78
+ "ax_only policy forbids XTEST and window focus",
79
+ ErrorCode::CoordinateBlocked,
80
+ )),
81
+ Self::Background => Err(err(
82
+ "Background policy forbids global XTEST input",
83
+ ErrorCode::ForegroundRequired,
84
+ )),
85
+ Self::Default | Self::Foreground => Ok(()),
86
+ }
87
+ }
88
+ }
89
+
90
+ #[derive(Debug, Clone, PartialEq)]
91
+ pub struct WindowInfo {
92
+ pub id: u32,
93
+ pub pid: u64,
94
+ pub title: String,
95
+ pub frame: Rect,
96
+ pub focused: bool,
97
+ pub minimized: bool,
98
+ pub z_order: usize,
99
+ }
100
+ pub struct Capture {
101
+ pub png_base64: String,
102
+ pub width: u16,
103
+ pub height: u16,
104
+ pub source: &'static str,
105
+ pub warnings: Vec<String>,
106
+ }
107
+ pub fn available() -> bool {
108
+ SessionKind::detect() == SessionKind::X11 && connect().is_ok()
109
+ }
110
+
111
+ pub fn list_windows() -> Result<Vec<WindowInfo>, ProtocolError> {
112
+ let (conn, screen) = connect()?;
113
+ let root = conn.setup().roots[screen].root;
114
+ let a = Atoms::new(&conn)?;
115
+ let active = property32(&conn, root, a.active, AtomEnum::WINDOW.into())
116
+ .first()
117
+ .copied();
118
+ let mut windows = property32(&conn, root, a.stacking, AtomEnum::WINDOW.into());
119
+ if windows.is_empty() {
120
+ windows = property32(&conn, root, a.clients, AtomEnum::WINDOW.into());
121
+ }
122
+ let mut out = Vec::new();
123
+ for (z_order, id) in windows.into_iter().rev().enumerate() {
124
+ let Ok(g) = conn
125
+ .get_geometry(id)
126
+ .ok()
127
+ .and_then(|c| c.reply().ok())
128
+ .ok_or(())
129
+ else {
130
+ continue;
131
+ };
132
+ let translated = conn
133
+ .translate_coordinates(id, root, 0, 0)
134
+ .ok()
135
+ .and_then(|c| c.reply().ok());
136
+ let states = property32(&conn, id, a.state, AtomEnum::ATOM.into());
137
+ out.push(WindowInfo {
138
+ id,
139
+ pid: property32(&conn, id, a.pid, AtomEnum::CARDINAL.into())
140
+ .first()
141
+ .copied()
142
+ .unwrap_or(0) as u64,
143
+ title: title(&conn, id, &a),
144
+ frame: Rect {
145
+ x: translated
146
+ .as_ref()
147
+ .map(|r| i32::from(r.dst_x))
148
+ .unwrap_or(i32::from(g.x)),
149
+ y: translated
150
+ .as_ref()
151
+ .map(|r| i32::from(r.dst_y))
152
+ .unwrap_or(i32::from(g.y)),
153
+ width: i32::from(g.width),
154
+ height: i32::from(g.height),
155
+ },
156
+ focused: active == Some(id),
157
+ minimized: states.contains(&a.hidden),
158
+ z_order,
159
+ });
160
+ }
161
+ Ok(out)
162
+ }
163
+
164
+ pub fn enrich_roots(roots: &mut [RootSnapshot], windows: &[WindowInfo]) {
165
+ let mut used = Vec::new();
166
+ for root in roots {
167
+ let best = windows
168
+ .iter()
169
+ .filter(|w| w.pid != 0 && w.pid == root.pid && !used.contains(&w.id))
170
+ .min_by_key(|w| {
171
+ let title = if !root.name.is_empty()
172
+ && (w.title.contains(&root.name) || root.name.contains(&w.title))
173
+ {
174
+ 0
175
+ } else {
176
+ 1_000_000
177
+ };
178
+ title + distance(root.frame.as_ref(), &w.frame)
179
+ });
180
+ if let Some(w) = best {
181
+ used.push(w.id);
182
+ root.x11_window = Some(w.id);
183
+ root.frame = Some(w.frame.clone());
184
+ root.is_focused = w.focused;
185
+ root.is_minimized = w.minimized;
186
+ root.z_order = Some(w.z_order);
187
+ if root.name.is_empty() {
188
+ root.name.clone_from(&w.title);
189
+ }
190
+ }
191
+ }
192
+ }
193
+ fn distance(a: Option<&Rect>, b: &Rect) -> i64 {
194
+ a.map(|a| {
195
+ i64::from(
196
+ (a.x - b.x).abs()
197
+ + (a.y - b.y).abs()
198
+ + (a.width - b.width).abs()
199
+ + (a.height - b.height).abs(),
200
+ )
201
+ })
202
+ .unwrap_or(0)
203
+ }
204
+
205
+ pub fn focus_window(window: Window, policy: PhysicalPolicy) -> Result<Value, ProtocolError> {
206
+ policy.require_physical(SessionKind::detect())?;
207
+ let (conn, screen) = connect()?;
208
+ let root = conn.setup().roots[screen].root;
209
+ let atom = intern(&conn, "_NET_ACTIVE_WINDOW")?;
210
+ let event = ClientMessageEvent {
211
+ response_type: CLIENT_MESSAGE_EVENT,
212
+ format: 32,
213
+ sequence: 0,
214
+ window,
215
+ type_: atom,
216
+ data: ClientMessageData::from([2, 0, 0, 0, 0]),
217
+ };
218
+ conn.send_event(
219
+ false,
220
+ root,
221
+ EventMask::SUBSTRUCTURE_REDIRECT | EventMask::SUBSTRUCTURE_NOTIFY,
222
+ event,
223
+ )
224
+ .map_err(xerr)?
225
+ .check()
226
+ .map_err(xerr)?;
227
+ conn.flush().map_err(xerr)?;
228
+ Ok(json!({"focused":true,"delivery":"ewmh","windowId":window}))
229
+ }
230
+
231
+ pub fn capture_window(
232
+ window: Window,
233
+ max_dimension: Option<u32>,
234
+ ) -> Result<Capture, ProtocolError> {
235
+ let (conn, _) = connect()?;
236
+ let g = conn
237
+ .get_geometry(window)
238
+ .map_err(xerr)?
239
+ .reply()
240
+ .map_err(xerr)?;
241
+ let pixels = u64::from(g.width) * u64::from(g.height);
242
+ if pixels == 0 || pixels > MAX_CAPTURE_PIXELS {
243
+ return Err(err(
244
+ format!(
245
+ "Refusing X11 capture of {}x{} window (64MP limit)",
246
+ g.width, g.height
247
+ ),
248
+ ErrorCode::CaptureFailed,
249
+ ));
250
+ }
251
+ let pixmap = conn.generate_id().map_err(xerr)?;
252
+ let mut warnings = Vec::new();
253
+ let (drawable, source) = match composite::name_window_pixmap(&conn, window, pixmap)
254
+ .ok()
255
+ .and_then(|c| c.check().ok())
256
+ .ok_or(())
257
+ {
258
+ Ok(()) => (pixmap as Drawable, "xcomposite"),
259
+ Err(_) => {
260
+ warnings.push(
261
+ "XComposite unavailable; GetImage fallback may contain stale or obscured pixels"
262
+ .into(),
263
+ );
264
+ (window as Drawable, "get_image")
265
+ }
266
+ };
267
+ let image = conn
268
+ .get_image(
269
+ ImageFormat::Z_PIXMAP,
270
+ drawable,
271
+ 0,
272
+ 0,
273
+ g.width,
274
+ g.height,
275
+ u32::MAX,
276
+ )
277
+ .map_err(xerr)?
278
+ .reply()
279
+ .map_err(|e| {
280
+ err(
281
+ format!("X11 GetImage failed: {e}"),
282
+ ErrorCode::CaptureFailed,
283
+ )
284
+ })?;
285
+ if source == "xcomposite" {
286
+ let _ = conn.free_pixmap(pixmap);
287
+ }
288
+ let rgba = decode(
289
+ &image.data,
290
+ g.width,
291
+ g.height,
292
+ image.depth,
293
+ conn.setup().image_byte_order,
294
+ )?;
295
+ let (output_width, output_height) = scaled_dimensions(g.width, g.height, max_dimension);
296
+ let rgba = resize_rgba(&rgba, g.width, g.height, output_width, output_height);
297
+ let mut encoded = Vec::new();
298
+ {
299
+ let mut encoder =
300
+ png::Encoder::new(&mut encoded, output_width.into(), output_height.into());
301
+ encoder.set_color(png::ColorType::Rgba);
302
+ encoder.set_depth(png::BitDepth::Eight);
303
+ let mut writer = encoder.write_header().map_err(capture_err)?;
304
+ writer.write_image_data(&rgba).map_err(capture_err)?;
305
+ }
306
+ Ok(Capture {
307
+ png_base64: base64::engine::general_purpose::STANDARD.encode(encoded),
308
+ width: output_width,
309
+ height: output_height,
310
+ source,
311
+ warnings,
312
+ })
313
+ }
314
+ fn scaled_dimensions(width: u16, height: u16, max_dimension: Option<u32>) -> (u16, u16) {
315
+ let largest = u32::from(width.max(height));
316
+ let Some(limit) = max_dimension.filter(|limit| *limit > 0 && *limit < largest) else {
317
+ return (width, height);
318
+ };
319
+ let scale = f64::from(limit) / f64::from(largest);
320
+ (
321
+ (f64::from(width) * scale).round().max(1.0) as u16,
322
+ (f64::from(height) * scale).round().max(1.0) as u16,
323
+ )
324
+ }
325
+
326
+ fn resize_rgba(
327
+ source: &[u8],
328
+ source_width: u16,
329
+ source_height: u16,
330
+ width: u16,
331
+ height: u16,
332
+ ) -> Vec<u8> {
333
+ if (source_width, source_height) == (width, height) {
334
+ return source.to_vec();
335
+ }
336
+ let mut output = vec![0; usize::from(width) * usize::from(height) * 4];
337
+ for y in 0..height {
338
+ let source_y = (u32::from(y) * u32::from(source_height) / u32::from(height)) as usize;
339
+ for x in 0..width {
340
+ let source_x = (u32::from(x) * u32::from(source_width) / u32::from(width)) as usize;
341
+ let from = (source_y * usize::from(source_width) + source_x) * 4;
342
+ let to = (usize::from(y) * usize::from(width) + usize::from(x)) * 4;
343
+ output[to..to + 4].copy_from_slice(&source[from..from + 4]);
344
+ }
345
+ }
346
+ output
347
+ }
348
+
349
+ fn decode(
350
+ data: &[u8],
351
+ w: u16,
352
+ h: u16,
353
+ depth: u8,
354
+ order: ImageOrder,
355
+ ) -> Result<Vec<u8>, ProtocolError> {
356
+ let expected = usize::from(w) * usize::from(h) * 4;
357
+ if !matches!(depth, 24 | 32) || data.len() < expected {
358
+ return Err(err(
359
+ format!(
360
+ "Unsupported X11 image layout: depth {depth}, {} bytes",
361
+ data.len()
362
+ ),
363
+ ErrorCode::CaptureFailed,
364
+ ));
365
+ }
366
+ let mut out = Vec::with_capacity(expected);
367
+ for p in data[..expected].chunks_exact(4) {
368
+ let (r, g, b) = if order == ImageOrder::LSB_FIRST {
369
+ (p[2], p[1], p[0])
370
+ } else {
371
+ (p[1], p[2], p[3])
372
+ };
373
+ out.extend_from_slice(&[r, g, b, 255]);
374
+ }
375
+ Ok(out)
376
+ }
377
+
378
+ pub struct Input {
379
+ conn: RustConnection,
380
+ root: Window,
381
+ target: Window,
382
+ }
383
+ impl Input {
384
+ pub fn connect(policy: PhysicalPolicy, target: Window) -> Result<Self, ProtocolError> {
385
+ policy.require_physical(SessionKind::detect())?;
386
+ let (conn, screen) = connect()?;
387
+ let root = conn.setup().roots[screen].root;
388
+ validate_target(&conn, root, target)?;
389
+ prepare_focus(&conn, root, target, policy)?;
390
+ xtest::get_version(&conn, 2, 2)
391
+ .map_err(xerr)?
392
+ .reply()
393
+ .map_err(|e| {
394
+ err(
395
+ format!("XTEST unavailable: {e}"),
396
+ ErrorCode::CapabilityDeferred,
397
+ )
398
+ })?;
399
+ Ok(Self { conn, root, target })
400
+ }
401
+ pub fn move_pointer(&self, x: i32, y: i32) -> Result<(), ProtocolError> {
402
+ self.preflight_point(x, y)?;
403
+ self.fake(MOTION_NOTIFY_EVENT, 0, x, y)
404
+ }
405
+ pub fn click(&self, x: i32, y: i32, button: &str, count: u64) -> Result<(), ProtocolError> {
406
+ let b = button_detail(button)?;
407
+ self.preflight_point(x, y)?;
408
+ self.fake(MOTION_NOTIFY_EVENT, 0, x, y)?;
409
+ for _ in 0..count.clamp(1, 3) {
410
+ self.fake(BUTTON_PRESS_EVENT, b, 0, 0)?;
411
+ self.fake(BUTTON_RELEASE_EVENT, b, 0, 0)?;
412
+ }
413
+ Ok(())
414
+ }
415
+ pub fn scroll(&self, x: i32, y: i32, dx: f64, dy: f64) -> Result<(), ProtocolError> {
416
+ self.preflight_point(x, y)?;
417
+ self.fake(MOTION_NOTIFY_EVENT, 0, x, y)?;
418
+ for _ in 0..dy.abs().ceil().clamp(0., 100.) as usize {
419
+ self.button(if dy < 0. { 4 } else { 5 })?;
420
+ }
421
+ for _ in 0..dx.abs().ceil().clamp(0., 100.) as usize {
422
+ self.button(if dx < 0. { 6 } else { 7 })?;
423
+ }
424
+ Ok(())
425
+ }
426
+ pub fn drag(&self, path: &[(i32, i32)], button: &str) -> Result<(), ProtocolError> {
427
+ if path.len() < 2 {
428
+ return Err(invalid("drag requires at least two points"));
429
+ }
430
+ let b = button_detail(button)?;
431
+ for &(x, y) in path {
432
+ self.preflight_point(x, y)?;
433
+ }
434
+ self.fake(MOTION_NOTIFY_EVENT, 0, path[0].0, path[0].1)?;
435
+ self.fake(BUTTON_PRESS_EVENT, b, 0, 0)?;
436
+ for &(x, y) in &path[1..] {
437
+ self.fake(MOTION_NOTIFY_EVENT, 0, x, y)?;
438
+ }
439
+ self.fake(BUTTON_RELEASE_EVENT, b, 0, 0)
440
+ }
441
+ pub fn type_text(&self, text: &str) -> Result<(), ProtocolError> {
442
+ for ch in text.chars() {
443
+ let (sym, shift) = char_keysym(ch)
444
+ .ok_or_else(|| invalid(format!("Unsupported XTEST character {ch:?}")))?;
445
+ if shift {
446
+ self.key(0xffe1, KEY_PRESS_EVENT)?;
447
+ }
448
+ self.key(sym, KEY_PRESS_EVENT)?;
449
+ self.key(sym, KEY_RELEASE_EVENT)?;
450
+ if shift {
451
+ self.key(0xffe1, KEY_RELEASE_EVENT)?;
452
+ }
453
+ }
454
+ Ok(())
455
+ }
456
+ pub fn keypress(&self, names: &[&str]) -> Result<(), ProtocolError> {
457
+ if names.is_empty() {
458
+ return Err(invalid("keypress requires keys"));
459
+ }
460
+ let syms = names
461
+ .iter()
462
+ .map(|n| named_keysym(n).ok_or_else(|| invalid(format!("Unsupported key '{n}'"))))
463
+ .collect::<Result<Vec<_>, _>>()?;
464
+ for &s in &syms {
465
+ self.key(s, KEY_PRESS_EVENT)?;
466
+ }
467
+ for &s in syms.iter().rev() {
468
+ self.key(s, KEY_RELEASE_EVENT)?;
469
+ }
470
+ Ok(())
471
+ }
472
+ fn preflight_point(&self, x: i32, y: i32) -> Result<(), ProtocolError> {
473
+ validate_target(&self.conn, self.root, self.target)?;
474
+ if active_window(&self.conn, self.root)? != Some(self.target) {
475
+ return Err(err(
476
+ "Refusing XTEST delivery because the owning window is no longer active",
477
+ ErrorCode::ForegroundRequired,
478
+ ));
479
+ }
480
+ let hit = self
481
+ .conn
482
+ .translate_coordinates(self.root, self.root, clamp_i16(x), clamp_i16(y))
483
+ .map_err(xerr)?
484
+ .reply()
485
+ .map_err(xerr)?
486
+ .child;
487
+ if hit == 0 || !windows_related(&self.conn, hit, self.target)? {
488
+ return Err(err(
489
+ "Refusing XTEST pointer delivery because the target point is outside or occluded",
490
+ ErrorCode::CoordinateBlocked,
491
+ ));
492
+ }
493
+ Ok(())
494
+ }
495
+ fn key(&self, sym: u32, event: u8) -> Result<(), ProtocolError> {
496
+ let code = self
497
+ .keycode(sym)?
498
+ .ok_or_else(|| invalid(format!("No keycode for keysym 0x{sym:x}")))?;
499
+ self.fake(event, code, 0, 0)
500
+ }
501
+ fn keycode(&self, sym: u32) -> Result<Option<Keycode>, ProtocolError> {
502
+ let setup = self.conn.setup();
503
+ let min = setup.min_keycode;
504
+ let count = setup.max_keycode - min + 1;
505
+ let m = self
506
+ .conn
507
+ .get_keyboard_mapping(min, count)
508
+ .map_err(xerr)?
509
+ .reply()
510
+ .map_err(xerr)?;
511
+ Ok(m.keysyms
512
+ .chunks(usize::from(m.keysyms_per_keycode))
513
+ .position(|s| s.contains(&sym))
514
+ .map(|i| min + i as u8))
515
+ }
516
+ fn button(&self, b: u8) -> Result<(), ProtocolError> {
517
+ self.fake(BUTTON_PRESS_EVENT, b, 0, 0)?;
518
+ self.fake(BUTTON_RELEASE_EVENT, b, 0, 0)
519
+ }
520
+ fn fake(&self, event: u8, detail: u8, x: i32, y: i32) -> Result<(), ProtocolError> {
521
+ validate_target(&self.conn, self.root, self.target)?;
522
+ if active_window(&self.conn, self.root)? != Some(self.target) {
523
+ return Err(err(
524
+ "Refusing XTEST delivery because the owning window is no longer active",
525
+ ErrorCode::ForegroundRequired,
526
+ ));
527
+ }
528
+ xtest::fake_input(
529
+ &self.conn,
530
+ event,
531
+ detail,
532
+ 0,
533
+ self.root,
534
+ clamp_i16(x),
535
+ clamp_i16(y),
536
+ 0,
537
+ )
538
+ .map_err(xerr)?
539
+ .check()
540
+ .map_err(xerr)?;
541
+ self.conn.flush().map_err(xerr)
542
+ }
543
+ }
544
+
545
+ #[derive(Debug, Clone, Copy, PartialEq, Eq)]
546
+ enum FocusPlan {
547
+ Confirm,
548
+ ActivateAndConfirm,
549
+ }
550
+
551
+ fn focus_plan(policy: PhysicalPolicy, active: bool) -> Result<FocusPlan, ProtocolError> {
552
+ match policy {
553
+ PhysicalPolicy::Foreground => Ok(FocusPlan::ActivateAndConfirm),
554
+ PhysicalPolicy::Default if active => Ok(FocusPlan::Confirm),
555
+ PhysicalPolicy::Default => Err(err(
556
+ "Default policy requires the owning X11 window to already be active",
557
+ ErrorCode::ForegroundRequired,
558
+ )),
559
+ PhysicalPolicy::AxOnly | PhysicalPolicy::Background => {
560
+ policy.require_physical(SessionKind::X11)?;
561
+ unreachable!()
562
+ }
563
+ }
564
+ }
565
+ fn prepare_focus(
566
+ c: &RustConnection,
567
+ root: Window,
568
+ target: Window,
569
+ policy: PhysicalPolicy,
570
+ ) -> Result<(), ProtocolError> {
571
+ let plan = focus_plan(policy, active_window(c, root)? == Some(target))?;
572
+ if plan == FocusPlan::ActivateAndConfirm {
573
+ request_active_window(c, root, target)?;
574
+ }
575
+ for _ in 0..20 {
576
+ if active_window(c, root)? == Some(target) {
577
+ return Ok(());
578
+ }
579
+ if plan == FocusPlan::Confirm {
580
+ break;
581
+ }
582
+ std::thread::sleep(std::time::Duration::from_millis(10));
583
+ }
584
+ Err(err(
585
+ "Window manager did not activate the owning X11 window; refusing XTEST delivery",
586
+ ErrorCode::ForegroundRequired,
587
+ ))
588
+ }
589
+ fn validate_target(c: &RustConnection, root: Window, target: Window) -> Result<(), ProtocolError> {
590
+ if target == 0 || target == root {
591
+ return Err(err(
592
+ "XTEST requires a specific owning X11 window",
593
+ ErrorCode::CoordinateBlocked,
594
+ ));
595
+ }
596
+ let attributes = c
597
+ .get_window_attributes(target)
598
+ .map_err(xerr)?
599
+ .reply()
600
+ .map_err(|_| {
601
+ err(
602
+ "Owning X11 window no longer exists",
603
+ ErrorCode::TargetNotFound,
604
+ )
605
+ })?;
606
+ let geometry = c.get_geometry(target).map_err(xerr)?.reply().map_err(|_| {
607
+ err(
608
+ "Owning X11 window has no usable geometry",
609
+ ErrorCode::CoordinateBlocked,
610
+ )
611
+ })?;
612
+ let atoms = Atoms::new(c)?;
613
+ if attributes.map_state != MapState::VIEWABLE
614
+ || geometry.width == 0
615
+ || geometry.height == 0
616
+ || property32(c, target, atoms.state, AtomEnum::ATOM.into()).contains(&atoms.hidden)
617
+ {
618
+ return Err(err(
619
+ "Owning X11 window is not mapped and visible",
620
+ ErrorCode::CoordinateBlocked,
621
+ ));
622
+ }
623
+ Ok(())
624
+ }
625
+ fn active_window(c: &RustConnection, root: Window) -> Result<Option<Window>, ProtocolError> {
626
+ let atom = intern(c, "_NET_ACTIVE_WINDOW")?;
627
+ Ok(property32(c, root, atom, AtomEnum::WINDOW.into())
628
+ .first()
629
+ .copied())
630
+ }
631
+ fn request_active_window(
632
+ c: &RustConnection,
633
+ root: Window,
634
+ target: Window,
635
+ ) -> Result<(), ProtocolError> {
636
+ let atom = intern(c, "_NET_ACTIVE_WINDOW")?;
637
+ let event = ClientMessageEvent {
638
+ response_type: CLIENT_MESSAGE_EVENT,
639
+ format: 32,
640
+ sequence: 0,
641
+ window: target,
642
+ type_: atom,
643
+ data: ClientMessageData::from([2, 0, 0, 0, 0]),
644
+ };
645
+ c.send_event(
646
+ false,
647
+ root,
648
+ EventMask::SUBSTRUCTURE_REDIRECT | EventMask::SUBSTRUCTURE_NOTIFY,
649
+ event,
650
+ )
651
+ .map_err(xerr)?
652
+ .check()
653
+ .map_err(xerr)?;
654
+ c.flush().map_err(xerr)
655
+ }
656
+ fn windows_related(c: &RustConnection, hit: Window, target: Window) -> Result<bool, ProtocolError> {
657
+ fn ancestors(c: &RustConnection, mut w: Window) -> Result<Vec<Window>, ProtocolError> {
658
+ let mut out = vec![w];
659
+ for _ in 0..64 {
660
+ let tree = c.query_tree(w).map_err(xerr)?.reply().map_err(xerr)?;
661
+ if tree.parent == 0 || tree.parent == w {
662
+ break;
663
+ }
664
+ w = tree.parent;
665
+ out.push(w);
666
+ }
667
+ Ok(out)
668
+ }
669
+ Ok(ancestors(c, hit)?.contains(&target) || ancestors(c, target)?.contains(&hit))
670
+ }
671
+ fn clamp_i16(value: i32) -> i16 {
672
+ value.clamp(i16::MIN.into(), i16::MAX.into()) as i16
673
+ }
674
+
675
+ fn button_detail(b: &str) -> Result<u8, ProtocolError> {
676
+ match b {
677
+ "left" => Ok(ButtonIndex::M1.into()),
678
+ "middle" => Ok(ButtonIndex::M2.into()),
679
+ "right" => Ok(ButtonIndex::M3.into()),
680
+ _ => Err(invalid(format!("Unsupported mouse button '{b}'"))),
681
+ }
682
+ }
683
+ fn char_keysym(c: char) -> Option<(u32, bool)> {
684
+ if c == '\n' {
685
+ return Some((0xff0d, false));
686
+ }
687
+ if c == '\t' {
688
+ return Some((0xff09, false));
689
+ }
690
+ if c == ' ' || c.is_ascii_lowercase() || c.is_ascii_digit() {
691
+ return Some((c as u32, false));
692
+ }
693
+ if c.is_ascii_uppercase() {
694
+ return Some((c.to_ascii_lowercase() as u32, true));
695
+ }
696
+ let shifted = "~!@#$%^&*()_+{}|:\"<>?";
697
+ let base = "`1234567890-=[]\\;',./";
698
+ shifted
699
+ .chars()
700
+ .position(|v| v == c)
701
+ .and_then(|i| base.chars().nth(i))
702
+ .map(|v| (v as u32, true))
703
+ }
704
+ fn named_keysym(n: &str) -> Option<u32> {
705
+ match n.to_ascii_lowercase().as_str() {
706
+ "enter" | "return" => Some(0xff0d),
707
+ "tab" => Some(0xff09),
708
+ "escape" | "esc" => Some(0xff1b),
709
+ "backspace" => Some(0xff08),
710
+ "delete" => Some(0xffff),
711
+ "space" => Some(0x20),
712
+ "left" => Some(0xff51),
713
+ "up" => Some(0xff52),
714
+ "right" => Some(0xff53),
715
+ "down" => Some(0xff54),
716
+ "home" => Some(0xff50),
717
+ "end" => Some(0xff57),
718
+ "pageup" => Some(0xff55),
719
+ "pagedown" => Some(0xff56),
720
+ "ctrl" | "control" => Some(0xffe3),
721
+ "shift" => Some(0xffe1),
722
+ "alt" | "option" => Some(0xffe9),
723
+ "meta" | "super" | "cmd" | "command" => Some(0xffeb),
724
+ v if v.len() == 1 => v.chars().next().map(|c| c as u32),
725
+ v if v.starts_with('f') => v[1..]
726
+ .parse::<u32>()
727
+ .ok()
728
+ .filter(|n| (1..=35).contains(n))
729
+ .map(|n| 0xffbd + n),
730
+ _ => None,
731
+ }
732
+ }
733
+
734
+ struct Atoms {
735
+ clients: Atom,
736
+ stacking: Atom,
737
+ active: Atom,
738
+ pid: Atom,
739
+ name: Atom,
740
+ state: Atom,
741
+ hidden: Atom,
742
+ utf8: Atom,
743
+ }
744
+ impl Atoms {
745
+ fn new(c: &RustConnection) -> Result<Self, ProtocolError> {
746
+ Ok(Self {
747
+ clients: intern(c, "_NET_CLIENT_LIST")?,
748
+ stacking: intern(c, "_NET_CLIENT_LIST_STACKING")?,
749
+ active: intern(c, "_NET_ACTIVE_WINDOW")?,
750
+ pid: intern(c, "_NET_WM_PID")?,
751
+ name: intern(c, "_NET_WM_NAME")?,
752
+ state: intern(c, "_NET_WM_STATE")?,
753
+ hidden: intern(c, "_NET_WM_STATE_HIDDEN")?,
754
+ utf8: intern(c, "UTF8_STRING")?,
755
+ })
756
+ }
757
+ }
758
+ fn connect() -> Result<(RustConnection, usize), ProtocolError> {
759
+ x11rb::connect(None).map_err(|e| {
760
+ err(
761
+ format!("X11 unavailable: {e}"),
762
+ ErrorCode::CapabilityDeferred,
763
+ )
764
+ })
765
+ }
766
+ fn intern(c: &RustConnection, n: &str) -> Result<Atom, ProtocolError> {
767
+ c.intern_atom(false, n.as_bytes())
768
+ .map_err(xerr)?
769
+ .reply()
770
+ .map(|r| r.atom)
771
+ .map_err(xerr)
772
+ }
773
+ fn property32(c: &RustConnection, w: Window, p: Atom, t: Atom) -> Vec<u32> {
774
+ c.get_property(false, w, p, t, 0, u32::MAX)
775
+ .ok()
776
+ .and_then(|v| v.reply().ok())
777
+ .and_then(|v| v.value32().map(Iterator::collect))
778
+ .unwrap_or_default()
779
+ }
780
+ fn title(c: &RustConnection, w: Window, a: &Atoms) -> String {
781
+ let utf = c
782
+ .get_property(false, w, a.name, a.utf8, 0, 4096)
783
+ .ok()
784
+ .and_then(|v| v.reply().ok())
785
+ .and_then(|v| String::from_utf8(v.value).ok())
786
+ .unwrap_or_default();
787
+ if !utf.is_empty() {
788
+ return utf;
789
+ }
790
+ c.get_property(false, w, AtomEnum::WM_NAME, AtomEnum::STRING, 0, 4096)
791
+ .ok()
792
+ .and_then(|v| v.reply().ok())
793
+ .map(|v| String::from_utf8_lossy(&v.value).into_owned())
794
+ .unwrap_or_default()
795
+ }
796
+ fn err(m: impl Into<String>, code: ErrorCode) -> ProtocolError {
797
+ ProtocolError::new(m, code)
798
+ }
799
+ fn invalid(m: impl Into<String>) -> ProtocolError {
800
+ err(m, ErrorCode::InvalidRequest)
801
+ }
802
+ fn xerr(e: impl std::fmt::Display) -> ProtocolError {
803
+ err(format!("X11 request failed: {e}"), ErrorCode::InternalError)
804
+ }
805
+ fn capture_err(e: impl std::fmt::Display) -> ProtocolError {
806
+ err(
807
+ format!("PNG encoding failed: {e}"),
808
+ ErrorCode::CaptureFailed,
809
+ )
810
+ }
811
+
812
+ #[cfg(test)]
813
+ mod tests {
814
+ use super::*;
815
+ #[test]
816
+ fn policy_guards() {
817
+ assert_eq!(
818
+ PhysicalPolicy::AxOnly
819
+ .require_physical(SessionKind::X11)
820
+ .unwrap_err()
821
+ .code,
822
+ ErrorCode::CoordinateBlocked
823
+ );
824
+ assert_eq!(
825
+ PhysicalPolicy::Background
826
+ .require_physical(SessionKind::X11)
827
+ .unwrap_err()
828
+ .code,
829
+ ErrorCode::ForegroundRequired
830
+ );
831
+ assert!(PhysicalPolicy::Foreground
832
+ .require_physical(SessionKind::X11)
833
+ .is_ok());
834
+ assert!(PhysicalPolicy::Default
835
+ .require_physical(SessionKind::Headless)
836
+ .is_err());
837
+ }
838
+ #[test]
839
+ fn focus_policy_is_deterministic() {
840
+ assert_eq!(
841
+ focus_plan(PhysicalPolicy::Foreground, false).unwrap(),
842
+ FocusPlan::ActivateAndConfirm
843
+ );
844
+ assert_eq!(
845
+ focus_plan(PhysicalPolicy::Foreground, true).unwrap(),
846
+ FocusPlan::ActivateAndConfirm
847
+ );
848
+ assert_eq!(
849
+ focus_plan(PhysicalPolicy::Default, true).unwrap(),
850
+ FocusPlan::Confirm
851
+ );
852
+ assert_eq!(
853
+ focus_plan(PhysicalPolicy::Default, false).unwrap_err().code,
854
+ ErrorCode::ForegroundRequired
855
+ );
856
+ assert_eq!(
857
+ focus_plan(PhysicalPolicy::Background, true)
858
+ .unwrap_err()
859
+ .code,
860
+ ErrorCode::ForegroundRequired
861
+ );
862
+ }
863
+ #[test]
864
+ fn coordinate_clamping_is_deterministic() {
865
+ assert_eq!(clamp_i16(i32::MIN), i16::MIN);
866
+ assert_eq!(clamp_i16(i32::MAX), i16::MAX);
867
+ assert_eq!(clamp_i16(42), 42);
868
+ }
869
+ #[test]
870
+ fn geometry_is_deterministic() {
871
+ let a = Rect {
872
+ x: 1,
873
+ y: 2,
874
+ width: 3,
875
+ height: 4,
876
+ };
877
+ assert_eq!(distance(Some(&a), &a), 0);
878
+ assert_eq!(distance(Some(&a), &Rect { x: 6, ..a.clone() }), 5);
879
+ }
880
+ #[test]
881
+ fn key_protocol() {
882
+ assert_eq!(named_keysym("Control"), Some(0xffe3));
883
+ assert_eq!(named_keysym("F12"), Some(0xffc9));
884
+ assert_eq!(char_keysym('A'), Some(('a' as u32, true)));
885
+ assert_eq!(char_keysym('!'), Some(('1' as u32, true)));
886
+ assert_eq!(char_keysym('é'), None);
887
+ }
888
+ #[test]
889
+ fn bounded_capture_dimensions_preserve_aspect_ratio() {
890
+ assert_eq!(scaled_dimensions(1920, 1080, Some(1000)), (1000, 563));
891
+ assert_eq!(scaled_dimensions(800, 600, Some(1000)), (800, 600));
892
+ assert_eq!(scaled_dimensions(1, 4000, Some(100)), (1, 100));
893
+ assert_eq!(scaled_dimensions(4000, 1, Some(100)), (100, 1));
894
+ }
895
+ #[test]
896
+ fn rgba_resize_is_bounded_and_deterministic() {
897
+ let source = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16];
898
+ assert_eq!(resize_rgba(&source, 2, 2, 1, 1), vec![1, 2, 3, 4]);
899
+ assert_eq!(resize_rgba(&source, 2, 2, 2, 2), source);
900
+ }
901
+ #[test]
902
+ fn decode_limits() {
903
+ assert_eq!(
904
+ decode(&[1, 2, 3, 0], 1, 1, 24, ImageOrder::LSB_FIRST).unwrap(),
905
+ vec![3, 2, 1, 255]
906
+ );
907
+ assert!(decode(&[0; 2], 1, 1, 16, ImageOrder::LSB_FIRST).is_err());
908
+ }
909
+ }