@forsion/tangu-computer-use 0.5.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +223 -0
- package/LICENSE +26 -0
- package/LICENSE.upstream +21 -0
- package/README.md +169 -0
- package/UPSTREAM.md +277 -0
- package/check.mjs +369 -0
- package/icon.png +0 -0
- package/install.sh +112 -0
- package/main.js +541 -0
- package/manifest.json +47 -0
- package/native/linux/bridge-rs/Cargo.lock +1204 -0
- package/native/linux/bridge-rs/Cargo.toml +18 -0
- package/native/linux/bridge-rs/src/atspi.rs +640 -0
- package/native/linux/bridge-rs/src/error.rs +65 -0
- package/native/linux/bridge-rs/src/lib.rs +11 -0
- package/native/linux/bridge-rs/src/main.rs +1074 -0
- package/native/linux/bridge-rs/src/protocol.rs +49 -0
- package/native/linux/bridge-rs/src/state.rs +293 -0
- package/native/linux/bridge-rs/src/wayland.rs +89 -0
- package/native/linux/bridge-rs/src/x11.rs +909 -0
- package/native/linux/bridge-rs/tests/protocol_tests.rs +73 -0
- package/native/macos/agent_cursor.swift +184 -0
- package/native/macos/agent_cursor_motion.swift +252 -0
- package/native/macos/agent_cursor_tests.swift +128 -0
- package/native/macos/agent_highlight.swift +229 -0
- package/native/macos/agent_highlight_tests.swift +99 -0
- package/native/macos/bridge.swift +3852 -0
- package/native/macos/foreground_activity.swift +57 -0
- package/native/macos/foreground_activity_tests.swift +31 -0
- package/native/macos/live_stream.swift +289 -0
- package/native/windows/bridge-rs/Cargo.lock +396 -0
- package/native/windows/bridge-rs/Cargo.toml +30 -0
- package/native/windows/bridge-rs/src/capture.rs +518 -0
- package/native/windows/bridge-rs/src/error.rs +81 -0
- package/native/windows/bridge-rs/src/input.rs +501 -0
- package/native/windows/bridge-rs/src/lib.rs +16 -0
- package/native/windows/bridge-rs/src/main.rs +1683 -0
- package/native/windows/bridge-rs/src/protocol.rs +53 -0
- package/native/windows/bridge-rs/src/refs.rs +237 -0
- package/native/windows/bridge-rs/src/state.rs +55 -0
- package/native/windows/bridge-rs/src/uia.rs +1252 -0
- package/native/windows/bridge-rs/src/window.rs +718 -0
- package/native/windows/bridge-rs/tests/protocol_tests.rs +252 -0
- package/native/windows/bridge-rs/tests/refs_tests.rs +86 -0
- package/native/windows/bridge-rs/tests/state_tests.rs +33 -0
- package/package.json +74 -0
- package/prebuilt/linux/arm64/linux-bridge +0 -0
- package/prebuilt/linux/x64/linux-bridge +0 -0
- package/prebuilt/macos/arm64/bridge +0 -0
- package/prebuilt/macos/arm64/tangu-computer-use.app.json +10 -0
- package/prebuilt/macos/arm64/tangu-computer-use.app.zip +0 -0
- package/prebuilt/macos/x64/bridge +0 -0
- package/prebuilt/macos/x64/tangu-computer-use.app.json +10 -0
- package/prebuilt/macos/x64/tangu-computer-use.app.zip +0 -0
- package/prebuilt/windows/windows-bridge.exe +0 -0
- package/scripts/blind-click.check.mjs +128 -0
- package/scripts/build-native.mjs +302 -0
- package/scripts/build.mjs +84 -0
- package/scripts/calc-fixture.mjs +78 -0
- package/scripts/cli-exit.check.mjs +49 -0
- package/scripts/helper-path.check.mjs +82 -0
- package/scripts/helper-refresh.check.mjs +62 -0
- package/scripts/helper-signal.check.mjs +59 -0
- package/scripts/highlight.check.mjs +32 -0
- package/scripts/keychain-free-install.check.mjs +140 -0
- package/scripts/live-view.check.mjs +124 -0
- package/scripts/macos-bundle.d.mts +1 -0
- package/scripts/macos-bundle.mjs +119 -0
- package/scripts/make-signing-cert.sh +57 -0
- package/scripts/mini-foreground.check.mjs +11 -0
- package/scripts/no-foreground.check.mjs +81 -0
- package/scripts/overlay-visible.check.mjs +221 -0
- package/scripts/package-macos-app.mjs +39 -0
- package/scripts/permissions.check.mjs +199 -0
- package/scripts/platform-contract.check.mjs +30 -0
- package/scripts/setup-helper.mjs +154 -0
- package/scripts/tangu-computer-use.entitlements +10 -0
- package/scripts/verify-macos-bundles.mjs +25 -0
- package/scripts/verify-package.mjs +74 -0
- package/skills/computer-use/SKILL.md +110 -0
- package/src/foregroundNote.ts +43 -0
- package/src/helperState.ts +31 -0
- package/src/index.ts +63 -0
- package/src/onboarding.ts +198 -0
- package/src/pi-compat.ts +49 -0
- package/src/settings.ts +11 -0
- package/src/setup.ts +97 -0
- package/src/tools.ts +289 -0
- package/src/vendor/actions.ts +130 -0
- package/src/vendor/bridge.ts +2405 -0
- package/src/vendor/cdp.ts +658 -0
- package/src/vendor/config.ts +113 -0
- package/src/vendor/contract.ts +104 -0
- package/src/vendor/note.ts +195 -0
- package/src/vendor/outline.ts +651 -0
- package/src/vendor/output.ts +134 -0
- package/src/vendor/permissions.ts +111 -0
- package/src/vendor/platform/architecture.ts +23 -0
- package/src/vendor/platform/coerce.ts +16 -0
- package/src/vendor/platform/index.ts +59 -0
- package/src/vendor/platform/linux/backend.ts +186 -0
- package/src/vendor/platform/linux/helper.ts +238 -0
- package/src/vendor/platform/macos/backend.ts +131 -0
- package/src/vendor/platform/macos/browser.ts +110 -0
- package/src/vendor/platform/macos/helper-path.d.mts +9 -0
- package/src/vendor/platform/macos/helper-path.mjs +34 -0
- package/src/vendor/platform/macos/helper.ts +291 -0
- package/src/vendor/platform/macos/permissions.ts +146 -0
- package/src/vendor/platform/types.ts +222 -0
- package/src/vendor/platform/windows/backend.ts +142 -0
- package/src/vendor/platform/windows/helper.ts +140 -0
- package/src/vendor/root-selection.ts +25 -0
- package/src/vendor/runtime.ts +129 -0
- package/src/vendor/state.ts +146 -0
- package/src/vendor/view.ts +147 -0
- package/tangu-plugins/computer-use/dist/foregroundNote.js +19 -0
- package/tangu-plugins/computer-use/dist/index.js +5672 -0
- package/tangu-plugins/computer-use/tangu-plugin.json +9 -0
- package/tsconfig.json +22 -0
- package/types/tangu-agent.d.ts +142 -0
|
@@ -0,0 +1,718 @@
|
|
|
1
|
+
//! Window discovery on Windows.
|
|
2
|
+
//!
|
|
3
|
+
//! On Windows this module enumerates top-level visible windows, extracts
|
|
4
|
+
//! metadata (title, PID, process name, bounds, focus state), classifies
|
|
5
|
+
//! known browsers, and returns the results with window refs scoped to a
|
|
6
|
+
//! fresh state ID.
|
|
7
|
+
//!
|
|
8
|
+
//! On non-Windows platforms all entry points return a deterministic
|
|
9
|
+
//! `unsupported_platform` error.
|
|
10
|
+
|
|
11
|
+
use serde_json::Value;
|
|
12
|
+
#[cfg(windows)]
|
|
13
|
+
use std::collections::{HashMap, VecDeque};
|
|
14
|
+
#[cfg(windows)]
|
|
15
|
+
use std::sync::atomic::{AtomicU64, Ordering};
|
|
16
|
+
#[cfg(windows)]
|
|
17
|
+
use std::sync::{Mutex, OnceLock};
|
|
18
|
+
|
|
19
|
+
use crate::error::{ErrorCode, ProtocolError};
|
|
20
|
+
use crate::refs::RefStore;
|
|
21
|
+
|
|
22
|
+
#[cfg(windows)]
|
|
23
|
+
use crate::refs::NativeHandle;
|
|
24
|
+
#[cfg(windows)]
|
|
25
|
+
use crate::state::StateId;
|
|
26
|
+
#[cfg(windows)]
|
|
27
|
+
use serde_json::json;
|
|
28
|
+
|
|
29
|
+
// ---------------------------------------------------------------------------
|
|
30
|
+
// Browser classification (cross-platform)
|
|
31
|
+
// ---------------------------------------------------------------------------
|
|
32
|
+
|
|
33
|
+
/// Classify a process name as one of the known browser families.
|
|
34
|
+
///
|
|
35
|
+
/// Returns `(is_browser, browser_family)` where `browser_family` is one of
|
|
36
|
+
/// `"chrome"`, `"edge"`, `"brave"`, or `None`.
|
|
37
|
+
///
|
|
38
|
+
/// Matching is case-insensitive and strips the `.exe` suffix when present.
|
|
39
|
+
pub fn classify_browser(process_name: &str) -> (bool, Option<&'static str>) {
|
|
40
|
+
// Example inputs: "chrome.exe", "msedge.exe", "NOTEPAD.EXE", "firefox"
|
|
41
|
+
let lower = process_name.to_lowercase();
|
|
42
|
+
let stem = lower.strip_suffix(".exe").unwrap_or(&lower);
|
|
43
|
+
match stem {
|
|
44
|
+
"chrome" | "chromium" => (true, Some("chrome")),
|
|
45
|
+
"msedge" | "edge" => (true, Some("edge")),
|
|
46
|
+
"brave" | "brave-browser" => (true, Some("brave")),
|
|
47
|
+
"firefox" => (true, Some("firefox")),
|
|
48
|
+
"vivaldi" => (true, Some("vivaldi")),
|
|
49
|
+
"opera" | "opera_gx" => (true, Some("opera")),
|
|
50
|
+
_ => (false, None),
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
// ---------------------------------------------------------------------------
|
|
55
|
+
// Window enumeration
|
|
56
|
+
// ---------------------------------------------------------------------------
|
|
57
|
+
|
|
58
|
+
/// Enumerate visible top-level windows and return a JSON response value.
|
|
59
|
+
///
|
|
60
|
+
/// On non-Windows this always returns `UnsupportedPlatform`.
|
|
61
|
+
/// On Windows it enumerates visible top-level windows, inserts their native
|
|
62
|
+
/// handles into `store`, and returns a fresh stateId along with per-window
|
|
63
|
+
/// metadata.
|
|
64
|
+
///
|
|
65
|
+
/// If `filter_pid` is `Some(pid)`, only windows belonging to the given
|
|
66
|
+
/// process are returned. When `filter_pid` is `None`, all visible windows
|
|
67
|
+
/// are enumerated (used by the TypeScript `listApps` fallback).
|
|
68
|
+
pub fn list_windows(store: &mut RefStore, filter_pid: Option<u64>) -> Result<Value, ProtocolError> {
|
|
69
|
+
#[cfg(not(windows))]
|
|
70
|
+
{
|
|
71
|
+
let _ = store; // unused on non-Windows
|
|
72
|
+
let _ = filter_pid;
|
|
73
|
+
Err(ProtocolError::new(
|
|
74
|
+
"Window discovery is only supported on Windows",
|
|
75
|
+
ErrorCode::UnsupportedPlatform,
|
|
76
|
+
))
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
#[cfg(windows)]
|
|
80
|
+
{
|
|
81
|
+
list_windows_impl(store, filter_pid)
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
// ---------------------------------------------------------------------------
|
|
86
|
+
// Windows-specific implementation
|
|
87
|
+
// ---------------------------------------------------------------------------
|
|
88
|
+
|
|
89
|
+
#[cfg(windows)]
|
|
90
|
+
use std::{thread, time::Duration};
|
|
91
|
+
#[cfg(windows)]
|
|
92
|
+
use windows::core::PWSTR;
|
|
93
|
+
#[cfg(windows)]
|
|
94
|
+
use windows::Win32::Foundation::{CloseHandle, BOOL, HWND, LPARAM, RECT, TRUE};
|
|
95
|
+
#[cfg(windows)]
|
|
96
|
+
use windows::Win32::System::Threading::{
|
|
97
|
+
OpenProcess, QueryFullProcessImageNameW, PROCESS_QUERY_LIMITED_INFORMATION,
|
|
98
|
+
};
|
|
99
|
+
#[cfg(windows)]
|
|
100
|
+
use windows::Win32::UI::Accessibility::{SetWinEventHook, HWINEVENTHOOK};
|
|
101
|
+
#[cfg(windows)]
|
|
102
|
+
use windows::Win32::UI::HiDpi::GetDpiForWindow;
|
|
103
|
+
#[cfg(windows)]
|
|
104
|
+
use windows::Win32::UI::WindowsAndMessaging::{
|
|
105
|
+
EnumWindows, GetAncestor, GetClassNameW, GetForegroundWindow, GetMessageW, GetWindow,
|
|
106
|
+
GetWindowLongPtrW, GetWindowRect, GetWindowTextW, GetWindowThreadProcessId, IsIconic,
|
|
107
|
+
IsWindowVisible, SetForegroundWindow, EVENT_OBJECT_CREATE, EVENT_OBJECT_DESTROY,
|
|
108
|
+
EVENT_OBJECT_HIDE, EVENT_SYSTEM_FOREGROUND, GA_ROOT, GWL_EXSTYLE, GW_OWNER, MSG,
|
|
109
|
+
WINEVENT_OUTOFCONTEXT, WS_EX_DLGMODALFRAME,
|
|
110
|
+
};
|
|
111
|
+
|
|
112
|
+
#[cfg(windows)]
|
|
113
|
+
const ROOT_EVENT_CAPACITY: usize = 256;
|
|
114
|
+
|
|
115
|
+
#[cfg(windows)]
|
|
116
|
+
#[derive(Clone)]
|
|
117
|
+
struct JournalEvent {
|
|
118
|
+
sequence: u64,
|
|
119
|
+
pid: u64,
|
|
120
|
+
payload: Value,
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
#[cfg(windows)]
|
|
124
|
+
#[derive(Default)]
|
|
125
|
+
struct RootEventJournal {
|
|
126
|
+
events: VecDeque<JournalEvent>,
|
|
127
|
+
roots: HashMap<isize, Value>,
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
#[cfg(windows)]
|
|
131
|
+
fn event_journal() -> &'static Mutex<RootEventJournal> {
|
|
132
|
+
static JOURNAL: OnceLock<Mutex<RootEventJournal>> = OnceLock::new();
|
|
133
|
+
JOURNAL.get_or_init(|| Mutex::new(RootEventJournal::default()))
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
#[cfg(windows)]
|
|
137
|
+
fn event_sequence() -> &'static AtomicU64 {
|
|
138
|
+
static SEQUENCE: AtomicU64 = AtomicU64::new(0);
|
|
139
|
+
&SEQUENCE
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/// Start the WinEvent message pump used as an acceleration and transient-root
|
|
143
|
+
/// journal. Final root snapshots remain authoritative.
|
|
144
|
+
pub fn start_root_event_journal() {
|
|
145
|
+
#[cfg(windows)]
|
|
146
|
+
std::thread::spawn(|| unsafe {
|
|
147
|
+
let foreground = SetWinEventHook(
|
|
148
|
+
EVENT_SYSTEM_FOREGROUND,
|
|
149
|
+
EVENT_SYSTEM_FOREGROUND,
|
|
150
|
+
None,
|
|
151
|
+
Some(root_event_callback),
|
|
152
|
+
0,
|
|
153
|
+
0,
|
|
154
|
+
WINEVENT_OUTOFCONTEXT,
|
|
155
|
+
);
|
|
156
|
+
let objects = SetWinEventHook(
|
|
157
|
+
EVENT_OBJECT_CREATE,
|
|
158
|
+
EVENT_OBJECT_HIDE,
|
|
159
|
+
None,
|
|
160
|
+
Some(root_event_callback),
|
|
161
|
+
0,
|
|
162
|
+
0,
|
|
163
|
+
WINEVENT_OUTOFCONTEXT,
|
|
164
|
+
);
|
|
165
|
+
if foreground.is_invalid() && objects.is_invalid() {
|
|
166
|
+
return;
|
|
167
|
+
}
|
|
168
|
+
let mut message = MSG::default();
|
|
169
|
+
while GetMessageW(&mut message, None, 0, 0).as_bool() {}
|
|
170
|
+
});
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
pub fn root_event_cursor() -> u64 {
|
|
174
|
+
#[cfg(windows)]
|
|
175
|
+
{
|
|
176
|
+
event_sequence().load(Ordering::Acquire)
|
|
177
|
+
}
|
|
178
|
+
#[cfg(not(windows))]
|
|
179
|
+
{
|
|
180
|
+
0
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
pub fn root_events_since(cursor: u64, pid: u64) -> Vec<Value> {
|
|
185
|
+
#[cfg(windows)]
|
|
186
|
+
{
|
|
187
|
+
event_journal()
|
|
188
|
+
.lock()
|
|
189
|
+
.map(|journal| {
|
|
190
|
+
journal
|
|
191
|
+
.events
|
|
192
|
+
.iter()
|
|
193
|
+
.filter(|event| event.sequence > cursor && event.pid == pid)
|
|
194
|
+
.map(|event| event.payload.clone())
|
|
195
|
+
.collect()
|
|
196
|
+
})
|
|
197
|
+
.unwrap_or_default()
|
|
198
|
+
}
|
|
199
|
+
#[cfg(not(windows))]
|
|
200
|
+
{
|
|
201
|
+
let _ = (cursor, pid);
|
|
202
|
+
Vec::new()
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
#[cfg(windows)]
|
|
207
|
+
unsafe extern "system" fn root_event_callback(
|
|
208
|
+
_hook: HWINEVENTHOOK,
|
|
209
|
+
event: u32,
|
|
210
|
+
hwnd: HWND,
|
|
211
|
+
object_id: i32,
|
|
212
|
+
child_id: i32,
|
|
213
|
+
_thread: u32,
|
|
214
|
+
_time: u32,
|
|
215
|
+
) {
|
|
216
|
+
if hwnd.0.is_null() || (event != EVENT_SYSTEM_FOREGROUND && (object_id != 0 || child_id != 0)) {
|
|
217
|
+
return;
|
|
218
|
+
}
|
|
219
|
+
if event != EVENT_OBJECT_DESTROY && GetAncestor(hwnd, GA_ROOT) != hwnd {
|
|
220
|
+
return;
|
|
221
|
+
}
|
|
222
|
+
let mut pid = 0u32;
|
|
223
|
+
GetWindowThreadProcessId(hwnd, Some(&mut pid));
|
|
224
|
+
if pid == 0 {
|
|
225
|
+
return;
|
|
226
|
+
}
|
|
227
|
+
let raw = hwnd.0 as isize;
|
|
228
|
+
let mut journal = match event_journal().lock() {
|
|
229
|
+
Ok(journal) => journal,
|
|
230
|
+
Err(_) => return,
|
|
231
|
+
};
|
|
232
|
+
let change = match event {
|
|
233
|
+
EVENT_OBJECT_DESTROY | EVENT_OBJECT_HIDE => "closed",
|
|
234
|
+
EVENT_SYSTEM_FOREGROUND => "focused",
|
|
235
|
+
_ => "appeared",
|
|
236
|
+
};
|
|
237
|
+
let root = if change == "closed" {
|
|
238
|
+
journal.roots.remove(&raw).unwrap_or_else(|| {
|
|
239
|
+
json!({
|
|
240
|
+
"kind": "window", "title": "", "pid": pid, "windowId": raw
|
|
241
|
+
})
|
|
242
|
+
})
|
|
243
|
+
} else {
|
|
244
|
+
let root = event_root_payload(hwnd, pid);
|
|
245
|
+
journal.roots.insert(raw, root.clone());
|
|
246
|
+
root
|
|
247
|
+
};
|
|
248
|
+
let sequence = event_sequence().fetch_add(1, Ordering::AcqRel) + 1;
|
|
249
|
+
let payload = json!({
|
|
250
|
+
"change": change,
|
|
251
|
+
"kind": root.get("kind").and_then(Value::as_str).unwrap_or("window"),
|
|
252
|
+
"title": root.get("title").and_then(Value::as_str).unwrap_or(""),
|
|
253
|
+
"pid": pid,
|
|
254
|
+
"isModal": root.get("isModal").and_then(Value::as_bool).unwrap_or(false),
|
|
255
|
+
"metadata": root.get("metadata").cloned().unwrap_or_else(|| json!({}))
|
|
256
|
+
});
|
|
257
|
+
journal.events.push_back(JournalEvent {
|
|
258
|
+
sequence,
|
|
259
|
+
pid: u64::from(pid),
|
|
260
|
+
payload,
|
|
261
|
+
});
|
|
262
|
+
while journal.events.len() > ROOT_EVENT_CAPACITY {
|
|
263
|
+
journal.events.pop_front();
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
#[cfg(windows)]
|
|
268
|
+
unsafe fn event_root_payload(hwnd: HWND, pid: u32) -> Value {
|
|
269
|
+
let class_name = get_window_class(hwnd);
|
|
270
|
+
let title = get_window_title(hwnd);
|
|
271
|
+
let owner = GetWindow(hwnd, GW_OWNER).ok();
|
|
272
|
+
let kind = if class_name == "#32768" {
|
|
273
|
+
"menu"
|
|
274
|
+
} else if class_name == "#32770" {
|
|
275
|
+
"dialog"
|
|
276
|
+
} else if owner.map(|owner| !owner.0.is_null()).unwrap_or(false) {
|
|
277
|
+
"popover"
|
|
278
|
+
} else {
|
|
279
|
+
"window"
|
|
280
|
+
};
|
|
281
|
+
let ex_style = GetWindowLongPtrW(hwnd, GWL_EXSTYLE) as u32;
|
|
282
|
+
json!({
|
|
283
|
+
"kind": kind,
|
|
284
|
+
"title": title,
|
|
285
|
+
"pid": pid,
|
|
286
|
+
"windowId": hwnd.0 as isize,
|
|
287
|
+
"isModal": class_name == "#32770" || (ex_style & WS_EX_DLGMODALFRAME.0) != 0,
|
|
288
|
+
"metadata": { "className": class_name, "exStyle": ex_style }
|
|
289
|
+
})
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
#[cfg(windows)]
|
|
293
|
+
unsafe extern "system" fn enum_windows_proc(hwnd: HWND, lparam: LPARAM) -> BOOL {
|
|
294
|
+
let handles = &mut *(lparam.0 as *mut Vec<HWND>);
|
|
295
|
+
handles.push(hwnd);
|
|
296
|
+
TRUE
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
#[cfg(windows)]
|
|
300
|
+
unsafe fn get_window_title(hwnd: HWND) -> String {
|
|
301
|
+
let mut buf = [0u16; 512];
|
|
302
|
+
let len = GetWindowTextW(hwnd, &mut buf);
|
|
303
|
+
if len > 0 {
|
|
304
|
+
String::from_utf16_lossy(&buf[..len as usize])
|
|
305
|
+
} else {
|
|
306
|
+
String::new()
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
#[cfg(windows)]
|
|
311
|
+
unsafe fn get_window_class(hwnd: HWND) -> String {
|
|
312
|
+
let mut buf = [0u16; 256];
|
|
313
|
+
let len = GetClassNameW(hwnd, &mut buf);
|
|
314
|
+
if len > 0 {
|
|
315
|
+
String::from_utf16_lossy(&buf[..len as usize])
|
|
316
|
+
} else {
|
|
317
|
+
String::new()
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
#[cfg(windows)]
|
|
322
|
+
unsafe fn get_process_name_by_pid(pid: u32) -> String {
|
|
323
|
+
let handle = match OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, false, pid) {
|
|
324
|
+
Ok(handle) => handle,
|
|
325
|
+
Err(_) => return String::new(),
|
|
326
|
+
};
|
|
327
|
+
|
|
328
|
+
let mut buf = [0u16; 260];
|
|
329
|
+
let mut size = buf.len() as u32;
|
|
330
|
+
let ok = QueryFullProcessImageNameW(
|
|
331
|
+
handle,
|
|
332
|
+
Default::default(),
|
|
333
|
+
PWSTR(buf.as_mut_ptr()),
|
|
334
|
+
&mut size,
|
|
335
|
+
)
|
|
336
|
+
.is_ok();
|
|
337
|
+
let _ = CloseHandle(handle);
|
|
338
|
+
|
|
339
|
+
if !ok || size == 0 {
|
|
340
|
+
return String::new();
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
let full_path = String::from_utf16_lossy(&buf[..size as usize]);
|
|
344
|
+
std::path::Path::new(&full_path)
|
|
345
|
+
.file_name()
|
|
346
|
+
.map(|n| n.to_string_lossy().to_string())
|
|
347
|
+
.unwrap_or(full_path)
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
#[cfg(windows)]
|
|
351
|
+
unsafe fn get_window_bounds_json(hwnd: HWND) -> Value {
|
|
352
|
+
let mut rect = RECT::default();
|
|
353
|
+
if GetWindowRect(hwnd, &mut rect).is_err() {
|
|
354
|
+
return json!({ "x": 0, "y": 0, "width": 0, "height": 0 });
|
|
355
|
+
}
|
|
356
|
+
json!({
|
|
357
|
+
"x": rect.left,
|
|
358
|
+
"y": rect.top,
|
|
359
|
+
"width": rect.right - rect.left,
|
|
360
|
+
"height": rect.bottom - rect.top,
|
|
361
|
+
})
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
#[cfg(windows)]
|
|
365
|
+
fn list_windows_impl(
|
|
366
|
+
store: &mut RefStore,
|
|
367
|
+
filter_pid: Option<u64>,
|
|
368
|
+
) -> Result<Value, ProtocolError> {
|
|
369
|
+
let mut hwnds: Vec<HWND> = Vec::new();
|
|
370
|
+
unsafe {
|
|
371
|
+
EnumWindows(
|
|
372
|
+
Some(enum_windows_proc),
|
|
373
|
+
LPARAM(&mut hwnds as *mut Vec<HWND> as isize),
|
|
374
|
+
)
|
|
375
|
+
.map_err(|e| {
|
|
376
|
+
ProtocolError::new(format!("EnumWindows failed: {e}"), ErrorCode::InternalError)
|
|
377
|
+
})?;
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
let foreground = unsafe { GetForegroundWindow() };
|
|
381
|
+
let mut windows_info = Vec::new();
|
|
382
|
+
|
|
383
|
+
for hwnd in hwnds {
|
|
384
|
+
if !unsafe { IsWindowVisible(hwnd).as_bool() } {
|
|
385
|
+
continue;
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
let title = unsafe { get_window_title(hwnd) };
|
|
389
|
+
let mut pid = 0u32;
|
|
390
|
+
unsafe {
|
|
391
|
+
GetWindowThreadProcessId(hwnd, Some(&mut pid));
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
if let Some(requested_pid) = filter_pid {
|
|
395
|
+
if u64::from(pid) != requested_pid {
|
|
396
|
+
continue;
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
let process_name = if pid != 0 {
|
|
401
|
+
unsafe { get_process_name_by_pid(pid) }
|
|
402
|
+
} else {
|
|
403
|
+
String::new()
|
|
404
|
+
};
|
|
405
|
+
let bounds = unsafe { get_window_bounds_json(hwnd) };
|
|
406
|
+
let class_name = unsafe { get_window_class(hwnd) };
|
|
407
|
+
let (is_browser, browser_family) = classify_browser(&process_name);
|
|
408
|
+
let wref = store.insert_window(NativeHandle::new(hwnd.0 as isize));
|
|
409
|
+
let dpi = unsafe { GetDpiForWindow(hwnd) };
|
|
410
|
+
let scale_factor = if dpi > 0 { f64::from(dpi) / 96.0 } else { 1.0 };
|
|
411
|
+
let owner = unsafe { GetWindow(hwnd, GW_OWNER).ok() };
|
|
412
|
+
let is_minimized = unsafe { IsIconic(hwnd).as_bool() };
|
|
413
|
+
let kind = if class_name == "#32768" {
|
|
414
|
+
"menu"
|
|
415
|
+
} else if class_name == "#32770" {
|
|
416
|
+
"dialog"
|
|
417
|
+
} else if owner.map(|hwnd| !hwnd.0.is_null()).unwrap_or(false) {
|
|
418
|
+
"popover"
|
|
419
|
+
} else {
|
|
420
|
+
"window"
|
|
421
|
+
};
|
|
422
|
+
let role = if kind == "menu" { "Menu" } else { "Window" };
|
|
423
|
+
let ex_style = unsafe { GetWindowLongPtrW(hwnd, GWL_EXSTYLE) } as u32;
|
|
424
|
+
let is_modal = class_name == "#32770" || (ex_style & WS_EX_DLGMODALFRAME.0) != 0;
|
|
425
|
+
|
|
426
|
+
windows_info.push(json!({
|
|
427
|
+
"kind": kind,
|
|
428
|
+
"rootRef": wref.to_string(),
|
|
429
|
+
"windowRef": wref.to_string(),
|
|
430
|
+
"ref": wref.to_string(),
|
|
431
|
+
"windowId": hwnd.0 as isize,
|
|
432
|
+
"title": title,
|
|
433
|
+
"pid": pid,
|
|
434
|
+
"appName": process_name.trim_end_matches(".exe"),
|
|
435
|
+
"processName": process_name,
|
|
436
|
+
"role": role,
|
|
437
|
+
"subrole": class_name,
|
|
438
|
+
"zOrder": windows_info.len(),
|
|
439
|
+
"framePoints": bounds,
|
|
440
|
+
"bounds": bounds,
|
|
441
|
+
"scaleFactor": scale_factor,
|
|
442
|
+
"isFocused": hwnd == foreground,
|
|
443
|
+
"isMain": hwnd == foreground,
|
|
444
|
+
"isMinimized": is_minimized,
|
|
445
|
+
"isOnscreen": !is_minimized,
|
|
446
|
+
"isModal": is_modal,
|
|
447
|
+
"metadata": { "className": class_name, "exStyle": ex_style, "isBrowser": is_browser, "browserFamily": browser_family },
|
|
448
|
+
"isBrowser": is_browser,
|
|
449
|
+
"browserFamily": browser_family,
|
|
450
|
+
}));
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
Ok(json!({
|
|
454
|
+
"stateId": StateId::fresh("w"),
|
|
455
|
+
"windows": windows_info,
|
|
456
|
+
"roots": windows_info,
|
|
457
|
+
}))
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
pub fn foreground_pid() -> Option<u64> {
|
|
461
|
+
#[cfg(not(windows))]
|
|
462
|
+
{
|
|
463
|
+
None
|
|
464
|
+
}
|
|
465
|
+
#[cfg(windows)]
|
|
466
|
+
{
|
|
467
|
+
let hwnd = unsafe { GetForegroundWindow() };
|
|
468
|
+
if hwnd.0.is_null() {
|
|
469
|
+
return None;
|
|
470
|
+
}
|
|
471
|
+
let mut pid = 0u32;
|
|
472
|
+
unsafe {
|
|
473
|
+
GetWindowThreadProcessId(hwnd, Some(&mut pid));
|
|
474
|
+
}
|
|
475
|
+
(pid != 0).then_some(u64::from(pid))
|
|
476
|
+
}
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
pub fn focus_window(
|
|
480
|
+
store: &RefStore,
|
|
481
|
+
target_ref: &crate::refs::WindowRef,
|
|
482
|
+
) -> Result<Value, ProtocolError> {
|
|
483
|
+
#[cfg(not(windows))]
|
|
484
|
+
{
|
|
485
|
+
let _ = store;
|
|
486
|
+
let _ = target_ref;
|
|
487
|
+
Err(ProtocolError::new(
|
|
488
|
+
"Window focus is only supported on Windows",
|
|
489
|
+
ErrorCode::UnsupportedPlatform,
|
|
490
|
+
))
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
#[cfg(windows)]
|
|
494
|
+
{
|
|
495
|
+
let native = store.get_window(target_ref).ok_or_else(|| {
|
|
496
|
+
ProtocolError::new(
|
|
497
|
+
format!("Window ref '{}' not found", target_ref),
|
|
498
|
+
ErrorCode::TargetNotFound,
|
|
499
|
+
)
|
|
500
|
+
})?;
|
|
501
|
+
let hwnd = HWND(native.raw() as *mut _);
|
|
502
|
+
let already_focused = unsafe { GetForegroundWindow() == hwnd };
|
|
503
|
+
let focused = already_focused || unsafe { SetForegroundWindow(hwnd).as_bool() };
|
|
504
|
+
Ok(json!({ "focused": focused, "alreadyFocused": already_focused }))
|
|
505
|
+
}
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
/// Bring an HWND to the foreground and confirm ownership before physical input.
|
|
509
|
+
/// Windows can reject SetForegroundWindow, so success is based on observation,
|
|
510
|
+
/// not the API's advisory return value.
|
|
511
|
+
pub fn ensure_foreground(hwnd: isize) -> Result<(), ProtocolError> {
|
|
512
|
+
#[cfg(not(windows))]
|
|
513
|
+
{
|
|
514
|
+
let _ = hwnd;
|
|
515
|
+
Err(ProtocolError::new(
|
|
516
|
+
"Window focus is only supported on Windows",
|
|
517
|
+
ErrorCode::UnsupportedPlatform,
|
|
518
|
+
))
|
|
519
|
+
}
|
|
520
|
+
#[cfg(windows)]
|
|
521
|
+
{
|
|
522
|
+
let target = HWND(hwnd as *mut _);
|
|
523
|
+
for _ in 0..4 {
|
|
524
|
+
if unsafe { GetForegroundWindow() == target } {
|
|
525
|
+
return Ok(());
|
|
526
|
+
}
|
|
527
|
+
let _ = unsafe { SetForegroundWindow(target) };
|
|
528
|
+
thread::sleep(Duration::from_millis(20));
|
|
529
|
+
}
|
|
530
|
+
if unsafe { GetForegroundWindow() == target } {
|
|
531
|
+
Ok(())
|
|
532
|
+
} else {
|
|
533
|
+
Err(ProtocolError::new(
|
|
534
|
+
"Windows refused to foreground the target; physical input was not sent",
|
|
535
|
+
ErrorCode::TargetNotFound,
|
|
536
|
+
))
|
|
537
|
+
}
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
|
|
541
|
+
// ---------------------------------------------------------------------------
|
|
542
|
+
// Tests
|
|
543
|
+
// ---------------------------------------------------------------------------
|
|
544
|
+
|
|
545
|
+
#[cfg(test)]
|
|
546
|
+
mod unit_tests {
|
|
547
|
+
use super::*;
|
|
548
|
+
use crate::error::ErrorCode;
|
|
549
|
+
|
|
550
|
+
// -- Browser classification (cross-platform) ----------------------------
|
|
551
|
+
|
|
552
|
+
#[test]
|
|
553
|
+
fn test_classify_browser_chrome_exe() {
|
|
554
|
+
let (is_browser, family) = classify_browser("chrome.exe");
|
|
555
|
+
assert!(is_browser);
|
|
556
|
+
assert_eq!(family, Some("chrome"));
|
|
557
|
+
}
|
|
558
|
+
|
|
559
|
+
#[test]
|
|
560
|
+
fn test_classify_browser_chromium_exe() {
|
|
561
|
+
let (is_browser, family) = classify_browser("chromium.exe");
|
|
562
|
+
assert!(is_browser);
|
|
563
|
+
assert_eq!(family, Some("chrome"));
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
#[test]
|
|
567
|
+
fn test_classify_browser_chrome_no_ext() {
|
|
568
|
+
// Some process names may not include .exe
|
|
569
|
+
let (is_browser, family) = classify_browser("chrome");
|
|
570
|
+
assert!(is_browser);
|
|
571
|
+
assert_eq!(family, Some("chrome"));
|
|
572
|
+
}
|
|
573
|
+
|
|
574
|
+
#[test]
|
|
575
|
+
fn test_classify_browser_edge_exe() {
|
|
576
|
+
let (is_browser, family) = classify_browser("msedge.exe");
|
|
577
|
+
assert!(is_browser);
|
|
578
|
+
assert_eq!(family, Some("edge"));
|
|
579
|
+
}
|
|
580
|
+
|
|
581
|
+
#[test]
|
|
582
|
+
fn test_classify_browser_edge_no_ext() {
|
|
583
|
+
let (is_browser, family) = classify_browser("edge");
|
|
584
|
+
assert!(is_browser);
|
|
585
|
+
assert_eq!(family, Some("edge"));
|
|
586
|
+
}
|
|
587
|
+
|
|
588
|
+
#[test]
|
|
589
|
+
fn test_classify_browser_brave_exe() {
|
|
590
|
+
let (is_browser, family) = classify_browser("brave.exe");
|
|
591
|
+
assert!(is_browser);
|
|
592
|
+
assert_eq!(family, Some("brave"));
|
|
593
|
+
}
|
|
594
|
+
|
|
595
|
+
#[test]
|
|
596
|
+
fn test_classify_browser_brave_browser_exe() {
|
|
597
|
+
let (is_browser, family) = classify_browser("brave-browser.exe");
|
|
598
|
+
assert!(is_browser);
|
|
599
|
+
assert_eq!(family, Some("brave"));
|
|
600
|
+
}
|
|
601
|
+
|
|
602
|
+
#[test]
|
|
603
|
+
fn test_classify_browser_not_a_browser() {
|
|
604
|
+
let (is_browser, family) = classify_browser("notepad.exe");
|
|
605
|
+
assert!(!is_browser);
|
|
606
|
+
assert_eq!(family, None);
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
#[test]
|
|
610
|
+
fn test_classify_browser_empty_string() {
|
|
611
|
+
let (is_browser, family) = classify_browser("");
|
|
612
|
+
assert!(!is_browser);
|
|
613
|
+
assert_eq!(family, None);
|
|
614
|
+
}
|
|
615
|
+
|
|
616
|
+
#[test]
|
|
617
|
+
fn test_classify_browser_case_insensitive() {
|
|
618
|
+
let (is_browser, family) = classify_browser("CHROME.EXE");
|
|
619
|
+
assert!(is_browser);
|
|
620
|
+
assert_eq!(family, Some("chrome"));
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
// -- Platform support check (non-Windows) -------------------------------
|
|
624
|
+
|
|
625
|
+
#[test]
|
|
626
|
+
#[cfg(not(windows))]
|
|
627
|
+
fn test_list_windows_unsupported_platform() {
|
|
628
|
+
let mut store = RefStore::new();
|
|
629
|
+
let result = list_windows(&mut store, None);
|
|
630
|
+
assert!(result.is_err());
|
|
631
|
+
let err = result.unwrap_err();
|
|
632
|
+
assert_eq!(err.code, ErrorCode::UnsupportedPlatform);
|
|
633
|
+
}
|
|
634
|
+
|
|
635
|
+
// -- JSON result shape --------------------------------------------------
|
|
636
|
+
|
|
637
|
+
#[test]
|
|
638
|
+
fn test_list_windows_result_shape_on_windows() {
|
|
639
|
+
// On non-Windows this just verifies the error path works.
|
|
640
|
+
#[cfg(not(windows))]
|
|
641
|
+
{
|
|
642
|
+
let mut store = RefStore::new();
|
|
643
|
+
let result = list_windows(&mut store, None);
|
|
644
|
+
assert!(result.is_err());
|
|
645
|
+
}
|
|
646
|
+
|
|
647
|
+
// On Windows we verify the result has the expected fields.
|
|
648
|
+
#[cfg(windows)]
|
|
649
|
+
{
|
|
650
|
+
let mut store = RefStore::new();
|
|
651
|
+
let result = list_windows(&mut store, None);
|
|
652
|
+
assert!(
|
|
653
|
+
result.is_ok(),
|
|
654
|
+
"list_windows should succeed on Windows: {:?}",
|
|
655
|
+
result.err()
|
|
656
|
+
);
|
|
657
|
+
let val = result.unwrap();
|
|
658
|
+
|
|
659
|
+
// Top-level fields
|
|
660
|
+
assert!(val.get("stateId").and_then(|v| v.as_str()).is_some());
|
|
661
|
+
assert!(val.get("windows").and_then(|v| v.as_array()).is_some());
|
|
662
|
+
|
|
663
|
+
if let Some(windows) = val["windows"].as_array() {
|
|
664
|
+
for w in windows {
|
|
665
|
+
assert!(w.get("ref").and_then(|v| v.as_str()).is_some());
|
|
666
|
+
assert!(w.get("title").is_some());
|
|
667
|
+
assert!(w.get("pid").and_then(|v| v.as_u64()).is_some());
|
|
668
|
+
assert!(w.get("processName").and_then(|v| v.as_str()).is_some());
|
|
669
|
+
assert!(w.get("bounds").and_then(|v| v.as_object()).is_some());
|
|
670
|
+
if let Some(bounds) = w["bounds"].as_object() {
|
|
671
|
+
assert!(bounds.contains_key("x"));
|
|
672
|
+
assert!(bounds.contains_key("y"));
|
|
673
|
+
assert!(bounds.contains_key("width"));
|
|
674
|
+
assert!(bounds.contains_key("height"));
|
|
675
|
+
}
|
|
676
|
+
assert!(w.get("isFocused").and_then(|v| v.as_bool()).is_some());
|
|
677
|
+
assert!(w.get("isBrowser").and_then(|v| v.as_bool()).is_some());
|
|
678
|
+
// browserFamily is Option<&str>, so it's either null or a string
|
|
679
|
+
assert!(w.get("browserFamily").is_some());
|
|
680
|
+
}
|
|
681
|
+
}
|
|
682
|
+
}
|
|
683
|
+
}
|
|
684
|
+
|
|
685
|
+
// -- Window refs and state IDs are fresh (Windows-only) -----------------
|
|
686
|
+
|
|
687
|
+
#[test]
|
|
688
|
+
#[cfg(windows)]
|
|
689
|
+
fn test_list_windows_produces_unique_state_ids() {
|
|
690
|
+
let mut store_a = RefStore::new();
|
|
691
|
+
let mut store_b = RefStore::new();
|
|
692
|
+
let result_a = list_windows(&mut store_a, None).unwrap();
|
|
693
|
+
let result_b = list_windows(&mut store_b, None).unwrap();
|
|
694
|
+
|
|
695
|
+
let id_a = result_a["stateId"].as_str().unwrap().to_owned();
|
|
696
|
+
let id_b = result_b["stateId"].as_str().unwrap().to_owned();
|
|
697
|
+
assert_ne!(
|
|
698
|
+
id_a, id_b,
|
|
699
|
+
"each listWindows call must produce a fresh stateId"
|
|
700
|
+
);
|
|
701
|
+
}
|
|
702
|
+
|
|
703
|
+
#[test]
|
|
704
|
+
#[cfg(windows)]
|
|
705
|
+
fn test_list_windows_window_ref_prefix() {
|
|
706
|
+
let mut store = RefStore::new();
|
|
707
|
+
let result = list_windows(&mut store, None).unwrap();
|
|
708
|
+
if let Some(windows) = result["windows"].as_array() {
|
|
709
|
+
for w in windows {
|
|
710
|
+
let wref = w["ref"].as_str().unwrap();
|
|
711
|
+
assert!(
|
|
712
|
+
wref.starts_with("@w"),
|
|
713
|
+
"window ref must start with @w, got {wref}"
|
|
714
|
+
);
|
|
715
|
+
}
|
|
716
|
+
}
|
|
717
|
+
}
|
|
718
|
+
}
|