@forsion/tangu-computer-use 0.6.0 → 0.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +35 -0
- package/UPSTREAM.md +3 -1
- package/icon.png +0 -0
- package/main.js +15 -0
- package/manifest.json +3 -3
- package/native/macos/bridge.swift +171 -7
- package/native/windows/bridge-rs/Cargo.lock +1 -0
- package/native/windows/bridge-rs/Cargo.toml +11 -0
- package/native/windows/bridge-rs/src/agent_marker.rs +97 -0
- package/native/windows/bridge-rs/src/lib.rs +4 -0
- package/native/windows/bridge-rs/src/main.rs +32 -0
- package/native/windows/bridge-rs/src/recorder.rs +1865 -0
- package/native/windows/bridge-rs/src/recorder_logic.rs +851 -0
- package/native/windows/bridge-rs/tests/recorder_logic_tests.rs +314 -0
- package/package.json +3 -2
- package/prebuilt/linux/x64/linux-bridge +0 -0
- package/prebuilt/macos/arm64/bridge +0 -0
- package/prebuilt/macos/arm64/tangu-computer-use.app.json +4 -4
- package/prebuilt/macos/arm64/tangu-computer-use.app.zip +0 -0
- package/prebuilt/macos/x64/bridge +0 -0
- package/prebuilt/macos/x64/tangu-computer-use.app.json +4 -4
- package/prebuilt/macos/x64/tangu-computer-use.app.zip +0 -0
- package/prebuilt/windows/windows-bridge.exe +0 -0
- package/scripts/exit-listeners.check.mjs +61 -0
- package/scripts/no-foreground.check.mjs +14 -1
- package/scripts/probe-recorder-windows.mjs +356 -0
- package/skills/computer-use/SKILL.md +4 -0
- package/src/foregroundNote.ts +16 -0
- package/src/index.ts +17 -5
- package/src/tools.ts +4 -3
- package/src/vendor/platform/macos/helper.ts +1 -1
- package/tangu-plugins/computer-use/dist/foregroundNote.js +10 -1
- package/tangu-plugins/computer-use/dist/index.js +25 -11
- package/tangu-plugins/computer-use/tangu-plugin.json +1 -1
- package/types/tangu-agent.d.ts +60 -1
|
@@ -0,0 +1,1865 @@
|
|
|
1
|
+
//! Tangu 新增:电脑历史(Computer History)采集器的 Windows 侧 —— `windows-bridge.exe serve --pipe <name>`。
|
|
2
|
+
//!
|
|
3
|
+
//! 与 macOS 的 `native/macos/activity_recorder.swift` 同一份合同(recordSubscribe,事件形状见 Genesis
|
|
4
|
+
//! `desktop/shared/computerHistory.ts`);纯逻辑在 `recorder_logic.rs`。只观测、不落盘。
|
|
5
|
+
//!
|
|
6
|
+
//! 进程模型:这是桌面端(Forsion 主进程)单独拉起的常驻进程,与引擎拉起的 stdio 助手互不相干 —— 桌面从自己的
|
|
7
|
+
//! 副本(`<computer-history>/bin/windows-bridge-<hash>.exe`)拉起它,管道名里带用户与版本哈希。单实例
|
|
8
|
+
//! (FILE_FLAG_FIRST_PIPE_INSTANCE),管道只给当前用户(显式 DACL)且拒绝远程客户端;最后一个订阅者走后 60 秒自行退出。
|
|
9
|
+
//!
|
|
10
|
+
//! 线程:
|
|
11
|
+
//! - 钩子线程:只泵消息 —— 低层鼠标 / 键盘钩子、前台与标题的 WinEvent、会话锁屏 / 睡眠的隐藏窗口。回调里只取
|
|
12
|
+
//! 几个数就转交 recorder 线程,**绝不**在这里做 UIA 调用(低层钩子回调慢了会拖住全系统输入,超时还会被系统摘掉);
|
|
13
|
+
//! - recorder 线程(MTA):持有 IUIAutomation 与全部状态、计时器、投递。UIA 事件处理器只投一条消息,焦点元素
|
|
14
|
+
//! 由 recorder 自己重新取(与 macOS「以 App 的 AXFocusedUIElement 为准」同一立场)。
|
|
15
|
+
//!
|
|
16
|
+
//! 性能纪律(ChatGPT 的 recorder 冻住过 IntelliJ):UIA 连接 / 事务超时压到 1 秒;只给焦点输入框挂值变化事件;
|
|
17
|
+
//! 浏览器网址只做有上限的遍历(≤400 节点、≤12 层、≤200ms),绝不展开网页内容(Document);之后按记住的地址栏快路重读。
|
|
18
|
+
//! 隐私闸在读 UIA **之前**:硬排除(密码管理器、凭据界面)与「所有订阅者都排除了的 App」不读标题、不看输入框;
|
|
19
|
+
//! 密码框(IsPassword)不读值,每次读值 / 报字前都重查(查不到也按密码框算);无痕窗口只留一条不带标题的切换。
|
|
20
|
+
//! 键盘钩子只看修饰键:没按 Ctrl / Win 的普通按键只转交一个「有输入」的信号(不带键值),用来触发对焦点框的差分读取。
|
|
21
|
+
|
|
22
|
+
use std::collections::HashMap;
|
|
23
|
+
use std::fs::File;
|
|
24
|
+
use std::io::{BufRead, BufReader, Read, Write};
|
|
25
|
+
use std::os::windows::io::{AsRawHandle, FromRawHandle};
|
|
26
|
+
use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
|
|
27
|
+
use std::sync::mpsc::{self, Receiver, RecvTimeoutError, Sender, SyncSender, TrySendError};
|
|
28
|
+
use std::sync::{Arc, Mutex, OnceLock};
|
|
29
|
+
use std::thread::{self, JoinHandle};
|
|
30
|
+
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
|
31
|
+
|
|
32
|
+
use serde_json::{json, Map, Value};
|
|
33
|
+
use windows::core::{implement, Interface, BSTR, HSTRING, PCWSTR, PWSTR, VARIANT};
|
|
34
|
+
use windows::Win32::Foundation::*;
|
|
35
|
+
use windows::Win32::Security::Authorization::{
|
|
36
|
+
ConvertSidToStringSidW, ConvertStringSecurityDescriptorToSecurityDescriptorW, SDDL_REVISION_1,
|
|
37
|
+
};
|
|
38
|
+
use windows::Win32::Security::{
|
|
39
|
+
GetTokenInformation, TokenUser, PSECURITY_DESCRIPTOR, SECURITY_ATTRIBUTES, TOKEN_QUERY, TOKEN_USER,
|
|
40
|
+
};
|
|
41
|
+
use windows::Win32::Storage::FileSystem::{
|
|
42
|
+
GetFileVersionInfoSizeW, GetFileVersionInfoW, VerQueryValueW, FILE_FLAG_FIRST_PIPE_INSTANCE,
|
|
43
|
+
PIPE_ACCESS_DUPLEX,
|
|
44
|
+
};
|
|
45
|
+
use windows::Win32::System::Com::{CoCreateInstance, CoInitializeEx, CLSCTX_INPROC_SERVER, COINIT_MULTITHREADED};
|
|
46
|
+
use windows::Win32::System::LibraryLoader::GetModuleHandleW;
|
|
47
|
+
use windows::Win32::System::Pipes::{
|
|
48
|
+
ConnectNamedPipe, CreateNamedPipeW, DisconnectNamedPipe, PeekNamedPipe, PIPE_READMODE_BYTE, PIPE_REJECT_REMOTE_CLIENTS,
|
|
49
|
+
PIPE_TYPE_BYTE, PIPE_UNLIMITED_INSTANCES, PIPE_WAIT,
|
|
50
|
+
};
|
|
51
|
+
use windows::Win32::System::RemoteDesktop::{
|
|
52
|
+
WTSRegisterSessionNotification, WTSUnRegisterSessionNotification, NOTIFY_FOR_THIS_SESSION,
|
|
53
|
+
};
|
|
54
|
+
use windows::Win32::System::StationsAndDesktops::{
|
|
55
|
+
CloseDesktop, GetUserObjectInformationW, OpenInputDesktop, DESKTOP_CONTROL_FLAGS, DESKTOP_READOBJECTS,
|
|
56
|
+
UOI_NAME,
|
|
57
|
+
};
|
|
58
|
+
use windows::Win32::System::Threading::{
|
|
59
|
+
GetCurrentProcess, GetCurrentProcessId, GetCurrentThreadId, OpenProcess, OpenProcessToken,
|
|
60
|
+
QueryFullProcessImageNameW, PROCESS_NAME_WIN32, PROCESS_QUERY_LIMITED_INFORMATION,
|
|
61
|
+
};
|
|
62
|
+
use windows::Win32::UI::Accessibility::*;
|
|
63
|
+
use windows::Win32::UI::Input::KeyboardAndMouse::{
|
|
64
|
+
GetAsyncKeyState, MapVirtualKeyW, MAPVK_VK_TO_CHAR, VK_CONTROL, VK_LWIN, VK_RMENU, VK_RWIN, VK_SHIFT,
|
|
65
|
+
VK_MENU,
|
|
66
|
+
};
|
|
67
|
+
use windows::Win32::UI::WindowsAndMessaging::*;
|
|
68
|
+
|
|
69
|
+
use crate::agent_marker;
|
|
70
|
+
use crate::recorder_logic::{self as logic, limit, Context, DeliveryState, Kind, PendingAction, Policy, PrivateVerdict, TextMode};
|
|
71
|
+
|
|
72
|
+
const IDLE_EXIT: Duration = Duration::from_secs(60);
|
|
73
|
+
const SUBSCRIBER_QUEUE: usize = 256;
|
|
74
|
+
const MAX_LINE_BYTES: u64 = 64 * 1024;
|
|
75
|
+
const PEEK_INTERVAL: Duration = Duration::from_millis(300);
|
|
76
|
+
|
|
77
|
+
const WINDOW_DEBOUNCE: Duration = Duration::from_millis(300);
|
|
78
|
+
const TEXT_DEBOUNCE: Duration = Duration::from_millis(1500);
|
|
79
|
+
const SAMPLE_DELAY: Duration = Duration::from_millis(250);
|
|
80
|
+
const WALK_NODE_CAP: usize = 400;
|
|
81
|
+
const WALK_DEPTH_CAP: usize = 12;
|
|
82
|
+
/// 拿到根元素之后的遍历预算(第一次进 Chrome 时根元素本身就可能要近 1 秒:它在那一刻才打开无障碍)。
|
|
83
|
+
const WALK_BUDGET: Duration = Duration::from_millis(300);
|
|
84
|
+
/// 没走完的窗口最多再遍历几次;一直走不完就一直按无痕处理(失败即关闭)。
|
|
85
|
+
const WALK_ATTEMPTS: u8 = 5;
|
|
86
|
+
const BROWSER_CACHE_CAP: usize = 16;
|
|
87
|
+
/// 打字时没跟踪到输入框(焦点通知丢了 / 被抢走),多久重新问一次焦点。
|
|
88
|
+
const REFOCUS_INTERVAL: Duration = Duration::from_secs(1);
|
|
89
|
+
/// 浏览器窗口判不了无痕时,隔多久自己再刷新一次(不等下一次切标签 / 改标题)。
|
|
90
|
+
const WALK_RETRY: Duration = Duration::from_secs(1);
|
|
91
|
+
const UIA_TIMEOUT_MS: u32 = 1000;
|
|
92
|
+
|
|
93
|
+
const WM_APP_RETARGET: u32 = WM_APP + 1;
|
|
94
|
+
const UIA_E_ELEMENTNOTAVAILABLE: i32 = 0x8004_0201_u32 as i32;
|
|
95
|
+
|
|
96
|
+
fn now_ms() -> i64 {
|
|
97
|
+
SystemTime::now()
|
|
98
|
+
.duration_since(UNIX_EPOCH)
|
|
99
|
+
.map(|d| d.as_millis() as i64)
|
|
100
|
+
.unwrap_or(0)
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
// ===========================================================================
|
|
104
|
+
// 订阅连接(命名管道)
|
|
105
|
+
// ===========================================================================
|
|
106
|
+
|
|
107
|
+
/// 一个订阅连接的出口。recorder 线程只管 try_send,永不因某个订阅者读得慢而卡住;积压满了就丢弃计数,
|
|
108
|
+
/// 下一条能入队时先补一条 `system/dropped`。写失败由写线程断开管道,读线程随之拿到 EOF 走唯一的退订路径。
|
|
109
|
+
pub struct Outbox {
|
|
110
|
+
tx: SyncSender<String>,
|
|
111
|
+
dropped: AtomicUsize,
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
impl Outbox {
|
|
115
|
+
fn send(&self, object: &Value) {
|
|
116
|
+
let Ok(mut line) = serde_json::to_string(object) else { return };
|
|
117
|
+
line.push('\n');
|
|
118
|
+
let dropped = self.dropped.load(Ordering::Acquire);
|
|
119
|
+
if dropped > 0 {
|
|
120
|
+
let marker = json!({ "ev": { "t": now_ms(), "kind": "system", "state": "dropped", "count": dropped } });
|
|
121
|
+
match self.tx.try_send(format!("{marker}\n")) {
|
|
122
|
+
Ok(()) => self.dropped.store(0, Ordering::Release),
|
|
123
|
+
Err(TrySendError::Full(_)) => {
|
|
124
|
+
self.dropped.fetch_add(1, Ordering::AcqRel);
|
|
125
|
+
return;
|
|
126
|
+
}
|
|
127
|
+
Err(TrySendError::Disconnected(_)) => return,
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
if let Err(TrySendError::Full(_)) = self.tx.try_send(line) {
|
|
131
|
+
self.dropped.fetch_add(1, Ordering::AcqRel);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
static SUBSCRIBERS: AtomicUsize = AtomicUsize::new(0);
|
|
137
|
+
static POSTED: AtomicU64 = AtomicU64::new(0);
|
|
138
|
+
/// 诊断:钩子装没装上(位 0 = 鼠标,1 = 键盘,2 = 前台 WinEvent,3 = 会话窗口)。
|
|
139
|
+
static HOOKS: AtomicUsize = AtomicUsize::new(0);
|
|
140
|
+
static HANDLED: AtomicU64 = AtomicU64::new(0);
|
|
141
|
+
|
|
142
|
+
/// 观测仪器:recorder 线程此刻在做哪一步、从什么时候开始(diagnostics 报出来;卡死时一眼看出卡在哪个 UIA 调用)。
|
|
143
|
+
/// TANGU_RECORDER_DEBUG=1 时每一步也写 stderr。
|
|
144
|
+
fn phase_slot() -> &'static Mutex<(&'static str, i64)> {
|
|
145
|
+
static PHASE: OnceLock<Mutex<(&'static str, i64)>> = OnceLock::new();
|
|
146
|
+
PHASE.get_or_init(|| Mutex::new(("idle", 0)))
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
fn debug_enabled() -> bool {
|
|
150
|
+
static DEBUG: OnceLock<bool> = OnceLock::new();
|
|
151
|
+
*DEBUG.get_or_init(|| std::env::var_os("TANGU_RECORDER_DEBUG").is_some_and(|v| v == "1"))
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
fn phase(name: &'static str) {
|
|
155
|
+
let now = now_ms();
|
|
156
|
+
if let Ok(mut slot) = phase_slot().lock() {
|
|
157
|
+
*slot = (name, now);
|
|
158
|
+
}
|
|
159
|
+
if debug_enabled() && name != "idle" {
|
|
160
|
+
eprintln!("[recorder] {now} {name}");
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
static LAST_ACTIVITY_MS: AtomicU64 = AtomicU64::new(0);
|
|
164
|
+
static NEXT_SUB_ID: AtomicU64 = AtomicU64::new(1);
|
|
165
|
+
|
|
166
|
+
fn touch_activity() {
|
|
167
|
+
LAST_ACTIVITY_MS.store(now_ms() as u64, Ordering::Release);
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/// `serve --pipe <name>` 的入口。返回进程退出码。
|
|
171
|
+
pub fn serve(pipe: &str) -> i32 {
|
|
172
|
+
if !logic::pipe_name_valid(pipe) {
|
|
173
|
+
eprintln!("[recorder] invalid pipe name");
|
|
174
|
+
return 2;
|
|
175
|
+
}
|
|
176
|
+
let security = match PipeSecurity::current_user() {
|
|
177
|
+
Ok(s) => s,
|
|
178
|
+
Err(e) => {
|
|
179
|
+
eprintln!("[recorder] pipe security: {e}");
|
|
180
|
+
return 1;
|
|
181
|
+
}
|
|
182
|
+
};
|
|
183
|
+
let name = HSTRING::from(pipe);
|
|
184
|
+
let mut first = match create_pipe(&name, true, &security) {
|
|
185
|
+
Ok(h) => Some(h),
|
|
186
|
+
Err(e) => {
|
|
187
|
+
// 同名管道已有服务者(另一个同版本实例):让它服务,我们安静退出。
|
|
188
|
+
eprintln!("[recorder] pipe busy: {e}");
|
|
189
|
+
return 0;
|
|
190
|
+
}
|
|
191
|
+
};
|
|
192
|
+
touch_activity();
|
|
193
|
+
start_recorder_thread();
|
|
194
|
+
thread::spawn(idle_watchdog);
|
|
195
|
+
loop {
|
|
196
|
+
let handle = first.take().or_else(|| create_pipe(&name, false, &security).ok());
|
|
197
|
+
let Some(handle) = handle else {
|
|
198
|
+
thread::sleep(Duration::from_millis(200));
|
|
199
|
+
continue;
|
|
200
|
+
};
|
|
201
|
+
let connected = unsafe { ConnectNamedPipe(handle, None) };
|
|
202
|
+
let ok = connected.is_ok() || unsafe { GetLastError() } == ERROR_PIPE_CONNECTED;
|
|
203
|
+
if !ok {
|
|
204
|
+
let _ = unsafe { CloseHandle(handle) };
|
|
205
|
+
continue;
|
|
206
|
+
}
|
|
207
|
+
let raw = handle.0 as isize;
|
|
208
|
+
thread::spawn(move || serve_client(raw));
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
fn idle_watchdog() {
|
|
213
|
+
loop {
|
|
214
|
+
thread::sleep(Duration::from_secs(5));
|
|
215
|
+
let idle = (now_ms() as u64).saturating_sub(LAST_ACTIVITY_MS.load(Ordering::Acquire));
|
|
216
|
+
if SUBSCRIBERS.load(Ordering::Acquire) == 0 && idle >= IDLE_EXIT.as_millis() as u64 {
|
|
217
|
+
std::process::exit(0);
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
/// 管道的安全描述符:只给当前用户(与 SYSTEM)。命名管道缺省 DACL 给 Everyone 读权限,这里收紧 ——
|
|
223
|
+
/// 相当于 macOS 那边按 getpeereid 只服务同一用户。
|
|
224
|
+
struct PipeSecurity {
|
|
225
|
+
descriptor: PSECURITY_DESCRIPTOR,
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
unsafe impl Send for PipeSecurity {}
|
|
229
|
+
unsafe impl Sync for PipeSecurity {}
|
|
230
|
+
|
|
231
|
+
impl PipeSecurity {
|
|
232
|
+
fn current_user() -> Result<Self, String> {
|
|
233
|
+
let sid = current_user_sid()?;
|
|
234
|
+
let sddl = HSTRING::from(format!("D:P(A;;GA;;;{sid})(A;;GA;;;SY)"));
|
|
235
|
+
let mut descriptor = PSECURITY_DESCRIPTOR::default();
|
|
236
|
+
unsafe {
|
|
237
|
+
ConvertStringSecurityDescriptorToSecurityDescriptorW(&sddl, SDDL_REVISION_1, &mut descriptor, None)
|
|
238
|
+
.map_err(|e| format!("ConvertStringSecurityDescriptor: {e}"))?;
|
|
239
|
+
}
|
|
240
|
+
Ok(Self { descriptor })
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
fn attributes(&self) -> SECURITY_ATTRIBUTES {
|
|
244
|
+
SECURITY_ATTRIBUTES {
|
|
245
|
+
nLength: std::mem::size_of::<SECURITY_ATTRIBUTES>() as u32,
|
|
246
|
+
lpSecurityDescriptor: self.descriptor.0,
|
|
247
|
+
bInheritHandle: FALSE,
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
fn current_user_sid() -> Result<String, String> {
|
|
253
|
+
unsafe {
|
|
254
|
+
let mut token = HANDLE::default();
|
|
255
|
+
OpenProcessToken(GetCurrentProcess(), TOKEN_QUERY, &mut token).map_err(|e| format!("OpenProcessToken: {e}"))?;
|
|
256
|
+
let mut size = 0u32;
|
|
257
|
+
let _ = GetTokenInformation(token, TokenUser, None, 0, &mut size);
|
|
258
|
+
let mut buffer = vec![0u8; size as usize];
|
|
259
|
+
let result = GetTokenInformation(token, TokenUser, Some(buffer.as_mut_ptr() as *mut _), size, &mut size);
|
|
260
|
+
let _ = CloseHandle(token);
|
|
261
|
+
result.map_err(|e| format!("GetTokenInformation: {e}"))?;
|
|
262
|
+
let user = &*(buffer.as_ptr() as *const TOKEN_USER);
|
|
263
|
+
let mut text = PWSTR::null();
|
|
264
|
+
ConvertSidToStringSidW(user.User.Sid, &mut text).map_err(|e| format!("ConvertSidToStringSid: {e}"))?;
|
|
265
|
+
let sid = text.to_string().map_err(|e| e.to_string());
|
|
266
|
+
let _ = LocalFree(HLOCAL(text.0 as *mut _));
|
|
267
|
+
sid
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
fn create_pipe(name: &HSTRING, first: bool, security: &PipeSecurity) -> Result<HANDLE, String> {
|
|
272
|
+
let attributes = security.attributes();
|
|
273
|
+
let mut mode = PIPE_ACCESS_DUPLEX;
|
|
274
|
+
if first {
|
|
275
|
+
mode |= FILE_FLAG_FIRST_PIPE_INSTANCE;
|
|
276
|
+
}
|
|
277
|
+
let handle = unsafe {
|
|
278
|
+
CreateNamedPipeW(
|
|
279
|
+
name,
|
|
280
|
+
mode,
|
|
281
|
+
PIPE_TYPE_BYTE | PIPE_READMODE_BYTE | PIPE_WAIT | PIPE_REJECT_REMOTE_CLIENTS,
|
|
282
|
+
PIPE_UNLIMITED_INSTANCES,
|
|
283
|
+
64 * 1024,
|
|
284
|
+
64 * 1024,
|
|
285
|
+
0,
|
|
286
|
+
Some(&attributes),
|
|
287
|
+
)
|
|
288
|
+
};
|
|
289
|
+
if handle.is_invalid() {
|
|
290
|
+
return Err(format!("CreateNamedPipe: {:?}", unsafe { GetLastError() }));
|
|
291
|
+
}
|
|
292
|
+
Ok(handle)
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
/// 一条连接:读第一行命令。recordSubscribe → 注册订阅者并一直读到对端关闭;diagnostics → 回一行就关。
|
|
296
|
+
fn serve_client(raw: isize) {
|
|
297
|
+
touch_activity();
|
|
298
|
+
// SAFETY: raw 是 ConnectNamedPipe 成功后的管道句柄,所有权交给 File(drop 时 CloseHandle)。
|
|
299
|
+
let file = unsafe { File::from_raw_handle(raw as *mut _) };
|
|
300
|
+
let Ok(read_half) = file.try_clone() else { return };
|
|
301
|
+
let mut reader = BufReader::new(read_half.take(MAX_LINE_BYTES));
|
|
302
|
+
let mut first = String::new();
|
|
303
|
+
if reader.read_line(&mut first).unwrap_or(0) == 0 {
|
|
304
|
+
return;
|
|
305
|
+
}
|
|
306
|
+
let request: Value = serde_json::from_str(first.trim()).unwrap_or(Value::Null);
|
|
307
|
+
let id = request.get("id").and_then(Value::as_str).unwrap_or("unknown").to_owned();
|
|
308
|
+
let mut writer = file;
|
|
309
|
+
match request.get("cmd").and_then(Value::as_str) {
|
|
310
|
+
Some("recordSubscribe") => {}
|
|
311
|
+
Some("diagnostics") => {
|
|
312
|
+
let reply = json!({ "id": id, "ok": true, "result": {
|
|
313
|
+
"protocolVersion": logic::RECORDER_PROTOCOL,
|
|
314
|
+
"pid": std::process::id(),
|
|
315
|
+
"subscribers": SUBSCRIBERS.load(Ordering::Acquire),
|
|
316
|
+
"os": "windows",
|
|
317
|
+
"phase": phase_slot().lock().map(|p| p.0).unwrap_or("?"),
|
|
318
|
+
"phaseMs": phase_slot().lock().map(|p| if p.1 > 0 { now_ms() - p.1 } else { 0 }).unwrap_or(-1),
|
|
319
|
+
"backlog": POSTED.load(Ordering::Acquire).saturating_sub(HANDLED.load(Ordering::Acquire)),
|
|
320
|
+
"hooks": HOOKS.load(Ordering::Acquire),
|
|
321
|
+
}});
|
|
322
|
+
let _ = writeln!(writer, "{reply}");
|
|
323
|
+
return;
|
|
324
|
+
}
|
|
325
|
+
other => {
|
|
326
|
+
let reply = json!({ "id": id, "ok": false, "error": {
|
|
327
|
+
"code": "unknown_command",
|
|
328
|
+
"message": format!("Unknown command '{}'", other.unwrap_or("")),
|
|
329
|
+
}});
|
|
330
|
+
let _ = writeln!(writer, "{reply}");
|
|
331
|
+
return;
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
let (tx, rx) = mpsc::sync_channel::<String>(SUBSCRIBER_QUEUE);
|
|
336
|
+
let outbox = Arc::new(Outbox { tx, dropped: AtomicUsize::new(0) });
|
|
337
|
+
// 回包排在这条连接写队列的第一位,之后才是事件。
|
|
338
|
+
outbox.send(&json!({ "id": id, "ok": true, "result": {
|
|
339
|
+
"subscribed": true, "protocolVersion": logic::RECORDER_PROTOCOL, "axTrusted": true,
|
|
340
|
+
}}));
|
|
341
|
+
let writer_raw = writer.as_raw_handle() as isize;
|
|
342
|
+
let writer_thread = thread::spawn(move || {
|
|
343
|
+
for line in rx {
|
|
344
|
+
if writer.write_all(line.as_bytes()).is_err() {
|
|
345
|
+
// 对端不读了 / 已关:断开管道,读线程拿到错误后退订。
|
|
346
|
+
let _ = unsafe { DisconnectNamedPipe(HANDLE(writer_raw as *mut _)) };
|
|
347
|
+
return;
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
});
|
|
351
|
+
let sub_id = NEXT_SUB_ID.fetch_add(1, Ordering::AcqRel);
|
|
352
|
+
SUBSCRIBERS.fetch_add(1, Ordering::AcqRel);
|
|
353
|
+
post(Msg::Subscribe { id: sub_id, policy: Policy::from_json(request.get("policy")), outbox: Arc::clone(&outbox) });
|
|
354
|
+
// 订阅之后对端发什么都不认,只判断它关没关。**不能**阻塞在 ReadFile 上等 EOF:同步打开的管道上,同一个文件对象
|
|
355
|
+
// (try_clone 复制出的句柄共享它)的 I/O 是串行的,挂着的读会把写线程的回包和事件全部卡住(CI 实测:回包永远到不了)。
|
|
356
|
+
// 所以按 PeekNamedPipe 轮询:出错 = 对端关了 / 我们断开了;有数据才读(不会阻塞),读了就丢。
|
|
357
|
+
let mut sink = reader.into_inner().into_inner();
|
|
358
|
+
let pipe = HANDLE(sink.as_raw_handle());
|
|
359
|
+
let mut buffer = [0u8; 4096];
|
|
360
|
+
loop {
|
|
361
|
+
let mut available = 0u32;
|
|
362
|
+
if unsafe { PeekNamedPipe(pipe, None, 0, None, Some(&mut available), None) }.is_err() {
|
|
363
|
+
break;
|
|
364
|
+
}
|
|
365
|
+
if available == 0 {
|
|
366
|
+
thread::sleep(PEEK_INTERVAL);
|
|
367
|
+
continue;
|
|
368
|
+
}
|
|
369
|
+
let want = (available as usize).min(buffer.len());
|
|
370
|
+
if !matches!(sink.read(&mut buffer[..want]), Ok(n) if n > 0) {
|
|
371
|
+
break;
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
post(Msg::Unsubscribe(sub_id));
|
|
375
|
+
drop(outbox);
|
|
376
|
+
SUBSCRIBERS.fetch_sub(1, Ordering::AcqRel);
|
|
377
|
+
touch_activity();
|
|
378
|
+
let _ = writer_thread.join();
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
// ===========================================================================
|
|
382
|
+
// recorder 线程
|
|
383
|
+
// ===========================================================================
|
|
384
|
+
|
|
385
|
+
enum Msg {
|
|
386
|
+
Subscribe { id: u64, policy: Policy, outbox: Arc<Outbox> },
|
|
387
|
+
Unsubscribe(u64),
|
|
388
|
+
Foreground(isize),
|
|
389
|
+
NameChange(isize),
|
|
390
|
+
Click { x: i32, y: i32, agent: bool },
|
|
391
|
+
Key { combo: String, agent: bool },
|
|
392
|
+
KeyActivity { agent: bool },
|
|
393
|
+
FocusChanged,
|
|
394
|
+
ValueChanged,
|
|
395
|
+
System(&'static str),
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
fn sender() -> &'static Mutex<Option<Sender<Msg>>> {
|
|
399
|
+
static SENDER: OnceLock<Mutex<Option<Sender<Msg>>>> = OnceLock::new();
|
|
400
|
+
SENDER.get_or_init(|| Mutex::new(None))
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
fn post(msg: Msg) {
|
|
404
|
+
if let Ok(guard) = sender().lock() {
|
|
405
|
+
if let Some(tx) = guard.as_ref() {
|
|
406
|
+
if tx.send(msg).is_ok() {
|
|
407
|
+
POSTED.fetch_add(1, Ordering::AcqRel);
|
|
408
|
+
}
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
fn start_recorder_thread() {
|
|
414
|
+
let (tx, rx) = mpsc::channel();
|
|
415
|
+
if let Ok(mut guard) = sender().lock() {
|
|
416
|
+
*guard = Some(tx);
|
|
417
|
+
}
|
|
418
|
+
thread::spawn(move || {
|
|
419
|
+
let hr = unsafe { CoInitializeEx(None, COINIT_MULTITHREADED) };
|
|
420
|
+
if hr.is_err() {
|
|
421
|
+
eprintln!("[recorder] CoInitializeEx: {hr:?}");
|
|
422
|
+
std::process::exit(1);
|
|
423
|
+
}
|
|
424
|
+
let uia: IUIAutomation = match unsafe { CoCreateInstance(&CUIAutomation8, None, CLSCTX_INPROC_SERVER) }
|
|
425
|
+
.or_else(|_| unsafe { CoCreateInstance(&CUIAutomation, None, CLSCTX_INPROC_SERVER) })
|
|
426
|
+
{
|
|
427
|
+
Ok(uia) => uia,
|
|
428
|
+
Err(e) => {
|
|
429
|
+
eprintln!("[recorder] IUIAutomation: {e}");
|
|
430
|
+
std::process::exit(1);
|
|
431
|
+
}
|
|
432
|
+
};
|
|
433
|
+
if let Ok(uia2) = uia.cast::<IUIAutomation2>() {
|
|
434
|
+
unsafe {
|
|
435
|
+
let _ = uia2.SetConnectionTimeout(UIA_TIMEOUT_MS);
|
|
436
|
+
let _ = uia2.SetTransactionTimeout(UIA_TIMEOUT_MS);
|
|
437
|
+
}
|
|
438
|
+
}
|
|
439
|
+
Recorder::new(uia).run(rx);
|
|
440
|
+
});
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
struct Sub {
|
|
444
|
+
id: u64,
|
|
445
|
+
policy: Policy,
|
|
446
|
+
delivery: DeliveryState,
|
|
447
|
+
outbox: Arc<Outbox>,
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
struct Focus {
|
|
451
|
+
element: IUIAutomationElement,
|
|
452
|
+
mode: TextMode,
|
|
453
|
+
el: Map<String, Value>,
|
|
454
|
+
sampling: bool,
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
struct Pending {
|
|
458
|
+
context: Context,
|
|
459
|
+
el: Map<String, Value>,
|
|
460
|
+
last_change: i64,
|
|
461
|
+
agent: bool,
|
|
462
|
+
latest: Option<String>,
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
enum FieldRead {
|
|
466
|
+
/// 密码框,或 IsPassword 查不到(证明不了不是)。
|
|
467
|
+
Secure,
|
|
468
|
+
Big,
|
|
469
|
+
/// 元素已销毁 / 值读不到。
|
|
470
|
+
Unreadable,
|
|
471
|
+
Value(String),
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
struct BrowserWindow {
|
|
475
|
+
address: Option<IUIAutomationElement>,
|
|
476
|
+
/// 工具栏上找到了无痕提示。
|
|
477
|
+
hint: bool,
|
|
478
|
+
/// 遍历走完了(没撞上节点数 / 时间上限)。
|
|
479
|
+
complete: bool,
|
|
480
|
+
attempts: u8,
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
struct HookThread {
|
|
484
|
+
tid: u32,
|
|
485
|
+
join: JoinHandle<()>,
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
struct Recorder {
|
|
489
|
+
uia: IUIAutomation,
|
|
490
|
+
self_pid: u32,
|
|
491
|
+
subs: Vec<Sub>,
|
|
492
|
+
running: bool,
|
|
493
|
+
locked: bool,
|
|
494
|
+
asleep: bool,
|
|
495
|
+
hooks: Option<HookThread>,
|
|
496
|
+
focus_handler: Option<IUIAutomationFocusChangedEventHandler>,
|
|
497
|
+
focus_cache: Option<IUIAutomationCacheRequest>,
|
|
498
|
+
value_handler: ValueHandlers,
|
|
499
|
+
/// 当前前台(解析过 UWP 宿主之后)的进程与窗口。
|
|
500
|
+
fg_pid: u32,
|
|
501
|
+
fg_hwnd: isize,
|
|
502
|
+
/// 情境是从哪个窗口读的、读时的原始标题(判断情境过没过时用;标题本身不另存)。
|
|
503
|
+
context: Option<Context>,
|
|
504
|
+
context_hwnd: isize,
|
|
505
|
+
context_raw_title: String,
|
|
506
|
+
window_deadline: Option<Instant>,
|
|
507
|
+
focus: Option<Focus>,
|
|
508
|
+
focus_is_password: bool,
|
|
509
|
+
pending: Option<Pending>,
|
|
510
|
+
text_deadline: Option<Instant>,
|
|
511
|
+
sample_deadline: Option<Instant>,
|
|
512
|
+
baseline: Option<String>,
|
|
513
|
+
browser_windows: HashMap<isize, BrowserWindow>,
|
|
514
|
+
last_refocus: Option<Instant>,
|
|
515
|
+
walk_retry: Option<Instant>,
|
|
516
|
+
app_names: HashMap<String, String>,
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
struct ValueHandlers {
|
|
520
|
+
property: IUIAutomationPropertyChangedEventHandler,
|
|
521
|
+
event: IUIAutomationEventHandler,
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
impl Recorder {
|
|
525
|
+
fn new(uia: IUIAutomation) -> Self {
|
|
526
|
+
let handler: IUIAutomationPropertyChangedEventHandler = ValueHandler.into();
|
|
527
|
+
let event: IUIAutomationEventHandler = handler.cast().expect("ValueHandler implements both interfaces");
|
|
528
|
+
let focus_cache = unsafe { uia.CreateCacheRequest() }.ok();
|
|
529
|
+
if let Some(cache) = &focus_cache {
|
|
530
|
+
for property in [
|
|
531
|
+
UIA_ControlTypePropertyId,
|
|
532
|
+
UIA_IsPasswordPropertyId,
|
|
533
|
+
UIA_ProcessIdPropertyId,
|
|
534
|
+
UIA_NamePropertyId,
|
|
535
|
+
UIA_IsValuePatternAvailablePropertyId,
|
|
536
|
+
UIA_ValueIsReadOnlyPropertyId,
|
|
537
|
+
UIA_IsTextPatternAvailablePropertyId,
|
|
538
|
+
] {
|
|
539
|
+
let _ = unsafe { cache.AddProperty(property) };
|
|
540
|
+
}
|
|
541
|
+
}
|
|
542
|
+
Self {
|
|
543
|
+
uia,
|
|
544
|
+
self_pid: unsafe { GetCurrentProcessId() },
|
|
545
|
+
subs: Vec::new(),
|
|
546
|
+
running: false,
|
|
547
|
+
locked: false,
|
|
548
|
+
asleep: false,
|
|
549
|
+
hooks: None,
|
|
550
|
+
focus_handler: None,
|
|
551
|
+
focus_cache,
|
|
552
|
+
value_handler: ValueHandlers { property: handler, event },
|
|
553
|
+
fg_pid: 0,
|
|
554
|
+
fg_hwnd: 0,
|
|
555
|
+
context: None,
|
|
556
|
+
context_hwnd: 0,
|
|
557
|
+
context_raw_title: String::new(),
|
|
558
|
+
window_deadline: None,
|
|
559
|
+
focus: None,
|
|
560
|
+
focus_is_password: false,
|
|
561
|
+
pending: None,
|
|
562
|
+
text_deadline: None,
|
|
563
|
+
sample_deadline: None,
|
|
564
|
+
baseline: None,
|
|
565
|
+
browser_windows: HashMap::new(),
|
|
566
|
+
last_refocus: None,
|
|
567
|
+
walk_retry: None,
|
|
568
|
+
app_names: HashMap::new(),
|
|
569
|
+
}
|
|
570
|
+
}
|
|
571
|
+
|
|
572
|
+
fn suspended(&self) -> bool {
|
|
573
|
+
self.locked || self.asleep
|
|
574
|
+
}
|
|
575
|
+
|
|
576
|
+
fn run(mut self, rx: Receiver<Msg>) {
|
|
577
|
+
loop {
|
|
578
|
+
let timeout = [self.window_deadline, self.text_deadline, self.sample_deadline, self.walk_retry]
|
|
579
|
+
.into_iter()
|
|
580
|
+
.flatten()
|
|
581
|
+
.min()
|
|
582
|
+
.map(|d| d.saturating_duration_since(Instant::now()))
|
|
583
|
+
.unwrap_or(Duration::from_secs(3600));
|
|
584
|
+
match rx.recv_timeout(timeout) {
|
|
585
|
+
Ok(msg) => self.handle(msg),
|
|
586
|
+
Err(RecvTimeoutError::Timeout) => {}
|
|
587
|
+
Err(RecvTimeoutError::Disconnected) => return,
|
|
588
|
+
}
|
|
589
|
+
self.fire_due();
|
|
590
|
+
phase("idle");
|
|
591
|
+
}
|
|
592
|
+
}
|
|
593
|
+
|
|
594
|
+
fn fire_due(&mut self) {
|
|
595
|
+
let now = Instant::now();
|
|
596
|
+
if self.window_deadline.is_some_and(|d| d <= now) || self.walk_retry.is_some_and(|d| d <= now) {
|
|
597
|
+
self.refresh_window();
|
|
598
|
+
}
|
|
599
|
+
if self.sample_deadline.is_some_and(|d| d <= now) {
|
|
600
|
+
self.sample_text();
|
|
601
|
+
}
|
|
602
|
+
if self.text_deadline.is_some_and(|d| d <= now) {
|
|
603
|
+
self.text_deadline = None;
|
|
604
|
+
self.refresh_if_stale();
|
|
605
|
+
self.flush_text();
|
|
606
|
+
}
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
fn handle(&mut self, msg: Msg) {
|
|
610
|
+
phase(match &msg {
|
|
611
|
+
Msg::Subscribe { .. } => "msg:subscribe",
|
|
612
|
+
Msg::Unsubscribe(_) => "msg:unsubscribe",
|
|
613
|
+
Msg::Foreground(_) => "msg:foreground",
|
|
614
|
+
Msg::NameChange(_) => "msg:namechange",
|
|
615
|
+
Msg::Click { .. } => "msg:click",
|
|
616
|
+
Msg::Key { .. } => "msg:key",
|
|
617
|
+
Msg::KeyActivity { .. } => "msg:keyactivity",
|
|
618
|
+
Msg::FocusChanged => "msg:focus",
|
|
619
|
+
Msg::ValueChanged => "msg:value",
|
|
620
|
+
Msg::System(_) => "msg:system",
|
|
621
|
+
});
|
|
622
|
+
self.dispatch(msg);
|
|
623
|
+
HANDLED.fetch_add(1, Ordering::AcqRel);
|
|
624
|
+
phase("idle");
|
|
625
|
+
}
|
|
626
|
+
|
|
627
|
+
fn dispatch(&mut self, msg: Msg) {
|
|
628
|
+
match msg {
|
|
629
|
+
Msg::Subscribe { id, policy, outbox } => self.subscribe(id, policy, outbox),
|
|
630
|
+
Msg::Unsubscribe(id) => {
|
|
631
|
+
self.subs.retain(|s| s.id != id);
|
|
632
|
+
self.reset_text();
|
|
633
|
+
if self.subs.is_empty() {
|
|
634
|
+
self.stop();
|
|
635
|
+
}
|
|
636
|
+
}
|
|
637
|
+
_ if !self.running => {}
|
|
638
|
+
Msg::System(state) => self.system_event(state),
|
|
639
|
+
_ if self.suspended() => {}
|
|
640
|
+
Msg::Foreground(hwnd) => self.on_foreground(hwnd),
|
|
641
|
+
Msg::NameChange(hwnd) => {
|
|
642
|
+
if hwnd == self.fg_hwnd && self.context.is_some() {
|
|
643
|
+
self.window_deadline = Some(Instant::now() + WINDOW_DEBOUNCE);
|
|
644
|
+
}
|
|
645
|
+
}
|
|
646
|
+
Msg::Click { x, y, agent } => self.clicked(x, y, agent),
|
|
647
|
+
Msg::Key { combo, agent } => self.keyed(combo, agent),
|
|
648
|
+
Msg::KeyActivity { agent } => {
|
|
649
|
+
// 在打字却没跟踪到输入框:焦点通知丢了或被别的元素抢走过,限频重新问一次焦点。
|
|
650
|
+
if self.focus.is_none() && self.last_refocus.is_none_or(|at| at.elapsed() >= REFOCUS_INTERVAL) {
|
|
651
|
+
self.last_refocus = Some(Instant::now());
|
|
652
|
+
self.focus_changed();
|
|
653
|
+
}
|
|
654
|
+
if self.focus.is_some() {
|
|
655
|
+
self.value_changed(agent);
|
|
656
|
+
}
|
|
657
|
+
}
|
|
658
|
+
Msg::FocusChanged => self.focus_changed(),
|
|
659
|
+
Msg::ValueChanged => {
|
|
660
|
+
if self.focus.is_some() {
|
|
661
|
+
self.value_changed(agent_marker::active());
|
|
662
|
+
}
|
|
663
|
+
}
|
|
664
|
+
}
|
|
665
|
+
}
|
|
666
|
+
|
|
667
|
+
fn subscribe(&mut self, id: u64, policy: Policy, outbox: Arc<Outbox>) {
|
|
668
|
+
self.subs.push(Sub { id, policy, delivery: DeliveryState::default(), outbox });
|
|
669
|
+
let was_running = self.running;
|
|
670
|
+
if was_running {
|
|
671
|
+
self.reset_text();
|
|
672
|
+
} else {
|
|
673
|
+
self.start();
|
|
674
|
+
}
|
|
675
|
+
if self.suspended() {
|
|
676
|
+
// 锁屏 / 睡眠时订阅:只给新来的补一条当前状态,不拍前台(人不在),解锁再接上。
|
|
677
|
+
let state = if self.locked { "locked" } else { "sleep" };
|
|
678
|
+
if let Some(sub) = self.subs.last() {
|
|
679
|
+
sub.outbox.send(&json!({ "ev": { "t": now_ms(), "kind": "system", "state": state } }));
|
|
680
|
+
}
|
|
681
|
+
return;
|
|
682
|
+
}
|
|
683
|
+
if was_running && self.context.is_some() {
|
|
684
|
+
let context = self.context.clone();
|
|
685
|
+
self.emit(Kind::App, context.as_ref(), None, Map::new(), false, true);
|
|
686
|
+
} else {
|
|
687
|
+
self.fg_pid = 0;
|
|
688
|
+
self.on_foreground(unsafe { GetForegroundWindow() }.0 as isize);
|
|
689
|
+
}
|
|
690
|
+
}
|
|
691
|
+
|
|
692
|
+
fn start(&mut self) {
|
|
693
|
+
if self.running {
|
|
694
|
+
return;
|
|
695
|
+
}
|
|
696
|
+
self.running = true;
|
|
697
|
+
self.locked = session_locked();
|
|
698
|
+
self.asleep = false;
|
|
699
|
+
self.hooks = start_hooks();
|
|
700
|
+
let handler: IUIAutomationFocusChangedEventHandler = FocusHandler.into();
|
|
701
|
+
phase("uia:AddFocusChangedEventHandler");
|
|
702
|
+
if unsafe { self.uia.AddFocusChangedEventHandler(None, &handler) }.is_ok() {
|
|
703
|
+
self.focus_handler = Some(handler);
|
|
704
|
+
}
|
|
705
|
+
}
|
|
706
|
+
|
|
707
|
+
fn stop(&mut self) {
|
|
708
|
+
if !self.running {
|
|
709
|
+
return;
|
|
710
|
+
}
|
|
711
|
+
self.running = false;
|
|
712
|
+
// 没人收了:丢掉未发的输入与基线,不再读任何东西。
|
|
713
|
+
self.pending = None;
|
|
714
|
+
self.text_deadline = None;
|
|
715
|
+
self.sample_deadline = None;
|
|
716
|
+
self.window_deadline = None;
|
|
717
|
+
self.release_focus();
|
|
718
|
+
if let Some(handler) = self.focus_handler.take() {
|
|
719
|
+
let _ = unsafe { self.uia.RemoveFocusChangedEventHandler(&handler) };
|
|
720
|
+
}
|
|
721
|
+
let _ = unsafe { self.uia.RemoveAllEventHandlers() };
|
|
722
|
+
if let Some(hooks) = self.hooks.take() {
|
|
723
|
+
let _ = unsafe { PostThreadMessageW(hooks.tid, WM_QUIT, WPARAM(0), LPARAM(0)) };
|
|
724
|
+
let _ = hooks.join.join();
|
|
725
|
+
}
|
|
726
|
+
self.context = None;
|
|
727
|
+
self.context_hwnd = 0;
|
|
728
|
+
self.fg_pid = 0;
|
|
729
|
+
self.fg_hwnd = 0;
|
|
730
|
+
self.baseline = None;
|
|
731
|
+
self.browser_windows.clear();
|
|
732
|
+
self.walk_retry = None;
|
|
733
|
+
}
|
|
734
|
+
|
|
735
|
+
/// 订阅者来去:未发的输入丢掉、差分基线作废,再按当前值重建(清除前打的字不能在清除后经旧基线冒出来)。
|
|
736
|
+
fn reset_text(&mut self) {
|
|
737
|
+
self.text_deadline = None;
|
|
738
|
+
self.sample_deadline = None;
|
|
739
|
+
self.pending = None;
|
|
740
|
+
self.baseline = None;
|
|
741
|
+
if !self.running || self.suspended() || self.focus.is_none() {
|
|
742
|
+
return;
|
|
743
|
+
}
|
|
744
|
+
if self.context.as_ref().is_some_and(|c| self.wants(Kind::Text, c)) {
|
|
745
|
+
self.prime_baseline();
|
|
746
|
+
}
|
|
747
|
+
}
|
|
748
|
+
|
|
749
|
+
// ── 投递 ──
|
|
750
|
+
|
|
751
|
+
fn emit(&mut self, kind: Kind, context: Option<&Context>, t: Option<i64>, fields: Map<String, Value>, agent: bool, dedupe: bool) {
|
|
752
|
+
let time = t.unwrap_or_else(now_ms);
|
|
753
|
+
for sub in &mut self.subs {
|
|
754
|
+
let delivered = if matches!(kind, Kind::App | Kind::Window) {
|
|
755
|
+
logic::context_delivery(kind, context, &sub.policy, &mut sub.delivery, dedupe)
|
|
756
|
+
} else {
|
|
757
|
+
logic::event_body(kind, context, &sub.policy)
|
|
758
|
+
};
|
|
759
|
+
let Some(mut body) = delivered else { continue };
|
|
760
|
+
for (name, value) in &fields {
|
|
761
|
+
body.insert(name.clone(), value.clone());
|
|
762
|
+
}
|
|
763
|
+
body.insert("t".into(), json!(time));
|
|
764
|
+
if agent {
|
|
765
|
+
body.insert("origin".into(), json!("agent"));
|
|
766
|
+
}
|
|
767
|
+
sub.outbox.send(&json!({ "ev": body }));
|
|
768
|
+
}
|
|
769
|
+
}
|
|
770
|
+
|
|
771
|
+
fn wants(&self, kind: Kind, context: &Context) -> bool {
|
|
772
|
+
self.subs.iter().any(|s| logic::event_body(kind, Some(context), &s.policy).is_some())
|
|
773
|
+
}
|
|
774
|
+
|
|
775
|
+
fn app_wanted(&self, bundle_id: &str) -> bool {
|
|
776
|
+
self.subs.iter().any(|s| s.policy.wants_app(bundle_id))
|
|
777
|
+
}
|
|
778
|
+
|
|
779
|
+
// ── 前台与情境 ──
|
|
780
|
+
|
|
781
|
+
fn on_foreground(&mut self, hwnd: isize) {
|
|
782
|
+
if hwnd == 0 || self.suspended() {
|
|
783
|
+
return;
|
|
784
|
+
}
|
|
785
|
+
let Some((pid, exe, class)) = window_app(hwnd) else { return };
|
|
786
|
+
if logic::is_ignored(&exe, &class) {
|
|
787
|
+
return;
|
|
788
|
+
}
|
|
789
|
+
if pid != self.fg_pid {
|
|
790
|
+
self.activated(hwnd, pid, &exe);
|
|
791
|
+
} else {
|
|
792
|
+
self.fg_hwnd = hwnd;
|
|
793
|
+
self.refresh_window();
|
|
794
|
+
}
|
|
795
|
+
}
|
|
796
|
+
|
|
797
|
+
fn activated(&mut self, hwnd: isize, pid: u32, exe: &str) {
|
|
798
|
+
self.flush_text();
|
|
799
|
+
self.release_focus();
|
|
800
|
+
self.window_deadline = None;
|
|
801
|
+
self.walk_retry = None;
|
|
802
|
+
self.fg_pid = pid;
|
|
803
|
+
self.fg_hwnd = hwnd;
|
|
804
|
+
let next = self.read_context(hwnd, pid, exe);
|
|
805
|
+
self.context = Some(next.clone());
|
|
806
|
+
self.context_hwnd = hwnd;
|
|
807
|
+
self.emit(Kind::App, Some(&next), None, Map::new(), agent_marker::active(), false);
|
|
808
|
+
self.focus_changed();
|
|
809
|
+
}
|
|
810
|
+
|
|
811
|
+
fn refresh_window(&mut self) {
|
|
812
|
+
self.window_deadline = None;
|
|
813
|
+
self.walk_retry = None;
|
|
814
|
+
if !self.running || self.suspended() || self.fg_hwnd == 0 {
|
|
815
|
+
return;
|
|
816
|
+
}
|
|
817
|
+
let hwnd = self.fg_hwnd;
|
|
818
|
+
let Some((pid, exe, _)) = window_app(hwnd) else { return };
|
|
819
|
+
if pid != self.fg_pid {
|
|
820
|
+
return self.activated(hwnd, pid, &exe);
|
|
821
|
+
}
|
|
822
|
+
let next = self.read_context(hwnd, pid, &exe);
|
|
823
|
+
let window_changed = hwnd != self.context_hwnd;
|
|
824
|
+
let url_changed = !window_changed
|
|
825
|
+
&& self.context.as_ref().is_some_and(|c| c.url != next.url || c.url_unknown != next.url_unknown);
|
|
826
|
+
match logic::pending_on_refresh(self.wants(Kind::Text, &next), window_changed, url_changed) {
|
|
827
|
+
PendingAction::Drop => self.drop_text(),
|
|
828
|
+
PendingAction::Flush => self.flush_text(),
|
|
829
|
+
PendingAction::Keep => {}
|
|
830
|
+
}
|
|
831
|
+
self.context = Some(next.clone());
|
|
832
|
+
self.context_hwnd = hwnd;
|
|
833
|
+
self.emit(Kind::Window, Some(&next), None, Map::new(), agent_marker::active(), true);
|
|
834
|
+
if window_changed {
|
|
835
|
+
self.focus_changed();
|
|
836
|
+
} else if self.focus.is_some() && !self.wants(Kind::Text, &next) {
|
|
837
|
+
// 同一窗口里换到了排除站点 / 无痕判定翻转:输入框撒手,不再挂着读值的事件。
|
|
838
|
+
self.release_focus();
|
|
839
|
+
}
|
|
840
|
+
}
|
|
841
|
+
|
|
842
|
+
/// 情境是不是已经过时(前台换了、标题变了而通知还在路上,或浏览器里标题没变而网址变了)→ 立刻同步刷新一次。
|
|
843
|
+
/// 读基线、开始一段输入、停手报字、记点击 / 快捷键之前都先过这一步。
|
|
844
|
+
fn refresh_if_stale(&mut self) {
|
|
845
|
+
if !self.running || self.suspended() {
|
|
846
|
+
return;
|
|
847
|
+
}
|
|
848
|
+
if self.window_deadline.is_some() {
|
|
849
|
+
return self.refresh_window();
|
|
850
|
+
}
|
|
851
|
+
let fg = unsafe { GetForegroundWindow() }.0 as isize;
|
|
852
|
+
if fg != self.context_hwnd {
|
|
853
|
+
return self.on_foreground(fg);
|
|
854
|
+
}
|
|
855
|
+
let Some(context) = self.context.clone() else { return };
|
|
856
|
+
// 硬排除 / 所有订阅者都排除的 App:标题从没读过,这里也不读来比对。
|
|
857
|
+
if context.hard_excluded || !self.app_wanted(&context.bundle_id) {
|
|
858
|
+
return;
|
|
859
|
+
}
|
|
860
|
+
if window_text(fg) != self.context_raw_title {
|
|
861
|
+
return self.refresh_window();
|
|
862
|
+
}
|
|
863
|
+
if !context.is_private && logic::is_browser(&context.bundle_id) {
|
|
864
|
+
let (url, resolved) = self.fast_url(fg);
|
|
865
|
+
if resolved.is_some_and(|resolved| url != context.url || resolved == context.url_unknown) {
|
|
866
|
+
self.refresh_window();
|
|
867
|
+
}
|
|
868
|
+
}
|
|
869
|
+
}
|
|
870
|
+
|
|
871
|
+
fn read_context(&mut self, hwnd: isize, pid: u32, exe: &str) -> Context {
|
|
872
|
+
let bundle_id = exe.to_lowercase();
|
|
873
|
+
let mut next = Context {
|
|
874
|
+
name: self.app_name(pid, exe),
|
|
875
|
+
bundle_id: bundle_id.clone(),
|
|
876
|
+
hard_excluded: pid == self.self_pid || logic::is_hard_excluded(exe) || bundle_id.starts_with("windows-bridge"),
|
|
877
|
+
..Context::default()
|
|
878
|
+
};
|
|
879
|
+
self.context_raw_title.clear();
|
|
880
|
+
if next.hard_excluded || !self.app_wanted(&bundle_id) {
|
|
881
|
+
return next;
|
|
882
|
+
}
|
|
883
|
+
let raw_title = window_text(hwnd);
|
|
884
|
+
let title = logic::title(&raw_title);
|
|
885
|
+
let mut is_private = title.as_deref().is_some_and(logic::has_private_marker);
|
|
886
|
+
let mut url = None;
|
|
887
|
+
let mut url_unknown = false;
|
|
888
|
+
if !is_private && logic::is_browser(exe) {
|
|
889
|
+
let (found, resolved, private) = self.browser_lookup(hwnd, exe, title.is_some());
|
|
890
|
+
url = found;
|
|
891
|
+
url_unknown = !resolved;
|
|
892
|
+
is_private = private;
|
|
893
|
+
}
|
|
894
|
+
self.context_raw_title = raw_title;
|
|
895
|
+
next.is_private = is_private;
|
|
896
|
+
if !is_private {
|
|
897
|
+
next.title = title;
|
|
898
|
+
next.url = url;
|
|
899
|
+
next.url_unknown = url_unknown;
|
|
900
|
+
}
|
|
901
|
+
next
|
|
902
|
+
}
|
|
903
|
+
|
|
904
|
+
fn app_name(&mut self, pid: u32, exe: &str) -> String {
|
|
905
|
+
let path = process_path(pid).unwrap_or_default();
|
|
906
|
+
if let Some(name) = self.app_names.get(&path) {
|
|
907
|
+
return name.clone();
|
|
908
|
+
}
|
|
909
|
+
let stem = exe.strip_suffix(".exe").or_else(|| exe.strip_suffix(".EXE")).unwrap_or(exe).to_owned();
|
|
910
|
+
let name = file_description(&path).and_then(|d| logic::label(&d)).unwrap_or(stem);
|
|
911
|
+
if self.app_names.len() > 256 {
|
|
912
|
+
self.app_names.clear();
|
|
913
|
+
}
|
|
914
|
+
self.app_names.insert(path, name.clone());
|
|
915
|
+
name
|
|
916
|
+
}
|
|
917
|
+
|
|
918
|
+
// ── 浏览器网址 / 无痕(有上限、绝不展开网页内容) ──
|
|
919
|
+
|
|
920
|
+
/// (网址, 读到没有, 无痕)。每个窗口走一遍有上限的遍历,记住地址栏与无痕判定;之后只按地址栏快路重读。
|
|
921
|
+
/// 失败即关闭:遍历没走完、又证明不了不是无痕的窗口按无痕处理(不带标题和网址),之后刷新时再走,最多 WALK_ATTEMPTS 次。
|
|
922
|
+
fn browser_lookup(&mut self, hwnd: isize, exe: &str, has_title: bool) -> (Option<String>, bool, bool) {
|
|
923
|
+
let previous = self.browser_windows.get(&hwnd);
|
|
924
|
+
if previous.is_none_or(|w| !w.complete && !w.hint && w.attempts < WALK_ATTEMPTS) {
|
|
925
|
+
let mut walked = self.walk_browser(hwnd);
|
|
926
|
+
if let Some(previous) = self.browser_windows.remove(&hwnd) {
|
|
927
|
+
walked.attempts = previous.attempts;
|
|
928
|
+
walked.address = walked.address.or(previous.address);
|
|
929
|
+
}
|
|
930
|
+
walked.attempts += 1;
|
|
931
|
+
if self.browser_windows.len() >= BROWSER_CACHE_CAP {
|
|
932
|
+
self.browser_windows.clear();
|
|
933
|
+
}
|
|
934
|
+
self.browser_windows.insert(hwnd, walked);
|
|
935
|
+
}
|
|
936
|
+
let Some(window) = self.browser_windows.get(&hwnd) else { return (None, true, true) };
|
|
937
|
+
// 标题读不到(空 / 读失败)就证明不了没有无痕标记:不信标题,只看遍历。
|
|
938
|
+
match logic::private_verdict(has_title && logic::title_marks_private(exe), window.hint, window.complete) {
|
|
939
|
+
PrivateVerdict::Normal => {}
|
|
940
|
+
PrivateVerdict::Private => return (None, true, true),
|
|
941
|
+
PrivateVerdict::Unknown => {
|
|
942
|
+
if window.attempts < WALK_ATTEMPTS {
|
|
943
|
+
self.walk_retry = Some(Instant::now() + WALK_RETRY);
|
|
944
|
+
}
|
|
945
|
+
return (None, true, true);
|
|
946
|
+
}
|
|
947
|
+
}
|
|
948
|
+
let (url, resolved) = self.fast_url(hwnd);
|
|
949
|
+
(url, resolved.unwrap_or(false), false)
|
|
950
|
+
}
|
|
951
|
+
|
|
952
|
+
/// 地址栏快路。返回 resolved = None:这个窗口没有记住的地址栏(判不了,不为此再走遍历)。
|
|
953
|
+
fn fast_url(&mut self, hwnd: isize) -> (Option<String>, Option<bool>) {
|
|
954
|
+
let Some(address) = self.browser_windows.get(&hwnd).and_then(|w| w.address.clone()) else {
|
|
955
|
+
return (None, None);
|
|
956
|
+
};
|
|
957
|
+
match element_value(&address) {
|
|
958
|
+
Ok(raw) => {
|
|
959
|
+
let (url, resolved) = logic::address_bar_url(&raw);
|
|
960
|
+
(url, Some(resolved))
|
|
961
|
+
}
|
|
962
|
+
Err(_) => {
|
|
963
|
+
// 地址栏元素失效(窗口重建了工具栏):下次重新遍历。
|
|
964
|
+
self.browser_windows.remove(&hwnd);
|
|
965
|
+
(None, Some(false))
|
|
966
|
+
}
|
|
967
|
+
}
|
|
968
|
+
}
|
|
969
|
+
|
|
970
|
+
fn walk_browser(&self, hwnd: isize) -> BrowserWindow {
|
|
971
|
+
phase("uia:walkBrowser");
|
|
972
|
+
let mut found = BrowserWindow { address: None, hint: false, complete: false, attempts: 0 };
|
|
973
|
+
let Ok(cache) = (unsafe { self.uia.CreateCacheRequest() }) else { return found };
|
|
974
|
+
for property in [UIA_ControlTypePropertyId, UIA_NamePropertyId, UIA_ClassNamePropertyId, UIA_AutomationIdPropertyId] {
|
|
975
|
+
let _ = unsafe { cache.AddProperty(property) };
|
|
976
|
+
}
|
|
977
|
+
let Ok(condition) = (unsafe { self.uia.ControlViewCondition() }) else { return found };
|
|
978
|
+
let started = Instant::now();
|
|
979
|
+
let Ok(root) = (unsafe { self.uia.ElementFromHandleBuildCache(HWND(hwnd as *mut _), &cache) }) else {
|
|
980
|
+
return found;
|
|
981
|
+
};
|
|
982
|
+
let root_ms = started.elapsed().as_millis();
|
|
983
|
+
let deadline = Instant::now() + WALK_BUDGET;
|
|
984
|
+
let mut queue = std::collections::VecDeque::from([(root, 0usize)]);
|
|
985
|
+
let mut nodes = 0usize;
|
|
986
|
+
let mut stop = "done";
|
|
987
|
+
while let Some((node, depth)) = queue.pop_front() {
|
|
988
|
+
if nodes >= WALK_NODE_CAP {
|
|
989
|
+
stop = "nodeCap";
|
|
990
|
+
break;
|
|
991
|
+
}
|
|
992
|
+
if Instant::now() >= deadline {
|
|
993
|
+
stop = "deadline";
|
|
994
|
+
break;
|
|
995
|
+
}
|
|
996
|
+
nodes += 1;
|
|
997
|
+
let control_type = unsafe { node.CachedControlType() }.map(|c| c.0).unwrap_or(0);
|
|
998
|
+
// 网页内容:绝不展开(会逼浏览器为整页建无障碍树)。
|
|
999
|
+
if control_type == logic::CT_DOCUMENT {
|
|
1000
|
+
continue;
|
|
1001
|
+
}
|
|
1002
|
+
let name = unsafe { node.CachedName() }.map(|b| b.to_string()).unwrap_or_default();
|
|
1003
|
+
if matches!(control_type, logic::CT_BUTTON | logic::CT_SPLITBUTTON | logic::CT_MENUITEM) {
|
|
1004
|
+
if debug_enabled() {
|
|
1005
|
+
eprintln!("[recorder] walk button {:?}", logic::truncate(&name, 60).0);
|
|
1006
|
+
}
|
|
1007
|
+
if logic::has_private_marker(&name) {
|
|
1008
|
+
found.hint = true;
|
|
1009
|
+
found.address = None;
|
|
1010
|
+
stop = "hint";
|
|
1011
|
+
break;
|
|
1012
|
+
}
|
|
1013
|
+
}
|
|
1014
|
+
if control_type == logic::CT_EDIT && found.address.is_none() {
|
|
1015
|
+
let class = unsafe { node.CachedClassName() }.map(|b| b.to_string()).unwrap_or_default();
|
|
1016
|
+
let automation_id = unsafe { node.CachedAutomationId() }.map(|b| b.to_string()).unwrap_or_default();
|
|
1017
|
+
if debug_enabled() {
|
|
1018
|
+
eprintln!("[recorder] walk edit name={:?} class={class:?} id={automation_id:?}", logic::truncate(&name, 60).0);
|
|
1019
|
+
}
|
|
1020
|
+
if logic::is_address_field(&name, &class, &automation_id) {
|
|
1021
|
+
found.address = Some(node.clone());
|
|
1022
|
+
}
|
|
1023
|
+
}
|
|
1024
|
+
// 子节点取不到、或深度到顶还有子节点:这棵子树没看过,遍历不算走完(无痕提示可能就在里面)。
|
|
1025
|
+
let Ok(children) = (unsafe { node.FindAllBuildCache(TreeScope_Children, &condition, &cache) }) else {
|
|
1026
|
+
stop = "childrenFailed";
|
|
1027
|
+
continue;
|
|
1028
|
+
};
|
|
1029
|
+
let count = unsafe { children.Length() }.unwrap_or(0);
|
|
1030
|
+
if depth + 1 >= WALK_DEPTH_CAP {
|
|
1031
|
+
if count > 0 {
|
|
1032
|
+
stop = "depthCap";
|
|
1033
|
+
}
|
|
1034
|
+
continue;
|
|
1035
|
+
}
|
|
1036
|
+
for index in 0..count {
|
|
1037
|
+
if let Ok(child) = unsafe { children.GetElement(index) } {
|
|
1038
|
+
queue.push_back((child, depth + 1));
|
|
1039
|
+
}
|
|
1040
|
+
}
|
|
1041
|
+
}
|
|
1042
|
+
found.complete = stop == "done";
|
|
1043
|
+
if debug_enabled() {
|
|
1044
|
+
eprintln!(
|
|
1045
|
+
"[recorder] walk {stop} nodes={nodes} rootMs={root_ms} walkMs={} address={} hint={}",
|
|
1046
|
+
started.elapsed().as_millis() - root_ms,
|
|
1047
|
+
found.address.is_some(),
|
|
1048
|
+
found.hint
|
|
1049
|
+
);
|
|
1050
|
+
}
|
|
1051
|
+
found
|
|
1052
|
+
}
|
|
1053
|
+
|
|
1054
|
+
// ── 焦点输入框与文本差分 ──
|
|
1055
|
+
|
|
1056
|
+
fn focus_changed(&mut self) {
|
|
1057
|
+
if !self.running || self.suspended() {
|
|
1058
|
+
return;
|
|
1059
|
+
}
|
|
1060
|
+
// 焦点通知可能比前台 / 标题通知先到:先把情境对齐,再按新情境决定读不读。
|
|
1061
|
+
self.refresh_if_stale();
|
|
1062
|
+
let Some(context) = self.context.clone() else { return self.release_and_flush() };
|
|
1063
|
+
// 无痕 / 排除 / 只记标题 / 都不要文字的 App:连焦点元素都不取(不读控件名、不挂事件)。
|
|
1064
|
+
// 按完整情境判(含排除站点、网址读不到时的失败即关闭),不只看 App。
|
|
1065
|
+
if context.hard_excluded || context.is_private || !self.wants(Kind::Text, &context) {
|
|
1066
|
+
return self.release_and_flush();
|
|
1067
|
+
}
|
|
1068
|
+
// 一次跨进程调用取齐判断要的属性(类型、密码框、进程、名字、两种模式可不可写)。
|
|
1069
|
+
phase("uia:GetFocusedElementBuildCache");
|
|
1070
|
+
let element = self.focus_cache.as_ref().and_then(|cache| unsafe { self.uia.GetFocusedElementBuildCache(cache) }.ok());
|
|
1071
|
+
self.set_focus(element, &context);
|
|
1072
|
+
}
|
|
1073
|
+
|
|
1074
|
+
fn release_and_flush(&mut self) {
|
|
1075
|
+
self.flush_text();
|
|
1076
|
+
self.release_focus();
|
|
1077
|
+
}
|
|
1078
|
+
|
|
1079
|
+
fn set_focus(&mut self, element: Option<IUIAutomationElement>, context: &Context) {
|
|
1080
|
+
if let (Some(element), Some(current)) = (&element, &self.focus) {
|
|
1081
|
+
phase("uia:CompareElements");
|
|
1082
|
+
let same = unsafe { self.uia.CompareElements(element, ¤t.element) }.is_ok_and(|b| b.as_bool());
|
|
1083
|
+
if same {
|
|
1084
|
+
// 同一个元素再次聚焦:网页可能已原地把它改成了密码框。复核一次,是就立刻撒手。
|
|
1085
|
+
if field_secure(element) {
|
|
1086
|
+
self.drop_secure_field();
|
|
1087
|
+
}
|
|
1088
|
+
return;
|
|
1089
|
+
}
|
|
1090
|
+
}
|
|
1091
|
+
self.flush_text();
|
|
1092
|
+
self.release_focus();
|
|
1093
|
+
let Some(element) = element else { return };
|
|
1094
|
+
let pid = unsafe { element.CachedProcessId() }.unwrap_or(0) as u32;
|
|
1095
|
+
let control_type = unsafe { element.CachedControlType() }.map(|c| c.0).unwrap_or(0);
|
|
1096
|
+
// 查不到按密码框算(证明不了不是)。
|
|
1097
|
+
let is_password = unsafe { element.CachedIsPassword() }.map(|b| b.as_bool()).unwrap_or(true);
|
|
1098
|
+
self.focus_is_password = is_password;
|
|
1099
|
+
// 焦点在别的进程(浮层 / 系统输入框)里:不看。
|
|
1100
|
+
if pid != self.fg_pid || is_password {
|
|
1101
|
+
if debug_enabled() {
|
|
1102
|
+
eprintln!("[recorder] focus skipped pid={pid} fg={} password={is_password}", self.fg_pid);
|
|
1103
|
+
}
|
|
1104
|
+
return;
|
|
1105
|
+
}
|
|
1106
|
+
let value_editable = (cached_bool(&element, UIA_IsValuePatternAvailablePropertyId) == Some(true))
|
|
1107
|
+
.then(|| cached_bool(&element, UIA_ValueIsReadOnlyPropertyId) == Some(false));
|
|
1108
|
+
let text_editable = (value_editable.is_none()
|
|
1109
|
+
&& control_type != logic::CT_COMBOBOX
|
|
1110
|
+
&& cached_bool(&element, UIA_IsTextPatternAvailablePropertyId) == Some(true))
|
|
1111
|
+
.then(|| text_range_editable(&element))
|
|
1112
|
+
.flatten();
|
|
1113
|
+
let mode = logic::text_mode(control_type, is_password, value_editable, text_editable);
|
|
1114
|
+
if debug_enabled() {
|
|
1115
|
+
eprintln!(
|
|
1116
|
+
"[recorder] focus type={} value={value_editable:?} text={text_editable:?} mode={mode:?}",
|
|
1117
|
+
logic::control_type_name(control_type)
|
|
1118
|
+
);
|
|
1119
|
+
}
|
|
1120
|
+
let Some(mode) = mode else { return };
|
|
1121
|
+
phase(if mode == TextMode::Value { "uia:AddPropertyChangedEventHandler" } else { "uia:AddAutomationEventHandler" });
|
|
1122
|
+
let registered = unsafe {
|
|
1123
|
+
match mode {
|
|
1124
|
+
TextMode::Value => self.uia.AddPropertyChangedEventHandlerNativeArray(
|
|
1125
|
+
&element,
|
|
1126
|
+
TreeScope_Element,
|
|
1127
|
+
None,
|
|
1128
|
+
&self.value_handler.property,
|
|
1129
|
+
&[UIA_ValueValuePropertyId],
|
|
1130
|
+
),
|
|
1131
|
+
TextMode::Text => self.uia.AddAutomationEventHandler(
|
|
1132
|
+
UIA_Text_TextChangedEventId,
|
|
1133
|
+
&element,
|
|
1134
|
+
TreeScope_Element,
|
|
1135
|
+
None,
|
|
1136
|
+
&self.value_handler.event,
|
|
1137
|
+
),
|
|
1138
|
+
}
|
|
1139
|
+
};
|
|
1140
|
+
// 挂不上值变化事件也照样跟踪(键盘活动信号兜底触发差分)。
|
|
1141
|
+
let _ = registered;
|
|
1142
|
+
let mut el = Map::new();
|
|
1143
|
+
el.insert("role".into(), json!(logic::control_type_name(control_type)));
|
|
1144
|
+
if let Some(label) = unsafe { element.CachedName() }.ok().and_then(|n| logic::label(&n.to_string())) {
|
|
1145
|
+
el.insert("label".into(), json!(label));
|
|
1146
|
+
}
|
|
1147
|
+
self.focus = Some(Focus { element, mode, el, sampling: false });
|
|
1148
|
+
if self.wants(Kind::Text, context) {
|
|
1149
|
+
self.prime_baseline();
|
|
1150
|
+
} else {
|
|
1151
|
+
self.baseline = None;
|
|
1152
|
+
}
|
|
1153
|
+
}
|
|
1154
|
+
|
|
1155
|
+
fn release_focus(&mut self) {
|
|
1156
|
+
self.sample_deadline = None;
|
|
1157
|
+
if let Some(focus) = self.focus.take() {
|
|
1158
|
+
phase("uia:RemoveValueHandler");
|
|
1159
|
+
unsafe {
|
|
1160
|
+
match focus.mode {
|
|
1161
|
+
TextMode::Value => {
|
|
1162
|
+
let _ = self.uia.RemovePropertyChangedEventHandler(&focus.element, &self.value_handler.property);
|
|
1163
|
+
}
|
|
1164
|
+
TextMode::Text => {
|
|
1165
|
+
let _ = self.uia.RemoveAutomationEventHandler(
|
|
1166
|
+
UIA_Text_TextChangedEventId,
|
|
1167
|
+
&focus.element,
|
|
1168
|
+
&self.value_handler.event,
|
|
1169
|
+
);
|
|
1170
|
+
}
|
|
1171
|
+
}
|
|
1172
|
+
}
|
|
1173
|
+
}
|
|
1174
|
+
self.focus_is_password = false;
|
|
1175
|
+
}
|
|
1176
|
+
|
|
1177
|
+
/// 聚焦时记下当前值作为差分基线(聚焦前就有的内容不算「打的字」)。
|
|
1178
|
+
fn prime_baseline(&mut self) {
|
|
1179
|
+
let Some(focus) = self.focus.as_mut() else { return };
|
|
1180
|
+
focus.sampling = false;
|
|
1181
|
+
match read_field(&focus.element, focus.mode) {
|
|
1182
|
+
FieldRead::Secure => self.drop_secure_field(),
|
|
1183
|
+
FieldRead::Big | FieldRead::Unreadable => self.baseline = None,
|
|
1184
|
+
FieldRead::Value(value) => {
|
|
1185
|
+
focus.sampling = value.chars().count() <= limit::SAMPLE_CHARS;
|
|
1186
|
+
self.baseline = Some(value);
|
|
1187
|
+
}
|
|
1188
|
+
}
|
|
1189
|
+
}
|
|
1190
|
+
|
|
1191
|
+
fn value_changed(&mut self, agent: bool) {
|
|
1192
|
+
// 一段新输入的开头:先确认情境没过时(同一个输入框所在的标签页可能刚切到排除站点)。
|
|
1193
|
+
if self.pending.is_none() && self.context.as_ref().is_some_and(|c| self.wants(Kind::Text, c)) {
|
|
1194
|
+
self.refresh_if_stale();
|
|
1195
|
+
}
|
|
1196
|
+
let (Some(_), Some(context)) = (&self.focus, self.context.clone()) else { return };
|
|
1197
|
+
if !self.wants(Kind::Text, &context) {
|
|
1198
|
+
// 这段期间的值不能进下一次差分:丢掉待发与基线。
|
|
1199
|
+
self.pending = None;
|
|
1200
|
+
self.text_deadline = None;
|
|
1201
|
+
self.baseline = None;
|
|
1202
|
+
return;
|
|
1203
|
+
}
|
|
1204
|
+
let now = now_ms();
|
|
1205
|
+
match self.pending.as_mut() {
|
|
1206
|
+
Some(pending) => {
|
|
1207
|
+
pending.last_change = now;
|
|
1208
|
+
pending.agent |= agent;
|
|
1209
|
+
}
|
|
1210
|
+
None => {
|
|
1211
|
+
let el = self.focus.as_ref().map(|f| f.el.clone()).unwrap_or_default();
|
|
1212
|
+
self.pending = Some(Pending { context, el, last_change: now, agent, latest: None });
|
|
1213
|
+
}
|
|
1214
|
+
}
|
|
1215
|
+
self.text_deadline = Some(Instant::now() + TEXT_DEBOUNCE);
|
|
1216
|
+
if self.focus.as_ref().is_some_and(|f| f.sampling) && self.sample_deadline.is_none() {
|
|
1217
|
+
self.sample_deadline = Some(Instant::now() + SAMPLE_DELAY);
|
|
1218
|
+
}
|
|
1219
|
+
}
|
|
1220
|
+
|
|
1221
|
+
/// 小输入框边打边采样(≤4 次/秒):抓住「发送后立刻被清空」之前的那一版。
|
|
1222
|
+
fn sample_text(&mut self) {
|
|
1223
|
+
self.sample_deadline = None;
|
|
1224
|
+
let (Some(focus), Some(pending), Some(live)) = (&self.focus, &self.pending, &self.context) else { return };
|
|
1225
|
+
if !self.wants(Kind::Text, &pending.context) || !self.wants(Kind::Text, live) {
|
|
1226
|
+
return;
|
|
1227
|
+
}
|
|
1228
|
+
let started = Instant::now();
|
|
1229
|
+
let read = read_field(&focus.element, focus.mode);
|
|
1230
|
+
let slow = started.elapsed().as_millis() > limit::SAMPLE_MS;
|
|
1231
|
+
let value = match read {
|
|
1232
|
+
FieldRead::Secure => return self.drop_secure_field(),
|
|
1233
|
+
FieldRead::Unreadable | FieldRead::Big => {
|
|
1234
|
+
if let Some(focus) = self.focus.as_mut() {
|
|
1235
|
+
focus.sampling = false;
|
|
1236
|
+
}
|
|
1237
|
+
return;
|
|
1238
|
+
}
|
|
1239
|
+
FieldRead::Value(text) => text,
|
|
1240
|
+
};
|
|
1241
|
+
if slow || value.chars().count() > limit::SAMPLE_CHARS {
|
|
1242
|
+
if let Some(focus) = self.focus.as_mut() {
|
|
1243
|
+
focus.sampling = false;
|
|
1244
|
+
}
|
|
1245
|
+
}
|
|
1246
|
+
let latest = pending.latest.clone();
|
|
1247
|
+
if value.is_empty() && latest.as_deref().is_some_and(|l| !l.is_empty()) {
|
|
1248
|
+
// 输入框被清空(聊天框发送的典型形态):清空前那版立刻当作一段输入报出去,基线归零。
|
|
1249
|
+
self.refresh_if_stale();
|
|
1250
|
+
if self.pending.is_none() || self.focus.is_none() {
|
|
1251
|
+
return;
|
|
1252
|
+
}
|
|
1253
|
+
if self.focus.as_ref().is_some_and(|f| field_secure(&f.element)) {
|
|
1254
|
+
return self.drop_secure_field();
|
|
1255
|
+
}
|
|
1256
|
+
let pending = self.pending.take().expect("checked above");
|
|
1257
|
+
if let Some(diff) = self
|
|
1258
|
+
.baseline
|
|
1259
|
+
.as_deref()
|
|
1260
|
+
.and_then(|base| logic::settle_text(base, latest.as_deref(), &value))
|
|
1261
|
+
.filter(|d| !d.inserted.is_empty())
|
|
1262
|
+
{
|
|
1263
|
+
let mut fields = logic::text_fields(&diff);
|
|
1264
|
+
fields.insert("el".into(), Value::Object(pending.el.clone()));
|
|
1265
|
+
self.emit(Kind::Text, Some(&pending.context), Some(pending.last_change), fields, pending.agent, false);
|
|
1266
|
+
}
|
|
1267
|
+
self.baseline = Some(value);
|
|
1268
|
+
self.text_deadline = None;
|
|
1269
|
+
return;
|
|
1270
|
+
}
|
|
1271
|
+
if let Some(pending) = self.pending.as_mut() {
|
|
1272
|
+
pending.latest = Some(value);
|
|
1273
|
+
}
|
|
1274
|
+
}
|
|
1275
|
+
|
|
1276
|
+
fn flush_text(&mut self) {
|
|
1277
|
+
self.text_deadline = None;
|
|
1278
|
+
self.sample_deadline = None;
|
|
1279
|
+
let Some(pending) = self.pending.take() else { return };
|
|
1280
|
+
let Some(focus) = &self.focus else {
|
|
1281
|
+
self.baseline = None;
|
|
1282
|
+
return;
|
|
1283
|
+
};
|
|
1284
|
+
let live_ok = self.context.as_ref().is_some_and(|c| self.wants(Kind::Text, c));
|
|
1285
|
+
if !self.wants(Kind::Text, &pending.context) || !live_ok {
|
|
1286
|
+
self.baseline = None;
|
|
1287
|
+
return;
|
|
1288
|
+
}
|
|
1289
|
+
let final_value = match read_field(&focus.element, focus.mode) {
|
|
1290
|
+
FieldRead::Secure => return self.drop_secure_field(),
|
|
1291
|
+
FieldRead::Big => {
|
|
1292
|
+
self.baseline = None;
|
|
1293
|
+
let mut fields = Map::new();
|
|
1294
|
+
fields.insert("el".into(), Value::Object(pending.el.clone()));
|
|
1295
|
+
fields.insert("bigEdit".into(), json!(true));
|
|
1296
|
+
return self.emit(Kind::Text, Some(&pending.context), Some(pending.last_change), fields, pending.agent, false);
|
|
1297
|
+
}
|
|
1298
|
+
// 元素没了 / 读不到:用最后一次采样兜底(有的聊天 App 发送后直接换掉输入框)。
|
|
1299
|
+
FieldRead::Unreadable => match pending.latest.clone() {
|
|
1300
|
+
Some(latest) => latest,
|
|
1301
|
+
None => return,
|
|
1302
|
+
},
|
|
1303
|
+
FieldRead::Value(text) => text,
|
|
1304
|
+
};
|
|
1305
|
+
// 没有基线(聚焦时读不到 / 刚从大文本缩回来):这次只建基线,不猜哪些是新打的。
|
|
1306
|
+
let Some(diff) = self
|
|
1307
|
+
.baseline
|
|
1308
|
+
.as_deref()
|
|
1309
|
+
.and_then(|base| logic::settle_text(base, pending.latest.as_deref(), &final_value))
|
|
1310
|
+
else {
|
|
1311
|
+
self.baseline = Some(final_value);
|
|
1312
|
+
return;
|
|
1313
|
+
};
|
|
1314
|
+
if field_secure(&focus.element) {
|
|
1315
|
+
return self.drop_secure_field();
|
|
1316
|
+
}
|
|
1317
|
+
self.baseline = Some(final_value);
|
|
1318
|
+
let mut fields = logic::text_fields(&diff);
|
|
1319
|
+
fields.insert("el".into(), Value::Object(pending.el.clone()));
|
|
1320
|
+
self.emit(Kind::Text, Some(&pending.context), Some(pending.last_change), fields, pending.agent, false);
|
|
1321
|
+
}
|
|
1322
|
+
|
|
1323
|
+
/// 丢掉待发输入(不报),并作废基线 —— 只丢待发不清基线的话,下一次差分会把丢掉的字又带出来。
|
|
1324
|
+
fn drop_text(&mut self) {
|
|
1325
|
+
self.text_deadline = None;
|
|
1326
|
+
self.sample_deadline = None;
|
|
1327
|
+
self.pending = None;
|
|
1328
|
+
self.baseline = None;
|
|
1329
|
+
}
|
|
1330
|
+
|
|
1331
|
+
/// 焦点框此刻是密码框:待发输入、基线、值变化事件全部立刻丢掉,不报。之后要等焦点换走再回来才会再看它。
|
|
1332
|
+
fn drop_secure_field(&mut self) {
|
|
1333
|
+
self.drop_text();
|
|
1334
|
+
self.release_focus();
|
|
1335
|
+
self.focus_is_password = true;
|
|
1336
|
+
}
|
|
1337
|
+
|
|
1338
|
+
// ── 点击 / 快捷键 / 系统 ──
|
|
1339
|
+
|
|
1340
|
+
fn clicked(&mut self, x: i32, y: i32, agent: bool) {
|
|
1341
|
+
if !self.context.as_ref().is_some_and(|c| self.wants(Kind::Click, c)) {
|
|
1342
|
+
return;
|
|
1343
|
+
}
|
|
1344
|
+
// 点的可能是刚打开的无痕窗口 / 刚切到的排除站点:先把情境对齐。
|
|
1345
|
+
self.refresh_if_stale();
|
|
1346
|
+
let Some(context) = self.context.clone() else { return };
|
|
1347
|
+
if !self.wants(Kind::Click, &context) {
|
|
1348
|
+
return;
|
|
1349
|
+
}
|
|
1350
|
+
// 点到别的 App 会先切前台;那一下不记。点在不抢前台的浮层(托盘、通知、开始菜单)上也不记。
|
|
1351
|
+
let point = POINT { x, y };
|
|
1352
|
+
let target = unsafe { GetAncestor(WindowFromPoint(point), GA_ROOT) };
|
|
1353
|
+
let Some((pid, _, _)) = window_app(target.0 as isize) else { return };
|
|
1354
|
+
if pid != self.fg_pid {
|
|
1355
|
+
return;
|
|
1356
|
+
}
|
|
1357
|
+
if self.pending.is_some() {
|
|
1358
|
+
self.flush_text();
|
|
1359
|
+
}
|
|
1360
|
+
let Ok(cache) = (unsafe { self.uia.CreateCacheRequest() }) else { return };
|
|
1361
|
+
for property in [UIA_ControlTypePropertyId, UIA_NamePropertyId, UIA_ProcessIdPropertyId] {
|
|
1362
|
+
let _ = unsafe { cache.AddProperty(property) };
|
|
1363
|
+
}
|
|
1364
|
+
phase("uia:ElementFromPointBuildCache");
|
|
1365
|
+
let Ok(hit) = (unsafe { self.uia.ElementFromPointBuildCache(point, &cache) }) else { return };
|
|
1366
|
+
if unsafe { hit.CachedProcessId() }.unwrap_or(0) as u32 != self.fg_pid {
|
|
1367
|
+
return;
|
|
1368
|
+
}
|
|
1369
|
+
let control_type = unsafe { hit.CachedControlType() }.map(|c| c.0).unwrap_or(0);
|
|
1370
|
+
if control_type == 0 {
|
|
1371
|
+
return;
|
|
1372
|
+
}
|
|
1373
|
+
let mut el = Map::new();
|
|
1374
|
+
el.insert("role".into(), json!(logic::control_type_name(control_type)));
|
|
1375
|
+
if logic::click_label_allowed(control_type) {
|
|
1376
|
+
if let Some(label) = unsafe { hit.CachedName() }.ok().and_then(|n| logic::label(&n.to_string())) {
|
|
1377
|
+
el.insert("label".into(), json!(label));
|
|
1378
|
+
}
|
|
1379
|
+
}
|
|
1380
|
+
let mut fields = Map::new();
|
|
1381
|
+
fields.insert("el".into(), Value::Object(el));
|
|
1382
|
+
self.emit(Kind::Click, Some(&context), None, fields, agent, false);
|
|
1383
|
+
}
|
|
1384
|
+
|
|
1385
|
+
fn keyed(&mut self, combo: String, agent: bool) {
|
|
1386
|
+
// 密码框里的组合键也不记(与 macOS 在 Secure Input 期间不报按键一致)。
|
|
1387
|
+
if self.focus_is_password || !self.context.as_ref().is_some_and(|c| self.wants(Kind::Key, c)) {
|
|
1388
|
+
return;
|
|
1389
|
+
}
|
|
1390
|
+
let fg = unsafe { GetForegroundWindow() }.0 as isize;
|
|
1391
|
+
if window_app(fg).map(|(pid, _, _)| pid) != Some(self.fg_pid) {
|
|
1392
|
+
return;
|
|
1393
|
+
}
|
|
1394
|
+
self.refresh_if_stale();
|
|
1395
|
+
let Some(context) = self.context.clone() else { return };
|
|
1396
|
+
if !self.wants(Kind::Key, &context) {
|
|
1397
|
+
return;
|
|
1398
|
+
}
|
|
1399
|
+
if self.pending.is_some() {
|
|
1400
|
+
self.flush_text();
|
|
1401
|
+
}
|
|
1402
|
+
let mut fields = Map::new();
|
|
1403
|
+
fields.insert("keys".into(), json!(combo));
|
|
1404
|
+
self.emit(Kind::Key, Some(&context), None, fields, agent, false);
|
|
1405
|
+
}
|
|
1406
|
+
|
|
1407
|
+
fn system_event(&mut self, state: &'static str) {
|
|
1408
|
+
let was_suspended = self.suspended();
|
|
1409
|
+
match state {
|
|
1410
|
+
"locked" if self.locked => return,
|
|
1411
|
+
"unlocked" if !self.locked => return,
|
|
1412
|
+
"sleep" if self.asleep => return,
|
|
1413
|
+
"wake" if !self.asleep => return,
|
|
1414
|
+
"locked" => self.locked = true,
|
|
1415
|
+
"unlocked" => self.locked = false,
|
|
1416
|
+
"sleep" => self.asleep = true,
|
|
1417
|
+
"wake" => self.asleep = false,
|
|
1418
|
+
_ => return,
|
|
1419
|
+
}
|
|
1420
|
+
if self.suspended() && !was_suspended {
|
|
1421
|
+
self.flush_text();
|
|
1422
|
+
self.release_focus();
|
|
1423
|
+
}
|
|
1424
|
+
let mut fields = Map::new();
|
|
1425
|
+
fields.insert("state".into(), json!(state));
|
|
1426
|
+
self.emit(Kind::System, None, None, fields, false, false);
|
|
1427
|
+
if was_suspended && !self.suspended() {
|
|
1428
|
+
// 回来了:重新拍一次前台。只清去重键;「没记」的标记留着(锁屏前在无痕窗口,解锁后回到普通页面照样带 resumed)。
|
|
1429
|
+
for sub in &mut self.subs {
|
|
1430
|
+
sub.delivery.last_key = None;
|
|
1431
|
+
}
|
|
1432
|
+
self.fg_pid = 0;
|
|
1433
|
+
self.on_foreground(unsafe { GetForegroundWindow() }.0 as isize);
|
|
1434
|
+
}
|
|
1435
|
+
}
|
|
1436
|
+
}
|
|
1437
|
+
|
|
1438
|
+
// ── UIA 读取小工具 ──
|
|
1439
|
+
|
|
1440
|
+
fn cached_bool(element: &IUIAutomationElement, property: UIA_PROPERTY_ID) -> Option<bool> {
|
|
1441
|
+
let value = unsafe { element.GetCachedPropertyValue(property) }.ok()?;
|
|
1442
|
+
bool::try_from(&value).ok()
|
|
1443
|
+
}
|
|
1444
|
+
|
|
1445
|
+
/// TextPattern 文档区间的 IsReadOnly(网页正文是只读的,富文本编辑区不是)。查不到 → None(不当输入框)。
|
|
1446
|
+
fn text_range_editable(element: &IUIAutomationElement) -> Option<bool> {
|
|
1447
|
+
unsafe {
|
|
1448
|
+
let pattern = element.GetCurrentPatternAs::<IUIAutomationTextPattern>(UIA_TextPatternId).ok()?;
|
|
1449
|
+
let read_only = pattern.DocumentRange().ok()?.GetAttributeValue(UIA_IsReadOnlyAttributeId).ok()?;
|
|
1450
|
+
bool::try_from(&read_only).ok().map(|read_only| !read_only)
|
|
1451
|
+
}
|
|
1452
|
+
}
|
|
1453
|
+
|
|
1454
|
+
fn element_value(element: &IUIAutomationElement) -> windows::core::Result<String> {
|
|
1455
|
+
unsafe {
|
|
1456
|
+
let pattern = element.GetCurrentPatternAs::<IUIAutomationValuePattern>(UIA_ValuePatternId)?;
|
|
1457
|
+
Ok(pattern.CurrentValue()?.to_string())
|
|
1458
|
+
}
|
|
1459
|
+
}
|
|
1460
|
+
|
|
1461
|
+
/// 报字 / 读值之前的复核:IsPassword 为真,或查不到(超时、App 不应答 —— 证明不了不是)→ 当密码框。
|
|
1462
|
+
/// 元素已销毁不算(没东西可读了)。
|
|
1463
|
+
fn field_secure(element: &IUIAutomationElement) -> bool {
|
|
1464
|
+
match unsafe { element.CurrentIsPassword() } {
|
|
1465
|
+
Ok(is_password) => is_password.as_bool(),
|
|
1466
|
+
Err(e) => e.code().0 != UIA_E_ELEMENTNOTAVAILABLE,
|
|
1467
|
+
}
|
|
1468
|
+
}
|
|
1469
|
+
|
|
1470
|
+
fn read_field(element: &IUIAutomationElement, mode: TextMode) -> FieldRead {
|
|
1471
|
+
phase("uia:readField");
|
|
1472
|
+
match unsafe { element.CurrentIsPassword() } {
|
|
1473
|
+
Ok(is_password) if is_password.as_bool() => return FieldRead::Secure,
|
|
1474
|
+
Ok(_) => {}
|
|
1475
|
+
Err(e) if e.code().0 == UIA_E_ELEMENTNOTAVAILABLE => return FieldRead::Unreadable,
|
|
1476
|
+
Err(_) => return FieldRead::Secure,
|
|
1477
|
+
}
|
|
1478
|
+
let value = match mode {
|
|
1479
|
+
TextMode::Value => element_value(element),
|
|
1480
|
+
TextMode::Text => unsafe {
|
|
1481
|
+
element
|
|
1482
|
+
.GetCurrentPatternAs::<IUIAutomationTextPattern>(UIA_TextPatternId)
|
|
1483
|
+
.and_then(|p| p.DocumentRange())
|
|
1484
|
+
.and_then(|r| r.GetText((limit::BIG_EDIT_CHARS + 1) as i32))
|
|
1485
|
+
.map(|b: BSTR| b.to_string())
|
|
1486
|
+
},
|
|
1487
|
+
};
|
|
1488
|
+
match value {
|
|
1489
|
+
Ok(text) if text.chars().count() > limit::BIG_EDIT_CHARS => FieldRead::Big,
|
|
1490
|
+
Ok(text) => FieldRead::Value(text),
|
|
1491
|
+
Err(_) => FieldRead::Unreadable,
|
|
1492
|
+
}
|
|
1493
|
+
}
|
|
1494
|
+
|
|
1495
|
+
// ── 窗口 / 进程小工具 ──
|
|
1496
|
+
|
|
1497
|
+
fn window_text(hwnd: isize) -> String {
|
|
1498
|
+
let mut buffer = [0u16; 512];
|
|
1499
|
+
let len = unsafe { GetWindowTextW(HWND(hwnd as *mut _), &mut buffer) };
|
|
1500
|
+
String::from_utf16_lossy(&buffer[..len.max(0) as usize])
|
|
1501
|
+
}
|
|
1502
|
+
|
|
1503
|
+
fn class_name(hwnd: HWND) -> String {
|
|
1504
|
+
let mut buffer = [0u16; 256];
|
|
1505
|
+
let len = unsafe { GetClassNameW(hwnd, &mut buffer) };
|
|
1506
|
+
String::from_utf16_lossy(&buffer[..len.max(0) as usize])
|
|
1507
|
+
}
|
|
1508
|
+
|
|
1509
|
+
fn window_pid(hwnd: HWND) -> u32 {
|
|
1510
|
+
let mut pid = 0u32;
|
|
1511
|
+
unsafe { GetWindowThreadProcessId(hwnd, Some(&mut pid)) };
|
|
1512
|
+
pid
|
|
1513
|
+
}
|
|
1514
|
+
|
|
1515
|
+
/// 前台窗口 → (App 进程 pid, exe 文件名, 前台窗口类名)。UWP App 的前台窗口属于 ApplicationFrameHost,
|
|
1516
|
+
/// 真正的 App 是它里面的 Windows.UI.Core.CoreWindow 子窗口的进程。
|
|
1517
|
+
fn window_app(hwnd: isize) -> Option<(u32, String, String)> {
|
|
1518
|
+
if hwnd == 0 {
|
|
1519
|
+
return None;
|
|
1520
|
+
}
|
|
1521
|
+
let handle = HWND(hwnd as *mut _);
|
|
1522
|
+
let class = class_name(handle);
|
|
1523
|
+
let mut pid = window_pid(handle);
|
|
1524
|
+
if pid == 0 {
|
|
1525
|
+
return None;
|
|
1526
|
+
}
|
|
1527
|
+
let mut exe = process_exe(pid)?;
|
|
1528
|
+
if exe.eq_ignore_ascii_case("applicationframehost.exe") {
|
|
1529
|
+
let core = unsafe { FindWindowExW(handle, None, &HSTRING::from("Windows.UI.Core.CoreWindow"), PCWSTR::null()) };
|
|
1530
|
+
if let Ok(core) = core {
|
|
1531
|
+
let inner = window_pid(core);
|
|
1532
|
+
if inner != 0 && inner != pid {
|
|
1533
|
+
if let Some(inner_exe) = process_exe(inner) {
|
|
1534
|
+
pid = inner;
|
|
1535
|
+
exe = inner_exe;
|
|
1536
|
+
}
|
|
1537
|
+
}
|
|
1538
|
+
}
|
|
1539
|
+
}
|
|
1540
|
+
Some((pid, exe, class))
|
|
1541
|
+
}
|
|
1542
|
+
|
|
1543
|
+
fn process_path(pid: u32) -> Option<String> {
|
|
1544
|
+
unsafe {
|
|
1545
|
+
let process = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, false, pid).ok()?;
|
|
1546
|
+
let mut buffer = [0u16; 1024];
|
|
1547
|
+
let mut size = buffer.len() as u32;
|
|
1548
|
+
let result = QueryFullProcessImageNameW(process, PROCESS_NAME_WIN32, PWSTR(buffer.as_mut_ptr()), &mut size);
|
|
1549
|
+
let _ = CloseHandle(process);
|
|
1550
|
+
result.ok()?;
|
|
1551
|
+
Some(String::from_utf16_lossy(&buffer[..size as usize]))
|
|
1552
|
+
}
|
|
1553
|
+
}
|
|
1554
|
+
|
|
1555
|
+
fn process_exe(pid: u32) -> Option<String> {
|
|
1556
|
+
let path = process_path(pid)?;
|
|
1557
|
+
path.rsplit(['\\', '/']).next().map(str::to_owned)
|
|
1558
|
+
}
|
|
1559
|
+
|
|
1560
|
+
/// 版本资源里的 FileDescription(「Google Chrome」「记事本」);取不到 → None。
|
|
1561
|
+
fn file_description(path: &str) -> Option<String> {
|
|
1562
|
+
if path.is_empty() {
|
|
1563
|
+
return None;
|
|
1564
|
+
}
|
|
1565
|
+
let wide = HSTRING::from(path);
|
|
1566
|
+
unsafe {
|
|
1567
|
+
let size = GetFileVersionInfoSizeW(&wide, None);
|
|
1568
|
+
if size == 0 {
|
|
1569
|
+
return None;
|
|
1570
|
+
}
|
|
1571
|
+
let mut data = vec![0u8; size as usize];
|
|
1572
|
+
GetFileVersionInfoW(&wide, 0, size, data.as_mut_ptr() as *mut _).ok()?;
|
|
1573
|
+
let mut pointer: *mut core::ffi::c_void = std::ptr::null_mut();
|
|
1574
|
+
let mut len = 0u32;
|
|
1575
|
+
if !VerQueryValueW(data.as_ptr() as *const _, &HSTRING::from("\\VarFileInfo\\Translation"), &mut pointer, &mut len).as_bool()
|
|
1576
|
+
|| len < 4
|
|
1577
|
+
{
|
|
1578
|
+
return None;
|
|
1579
|
+
}
|
|
1580
|
+
let translation = *(pointer as *const [u16; 2]);
|
|
1581
|
+
let key = format!("\\StringFileInfo\\{:04x}{:04x}\\FileDescription", translation[0], translation[1]);
|
|
1582
|
+
if !VerQueryValueW(data.as_ptr() as *const _, &HSTRING::from(key), &mut pointer, &mut len).as_bool() || len == 0 {
|
|
1583
|
+
return None;
|
|
1584
|
+
}
|
|
1585
|
+
let text = std::slice::from_raw_parts(pointer as *const u16, len as usize);
|
|
1586
|
+
let end = text.iter().position(|&c| c == 0).unwrap_or(text.len());
|
|
1587
|
+
Some(String::from_utf16_lossy(&text[..end]))
|
|
1588
|
+
}
|
|
1589
|
+
}
|
|
1590
|
+
|
|
1591
|
+
/// 开始订阅时会话是不是锁着:输入桌面打不开或不是 Default(锁屏 / UAC 安全桌面)。
|
|
1592
|
+
fn session_locked() -> bool {
|
|
1593
|
+
unsafe {
|
|
1594
|
+
let Ok(desktop) = OpenInputDesktop(DESKTOP_CONTROL_FLAGS(0), false, DESKTOP_READOBJECTS) else { return true };
|
|
1595
|
+
let mut buffer = [0u16; 64];
|
|
1596
|
+
let mut needed = 0u32;
|
|
1597
|
+
let ok = GetUserObjectInformationW(
|
|
1598
|
+
HANDLE(desktop.0),
|
|
1599
|
+
UOI_NAME,
|
|
1600
|
+
Some(buffer.as_mut_ptr() as *mut _),
|
|
1601
|
+
(buffer.len() * 2) as u32,
|
|
1602
|
+
Some(&mut needed),
|
|
1603
|
+
)
|
|
1604
|
+
.is_ok();
|
|
1605
|
+
let _ = CloseDesktop(desktop);
|
|
1606
|
+
if !ok {
|
|
1607
|
+
return true;
|
|
1608
|
+
}
|
|
1609
|
+
let end = buffer.iter().position(|&c| c == 0).unwrap_or(buffer.len());
|
|
1610
|
+
!String::from_utf16_lossy(&buffer[..end]).eq_ignore_ascii_case("Default")
|
|
1611
|
+
}
|
|
1612
|
+
}
|
|
1613
|
+
|
|
1614
|
+
// ===========================================================================
|
|
1615
|
+
// UIA 事件处理器(只投消息)
|
|
1616
|
+
// ===========================================================================
|
|
1617
|
+
|
|
1618
|
+
#[implement(IUIAutomationFocusChangedEventHandler)]
|
|
1619
|
+
struct FocusHandler;
|
|
1620
|
+
|
|
1621
|
+
impl IUIAutomationFocusChangedEventHandler_Impl for FocusHandler_Impl {
|
|
1622
|
+
fn HandleFocusChangedEvent(&self, _sender: Option<&IUIAutomationElement>) -> windows::core::Result<()> {
|
|
1623
|
+
post(Msg::FocusChanged);
|
|
1624
|
+
Ok(())
|
|
1625
|
+
}
|
|
1626
|
+
}
|
|
1627
|
+
|
|
1628
|
+
#[implement(IUIAutomationPropertyChangedEventHandler, IUIAutomationEventHandler)]
|
|
1629
|
+
struct ValueHandler;
|
|
1630
|
+
|
|
1631
|
+
impl IUIAutomationPropertyChangedEventHandler_Impl for ValueHandler_Impl {
|
|
1632
|
+
fn HandlePropertyChangedEvent(
|
|
1633
|
+
&self,
|
|
1634
|
+
_sender: Option<&IUIAutomationElement>,
|
|
1635
|
+
_property: UIA_PROPERTY_ID,
|
|
1636
|
+
_value: &VARIANT,
|
|
1637
|
+
) -> windows::core::Result<()> {
|
|
1638
|
+
post(Msg::ValueChanged);
|
|
1639
|
+
Ok(())
|
|
1640
|
+
}
|
|
1641
|
+
}
|
|
1642
|
+
|
|
1643
|
+
impl IUIAutomationEventHandler_Impl for ValueHandler_Impl {
|
|
1644
|
+
fn HandleAutomationEvent(&self, _sender: Option<&IUIAutomationElement>, _event: UIA_EVENT_ID) -> windows::core::Result<()> {
|
|
1645
|
+
post(Msg::ValueChanged);
|
|
1646
|
+
Ok(())
|
|
1647
|
+
}
|
|
1648
|
+
}
|
|
1649
|
+
|
|
1650
|
+
// ===========================================================================
|
|
1651
|
+
// 钩子线程(只泵消息、只转交)
|
|
1652
|
+
// ===========================================================================
|
|
1653
|
+
|
|
1654
|
+
thread_local! {
|
|
1655
|
+
static NAME_HOOK: std::cell::Cell<(isize, u32)> = const { std::cell::Cell::new((0, 0)) };
|
|
1656
|
+
}
|
|
1657
|
+
|
|
1658
|
+
fn start_hooks() -> Option<HookThread> {
|
|
1659
|
+
let (ready_tx, ready_rx) = mpsc::channel::<u32>();
|
|
1660
|
+
let join = thread::spawn(move || unsafe {
|
|
1661
|
+
let tid = GetCurrentThreadId();
|
|
1662
|
+
let module = GetModuleHandleW(None).unwrap_or_default();
|
|
1663
|
+
let instance = HINSTANCE(module.0);
|
|
1664
|
+
let mouse = SetWindowsHookExW(WH_MOUSE_LL, Some(mouse_proc), instance, 0).ok();
|
|
1665
|
+
let keyboard = SetWindowsHookExW(WH_KEYBOARD_LL, Some(keyboard_proc), instance, 0).ok();
|
|
1666
|
+
let foreground = SetWinEventHook(
|
|
1667
|
+
EVENT_SYSTEM_FOREGROUND,
|
|
1668
|
+
EVENT_SYSTEM_FOREGROUND,
|
|
1669
|
+
None,
|
|
1670
|
+
Some(win_event_proc),
|
|
1671
|
+
0,
|
|
1672
|
+
0,
|
|
1673
|
+
WINEVENT_OUTOFCONTEXT | WINEVENT_SKIPOWNPROCESS,
|
|
1674
|
+
);
|
|
1675
|
+
let session_window = create_session_window(instance);
|
|
1676
|
+
if let Some(hwnd) = session_window {
|
|
1677
|
+
let _ = WTSRegisterSessionNotification(hwnd, NOTIFY_FOR_THIS_SESSION);
|
|
1678
|
+
}
|
|
1679
|
+
let installed = usize::from(mouse.is_some())
|
|
1680
|
+
| usize::from(keyboard.is_some()) << 1
|
|
1681
|
+
| usize::from(!foreground.is_invalid()) << 2
|
|
1682
|
+
| usize::from(session_window.is_some()) << 3;
|
|
1683
|
+
HOOKS.store(installed, Ordering::Release);
|
|
1684
|
+
if debug_enabled() {
|
|
1685
|
+
eprintln!("[recorder] hooks installed: {installed:#06b}");
|
|
1686
|
+
}
|
|
1687
|
+
let _ = ready_tx.send(tid);
|
|
1688
|
+
retarget_name_hook(window_pid(GetForegroundWindow()));
|
|
1689
|
+
let mut message = MSG::default();
|
|
1690
|
+
while GetMessageW(&mut message, None, 0, 0).0 > 0 {
|
|
1691
|
+
if message.hwnd.0.is_null() && message.message == WM_APP_RETARGET {
|
|
1692
|
+
retarget_name_hook(message.wParam.0 as u32);
|
|
1693
|
+
continue;
|
|
1694
|
+
}
|
|
1695
|
+
let _ = TranslateMessage(&message);
|
|
1696
|
+
DispatchMessageW(&message);
|
|
1697
|
+
}
|
|
1698
|
+
let (name_hook, _) = NAME_HOOK.with(|h| h.get());
|
|
1699
|
+
if name_hook != 0 {
|
|
1700
|
+
let _ = UnhookWinEvent(HWINEVENTHOOK(name_hook as *mut _));
|
|
1701
|
+
}
|
|
1702
|
+
if !foreground.is_invalid() {
|
|
1703
|
+
let _ = UnhookWinEvent(foreground);
|
|
1704
|
+
}
|
|
1705
|
+
if let Some(hook) = mouse {
|
|
1706
|
+
let _ = UnhookWindowsHookEx(hook);
|
|
1707
|
+
}
|
|
1708
|
+
if let Some(hook) = keyboard {
|
|
1709
|
+
let _ = UnhookWindowsHookEx(hook);
|
|
1710
|
+
}
|
|
1711
|
+
if let Some(hwnd) = session_window {
|
|
1712
|
+
let _ = WTSUnRegisterSessionNotification(hwnd);
|
|
1713
|
+
let _ = DestroyWindow(hwnd);
|
|
1714
|
+
}
|
|
1715
|
+
HOOKS.store(0, Ordering::Release);
|
|
1716
|
+
});
|
|
1717
|
+
let tid = ready_rx.recv_timeout(Duration::from_secs(5)).ok()?;
|
|
1718
|
+
Some(HookThread { tid, join })
|
|
1719
|
+
}
|
|
1720
|
+
|
|
1721
|
+
/// 标题变化只订前台窗口所在进程的(全系统订会把每个 App 里每个元素的改名都编组过来)。只在钩子线程上调用。
|
|
1722
|
+
unsafe fn retarget_name_hook(pid: u32) {
|
|
1723
|
+
let (current, current_pid) = NAME_HOOK.with(|h| h.get());
|
|
1724
|
+
if pid == current_pid && current != 0 {
|
|
1725
|
+
return;
|
|
1726
|
+
}
|
|
1727
|
+
if current != 0 {
|
|
1728
|
+
let _ = UnhookWinEvent(HWINEVENTHOOK(current as *mut _));
|
|
1729
|
+
}
|
|
1730
|
+
let hook = if pid == 0 {
|
|
1731
|
+
HWINEVENTHOOK::default()
|
|
1732
|
+
} else {
|
|
1733
|
+
SetWinEventHook(EVENT_OBJECT_NAMECHANGE, EVENT_OBJECT_NAMECHANGE, None, Some(win_event_proc), pid, 0, WINEVENT_OUTOFCONTEXT)
|
|
1734
|
+
};
|
|
1735
|
+
NAME_HOOK.with(|h| h.set((hook.0 as isize, pid)));
|
|
1736
|
+
}
|
|
1737
|
+
|
|
1738
|
+
unsafe extern "system" fn win_event_proc(
|
|
1739
|
+
_hook: HWINEVENTHOOK,
|
|
1740
|
+
event: u32,
|
|
1741
|
+
hwnd: HWND,
|
|
1742
|
+
object_id: i32,
|
|
1743
|
+
child_id: i32,
|
|
1744
|
+
_thread: u32,
|
|
1745
|
+
_time: u32,
|
|
1746
|
+
) {
|
|
1747
|
+
if hwnd.0.is_null() {
|
|
1748
|
+
return;
|
|
1749
|
+
}
|
|
1750
|
+
if event == EVENT_SYSTEM_FOREGROUND {
|
|
1751
|
+
post(Msg::Foreground(hwnd.0 as isize));
|
|
1752
|
+
let _ = PostThreadMessageW(GetCurrentThreadId(), WM_APP_RETARGET, WPARAM(window_pid(hwnd) as usize), LPARAM(0));
|
|
1753
|
+
} else if event == EVENT_OBJECT_NAMECHANGE
|
|
1754
|
+
&& object_id == OBJID_WINDOW.0
|
|
1755
|
+
&& child_id == CHILDID_SELF as i32
|
|
1756
|
+
&& GetAncestor(hwnd, GA_ROOT) == hwnd
|
|
1757
|
+
{
|
|
1758
|
+
post(Msg::NameChange(hwnd.0 as isize));
|
|
1759
|
+
}
|
|
1760
|
+
}
|
|
1761
|
+
|
|
1762
|
+
fn key_down(vk: u16) -> bool {
|
|
1763
|
+
(unsafe { GetAsyncKeyState(vk as i32) }) < 0
|
|
1764
|
+
}
|
|
1765
|
+
|
|
1766
|
+
unsafe extern "system" fn mouse_proc(code: i32, wparam: WPARAM, lparam: LPARAM) -> LRESULT {
|
|
1767
|
+
if code == HC_ACTION as i32 && wparam.0 as u32 == WM_LBUTTONDOWN {
|
|
1768
|
+
let info = &*(lparam.0 as *const MSLLHOOKSTRUCT);
|
|
1769
|
+
post(Msg::Click { x: info.pt.x, y: info.pt.y, agent: agent_marker::active() });
|
|
1770
|
+
}
|
|
1771
|
+
CallNextHookEx(None, code, wparam, lparam)
|
|
1772
|
+
}
|
|
1773
|
+
|
|
1774
|
+
unsafe extern "system" fn keyboard_proc(code: i32, wparam: WPARAM, lparam: LPARAM) -> LRESULT {
|
|
1775
|
+
let message = wparam.0 as u32;
|
|
1776
|
+
if code == HC_ACTION as i32 && (message == WM_KEYDOWN || message == WM_SYSKEYDOWN) {
|
|
1777
|
+
let info = &*(lparam.0 as *const KBDLLHOOKSTRUCT);
|
|
1778
|
+
let vk = info.vkCode;
|
|
1779
|
+
// 修饰键本身不算一次按键。
|
|
1780
|
+
let modifier = matches!(vk, 0x10..=0x12 | 0xA0..=0xA5 | 0x5B | 0x5C);
|
|
1781
|
+
if !modifier {
|
|
1782
|
+
let agent = agent_marker::active();
|
|
1783
|
+
let ctrl = key_down(VK_CONTROL.0);
|
|
1784
|
+
let win = key_down(VK_LWIN.0) || key_down(VK_RWIN.0);
|
|
1785
|
+
// AltGr = 左 Ctrl + 右 Alt:那是在打字符(德语键盘的 @ 等),不是快捷键,绝不按快捷键报出。
|
|
1786
|
+
let alt_gr = key_down(VK_RMENU.0) && !win;
|
|
1787
|
+
if (ctrl || win) && !alt_gr {
|
|
1788
|
+
let character = match vk {
|
|
1789
|
+
0x30..=0x39 | 0x41..=0x5A | 0x08..=0x2E | 0x60..=0x87 => None,
|
|
1790
|
+
_ => {
|
|
1791
|
+
let mapped = MapVirtualKeyW(vk, MAPVK_VK_TO_CHAR);
|
|
1792
|
+
// 高位 = 死键;不报。
|
|
1793
|
+
(mapped & 0x8000_0000 == 0).then(|| char::from_u32(mapped & 0xFFFF)).flatten()
|
|
1794
|
+
}
|
|
1795
|
+
};
|
|
1796
|
+
if let Some(combo) = logic::key_name(vk, character)
|
|
1797
|
+
.and_then(|key| logic::key_combo(ctrl, key_down(VK_MENU.0), key_down(VK_SHIFT.0), win, &key))
|
|
1798
|
+
{
|
|
1799
|
+
post(Msg::Key { combo, agent });
|
|
1800
|
+
}
|
|
1801
|
+
}
|
|
1802
|
+
// 只告诉 recorder「有输入」,不带键值:用来触发对焦点输入框的差分读取。
|
|
1803
|
+
post(Msg::KeyActivity { agent });
|
|
1804
|
+
}
|
|
1805
|
+
}
|
|
1806
|
+
CallNextHookEx(None, code, wparam, lparam)
|
|
1807
|
+
}
|
|
1808
|
+
|
|
1809
|
+
// ── 会话锁屏 / 睡眠:隐藏的顶层窗口(WM_POWERBROADCAST 不发给仅消息窗口) ──
|
|
1810
|
+
|
|
1811
|
+
const WTS_CONSOLE_DISCONNECT: usize = 0x2;
|
|
1812
|
+
const WTS_REMOTE_DISCONNECT: usize = 0x4;
|
|
1813
|
+
const WTS_SESSION_LOCK: usize = 0x7;
|
|
1814
|
+
const WTS_SESSION_UNLOCK: usize = 0x8;
|
|
1815
|
+
const PBT_APMSUSPEND: usize = 0x4;
|
|
1816
|
+
const PBT_APMRESUMESUSPEND: usize = 0x7;
|
|
1817
|
+
const PBT_APMRESUMEAUTOMATIC: usize = 0x12;
|
|
1818
|
+
|
|
1819
|
+
fn create_session_window(instance: HINSTANCE) -> Option<HWND> {
|
|
1820
|
+
static REGISTERED: AtomicBool = AtomicBool::new(false);
|
|
1821
|
+
let class = HSTRING::from("TanguComputerHistorySession");
|
|
1822
|
+
unsafe {
|
|
1823
|
+
if !REGISTERED.swap(true, Ordering::AcqRel) {
|
|
1824
|
+
let wc = WNDCLASSW {
|
|
1825
|
+
lpfnWndProc: Some(session_window_proc),
|
|
1826
|
+
hInstance: instance,
|
|
1827
|
+
lpszClassName: PCWSTR(class.as_ptr()),
|
|
1828
|
+
..Default::default()
|
|
1829
|
+
};
|
|
1830
|
+
RegisterClassW(&wc);
|
|
1831
|
+
}
|
|
1832
|
+
CreateWindowExW(
|
|
1833
|
+
WINDOW_EX_STYLE(0),
|
|
1834
|
+
&class,
|
|
1835
|
+
&HSTRING::from("Tangu Computer History"),
|
|
1836
|
+
WS_OVERLAPPED,
|
|
1837
|
+
0,
|
|
1838
|
+
0,
|
|
1839
|
+
0,
|
|
1840
|
+
0,
|
|
1841
|
+
None,
|
|
1842
|
+
None,
|
|
1843
|
+
instance,
|
|
1844
|
+
None,
|
|
1845
|
+
)
|
|
1846
|
+
.ok()
|
|
1847
|
+
}
|
|
1848
|
+
}
|
|
1849
|
+
|
|
1850
|
+
unsafe extern "system" fn session_window_proc(hwnd: HWND, message: u32, wparam: WPARAM, lparam: LPARAM) -> LRESULT {
|
|
1851
|
+
match message {
|
|
1852
|
+
WM_WTSSESSION_CHANGE => match wparam.0 {
|
|
1853
|
+
WTS_SESSION_LOCK | WTS_CONSOLE_DISCONNECT | WTS_REMOTE_DISCONNECT => post(Msg::System("locked")),
|
|
1854
|
+
WTS_SESSION_UNLOCK => post(Msg::System("unlocked")),
|
|
1855
|
+
_ => {}
|
|
1856
|
+
},
|
|
1857
|
+
WM_POWERBROADCAST => match wparam.0 {
|
|
1858
|
+
PBT_APMSUSPEND => post(Msg::System("sleep")),
|
|
1859
|
+
PBT_APMRESUMESUSPEND | PBT_APMRESUMEAUTOMATIC => post(Msg::System("wake")),
|
|
1860
|
+
_ => {}
|
|
1861
|
+
},
|
|
1862
|
+
_ => {}
|
|
1863
|
+
}
|
|
1864
|
+
DefWindowProcW(hwnd, message, wparam, lparam)
|
|
1865
|
+
}
|