@forsion/tangu-computer-use 0.5.8 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1806 @@
1
+ import AppKit
2
+ import ApplicationServices
3
+ import Carbon
4
+ import Foundation
5
+
6
+ // Tangu 新增:电脑历史(Computer History)采集器 —— `recordSubscribe`(协议 13)的 helper 侧。
7
+ // 只观测、不落盘:事件按每个订阅者自己的策略过滤后,一行一条 `{"ev":{…}}` 推给订阅连接;
8
+ // 落盘 / 保留 / 清除 / 暂停全归 Forsion 桌面主进程(形状见 Genesis desktop/shared/computerHistory.ts,三处同步)。
9
+ //
10
+ // 性能纪律(ChatGPT 的 recorder 冻住过 IntelliJ 10–13 秒):
11
+ // - 只给前台 App 建**一个** AXObserver;切走时先逐条 AXObserverRemoveNotification,再摘 run-loop source;
12
+ // - 每个 AX 元素引用各自设 0.3s 消息超时。**不**设在 system-wide 元素上 —— 那会改掉整个进程的全局默认,
13
+ // CU 的 look/act 也跟着变;
14
+ // - 绝不打开 AXEnhancedUserInterface / AXManualAccessibility(不走 Bridge.ensureEnhancedAccessibility),
15
+ // 绝不整树遍历:浏览器 URL 只做有上限的广度遍历(≤400 节点、≤12 层、≤200ms),且不进 AXWebArea
16
+ // (每个节点先读角色,再决定读不读子节点 —— 绝不向网页区域要 AXChildren);之后按记住的快路重读;
17
+ // - AX 读取全在独立的 recorder 线程(自己的 CFRunLoop)上做,不占 NSApp 主线程(agent 光标 / 边缘光效在那画)。
18
+ // NSEvent 全局监听与 NSWorkspace / 分布式通知只能挂在主线程,回调里只取几个数就转交 recorder 线程。
19
+ // 隐私闸在读 AX **之前**:硬排除名单(密码管理器等)与「所有订阅者都排除了的 App」不建 observer、不读标题;
20
+ // 安全输入框 / Secure Input 期间不读值,每次读值 / 报字前都重查(网页能把已聚焦的输入框原地改成密码框;
21
+ // 角色或子角色没查到也按安全输入框算,证明不了不是就不读);
22
+ // 无痕窗口只留激活时那一条不带标题的 App 切换,不发 window 事件(Safari 判不出,整个按无痕处理)。
23
+ // 没记的情境(无痕 / 排除)之后第一条可记录的情境事件带 `resumed: true`,折叠方据此切段(recorderContextDelivery)。
24
+ // 情境(无痕 / 排除站点)在读基线、开始一段输入、停手报字、记点击 / 快捷键之前先对齐(refreshIfStale;
25
+ // 浏览器里标题没变也按快路重读网址,同标题换站点算换了情境),不拿上一个窗口 / 标签页 / 页面的情境去判;
26
+ // 点击另按窗口列表确认落在前台 App 自己的窗口上。
27
+
28
+ // MARK: - 纯函数(activity_recorder_tests.swift 覆盖)
29
+
30
+ enum RecorderLimit {
31
+ static let title = 200
32
+ static let url = 300
33
+ static let label = 80
34
+ static let text = 500
35
+ /// 超过这个字符数的输入框不读值、不做差分,只报 bigEdit。
36
+ static let bigEditChars = 20_000
37
+ /// 不超过这个长度的输入框才边打边采样(聊天框、搜索框);更大的只在停手时读一次。
38
+ static let sampleChars = 2_000
39
+ /// 一次采样(整次读:角色 / 子角色复核 + 值)超过这么久的 App 不再边打边采样。
40
+ static let sampleSeconds = 0.05
41
+ }
42
+
43
+ /// 始终排除:不建 observer、不读标题,只发一条 `app.excluded` 的切换。桌面下发的排除表叠加在这之上,改不掉这份。
44
+ let recorderHardExcludedBundleIds: Set<String> = [
45
+ "com.1password.1password", "com.agilebits.onepassword7", "com.agilebits.onepassword-osx",
46
+ "com.bitwarden.desktop", "com.lastpass.LastPass", "com.dashlane.dashlanephonefinal",
47
+ "com.nordsec.nordpass", "me.proton.pass.electron",
48
+ "com.apple.keychainaccess", "com.apple.Passwords", "org.keepassxc.keepassxc",
49
+ "com.apple.SecurityAgent", "com.apple.ScreenContinuity",
50
+ "com.forsion.tangu-computer-use",
51
+ ]
52
+
53
+ /// 系统浮层:激活了也当没发生(不切 observer、不发事件),免得时间线被调度中心 / 通知中心切碎。
54
+ let recorderIgnoredBundleIds: Set<String> = [
55
+ "com.apple.WindowManager", "com.apple.controlcenter", "com.apple.notificationcenterui",
56
+ "com.apple.LocalAuthentication.UIAgent",
57
+ ]
58
+
59
+ /// 无痕 / 私密窗口的标题标记(小写比较,多语言)。宁可误判成无痕少记,也不把无痕窗口记进去。
60
+ let recorderPrivateMarkers: [String] = [
61
+ "private browsing", "private window", "incognito", "inprivate",
62
+ "privater modus", "navigation privée", "navegación privada", "navegação privada", "navigazione anonima",
63
+ "инкогнито", "приватный просмотр", "プライベートブラウズ", "シークレット", "시크릿",
64
+ "无痕", "無痕", "隐私浏览", "隱私瀏覽", "隐身", "隱身", "私密浏览", "私密瀏覽",
65
+ ]
66
+
67
+ func recorderHasPrivateMarker(_ text: String) -> Bool {
68
+ let lower = text.lowercased()
69
+ return recorderPrivateMarkers.contains { lower.contains($0) }
70
+ }
71
+
72
+ /// 无痕窗口**没有**可靠标记、又还没验证出别的判定办法的 App:整个 App 按无痕处理(只留不带标题的切换,
73
+ /// 连标题都不读)。Safari 的私密窗口标题就是网页标题;Window 菜单里「移到新的私密窗口」那一项随语言变、
74
+ /// 切窗口后是否及时刷新也没验证过。等 `node scripts/probe-recorder.mjs` 在真机上找到信号再从这里移除。
75
+ let recorderPrivateUnverifiedBundleIds: Set<String> = ["com.apple.safari", "com.apple.safaritechnologypreview"]
76
+
77
+ func recorderPrivateUnverified(_ bundleId: String) -> Bool {
78
+ recorderPrivateUnverifiedBundleIds.contains(bundleId.lowercased())
79
+ }
80
+
81
+ /// 按浏览器处理(读网址、找无痕提示、排除站点读不到网址就按排除算)的 App。CU 自己那份表只有正式渠道,
82
+ /// 这里补上 Beta / Dev / Canary 等渠道与其他常见浏览器;前缀按「.」边界匹配(不把 Thunderbird 之类算进来)。
83
+ let recorderBrowserBundleIds: Set<String> = [
84
+ "com.apple.safari", "com.apple.safaritechnologypreview",
85
+ "org.chromium.chromium", "net.imput.helium", "app.zen-browser.zen", "com.kagi.kfmac",
86
+ "com.duckduckgo.macos.browser", "org.mozilla.firefox", "org.mozilla.firefoxdeveloperedition", "org.mozilla.nightly",
87
+ ]
88
+ let recorderBrowserBundlePrefixes: [String] = [
89
+ "com.google.chrome", "com.microsoft.edgemac", "com.brave.browser", "com.vivaldi.vivaldi",
90
+ "com.operasoftware", "company.thebrowser",
91
+ ]
92
+
93
+ func recorderIsBrowser(_ bundleId: String) -> Bool {
94
+ let id = bundleId.lowercased()
95
+ return recorderBrowserBundleIds.contains(id) || recorderBrowserBundlePrefixes.contains { id == $0 || id.hasPrefix($0 + ".") }
96
+ }
97
+
98
+ /// 按字符(字素簇,不会切出半个汉字 / emoji)截到 max;第二项 = 是否截过。
99
+ func recorderTruncate(_ value: String, max: Int) -> (String, Bool) {
100
+ guard value.count > max else { return (value, false) }
101
+ return (String(value.prefix(max)), true)
102
+ }
103
+
104
+ private func recorderCollapse(_ raw: String?) -> String? {
105
+ guard let raw else { return nil }
106
+ let collapsed = raw.split(whereSeparator: { $0.isWhitespace || $0.isNewline }).joined(separator: " ")
107
+ return collapsed.isEmpty ? nil : collapsed
108
+ }
109
+
110
+ /// 界面标签(控件名):空白与换行折叠成一个空格,≤80;空串 → nil。
111
+ func recorderLabel(_ raw: String?) -> String? {
112
+ recorderCollapse(raw).map { recorderTruncate($0, max: RecorderLimit.label).0 }
113
+ }
114
+
115
+ /// 窗口标题:同上,≤200。
116
+ func recorderTitle(_ raw: String?) -> String? {
117
+ recorderCollapse(raw).map { recorderTruncate($0, max: RecorderLimit.title).0 }
118
+ }
119
+
120
+ func recorderFirstNonEmpty(_ values: String?...) -> String? {
121
+ values.first { !($0?.isEmpty ?? true) } ?? nil
122
+ }
123
+
124
+ struct RecorderTextDiff: Equatable {
125
+ /// 新值独有的那段(替换 / 插入);纯删除时为空串。
126
+ let inserted: String
127
+ /// 旧值被删掉的字符数。
128
+ let deleted: Int
129
+ }
130
+
131
+ /// 两版值剥掉公共前缀 / 后缀,剩下中间那段。相同 → nil。按 Character 比较,不会把一个字拆成两半。
132
+ func recorderTextDiff(old: String, new: String) -> RecorderTextDiff? {
133
+ if old == new { return nil }
134
+ let before = Array(old), after = Array(new)
135
+ let limit = min(before.count, after.count)
136
+ var prefix = 0
137
+ while prefix < limit && before[prefix] == after[prefix] { prefix += 1 }
138
+ var suffix = 0
139
+ while suffix < limit - prefix && before[before.count - 1 - suffix] == after[after.count - 1 - suffix] { suffix += 1 }
140
+ return RecorderTextDiff(inserted: String(after[prefix..<(after.count - suffix)]), deleted: before.count - prefix - suffix)
141
+ }
142
+
143
+ /// 一段输入停手后该报什么。baseline = 上次报过时的值;latest = 期间最后一次采样;final = 停手后读到的值。
144
+ /// 正常按 baseline→final;若 final 相对 baseline 没有新增(发送后被清空、打完又删回去)而 latest 有,
145
+ /// 就按 baseline→latest —— 不然聊天框里「打字 + 回车」这种最常见的输入整段都会丢。
146
+ func recorderSettleText(baseline: String, latest: String?, final: String) -> RecorderTextDiff? {
147
+ let direct = recorderTextDiff(old: baseline, new: final)
148
+ if let direct, !direct.inserted.isEmpty { return direct }
149
+ if let latest, let mid = recorderTextDiff(old: baseline, new: latest), !mid.inserted.isEmpty { return mid }
150
+ return direct
151
+ }
152
+
153
+ /// 差分 → text 事件字段:插入段 ≤500(超了带 truncated),纯删除只报 deleted:<n>。
154
+ func recorderTextFields(_ diff: RecorderTextDiff) -> [String: Any] {
155
+ var fields: [String: Any] = [:]
156
+ if !diff.inserted.isEmpty {
157
+ let (text, truncated) = recorderTruncate(diff.inserted, max: RecorderLimit.text)
158
+ fields["text"] = text
159
+ if truncated { fields["truncated"] = true }
160
+ }
161
+ if diff.deleted > 0 { fields["deleted"] = diff.deleted }
162
+ return fields
163
+ }
164
+
165
+ func recorderIsSecureField(role: String, subrole: String) -> Bool {
166
+ role == "AXSecureTextField" || subrole == "AXSecureTextField"
167
+ }
168
+
169
+ /// 焦点元素是不是「可编辑文本」:只有这类的 ValueChanged 才处理。安全输入框一律不是。
170
+ func recorderIsEditableText(role: String, subrole: String, editable: Bool) -> Bool {
171
+ if recorderIsSecureField(role: role, subrole: subrole) { return false }
172
+ return ["AXTextField", "AXTextArea", "AXComboBox", "AXSearchField"].contains(role) || subrole == "AXSearchField" || editable
173
+ }
174
+
175
+ /// 点击只给按钮 / 链接 / 菜单项 / 标签页这类控件带标签;文本框的值绝不当标签。
176
+ func recorderClickLabelAllowed(role: String, subrole: String) -> Bool {
177
+ if role == "AXSecureTextField" || subrole == "AXSecureTextField" { return false }
178
+ return ["AXButton", "AXPopUpButton", "AXMenuButton", "AXLink", "AXMenuItem", "AXMenuBarItem", "AXRadioButton", "AXTab", "AXCheckBox", "AXDisclosureTriangle"].contains(role)
179
+ || subrole == "AXTabButton"
180
+ }
181
+
182
+ /// 点击命中的控件一次要读的属性(一次 IPC)。**不含 AXValue**:点到文本框 / 安全输入框时连值都不读;
183
+ /// 标签只从标题 / 描述来(弹出按钮这类只有值、没有标题 / 描述的控件因此不带标签)。
184
+ let recorderClickAttributes: [String] = [kAXRoleAttribute, kAXSubroleAttribute, kAXTitleAttribute, kAXDescriptionAttribute]
185
+
186
+ /// 一个 AX 属性的读数。安全输入框的复核靠它分清「这个框本来就没有子角色」与「子角色没查到」——
187
+ /// 后者证明不了不是密码框(Codex 第三轮 P1:子角色查询失败被折成空串,角色仍是 AXTextField 就当普通框读了值)。
188
+ enum RecorderAXAttr: Equatable {
189
+ case value(String)
190
+ /// 确实没有这个属性(AttributeUnsupported / NoValue / CFNull / 空串)。
191
+ case absent
192
+ /// 元素已销毁(InvalidUIElement)。
193
+ case gone
194
+ /// 查询失败:超时(CannotComplete)、App 不应答、API 关了等其他一切错误。
195
+ case failed
196
+ }
197
+
198
+ func recorderAXAttr(error: AXError) -> RecorderAXAttr {
199
+ switch error {
200
+ case .attributeUnsupported, .noValue: return .absent
201
+ case .invalidUIElement: return .gone
202
+ default: return .failed
203
+ }
204
+ }
205
+
206
+ /// AXUIElementCopyMultipleAttributeValues 返回数组里的一项 → 读数。缺的属性是 AXValue(axError)或 CFNull;
207
+ /// 不是字符串的值按查询失败算(角色 / 子角色只会是字符串)。
208
+ func recorderAXAttr(_ value: AnyObject) -> RecorderAXAttr {
209
+ let ref = value as CFTypeRef
210
+ if CFGetTypeID(ref) == CFNullGetTypeID() { return .absent }
211
+ if CFGetTypeID(ref) == AXValueGetTypeID() {
212
+ let axValue = unsafeBitCast(ref, to: AXValue.self)
213
+ guard AXValueGetType(axValue) == .axError else { return .failed }
214
+ var raw: Int32 = 0
215
+ guard AXValueGetValue(axValue, .axError, &raw), let error = AXError(rawValue: raw) else { return .failed }
216
+ return recorderAXAttr(error: error)
217
+ }
218
+ guard let text = value as? String else { return .failed }
219
+ return text.isEmpty ? .absent : .value(text)
220
+ }
221
+
222
+ enum RecorderFieldState: Equatable {
223
+ case plain
224
+ /// 安全输入框,或 Secure Input 开着。
225
+ case secure
226
+ /// 角色或子角色查询失败(超时 / App 不应答):证明不了不是密码框。不读值,待发输入丢掉。
227
+ case unconfirmed
228
+ /// 元素已销毁,或根本没有角色:不是(或已不再是)能读值的输入框。
229
+ case unreadable
230
+ }
231
+
232
+ func recorderFieldState(secureInput: Bool, role: RecorderAXAttr, subrole: RecorderAXAttr) -> RecorderFieldState {
233
+ if secureInput { return .secure }
234
+ // 读到的任何一项是安全输入框就是安全输入框(另一项失败了也一样)。
235
+ if role == .value("AXSecureTextField") || subrole == .value("AXSecureTextField") { return .secure }
236
+ if role == .failed || subrole == .failed { return .unconfirmed }
237
+ guard case .value = role, subrole != .gone else { return .unreadable }
238
+ return .plain
239
+ }
240
+
241
+ /// 报字前复核(只查 Secure Input 与角色 / 子角色,不读值)的结论:true = 丢掉。安全输入框、或角色 / 子角色没查到 → 丢。
242
+ /// 元素已销毁 / 没有角色(.unreadable)不丢:要报的内容是它还确认是普通框时读的,销毁了也不会再变成密码框
243
+ /// (有的聊天 App 发送后直接换掉输入框,停手时靠最后一次采样兜底)。
244
+ func recorderRecheckDrops(_ state: RecorderFieldState) -> Bool {
245
+ state == .secure || state == .unconfirmed
246
+ }
247
+
248
+ enum RecorderFieldRead: Equatable {
249
+ /// 此刻是安全输入框 / Secure Input 开着,或角色 / 子角色查询失败(证明不了不是):值没读。
250
+ /// 调用方丢掉待发、基线与这个框的 ValueChanged 观察(要等焦点换走再回来才重新看它)。
251
+ case secure
252
+ /// 元素已销毁 / 没有角色,或值读不到:值没读到(停手时退回最后一次采样)。
253
+ case unreadable
254
+ /// 超过 bigEditChars:只能报 bigEdit。
255
+ case big
256
+ case value(String)
257
+ }
258
+
259
+ /// 读焦点框的值(基线、采样、停手)的唯一入口。先查 Secure Input,再重查角色 / 子角色(与字数同一次 IPC)——
260
+ /// 网页能把已聚焦的普通输入框原地改成密码框而 AX 身份不变,聚焦时查过的那一次不算数。
261
+ /// 安全输入框 / Secure Input / 角色或子角色没查到 → .secure,值根本不读;元素已销毁 / 没有角色 → .unreadable,也不读值。
262
+ /// 读数经闭包注入(真路径是 AX,单测是假读数)。
263
+ func recorderReadField(
264
+ secureInput: () -> Bool,
265
+ probe: () -> (role: RecorderAXAttr, subrole: RecorderAXAttr, count: Int?),
266
+ value: () -> String?
267
+ ) -> RecorderFieldRead {
268
+ if secureInput() { return .secure }
269
+ let (role, subrole, count) = probe()
270
+ switch recorderFieldState(secureInput: false, role: role, subrole: subrole) {
271
+ case .secure, .unconfirmed: return .secure
272
+ case .unreadable: return .unreadable
273
+ case .plain: break
274
+ }
275
+ if let count, count > RecorderLimit.bigEditChars { return .big }
276
+ guard let text = value() else { return .unreadable }
277
+ return text.utf16.count > RecorderLimit.bigEditChars ? .big : .value(text)
278
+ }
279
+
280
+ /// 边打边采样的一次读:整次读(Secure Input、角色 / 子角色复核、值)计时。超过 50ms 或文本大了 → 之后不再采样,
281
+ /// 退回只在停手时读 —— 停手那次照常先复核 Secure Input 与角色 / 子角色,报字前再核一次(flushText),与采不采样无关。
282
+ /// 只给读值计时的话,角色查询慢(近 300ms 超时)、读值快的 App(IntelliJ 等)会被连续输入反复慢查询(Codex 第三轮 P2)。
283
+ func recorderSampleField(
284
+ clock: () -> Double,
285
+ secureInput: () -> Bool,
286
+ probe: () -> (role: RecorderAXAttr, subrole: RecorderAXAttr, count: Int?),
287
+ value: () -> String?
288
+ ) -> (read: RecorderFieldRead, keepSampling: Bool) {
289
+ let started = clock()
290
+ let read = recorderReadField(secureInput: secureInput, probe: probe, value: value)
291
+ let fast = clock() - started <= RecorderLimit.sampleSeconds
292
+ switch read {
293
+ case .big: return (read, false)
294
+ case .value(let text): return (read, fast && text.utf16.count <= RecorderLimit.sampleChars)
295
+ case .secure, .unreadable: return (read, fast)
296
+ }
297
+ }
298
+
299
+ /// 地址栏的描述 / 标题(浏览器 URL 的兜底来源)。
300
+ func recorderIsAddressFieldLabel(_ label: String) -> Bool {
301
+ let lower = label.lowercased()
302
+ return ["address", "url", "地址", "search or enter", "搜索或输入", "location"].contains { lower.contains($0) }
303
+ }
304
+
305
+ /// 浏览器 URL 出 helper 前的形状:只收 http(s),去掉 query / fragment / 用户名密码,≤300。
306
+ /// 地址栏里没写 scheme 的(example.com/path)按 https 补上;带空白的(搜索词)不是 URL → nil。
307
+ func recorderSanitizeURL(_ raw: String) -> String? {
308
+ var text = raw.trimmingCharacters(in: .whitespacesAndNewlines)
309
+ guard !text.isEmpty, !text.contains(where: { $0.isWhitespace }) else { return nil }
310
+ if !text.contains("://") {
311
+ let host = text.split(separator: "/", maxSplits: 1).first.map(String.init) ?? ""
312
+ guard host.contains("."), !host.hasPrefix("."), !host.hasSuffix(".") else { return nil }
313
+ text = "https://" + text
314
+ }
315
+ guard var components = URLComponents(string: text),
316
+ let scheme = components.scheme?.lowercased(), scheme == "http" || scheme == "https",
317
+ let host = components.host, !host.isEmpty
318
+ else { return nil }
319
+ components.user = nil
320
+ components.password = nil
321
+ components.query = nil
322
+ components.fragment = nil
323
+ guard let clean = components.string else { return nil }
324
+ return recorderTruncate(clean, max: RecorderLimit.url).0
325
+ }
326
+
327
+ func recorderURLHost(_ url: String) -> String? {
328
+ URLComponents(string: url)?.host?.lowercased()
329
+ }
330
+
331
+ /// 浏览器内置页(新标签页、设置页、本地文件):读到了网址、但不属于任何站点,排除站点管不到它们。
332
+ /// 能包住别的网址的 scheme(view-source: / blob: / 扩展页)不在这里 —— 那类读到了也按「没读到」算。
333
+ private let recorderInternalSchemes: Set<String> = ["about", "chrome", "edge", "brave", "vivaldi", "opera", "arc", "file"]
334
+
335
+ /// AXWebArea 的 AXURL → (出 helper 的网址, 算不算读到了)。空串、解析不了、或包着别的网址的 scheme → 没读到。
336
+ func recorderWebAreaURL(_ raw: String?) -> (url: String?, resolved: Bool) {
337
+ guard let raw = raw?.trimmingCharacters(in: .whitespacesAndNewlines), !raw.isEmpty else { return (nil, false) }
338
+ if let url = recorderSanitizeURL(raw) { return (url, true) }
339
+ guard let scheme = URLComponents(string: raw)?.scheme?.lowercased(), recorderInternalSchemes.contains(scheme) else { return (nil, false) }
340
+ return (nil, true)
341
+ }
342
+
343
+ /// 排除站点条目归一成裸域名:去 scheme / 路径 / 用户名 / 端口 / 前导「*.」「.」/ 末尾点,小写。不像域名 → nil。
344
+ func recorderNormalizeDomain(_ raw: String) -> String? {
345
+ var text = raw.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
346
+ if let range = text.range(of: "://") { text = String(text[range.upperBound...]) }
347
+ text = String(text.split(separator: "/", maxSplits: 1, omittingEmptySubsequences: false).first ?? "")
348
+ if let at = text.lastIndex(of: "@") { text = String(text[text.index(after: at)...]) }
349
+ if let colon = text.firstIndex(of: ":") { text = String(text[..<colon]) }
350
+ while text.hasPrefix("*.") { text.removeFirst(2) }
351
+ while text.hasPrefix(".") { text.removeFirst() }
352
+ while text.hasSuffix(".") { text.removeLast() }
353
+ guard !text.isEmpty, !text.contains(where: { $0.isWhitespace || $0 == "*" }) else { return nil }
354
+ return text
355
+ }
356
+
357
+ /// host 等于该域名或是它的子域(`docs.example.com` 命中 `example.com`,`badexample.com` 不命中)。
358
+ func recorderHostMatches(_ host: String, domain: String) -> Bool {
359
+ var normalized = host.lowercased()
360
+ while normalized.hasSuffix(".") { normalized.removeLast() }
361
+ return normalized == domain || normalized.hasSuffix("." + domain)
362
+ }
363
+
364
+ /// bundle id 匹配(不分大小写):精确,或以「.*」结尾的前缀通配(桌面默认只记标题的 `com.forsion.*`)。
365
+ func recorderBundleMatches(_ bundleId: String, pattern: String) -> Bool {
366
+ let id = bundleId.lowercased(), rule = pattern.lowercased()
367
+ if rule.hasSuffix(".*") { return id.hasPrefix(String(rule.dropLast())) }
368
+ return id == rule
369
+ }
370
+
371
+ private let recorderSpecialKeys: [UInt16: String] = [
372
+ 36: "↩", 76: "⌤", 48: "⇥", 49: "Space", 51: "⌫", 117: "⌦", 53: "⎋",
373
+ 123: "←", 124: "→", 125: "↓", 126: "↑", 115: "↖", 119: "↘", 116: "⇞", 121: "⇟",
374
+ 122: "F1", 120: "F2", 99: "F3", 118: "F4", 96: "F5", 97: "F6", 98: "F7", 100: "F8",
375
+ 101: "F9", 109: "F10", 103: "F11", 111: "F12",
376
+ ]
377
+
378
+ /// 快捷键的主键名:特殊键查表,其余取不计修饰键的字符并大写;控制字符 / 私有区功能键码 → nil(不报)。
379
+ func recorderKeyName(keyCode: UInt16, characters: String?) -> String? {
380
+ if let special = recorderSpecialKeys[keyCode] { return special }
381
+ guard let first = characters?.first else { return nil }
382
+ guard !first.unicodeScalars.contains(where: { $0.value < 0x20 || $0.value == 0x7F || (0xF700...0xF8FF).contains($0.value) }) else { return nil }
383
+ return String(first).uppercased()
384
+ }
385
+
386
+ /// 快捷键组合串,如「⌘⇧P」「⌃C」。没按 ⌘ / ⌃ 的一律不算快捷键 → nil。
387
+ func recorderKeyCombo(command: Bool, control: Bool, option: Bool, shift: Bool, key: String) -> String? {
388
+ guard command || control, !key.isEmpty else { return nil }
389
+ return (control ? "⌃" : "") + (option ? "⌥" : "") + (command ? "⌘" : "") + (shift ? "⇧" : "") + key
390
+ }
391
+
392
+ /// 订阅者随 recordSubscribe 下发的策略。全部可选;硬排除名单与 Secure Input 在它之上生效。
393
+ struct RecorderPolicy {
394
+ var excludeBundleIds: [String] = []
395
+ var titleOnlyBundleIds: [String] = []
396
+ /// 已归一的裸域名。
397
+ var excludeDomains: [String] = []
398
+ var text = true
399
+ var clicks = true
400
+ var keys = true
401
+
402
+ init() {}
403
+
404
+ init(json: Any?) {
405
+ let object = json as? [String: Any] ?? [:]
406
+ func list(_ key: String) -> [String] {
407
+ Array(((object[key] as? [Any]) ?? []).compactMap { $0 as? String }.prefix(1000))
408
+ }
409
+ excludeBundleIds = list("excludeBundleIds")
410
+ titleOnlyBundleIds = list("titleOnlyBundleIds")
411
+ excludeDomains = list("excludeDomains").compactMap(recorderNormalizeDomain)
412
+ text = object["text"] as? Bool ?? true
413
+ clicks = object["clicks"] as? Bool ?? true
414
+ keys = object["keys"] as? Bool ?? true
415
+ }
416
+
417
+ func excludes(bundleId: String) -> Bool { excludeBundleIds.contains { recorderBundleMatches(bundleId, pattern: $0) } }
418
+ func titleOnly(bundleId: String) -> Bool { titleOnlyBundleIds.contains { recorderBundleMatches(bundleId, pattern: $0) } }
419
+ }
420
+
421
+ /// 一次观测时的前台情境。纯数据(不含 AX 引用),过滤只看它 —— 所以能单测。
422
+ struct RecorderContext: Equatable {
423
+ var name: String
424
+ var bundleId: String
425
+ /// 硬排除(密码管理器 / 钥匙串 / helper 自己):连标题都没读。
426
+ var hardExcluded = false
427
+ /// 无痕 / 私密窗口时恒为 nil(标题与 URL 根本不进这个结构)。
428
+ var title: String?
429
+ var url: String?
430
+ /// 浏览器但网址没读到(超时 / 没找到网页区域与地址栏)。订阅者设了排除站点时按排除处理。
431
+ var urlUnknown = false
432
+ var isPrivate = false
433
+ }
434
+
435
+ enum RecorderKind: String {
436
+ case app, window, text, click, key, system
437
+ }
438
+
439
+ /// 按某个订阅者的策略,把一次观测变成事件体(不含 t / origin / 各 kind 自己的字段)。nil = 这个订阅者不该收到。
440
+ /// - 排除 App(硬排除 / 策略排除):只有切到它的那一条 app 事件,只带 `app.excluded` —— App 里的窗口切换
441
+ /// 一概不发,连时间点和次数都不外泄;
442
+ /// - 排除站点(当前 URL 命中 / 设了排除站点而浏览器网址没读到):app 切换与 window 标记,只带 `app.excluded`
443
+ /// (同一浏览器里从允许的页面进到排除站点,靠这条 window 标记结束上一段的计时;连着的标记由 emit 按情境键去重);
444
+ /// - 无痕窗口:只有激活时那一条 app 事件,不带标题与 URL;window 事件一概不发,连进出无痕窗口的时间点都不外泄
445
+ /// (同一 App 里切走后的下一条情境事件照常发出,见 recorderContextDelivery);
446
+ /// - 只记标题的 App:没有 text / click / key。
447
+ func recorderEventBody(kind: RecorderKind, context: RecorderContext?, policy: RecorderPolicy) -> [String: Any]? {
448
+ var body: [String: Any] = ["kind": kind.rawValue]
449
+ guard let context else { return kind == .system ? body : nil }
450
+ var app: [String: Any] = ["name": context.name]
451
+ if !context.bundleId.isEmpty { app["bundleId"] = context.bundleId }
452
+ let domainExcluded: Bool
453
+ if let host = context.url.flatMap(recorderURLHost) {
454
+ domainExcluded = policy.excludeDomains.contains { recorderHostMatches(host, domain: $0) }
455
+ } else {
456
+ // 读不到网址就判断不了是不是排除站点:宁可少记(连标题都不带),也不把排除站点记进去。
457
+ domainExcluded = context.urlUnknown && !policy.excludeDomains.isEmpty
458
+ }
459
+ let appExcluded = context.hardExcluded || policy.excludes(bundleId: context.bundleId)
460
+ if appExcluded || domainExcluded {
461
+ guard kind == .app || (kind == .window && !appExcluded) else { return nil }
462
+ app["excluded"] = true
463
+ body["app"] = app
464
+ return body
465
+ }
466
+ body["app"] = app
467
+ if context.isPrivate {
468
+ guard kind == .app else { return nil }
469
+ return body
470
+ }
471
+ switch kind {
472
+ case .text: if !policy.text { return nil }
473
+ case .click: if !policy.clicks { return nil }
474
+ case .key: if !policy.keys { return nil }
475
+ default: break
476
+ }
477
+ if kind == .text || kind == .click || kind == .key, policy.titleOnly(bundleId: context.bundleId) { return nil }
478
+ if let title = context.title, !title.isEmpty { body["title"] = title }
479
+ if let url = context.url { body["url"] = url }
480
+ return body
481
+ }
482
+
483
+ /// app / window 事件的去重键:同一订阅者连着收到两条同键的情境事件是冗余的。
484
+ func recorderContextKey(_ body: [String: Any]) -> String {
485
+ let app = body["app"] as? [String: Any] ?? [:]
486
+ return [app["bundleId"] as? String ?? app["name"] as? String ?? "",
487
+ (app["excluded"] as? Bool) == true ? "x" : "",
488
+ body["title"] as? String ?? "", body["url"] as? String ?? ""].joined(separator: "\u{1F}")
489
+ }
490
+
491
+ /// 一个订阅者的情境投递状态。只在 recorder 线程读写。
492
+ struct RecorderDeliveryState: Equatable {
493
+ /// 上一次发给它的 app / window 情境键(去重 + 判断要不要补发快照)。
494
+ var lastKey: String?
495
+ /// 最近处理过的情境对它没记(无痕窗口 / 排除 App / 排除站点):下一条可记录的情境事件带 `resumed: true`。
496
+ var unrecorded = false
497
+ }
498
+
499
+ /// 这个情境对该订阅者算不算「记了」:标题 / 网址照常进事件。无痕窗口、排除 App、排除站点(含设了排除站点而网址没读到)不算。
500
+ func recorderContextRecorded(_ context: RecorderContext, policy: RecorderPolicy) -> Bool {
501
+ guard let body = recorderEventBody(kind: .window, context: context, policy: policy) else { return false }
502
+ return (body["app"] as? [String: Any])?["excluded"] as? Bool != true
503
+ }
504
+
505
+ /// 一条 app / window 情境事件对某个订阅者:发什么(nil = 不发),并更新它的投递状态。
506
+ /// - dedupe 且与上一条同键 → 不发;
507
+ /// - 被滤掉的 window 事件(无痕窗口;排除 App 里的窗口变化):不发,但键记成这个情境的 app 事件的键。
508
+ /// 同一浏览器里从普通窗口 A 切进无痕窗口、再切回 A 时,回来的那条必须照常发出 —— 键若还停在 A,它会被当成重复吞掉,
509
+ /// 中间那段就全算进了 A。记成无痕情境的键(不带标题),还在无痕窗口时同一 App 重新激活的那条 app 事件照样去重。
510
+ /// 排除 App 的键本来就是切进来时那条,记不记都一样;
511
+ /// - 断点(`resumed: true`,形状见 Genesis desktop/shared/computerHistory.ts):没记的情境(无痕窗口、排除 App / 站点)之后,
512
+ /// 第一条可记录的情境事件带上它 —— 只是一个标记,不带那段的任何时间或内容;折叠方见到它就切段,不跨过它合并,
513
+ /// 没记的那段就不会被算进前后的普通页面。它照发不去重(无标题的普通窗口与无痕情境同键,去重会把回来的那条吞掉)。
514
+ /// 切进排除 App 那条不带标题的切换已经标出了没记那段的开头,之后切到别的 App 照样带:合同就是「没记之后的第一条」,
515
+ /// 折叠方不必分辨哪种没记的情境发过标记;把排除标记滤掉再显示的消费方,A → 排除 App → A 也不会被并成一段。
516
+ /// 无痕窗口那条不带标题的切换与排除标记本身都不是可记录的情境,不带它。
517
+ func recorderContextDelivery(kind: RecorderKind, context: RecorderContext?, policy: RecorderPolicy, state: inout RecorderDeliveryState, dedupe: Bool) -> [String: Any]? {
518
+ guard let context else { return nil }
519
+ let recorded = recorderContextRecorded(context, policy: policy)
520
+ if !recorded { state.unrecorded = true }
521
+ guard var body = recorderEventBody(kind: kind, context: context, policy: policy) else {
522
+ if kind == .window, let app = recorderEventBody(kind: .app, context: context, policy: policy) { state.lastKey = recorderContextKey(app) }
523
+ return nil
524
+ }
525
+ let key = recorderContextKey(body)
526
+ let resumed = recorded && state.unrecorded
527
+ if dedupe && key == state.lastKey && !resumed { return nil }
528
+ if resumed {
529
+ body["resumed"] = true
530
+ state.unrecorded = false
531
+ }
532
+ state.lastKey = key
533
+ return body
534
+ }
535
+
536
+ func recorderJSONLine(_ object: [String: Any]) -> Data? {
537
+ guard JSONSerialization.isValidJSONObject(object),
538
+ var data = try? JSONSerialization.data(withJSONObject: object, options: [.withoutEscapingSlashes])
539
+ else { return nil }
540
+ data.append(0x0A)
541
+ return data
542
+ }
543
+
544
+ func recorderNowMs() -> Double { Date().timeIntervalSince1970 * 1000 }
545
+
546
+ /// Secure Input(密码框、「安全键盘输入」)开着就不读值、不报按键。两个来源都看:Carbon 的全局开关,
547
+ /// 以及 WindowServer 会话字典里的开启者 pid(ChatGPT 用的就是后者)。
548
+ func recorderSecureInputActive() -> Bool {
549
+ if IsSecureEventInputEnabled() { return true }
550
+ if let session = CGSessionCopyCurrentDictionary() as? [String: Any],
551
+ let pid = (session["kCGSSessionSecureInputPID"] as? NSNumber)?.intValue, pid != 0 { return true }
552
+ return false
553
+ }
554
+
555
+ /// 当前会话是不是「人不在」:屏幕已锁,或快速切换用户后这个会话不在控制台上。开始订阅时据此定初始状态。
556
+ func recorderSessionLocked(_ session: [String: Any]?) -> Bool {
557
+ guard let session else { return false }
558
+ if (session["CGSSessionScreenIsLocked"] as? NSNumber)?.boolValue == true { return true }
559
+ if let onConsole = session["kCGSSessionOnConsoleKey"] as? NSNumber, !onConsole.boolValue { return true }
560
+ return false
561
+ }
562
+
563
+ /// 屏幕上这一点最上面的窗口属于哪个进程:CGWindowList(从前往后)里第一个包含该点的窗口,跳过全透明的
564
+ /// 与 skipPid(helper 自己的 agent 光标 / 边缘光效浮层)。只用属主 pid / 边界 / 透明度,不需要屏幕录制授权。
565
+ /// 找不到 → nil,调用方按「不记」处理。
566
+ func recorderWindowOwner(at point: CGPoint, windows: [[String: Any]], skipPid: pid_t) -> pid_t? {
567
+ for entry in windows {
568
+ guard let owner = (entry[kCGWindowOwnerPID as String] as? NSNumber)?.int32Value, owner != skipPid else { continue }
569
+ if let alpha = (entry[kCGWindowAlpha as String] as? NSNumber)?.doubleValue, alpha <= 0 { continue }
570
+ guard let boundsDict = entry[kCGWindowBounds as String] as? [String: Any],
571
+ let bounds = CGRect(dictionaryRepresentation: boundsDict as CFDictionary),
572
+ bounds.contains(point)
573
+ else { continue }
574
+ return owner
575
+ }
576
+ return nil
577
+ }
578
+
579
+ enum RecorderPendingAction: Equatable {
580
+ case keep, flush, drop
581
+ }
582
+
583
+ /// 窗口情境刷新时,待发那段输入怎么办:
584
+ /// - 新情境没有订阅者要文字(无痕窗口 / 排除站点 / 排除 App)→ 丢,连基线一起作废 —— 这段可能正是在新窗口、
585
+ /// 新标签页里打的(焦点通知比窗口 / 标题通知先到),不能按旧情境报出去;
586
+ /// - 换了窗口,或同一窗口里网址变了(导航,含标题不变的换站点)→ 按它自己的情境先报;
587
+ /// - 同一窗口、同一网址只是标题变了 → 留着,停手时照常报。
588
+ func recorderPendingOnRefresh(nextWantsText: Bool, windowChanged: Bool, urlChanged: Bool) -> RecorderPendingAction {
589
+ if !nextWantsText { return .drop }
590
+ return windowChanged || urlChanged ? .flush : .keep
591
+ }
592
+
593
+ /// 快路重读到的网址(fresh)与情境里记的是否不同 —— 同一窗口、同一标题下的换站点 / 站内导航。
594
+ /// fresh = nil:这个窗口没有记住的快路,判不了 → 不算变(不为此走遍历,等标题 / 焦点变化)。
595
+ /// 现在读不到了(resolved = false)而情境里读到过 → 算变:刷新后按「网址没读到」处理,设了排除站点的订阅者据此失败即关闭。
596
+ func recorderURLChanged(_ context: RecorderContext, fresh: (url: String?, resolved: Bool)?) -> Bool {
597
+ guard let fresh else { return false }
598
+ return fresh.url != context.url || fresh.resolved == context.urlUnknown
599
+ }
600
+
601
+ /// 无痕提示可能出现在的控件角色(Chromium 系工具栏上的头像按钮写着「Incognito / InPrivate」)。
602
+ let recorderPrivateHintRoles: Set<String> = ["AXButton", "AXMenuButton", "AXPopUpButton"]
603
+
604
+ /// 有上限的遍历在哪儿停了手 —— 也就是无痕扫描没看完的原因。
605
+ enum RecorderWalkStop: String, CaseIterable {
606
+ /// 找到网页区域后,剩下的非网页节点不再展开(更深处的工具栏节点没看;子节点没读,所以不知道有没有)。
607
+ case afterWebArea
608
+ /// 到了深度上限还有没展开的节点(同上,不知道有没有子节点)。
609
+ case depthCap
610
+ /// 节点数到上限,还有没排进队的子节点。
611
+ case nodeCap
612
+ /// 时间预算用完,还有排着队没看的节点。
613
+ case deadline
614
+ }
615
+
616
+ struct RecorderWalkResult<Node> {
617
+ var url: String?
618
+ /// 网页区域或地址栏读到了网址(含没有网址的内置页)。
619
+ var resolved = false
620
+ var isPrivate = false
621
+ /// 读到网址的那个网页区域的父节点(之后的快路)。
622
+ var webContainer: Node?
623
+ var addressField: Node?
624
+ var visited = 0
625
+ /// 空 = 窗口外框(网页内容本就不进)全看完了;非空 = 无痕扫描不完整,这些是停手的原因。
626
+ var stops: Set<RecorderWalkStop> = []
627
+ var complete: Bool { stops.isEmpty }
628
+ }
629
+
630
+ /// 有上限的广度遍历:找浏览器网址(网页区域的 AXURL,或地址栏)与工具栏上的无痕提示。读数经闭包注入
631
+ /// (真路径是 AX 调用,单测是假树):
632
+ /// - describe(node):一次取角色与标签(description + title)。**先知道角色,再决定读不读子节点**;
633
+ /// - webURL(node):只对 AXWebArea 调用,只读它自己的 AXURL;
634
+ /// - children(node):只对准备展开的非网页节点调用 —— 绝不向 AXWebArea 要 AXChildren,那会逼浏览器
635
+ /// 把整棵网页无障碍树生成出来(复杂页面上万节点,拖慢目标 App);
636
+ /// - fieldValue(node):没读到网页区域网址时,读地址栏的值兜底。
637
+ /// 访问顺序与覆盖范围和以前一致(找到网页区域后不再展开,但把已排进队列的节点看完);无痕扫描不完整时
638
+ /// 仍按普通窗口处理(行为暂不改,等真机探针的数据),只在 stops 里如实报出来。
639
+ func recorderBoundedWalk<Node>(
640
+ root: Node, nodeCap: Int, depthCap: Int, expired: () -> Bool,
641
+ describe: (Node) -> (role: String, label: String),
642
+ webURL: (Node) -> (url: String?, resolved: Bool),
643
+ fieldValue: (Node) -> String?,
644
+ children: (Node) -> [Node]
645
+ ) -> RecorderWalkResult<Node> {
646
+ var result = RecorderWalkResult<Node>()
647
+ var queue: [(node: Node, parent: Node?, depth: Int)] = [(root, nil, 0)]
648
+ var head = 0
649
+ while head < queue.count {
650
+ if expired() { result.stops.insert(.deadline); break }
651
+ let (node, parent, depth) = queue[head]
652
+ head += 1
653
+ result.visited += 1
654
+ let (role, label) = describe(node)
655
+ if role == "AXWebArea" {
656
+ // 只读网页区域自己的 AXURL,绝不往里走(也不读它的子节点)。
657
+ if !result.resolved {
658
+ let found = webURL(node)
659
+ if found.resolved {
660
+ result.url = found.url
661
+ result.resolved = true
662
+ result.webContainer = parent
663
+ }
664
+ }
665
+ continue
666
+ }
667
+ if recorderPrivateHintRoles.contains(role) && recorderHasPrivateMarker(label) { result.isPrivate = true }
668
+ if result.addressField == nil && (role == "AXTextField" || role == "AXComboBox") && recorderIsAddressFieldLabel(label) { result.addressField = node }
669
+ // 找到网页区域后不再往下展开,但把已排进队列的节点看完:工具栏(无痕头像按钮)不一定排在网页区域前面。
670
+ if result.resolved { result.stops.insert(.afterWebArea); continue }
671
+ if depth >= depthCap { result.stops.insert(.depthCap); continue }
672
+ for child in children(node) {
673
+ guard queue.count < nodeCap else { result.stops.insert(.nodeCap); break }
674
+ queue.append((child, node, depth + 1))
675
+ }
676
+ }
677
+ if !result.resolved, let field = result.addressField, let url = fieldValue(field).flatMap(recorderSanitizeURL) {
678
+ result.url = url
679
+ result.resolved = true
680
+ }
681
+ return result
682
+ }
683
+
684
+ /// 无痕扫描(有上限的遍历)的计数:走了几次、几次没看完、各因什么停手。只进 diagnostics(只有计数,不含任何内容),
685
+ /// 给真机探针判断「扫描不完整时仍按普通窗口记」这条要不要改(Codex #3)。任意线程。
686
+ final class RecorderScanStats {
687
+ private let lock = NSLock()
688
+ private var walks = 0
689
+ private var incomplete = 0
690
+ private var stops: [RecorderWalkStop: Int] = [:]
691
+
692
+ func record(_ reasons: Set<RecorderWalkStop>) {
693
+ lock.lock(); defer { lock.unlock() }
694
+ walks += 1
695
+ if !reasons.isEmpty { incomplete += 1 }
696
+ for reason in reasons { stops[reason, default: 0] += 1 }
697
+ }
698
+
699
+ /// stops 的四个原因总是都在(没发生过为 0),形状稳定。
700
+ func snapshot() -> (walks: Int, incomplete: Int, stops: [String: Int]) {
701
+ lock.lock(); defer { lock.unlock() }
702
+ var byName: [String: Int] = [:]
703
+ for reason in RecorderWalkStop.allCases { byName[reason.rawValue] = stops[reason] ?? 0 }
704
+ return (walks, incomplete, byName)
705
+ }
706
+ }
707
+
708
+ /// 浏览器窗口的网址读取:按窗口记住上次遍历找到的网页区域父节点 / 地址栏(快路)与那次的无痕结论。
709
+ /// 读数经闭包注入(真路径是 AX 调用,单测是假读数)。只在 recorder 线程用。
710
+ /// - lookup 每次都先走快路重读网址,**不**按标题命中缓存:同标题换站点 / 不改标题的站内导航时,
711
+ /// 标题没变、网址变了(Codex #2)。这里根本不存网址 —— 网址的真源永远是这一次的读数;
712
+ /// - 快路没有或读不到时才考虑遍历:只在标题变了(或第一次见这个窗口)时走;同一窗口 30 秒内走过一遍
713
+ /// 且一无所获(没留下快路)就不再重走。标题没变而快路读不到 → 按没读到返回(排除站点失败即关闭),不走遍历。
714
+ final class RecorderBrowserURLs {
715
+ struct Walked {
716
+ var url: String?
717
+ var resolved: Bool
718
+ var isPrivate: Bool
719
+ var webContainer: AXUIElement?
720
+ var addressField: AXUIElement?
721
+ }
722
+
723
+ private struct Entry {
724
+ let window: AXUIElement
725
+ var title: String
726
+ /// 无痕与否按窗口定:一个窗口不会从普通变成无痕(遍历过一次认出来就一直算)。
727
+ var isPrivate: Bool
728
+ var webContainer: AXUIElement?
729
+ var addressField: AXUIElement?
730
+ var walkedAt: Double
731
+ var hasFastPath: Bool { webContainer != nil || addressField != nil }
732
+ }
733
+
734
+ static let rewalkAfter: Double = 30
735
+ static let capacity = 32
736
+
737
+ private var entries: [CFHashCode: Entry] = [:]
738
+ private let fastRead: (_ webContainer: AXUIElement?, _ addressField: AXUIElement?) -> (url: String?, resolved: Bool)
739
+ private let walk: (AXUIElement) -> Walked
740
+ private let now: () -> Double
741
+
742
+ init(
743
+ fastRead: @escaping (_ webContainer: AXUIElement?, _ addressField: AXUIElement?) -> (url: String?, resolved: Bool),
744
+ walk: @escaping (AXUIElement) -> Walked,
745
+ now: @escaping () -> Double = { CFAbsoluteTimeGetCurrent() }
746
+ ) {
747
+ self.fastRead = fastRead
748
+ self.walk = walk
749
+ self.now = now
750
+ }
751
+
752
+ /// 这个窗口现在的网址与无痕结论(readContext 用)。
753
+ func lookup(window: AXUIElement, title: String) -> (url: String?, resolved: Bool, isPrivate: Bool) {
754
+ let hash = CFHash(window)
755
+ let cached = entry(for: window)
756
+ var found = cached.flatMap(fast) ?? (url: nil, resolved: false)
757
+ var next = cached ?? Entry(window: window, title: title, isPrivate: false, webContainer: nil, addressField: nil, walkedAt: -Double.infinity)
758
+ let mayWalk = cached.map { $0.title != title && ($0.hasFastPath || now() - $0.walkedAt > Self.rewalkAfter) } ?? true
759
+ if !found.resolved && mayWalk {
760
+ let walked = walk(window)
761
+ found = (walked.url, walked.resolved)
762
+ next.isPrivate = next.isPrivate || walked.isPrivate
763
+ next.webContainer = walked.webContainer
764
+ next.addressField = walked.addressField
765
+ next.walkedAt = now()
766
+ }
767
+ next.title = title
768
+ if entries[hash] == nil && entries.count >= Self.capacity { entries.removeAll() }
769
+ entries[hash] = next
770
+ return (found.url, found.resolved, next.isPrivate)
771
+ }
772
+
773
+ /// 只走快路重读一次(refreshIfStale 用,1–2 次 AX 读)。这个窗口没有记住的快路 → nil,绝不在这里走遍历。
774
+ func fastURL(window: AXUIElement) -> (url: String?, resolved: Bool)? {
775
+ entry(for: window).flatMap(fast)
776
+ }
777
+
778
+ func removeAll() { entries = [:] }
779
+
780
+ private func entry(for window: AXUIElement) -> Entry? {
781
+ entries[CFHash(window)].flatMap { CFEqual($0.window, window) ? $0 : nil }
782
+ }
783
+
784
+ private func fast(_ entry: Entry) -> (url: String?, resolved: Bool)? {
785
+ guard entry.hasFastPath else { return nil }
786
+ return fastRead(entry.webContainer, entry.addressField)
787
+ }
788
+ }
789
+
790
+ /// 订阅只给同一用户的进程(socket 在用户自己的 Caches 下,这是纵深防御)。不校验对端签名,helper 也不知道桌面端
791
+ /// 开没开电脑历史:helper 在运行时,同一用户的任何进程都能带自己的策略订阅,桌面端关闭 / 暂停 / 排除都管不到它
792
+ /// (内置的硬排除、Secure Input、无痕窗口照常生效)。与它已经能在同一 socket 上调 getUserContext / axReadText /
793
+ /// focusedElement / look 读屏幕内容是同一暴露面,0.6.0 接受这个风险;已知限制与后续计划记在 CHANGELOG 0.6.0。
794
+ func recorderPeerIsSameUser(_ fd: Int32) -> Bool {
795
+ var uid: uid_t = 0
796
+ var gid: gid_t = 0
797
+ return getpeereid(fd, &uid, &gid) == 0 && uid == getuid()
798
+ }
799
+
800
+ /// agent 动作的深度计数:act / actBatch 等执行期间,以及结束后一小段(AX 通知是异步到的),
801
+ /// 观测到的事件打 `origin:"agent"`。与 ForegroundActivity 无关 —— 那个只在物理输入时亮。
802
+ final class RecorderAgentGate {
803
+ private let lock = NSLock()
804
+ private var depth = 0
805
+ private var lastLeave = -Double.infinity
806
+ private let grace: Double
807
+ private let now: () -> Double
808
+
809
+ init(grace: Double = 0.75, now: @escaping () -> Double = { CFAbsoluteTimeGetCurrent() }) {
810
+ self.grace = grace
811
+ self.now = now
812
+ }
813
+
814
+ func enter() { lock.lock(); depth += 1; lock.unlock() }
815
+ func leave() {
816
+ lock.lock()
817
+ if depth > 0 {
818
+ depth -= 1
819
+ if depth == 0 { lastLeave = now() }
820
+ }
821
+ lock.unlock()
822
+ }
823
+ func active() -> Bool {
824
+ lock.lock(); defer { lock.unlock() }
825
+ return depth > 0 || now() - lastLeave < grace
826
+ }
827
+ }
828
+
829
+ // MARK: - 订阅连接
830
+
831
+ /// 一个订阅连接。写入走它自己的串行队列:recorder 线程只管入队,永远不因某个订阅者读得慢而卡住;
832
+ /// 积压到上限就丢弃并计数,下一条能入队时先补一条 `system/dropped`。写失败(对端关了 / 超时)→ shutdown 套接字,
833
+ /// 让读侧(Bridge.processClient)拿到 EOF,走唯一的退订路径。
834
+ final class RecorderSubscriber {
835
+ let fd: Int32
836
+ let policy: RecorderPolicy
837
+ /// 只在 recorder 线程读写:上一次发给它的 app/window 情境键,以及要不要在下一条可记录的情境事件上带 resumed。
838
+ var delivery = RecorderDeliveryState()
839
+ private let queue: DispatchQueue
840
+ private let lock = NSLock()
841
+ private let maxPending: Int
842
+ private var pending = 0
843
+ private var dropped = 0
844
+ private var failed = false
845
+ private var closed = false
846
+
847
+ init(fd: Int32, policy: RecorderPolicy, maxPending: Int = 256, sendTimeout: Int = 2) {
848
+ self.fd = fd
849
+ self.policy = policy
850
+ self.maxPending = maxPending
851
+ queue = DispatchQueue(label: "forsion.recorder.subscriber.\(fd)")
852
+ // 写阻塞上限:对端卡住不读时,写队列最多等这么久就判失败断开,不会永远挂着。
853
+ var timeout = timeval(tv_sec: sendTimeout, tv_usec: 0)
854
+ _ = setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, &timeout, socklen_t(MemoryLayout<timeval>.size))
855
+ var bufferSize: Int32 = 256 * 1024
856
+ _ = setsockopt(fd, SOL_SOCKET, SO_SNDBUF, &bufferSize, socklen_t(MemoryLayout<Int32>.size))
857
+ }
858
+
859
+ var isFailed: Bool { lock.lock(); defer { lock.unlock() }; return failed }
860
+
861
+ /// 入队一行 JSON。永不阻塞调用方。
862
+ func send(_ object: [String: Any]) {
863
+ guard let line = recorderJSONLine(object) else { return }
864
+ lock.lock()
865
+ if failed || closed { lock.unlock(); return }
866
+ if pending >= maxPending {
867
+ dropped += 1
868
+ lock.unlock()
869
+ return
870
+ }
871
+ var lines = [line]
872
+ if dropped > 0 {
873
+ let marker: [String: Any] = ["ev": ["t": Int64(recorderNowMs()), "kind": "system", "state": "dropped", "count": dropped]]
874
+ if let markerLine = recorderJSONLine(marker) { lines.insert(markerLine, at: 0) }
875
+ dropped = 0
876
+ }
877
+ pending += 1
878
+ lock.unlock()
879
+ queue.async { [self] in
880
+ for next in lines { deliver(next) }
881
+ lock.lock(); pending -= 1; lock.unlock()
882
+ }
883
+ }
884
+
885
+ /// 退订:之后的入队全部丢弃,并等写队列排空 —— 调用方随后才能关 fd(否则排队的写会落到被复用的 fd 上)。
886
+ func close() {
887
+ lock.lock(); closed = true; lock.unlock()
888
+ queue.sync {}
889
+ }
890
+
891
+ private func deliver(_ data: Data) {
892
+ lock.lock()
893
+ let skip = failed || closed
894
+ lock.unlock()
895
+ guard !skip else { return }
896
+ if !writeAll(data) {
897
+ lock.lock(); failed = true; lock.unlock()
898
+ Darwin.shutdown(fd, SHUT_RDWR)
899
+ }
900
+ }
901
+
902
+ private func writeAll(_ data: Data) -> Bool {
903
+ data.withUnsafeBytes { raw -> Bool in
904
+ guard let base = raw.baseAddress else { return true }
905
+ var offset = 0
906
+ while offset < raw.count {
907
+ let sent = Darwin.send(fd, base.advanced(by: offset), raw.count - offset, 0)
908
+ if sent > 0 { offset += sent; continue }
909
+ if sent < 0 && errno == EINTR { continue }
910
+ return false
911
+ }
912
+ return true
913
+ }
914
+ }
915
+ }
916
+
917
+ // MARK: - 采集器
918
+
919
+ private func recorderAXCallback(_ observer: AXObserver, _ element: AXUIElement, _ notification: CFString, _ refcon: UnsafeMutableRawPointer?) {
920
+ guard let refcon else { return }
921
+ Unmanaged<ActivityRecorder>.fromOpaque(refcon).takeUnretainedValue().handleAX(observer: observer, element: element, notification: notification as String)
922
+ }
923
+
924
+ final class ActivityRecorder {
925
+ static let shared = ActivityRecorder()
926
+
927
+ private static let axTimeout: Float = 0.3
928
+ private static let textDebounce: TimeInterval = 1.5
929
+ private static let sampleDelay: TimeInterval = 0.25
930
+ private static let windowDebounce: TimeInterval = 0.3
931
+ private static let walkNodeCap = 400
932
+ private static let walkDepthCap = 12
933
+ private static let walkBudget: TimeInterval = 0.2
934
+
935
+ let agentGate = RecorderAgentGate()
936
+ /// 无痕扫描计数(diagnostics 读,任意线程)。
937
+ let scanStats = RecorderScanStats()
938
+ /// 测试缝:单测注入「已授权」,好在没有辅助功能授权的机器上跑通订阅流(未授权时观察者本身照样建不起来)。
939
+ var accessibilityTrusted: () -> Bool = { AXIsProcessTrusted() }
940
+ /// 测试缝:开始订阅时会话是不是锁着(单测两种都要跑,不能取决于跑测试时屏幕锁没锁)。
941
+ var sessionLocked: () -> Bool = { recorderSessionLocked(CGSessionCopyCurrentDictionary() as? [String: Any]) }
942
+
943
+ // 订阅者注册表(任意线程,registryLock)。
944
+ private let registryLock = NSLock()
945
+ private var subscribers: [Int32: RecorderSubscriber] = [:]
946
+
947
+ // recorder 线程本身。
948
+ private let threadLock = NSLock()
949
+ private var runLoop: CFRunLoop?
950
+
951
+ // ↓ 以下只在 recorder 线程读写。
952
+ private var running = false
953
+ private var locked = false
954
+ private var asleep = false
955
+ private var suspended: Bool { locked || asleep }
956
+ private var browserIds: Set<String> = []
957
+ private var observer: AXObserver?
958
+ private var observedApp: AXUIElement?
959
+ private var observedRunningApp: NSRunningApplication?
960
+ private var observedPid: pid_t = 0
961
+ private var appNotifications: [String] = []
962
+ private var context: RecorderContext?
963
+ private var focusedWindow: AXUIElement?
964
+ /// 焦点窗口标题的哈希(不存标题本身):判断情境是不是已经过时(标题变了、通知还没到 / 还在去抖)。
965
+ private var focusedTitleKey: Int?
966
+ private var windowTimer: Timer?
967
+ private var retryTimer: Timer?
968
+ private var focusElement: AXUIElement?
969
+ private var focusEl: [String: Any] = [:]
970
+ private var focusSampling = false
971
+ private var pending: PendingText?
972
+ private var textTimer: Timer?
973
+ private var sampleTimer: Timer?
974
+ private var baselines: [(element: AXUIElement, value: String)] = []
975
+ private lazy var browserURLs = RecorderBrowserURLs(
976
+ fastRead: { [unowned self] container, field in fastBrowserURL(container: container, field: field) },
977
+ walk: { [unowned self] window in walkBrowserWindow(window) }
978
+ )
979
+
980
+ // ↓ 以下只在主线程读写。
981
+ private var mainObservers: [(NotificationCenter, NSObjectProtocol)] = []
982
+ private var mouseMonitor: Any?
983
+ private var keyMonitor: Any?
984
+ /// 只给测试:主线程观察者一共收到过几条通知。拆掉之后再发通知,这个数不该再涨(只数数组长度证明不了真的摘掉了)。
985
+ private(set) var debugMainDeliveries = 0
986
+
987
+ private struct PendingText {
988
+ let element: AXUIElement
989
+ let context: RecorderContext
990
+ let el: [String: Any]
991
+ var lastChange: Double
992
+ var agent: Bool
993
+ var latest: String?
994
+ }
995
+
996
+ // MARK: 对外(任意线程)
997
+
998
+ var subscriberCount: Int {
999
+ registryLock.lock(); defer { registryLock.unlock() }
1000
+ return subscribers.count
1001
+ }
1002
+
1003
+ /// 在 serve socket 的专用连接上开始订阅。成功 → nil,回包已排在这条连接写队列的第一位;
1004
+ /// 失败 → 错误码与信息,由调用方回错误包(连接照常当普通连接用)。
1005
+ func subscribe(fd: Int32, id: String, policy: Any?, protocolVersion: Int, browserBundleIds: Set<String>) -> (code: String, message: String)? {
1006
+ guard recorderPeerIsSameUser(fd) else {
1007
+ return ("forbidden", "recordSubscribe is only served to processes of the same user")
1008
+ }
1009
+ guard accessibilityTrusted() else {
1010
+ return ("accessibility_denied", "The helper does not have the Accessibility permission")
1011
+ }
1012
+ let subscriber = RecorderSubscriber(fd: fd, policy: RecorderPolicy(json: policy))
1013
+ subscriber.send(["id": id, "ok": true, "result": ["subscribed": true, "protocolVersion": protocolVersion, "axTrusted": true]])
1014
+ registryLock.lock()
1015
+ subscribers[fd] = subscriber
1016
+ registryLock.unlock()
1017
+ perform { [self] in
1018
+ browserIds = browserBundleIds
1019
+ let wasRunning = running
1020
+ if wasRunning { resetText() } else { start() }
1021
+ if suspended {
1022
+ // 锁屏 / 睡眠 / 会话切走时订阅:只给新来的补一条当前状态,不拍前台(人不在),解锁通知来了再接上。
1023
+ subscriber.send(["ev": ["t": Int64(recorderNowMs()), "kind": "system", "state": locked ? "locked" : "sleep"]])
1024
+ return
1025
+ }
1026
+ guard wasRunning else { return activated(NSWorkspace.shared.frontmostApplication) }
1027
+ if observer == nil, let app = observedRunningApp, !isHardExcluded(app), appWanted(app.bundleIdentifier ?? "") {
1028
+ // 之前的订阅者都排除了当前 App(所以没建 observer),或上次没挂上:新来的要它,重新接上。
1029
+ observedPid = 0
1030
+ activated(app)
1031
+ } else {
1032
+ emitSnapshot()
1033
+ }
1034
+ }
1035
+ return nil
1036
+ }
1037
+
1038
+ /// 连接读到 EOF(对端关闭,或写失败后我们自己 shutdown 了)时调用。最后一个订阅者走了就全部拆掉。
1039
+ func unsubscribe(fd: Int32) {
1040
+ registryLock.lock()
1041
+ let subscriber = subscribers.removeValue(forKey: fd)
1042
+ registryLock.unlock()
1043
+ guard let subscriber else { return }
1044
+ subscriber.close()
1045
+ perform { [self] in
1046
+ resetText()
1047
+ stopIfIdle()
1048
+ }
1049
+ }
1050
+
1051
+ /// 只给测试:采集器拆没拆干净。须在主线程调用(recorder 线程上的字段经 perform 同步取)。
1052
+ struct DebugState: Equatable {
1053
+ var running = false
1054
+ var observer = false
1055
+ var focus = false
1056
+ var pendingText = false
1057
+ var timers = false
1058
+ var baselines = 0
1059
+ var mainObservers = 0
1060
+ var mouseMonitor = false
1061
+ var keyMonitor = false
1062
+ }
1063
+
1064
+ func debugState() -> DebugState {
1065
+ precondition(Thread.isMainThread, "debugState reads main-thread fields")
1066
+ var state = DebugState()
1067
+ let done = DispatchSemaphore(value: 0)
1068
+ perform { [self] in
1069
+ state.running = running
1070
+ state.observer = observer != nil || observedApp != nil
1071
+ state.focus = focusElement != nil
1072
+ state.pendingText = pending != nil
1073
+ state.timers = windowTimer != nil || retryTimer != nil || textTimer != nil || sampleTimer != nil
1074
+ state.baselines = baselines.count
1075
+ done.signal()
1076
+ }
1077
+ done.wait()
1078
+ state.mainObservers = mainObservers.count
1079
+ state.mouseMonitor = mouseMonitor != nil
1080
+ state.keyMonitor = keyMonitor != nil
1081
+ return state
1082
+ }
1083
+
1084
+ func agentEnter() { agentGate.enter() }
1085
+ func agentLeave() { agentGate.leave() }
1086
+
1087
+ // MARK: recorder 线程
1088
+
1089
+ private func ensureThread() -> CFRunLoop? {
1090
+ threadLock.lock(); defer { threadLock.unlock() }
1091
+ if let runLoop { return runLoop }
1092
+ let ready = DispatchSemaphore(value: 0)
1093
+ var created: CFRunLoop?
1094
+ let thread = Thread {
1095
+ created = CFRunLoopGetCurrent()
1096
+ // 常驻一个空端口,run loop 才不会因为「没有源」立刻返回。没有订阅者时这个线程只是睡在 mach_msg 里。
1097
+ RunLoop.current.add(NSMachPort(), forMode: .default)
1098
+ ready.signal()
1099
+ while true {
1100
+ autoreleasepool { _ = RunLoop.current.run(mode: .default, before: .distantFuture) }
1101
+ }
1102
+ }
1103
+ thread.name = "forsion.activity-recorder"
1104
+ thread.qualityOfService = .utility
1105
+ thread.start()
1106
+ ready.wait()
1107
+ runLoop = created
1108
+ return created
1109
+ }
1110
+
1111
+ private func perform(_ block: @escaping () -> Void) {
1112
+ guard let runLoop = ensureThread() else { return }
1113
+ CFRunLoopPerformBlock(runLoop, CFRunLoopMode.defaultMode.rawValue, block)
1114
+ CFRunLoopWakeUp(runLoop)
1115
+ }
1116
+
1117
+ private func makeTimer(_ interval: TimeInterval, _ block: @escaping () -> Void) -> Timer {
1118
+ let timer = Timer(timeInterval: interval, repeats: false) { _ in block() }
1119
+ RunLoop.current.add(timer, forMode: .default)
1120
+ return timer
1121
+ }
1122
+
1123
+ private func subscriberSnapshot() -> [RecorderSubscriber] {
1124
+ registryLock.lock(); defer { registryLock.unlock() }
1125
+ return Array(subscribers.values)
1126
+ }
1127
+
1128
+ /// 只装观察者、定初始状态,不发事件 —— 首条事件由 subscribe 按是否「人不在」决定(快照或 system 标记)。
1129
+ private func start() {
1130
+ guard !running else { return }
1131
+ running = true
1132
+ // 锁屏 / 会话切走时开始订阅(helper 在锁屏期间重启、桌面端退避重连):照实记下,等解锁通知再拍前台。
1133
+ locked = sessionLocked()
1134
+ asleep = false
1135
+ DispatchQueue.main.async { [self] in installMainObservers() }
1136
+ }
1137
+
1138
+ private func stopIfIdle() {
1139
+ guard running, subscriberCount == 0 else { return }
1140
+ running = false
1141
+ // 没人收了:丢掉未发的输入与所有基线,不再读任何东西。
1142
+ pending = nil
1143
+ textTimer?.invalidate(); textTimer = nil
1144
+ sampleTimer?.invalidate(); sampleTimer = nil
1145
+ teardownObserver()
1146
+ context = nil
1147
+ baselines = []
1148
+ browserURLs.removeAll()
1149
+ DispatchQueue.main.async { [self] in removeMainObservers() }
1150
+ }
1151
+
1152
+ /// 订阅者来去时:未发的输入丢掉(不报)、差分基线全部作废,再按当前值重建焦点框的基线。
1153
+ /// 桌面端「清除」后会断开重订阅 —— 不这样的话,清除前打的字会作为待发输入、或经旧基线的差分在清除后重新冒出来。
1154
+ /// 多个订阅者(dev + 正式版)并存时基线是共用的,所以任何一方来去都做,不只最后一个走的时候。
1155
+ private func resetText() {
1156
+ textTimer?.invalidate(); textTimer = nil
1157
+ sampleTimer?.invalidate(); sampleTimer = nil
1158
+ pending = nil
1159
+ baselines = []
1160
+ guard running, !suspended, let focusElement, let context, wants(.text, context) else { return }
1161
+ primeBaseline(focusElement)
1162
+ }
1163
+
1164
+ private func emit(_ kind: RecorderKind, context: RecorderContext?, t: Double? = nil, fields: [String: Any] = [:], agent: Bool = false, dedupe: Bool = false) {
1165
+ let time = Int64(t ?? recorderNowMs())
1166
+ for subscriber in subscriberSnapshot() {
1167
+ let delivered: [String: Any]?
1168
+ if kind == .app || kind == .window {
1169
+ delivered = recorderContextDelivery(kind: kind, context: context, policy: subscriber.policy, state: &subscriber.delivery, dedupe: dedupe)
1170
+ } else {
1171
+ delivered = recorderEventBody(kind: kind, context: context, policy: subscriber.policy)
1172
+ }
1173
+ guard var body = delivered else { continue }
1174
+ for (name, value) in fields { body[name] = value }
1175
+ body["t"] = time
1176
+ if agent { body["origin"] = "agent" }
1177
+ subscriber.send(["ev": body])
1178
+ }
1179
+ }
1180
+
1181
+ private func wants(_ kind: RecorderKind, _ context: RecorderContext) -> Bool {
1182
+ subscriberSnapshot().contains { recorderEventBody(kind: kind, context: context, policy: $0.policy) != nil }
1183
+ }
1184
+
1185
+ /// 有没有订阅者需要这个 App 的标题以上的信息。都排除了 → 连 observer 都不建(用户的排除表也在读 AX 之前生效)。
1186
+ private func appWanted(_ bundleId: String) -> Bool {
1187
+ subscriberSnapshot().contains { !$0.policy.excludes(bundleId: bundleId) }
1188
+ }
1189
+
1190
+ private func textWanted(_ bundleId: String) -> Bool {
1191
+ subscriberSnapshot().contains { $0.policy.text && !$0.policy.excludes(bundleId: bundleId) && !$0.policy.titleOnly(bundleId: bundleId) }
1192
+ }
1193
+
1194
+ private func emitSnapshot() {
1195
+ guard running, !suspended, let context else { return }
1196
+ emit(.app, context: context, dedupe: true)
1197
+ }
1198
+
1199
+ // MARK: 前台 App 与 observer
1200
+
1201
+ private func isHardExcluded(_ app: NSRunningApplication) -> Bool {
1202
+ app.processIdentifier == getpid() || recorderHardExcludedBundleIds.contains(app.bundleIdentifier ?? "")
1203
+ }
1204
+
1205
+ fileprivate func activated(_ app: NSRunningApplication?) {
1206
+ guard running, !suspended, let app else { return }
1207
+ if recorderIgnoredBundleIds.contains(app.bundleIdentifier ?? "") { return }
1208
+ flushText()
1209
+ let samePid = app.processIdentifier == observedPid
1210
+ if !samePid { retarget(app) }
1211
+ let (next, window, titleKey) = readContext(app)
1212
+ let windowChanged = !sameElement(window, focusedWindow)
1213
+ context = next
1214
+ focusedWindow = window
1215
+ focusedTitleKey = titleKey
1216
+ emit(.app, context: next, agent: agentGate.active(), dedupe: samePid)
1217
+ if !samePid || windowChanged { setFocus(focusedUIElement()) }
1218
+ }
1219
+
1220
+ private static let retargetRetryDelays: [TimeInterval] = [0.5, 1, 2]
1221
+
1222
+ private func retarget(_ app: NSRunningApplication, attempt: Int = 0) {
1223
+ teardownObserver()
1224
+ observedRunningApp = app
1225
+ observedPid = app.processIdentifier
1226
+ guard !isHardExcluded(app), appWanted(app.bundleIdentifier ?? ""), let runLoop else { return }
1227
+ let appElement = AXUIElementCreateApplication(app.processIdentifier)
1228
+ AXUIElementSetMessagingTimeout(appElement, Self.axTimeout)
1229
+ observedApp = appElement
1230
+ var created: AXObserver?
1231
+ if AXObserverCreate(app.processIdentifier, recorderAXCallback, &created) == .success, let created {
1232
+ let refcon = Unmanaged.passUnretained(self).toOpaque()
1233
+ for name in [kAXFocusedWindowChangedNotification, kAXMainWindowChangedNotification, kAXTitleChangedNotification, kAXFocusedUIElementChangedNotification] {
1234
+ let status = AXObserverAddNotification(created, appElement, name as CFString, refcon)
1235
+ if status == .success || status == .notificationAlreadyRegistered { appNotifications.append(name) }
1236
+ }
1237
+ if !appNotifications.isEmpty {
1238
+ CFRunLoopAddSource(runLoop, AXObserverGetRunLoopSource(created), .defaultMode)
1239
+ observer = created
1240
+ return
1241
+ }
1242
+ }
1243
+ // 一个通知都没挂上:刚启动的 App 常常还没开始应答辅助功能(cannotComplete)。它还在前台就隔一会儿重来,
1244
+ // 最多 3 次(0.5 / 1 / 2 秒);不然它在前台多久,这段时间就一条窗口 / 输入都记不到。
1245
+ guard attempt < Self.retargetRetryDelays.count else { return }
1246
+ let pid = app.processIdentifier
1247
+ retryTimer = makeTimer(Self.retargetRetryDelays[attempt]) { [self] in
1248
+ retryTimer = nil
1249
+ guard running, !suspended, observer == nil, observedPid == pid else { return }
1250
+ retarget(app, attempt: attempt + 1)
1251
+ // 挂上了:按现在的窗口补一次情境(首次读时 App 可能还没应答,标题是空的)。
1252
+ if observer != nil { refreshWindow() }
1253
+ }
1254
+ }
1255
+
1256
+ private func teardownObserver() {
1257
+ windowTimer?.invalidate(); windowTimer = nil
1258
+ retryTimer?.invalidate(); retryTimer = nil
1259
+ releaseFocus()
1260
+ if let observer, let appElement = observedApp {
1261
+ // 顺序要紧:先逐条摘通知,再摘 run-loop source(反过来会漏 wired memory,aw-watcher #139)。
1262
+ for name in appNotifications { AXObserverRemoveNotification(observer, appElement, name as CFString) }
1263
+ if let runLoop { CFRunLoopRemoveSource(runLoop, AXObserverGetRunLoopSource(observer), .defaultMode) }
1264
+ }
1265
+ observer = nil
1266
+ observedApp = nil
1267
+ observedRunningApp = nil
1268
+ observedPid = 0
1269
+ appNotifications = []
1270
+ focusedWindow = nil
1271
+ focusedTitleKey = nil
1272
+ }
1273
+
1274
+ fileprivate func handleAX(observer: AXObserver, element: AXUIElement, notification: String) {
1275
+ guard running, !suspended, let current = self.observer, CFEqual(current, observer) else { return }
1276
+ switch notification {
1277
+ case kAXValueChangedNotification:
1278
+ valueChanged(element)
1279
+ case kAXFocusedUIElementChangedNotification:
1280
+ // 焦点通知可能比窗口 / 标题通知先到(或那边还在去抖):先把情境对齐,再按新情境决定读不读基线 ——
1281
+ // 不然刚打开的无痕窗口、刚切到的排除站点标签页,会拿上一个窗口的情境读值、记字。
1282
+ refreshIfStale()
1283
+ // 回调给的元素在少数 App 里是 App 本身;以 App 的 AXFocusedUIElement 为准。
1284
+ setFocus(focusedUIElement())
1285
+ case kAXUIElementDestroyedNotification:
1286
+ if let focus = focusElement, CFEqual(focus, element) {
1287
+ flushText()
1288
+ releaseFocus()
1289
+ }
1290
+ case kAXTitleChangedNotification:
1291
+ // App 级订阅会收到所有元素的标题变化;只关心焦点窗口的。
1292
+ if focusedWindow == nil || CFEqual(focusedWindow!, element) { scheduleWindowRefresh() }
1293
+ default:
1294
+ scheduleWindowRefresh()
1295
+ }
1296
+ }
1297
+
1298
+ private func scheduleWindowRefresh() {
1299
+ windowTimer?.invalidate()
1300
+ windowTimer = makeTimer(Self.windowDebounce) { [self] in refreshWindow() }
1301
+ }
1302
+
1303
+ private func refreshWindow() {
1304
+ windowTimer?.invalidate(); windowTimer = nil
1305
+ guard running, !suspended, let app = observedRunningApp else { return }
1306
+ let (next, window, titleKey) = readContext(app)
1307
+ let windowChanged = !sameElement(window, focusedWindow)
1308
+ // 同一窗口里网址变了(导航,含标题不变的换站点)也算换了情境:待发输入先按旧情境报,新情境不收文字就丢。
1309
+ let urlChanged = !windowChanged && context.map { $0.url != next.url || $0.urlUnknown != next.urlUnknown } == true
1310
+ switch recorderPendingOnRefresh(nextWantsText: wants(.text, next), windowChanged: windowChanged, urlChanged: urlChanged) {
1311
+ case .drop: dropText()
1312
+ case .flush: flushText()
1313
+ case .keep: break
1314
+ }
1315
+ context = next
1316
+ focusedWindow = window
1317
+ focusedTitleKey = titleKey
1318
+ emit(.window, context: next, agent: agentGate.active(), dedupe: true)
1319
+ if windowChanged { setFocus(focusedUIElement()) }
1320
+ }
1321
+
1322
+ /// 情境是不是已经过时:窗口刷新还在去抖、焦点窗口换了、它的标题变了(通知还在路上),或浏览器里标题没变
1323
+ /// 而网址变了(同标题换站点 / 站内导航)→ 立刻同步刷新一次。读基线、开始一段输入、停手报字、记点击 / 快捷键之前
1324
+ /// 都先过这一步;平时只多两次轻量 AX 读(焦点窗口、标题),浏览器里再多 1–2 次(记住的快路,绝不在这里走遍历)。
1325
+ private func refreshIfStale() {
1326
+ guard running, !suspended else { return }
1327
+ if windowTimer != nil { return refreshWindow() }
1328
+ guard let appElement = observedApp else { return }
1329
+ let window = axElement(axCopy(appElement, kAXFocusedWindowAttribute))
1330
+ guard sameElement(window, focusedWindow) else { return refreshWindow() }
1331
+ guard let window, let context, !context.hardExcluded, !recorderPrivateUnverified(context.bundleId) else { return }
1332
+ if titleKey(window) != focusedTitleKey { return refreshWindow() }
1333
+ // 这个窗口没有快路(上次遍历没找到网页区域与地址栏)→ fastURL 为 nil、不刷新:那时情境本就是「网址没读到」,
1334
+ // 设了排除站点的订阅者已经失败即关闭。焦点窗口的缓存项只会在换了焦点窗口之后被挤掉,换回来时 refreshWindow 会重建。
1335
+ if !context.isPrivate, isBrowser(context.bundleId), recorderURLChanged(context, fresh: browserURLs.fastURL(window: window)) {
1336
+ refreshWindow()
1337
+ }
1338
+ }
1339
+
1340
+ private func titleKey(_ window: AXUIElement) -> Int? {
1341
+ AXUIElementSetMessagingTimeout(window, Self.axTimeout)
1342
+ return (axCopy(window, kAXTitleAttribute) as? String)?.hashValue
1343
+ }
1344
+
1345
+ private func sameElement(_ lhs: AXUIElement?, _ rhs: AXUIElement?) -> Bool {
1346
+ switch (lhs, rhs) {
1347
+ case (nil, nil): return true
1348
+ case let (lhs?, rhs?): return CFEqual(lhs, rhs)
1349
+ default: return false
1350
+ }
1351
+ }
1352
+
1353
+ private func isBrowser(_ bundleId: String) -> Bool {
1354
+ browserIds.contains(bundleId) || recorderIsBrowser(bundleId)
1355
+ }
1356
+
1357
+ /// 读前台窗口的情境。第三项 = 标题哈希(refreshIfStale 比对用;标题本身不留)。
1358
+ private func readContext(_ app: NSRunningApplication) -> (RecorderContext, AXUIElement?, Int?) {
1359
+ let bundleId = app.bundleIdentifier ?? ""
1360
+ var next = RecorderContext(name: app.localizedName ?? bundleId, bundleId: bundleId)
1361
+ next.hardExcluded = isHardExcluded(app)
1362
+ guard !next.hardExcluded, let appElement = observedApp,
1363
+ let window = axElement(axCopy(appElement, kAXFocusedWindowAttribute))
1364
+ else { return (next, nil, nil) }
1365
+ // Safari 等判不出无痕窗口的 App:整个按无痕处理,标题都不读(见 recorderPrivateUnverifiedBundleIds)。
1366
+ if recorderPrivateUnverified(bundleId) {
1367
+ next.isPrivate = true
1368
+ return (next, window, nil)
1369
+ }
1370
+ AXUIElementSetMessagingTimeout(window, Self.axTimeout)
1371
+ let rawTitle = axCopy(window, kAXTitleAttribute) as? String
1372
+ let title = recorderTitle(rawTitle)
1373
+ var isPrivate = title.map(recorderHasPrivateMarker) ?? false
1374
+ var url: String?
1375
+ var urlUnknown = false
1376
+ if !isPrivate && isBrowser(bundleId) {
1377
+ let info = browserURLs.lookup(window: window, title: title ?? "")
1378
+ url = info.url
1379
+ urlUnknown = !info.resolved
1380
+ isPrivate = info.isPrivate
1381
+ }
1382
+ next.isPrivate = isPrivate
1383
+ if !isPrivate {
1384
+ next.title = title
1385
+ next.url = url
1386
+ next.urlUnknown = urlUnknown
1387
+ }
1388
+ return (next, window, rawTitle?.hashValue)
1389
+ }
1390
+
1391
+ // MARK: 浏览器 URL(有上限、绝不进网页内容;缓存与读取策略见 RecorderBrowserURLs)
1392
+
1393
+ /// 快路:上次遍历记下的网页区域父节点 / 地址栏。父节点只读它的直接子节点,每个子节点一次取角色 + AXURL;
1394
+ /// 通常 2 次 AX 调用。父节点本身绝不是网页区域(遍历从不展开网页区域),所以这里也不会向网页区域要子节点。
1395
+ private func fastBrowserURL(container: AXUIElement?, field: AXUIElement?) -> (url: String?, resolved: Bool) {
1396
+ if let container {
1397
+ for element in axChildren(container) {
1398
+ AXUIElementSetMessagingTimeout(element, Self.axTimeout)
1399
+ let values = axMulti(element, [kAXRoleAttribute, "AXURL"])
1400
+ guard values[0] as? String == "AXWebArea" else { continue }
1401
+ let found = recorderWebAreaURL(rawURL(values[1]))
1402
+ if found.resolved { return found }
1403
+ }
1404
+ }
1405
+ if let field, let url = (axCopy(field, kAXValueAttribute) as? String).flatMap(recorderSanitizeURL) { return (url, true) }
1406
+ return (nil, false)
1407
+ }
1408
+
1409
+ private func rawURL(_ value: AnyObject?) -> String? {
1410
+ (value as? URL)?.absoluteString ?? (value as? String)
1411
+ }
1412
+
1413
+ /// 慢路:有上限的广度遍历(≤400 节点、≤12 层、≤200ms)。先读角色、再决定读不读子节点,绝不向 AXWebArea 要 AXChildren。
1414
+ private func walkBrowserWindow(_ window: AXUIElement) -> RecorderBrowserURLs.Walked {
1415
+ let deadline = CFAbsoluteTimeGetCurrent() + Self.walkBudget
1416
+ let walked = recorderBoundedWalk(
1417
+ root: window, nodeCap: Self.walkNodeCap, depthCap: Self.walkDepthCap,
1418
+ expired: { CFAbsoluteTimeGetCurrent() >= deadline },
1419
+ describe: { [self] element in
1420
+ AXUIElementSetMessagingTimeout(element, Self.axTimeout)
1421
+ let values = axMulti(element, [kAXRoleAttribute, kAXDescriptionAttribute, kAXTitleAttribute])
1422
+ return (values[0] as? String ?? "", [values[1] as? String, values[2] as? String].compactMap { $0 }.joined(separator: " "))
1423
+ },
1424
+ webURL: { [self] element in recorderWebAreaURL(rawURL(axCopy(element, "AXURL"))) },
1425
+ fieldValue: { [self] element in axCopy(element, kAXValueAttribute) as? String },
1426
+ children: { [self] element in axChildren(element) }
1427
+ )
1428
+ scanStats.record(walked.stops)
1429
+ return RecorderBrowserURLs.Walked(url: walked.url, resolved: walked.resolved, isPrivate: walked.isPrivate, webContainer: walked.webContainer, addressField: walked.addressField)
1430
+ }
1431
+
1432
+ // MARK: 焦点输入框与文本差分
1433
+
1434
+ private func focusedUIElement() -> AXUIElement? {
1435
+ guard let observedApp else { return nil }
1436
+ return axElement(axCopy(observedApp, kAXFocusedUIElementAttribute))
1437
+ }
1438
+
1439
+ private func setFocus(_ element: AXUIElement?) {
1440
+ if let element, let current = focusElement, CFEqual(element, current) {
1441
+ // 同一个 AX 元素再次聚焦:网页可能已原地把它改成了密码框(身份不变)。复核一次,是就立刻撒手。
1442
+ if fieldSecure(current) { dropSecureField(current) }
1443
+ return
1444
+ }
1445
+ flushText()
1446
+ releaseFocus()
1447
+ // 无痕窗口(含整个 Safari)里连焦点框都不看:不读控件名、不挂 ValueChanged(一个窗口的无痕与否不会变,换窗口会重新 setFocus)。
1448
+ guard let element, let observer, let context, !context.hardExcluded, !context.isPrivate, textWanted(context.bundleId) else { return }
1449
+ AXUIElementSetMessagingTimeout(element, Self.axTimeout)
1450
+ let values = axMulti(element, [kAXRoleAttribute, kAXSubroleAttribute, kAXTitleAttribute, kAXDescriptionAttribute, "AXPlaceholderValue", "AXEditable"])
1451
+ let role = values[0] as? String ?? ""
1452
+ let subrole = values[1] as? String ?? ""
1453
+ guard recorderIsEditableText(role: role, subrole: subrole, editable: (values[5] as? NSNumber)?.boolValue ?? false) else { return }
1454
+ // ValueChanged 只挂在焦点输入框上:挂在 App 上会收到进度条、滑块、网页内容的全部值变化。
1455
+ let refcon = Unmanaged.passUnretained(self).toOpaque()
1456
+ guard AXObserverAddNotification(observer, element, kAXValueChangedNotification as CFString, refcon) == .success else { return }
1457
+ _ = AXObserverAddNotification(observer, element, kAXUIElementDestroyedNotification as CFString, refcon)
1458
+ focusElement = element
1459
+ var el: [String: Any] = ["role": role]
1460
+ if let label = recorderLabel(recorderFirstNonEmpty(values[2] as? String, values[3] as? String, values[4] as? String)) { el["label"] = label }
1461
+ focusEl = el
1462
+ // 无痕窗口 / 排除站点里连基线都不读(值不进 helper 内存;调用方已先 refreshIfStale,情境是新的);
1463
+ // 之后变成想要时,第一段输入只建基线。
1464
+ if wants(.text, context) { primeBaseline(element) } else { setBaseline(element, nil) }
1465
+ }
1466
+
1467
+ private func releaseFocus() {
1468
+ sampleTimer?.invalidate(); sampleTimer = nil
1469
+ if let element = focusElement, let observer {
1470
+ AXObserverRemoveNotification(observer, element, kAXValueChangedNotification as CFString)
1471
+ AXObserverRemoveNotification(observer, element, kAXUIElementDestroyedNotification as CFString)
1472
+ }
1473
+ focusElement = nil
1474
+ focusEl = [:]
1475
+ focusSampling = false
1476
+ }
1477
+
1478
+ /// 聚焦时记下当前值作为差分基线(聚焦前就有的内容不算「打的字」)。
1479
+ private func primeBaseline(_ element: AXUIElement) {
1480
+ focusSampling = false
1481
+ switch readField(element, withCount: true) {
1482
+ case .secure: dropSecureField(element)
1483
+ case .big, .unreadable: setBaseline(element, nil)
1484
+ case .value(let value):
1485
+ setBaseline(element, value)
1486
+ focusSampling = value.utf16.count <= RecorderLimit.sampleChars
1487
+ }
1488
+ }
1489
+
1490
+ /// recorderSampleField 的 AX 版(整次读计时 + 读数)。withCount:基线 / 停手把字数与角色放在同一次 IPC 里取
1491
+ /// (与以前读字数 + 读值一样是两次);采样不取字数(比以前多一次角色读取,计进 50ms 上限)。
1492
+ private func sampleField(_ element: AXUIElement, withCount: Bool) -> (read: RecorderFieldRead, keepSampling: Bool) {
1493
+ recorderSampleField(
1494
+ clock: { CFAbsoluteTimeGetCurrent() },
1495
+ secureInput: recorderSecureInputActive,
1496
+ probe: { [self] in fieldIdentity(element, withCount: withCount) },
1497
+ value: { [self] in axCopy(element, kAXValueAttribute) as? String }
1498
+ )
1499
+ }
1500
+
1501
+ private func readField(_ element: AXUIElement, withCount: Bool) -> RecorderFieldRead {
1502
+ sampleField(element, withCount: withCount).read
1503
+ }
1504
+
1505
+ /// 一次 IPC 取角色、子角色(与字数),逐项分清「确实没有」与「查询失败」。整次调用失败 → 两项都按那次的错误算
1506
+ /// (元素已销毁 → .gone,其余一律 .failed;整次调用的「没有这个属性」说明不了某一项)。
1507
+ private func fieldIdentity(_ element: AXUIElement, withCount: Bool) -> (role: RecorderAXAttr, subrole: RecorderAXAttr, count: Int?) {
1508
+ let attributes = withCount ? [kAXRoleAttribute, kAXSubroleAttribute, kAXNumberOfCharactersAttribute] : [kAXRoleAttribute, kAXSubroleAttribute]
1509
+ var values: CFArray?
1510
+ let status = AXUIElementCopyMultipleAttributeValues(element, attributes as CFArray, AXCopyMultipleAttributeOptions(rawValue: 0), &values)
1511
+ guard status == .success, let array = values as? [AnyObject], array.count == attributes.count else {
1512
+ let failed: RecorderAXAttr = status == .invalidUIElement ? .gone : .failed
1513
+ return (failed, failed, nil)
1514
+ }
1515
+ return (recorderAXAttr(array[0]), recorderAXAttr(array[1]), withCount ? (array[2] as? NSNumber)?.intValue : nil)
1516
+ }
1517
+
1518
+ /// 报字之前再核一次(只查角色 / 子角色与 Secure Input,不读值):读值与报出之间,这个框可能刚被改成密码框。
1519
+ /// 角色或子角色没查到 → 证明不了不是,按安全输入框丢;元素已销毁不算(见 recorderRecheckDrops)。
1520
+ private func fieldSecure(_ element: AXUIElement) -> Bool {
1521
+ if recorderSecureInputActive() { return true }
1522
+ let (role, subrole, _) = fieldIdentity(element, withCount: false)
1523
+ return recorderRecheckDrops(recorderFieldState(secureInput: false, role: role, subrole: subrole))
1524
+ }
1525
+
1526
+ /// 焦点框此刻是安全输入框(或 Secure Input 开着):待发输入、差分基线、这个框的 ValueChanged 观察全部立刻丢掉,不报。
1527
+ /// 之后要等焦点换走再回来(setFocus 重新按角色判断)才会再看它。调用方在这之后不能再碰 focusElement。
1528
+ private func dropSecureField(_ element: AXUIElement) {
1529
+ textTimer?.invalidate(); textTimer = nil
1530
+ sampleTimer?.invalidate(); sampleTimer = nil
1531
+ if let pendingElement = pending?.element { setBaseline(pendingElement, nil) }
1532
+ pending = nil
1533
+ setBaseline(element, nil)
1534
+ if let focus = focusElement, CFEqual(focus, element) { releaseFocus() }
1535
+ }
1536
+
1537
+ private func baseline(_ element: AXUIElement) -> String? {
1538
+ baselines.first { CFEqual($0.element, element) }?.value
1539
+ }
1540
+
1541
+ private func setBaseline(_ element: AXUIElement, _ value: String?) {
1542
+ baselines.removeAll { CFEqual($0.element, element) }
1543
+ guard let value else { return }
1544
+ baselines.append((element, value))
1545
+ if baselines.count > 8 { baselines.removeFirst(baselines.count - 8) }
1546
+ }
1547
+
1548
+ private func valueChanged(_ element: AXUIElement) {
1549
+ guard let focus = focusElement, CFEqual(element, focus) else { return }
1550
+ // 一段新输入的开头:先确认情境没过时(同一个输入框所在的标签页可能刚切到排除站点)。刷新若换了焦点,这次作废。
1551
+ // 当前情境本就不收文字时不查:过时只会让这段多丢,不会多记(也免得无痕窗口里每敲一下都多两次 AX 读)。
1552
+ if pending == nil, let context, wants(.text, context) { refreshIfStale() }
1553
+ guard let focus = focusElement, CFEqual(element, focus), let context else { return }
1554
+ if recorderSecureInputActive() { return dropSecureField(focus) }
1555
+ guard wants(.text, context) else {
1556
+ // 这段期间的值不能进下一次差分:丢掉待发与基线。
1557
+ pending = nil
1558
+ textTimer?.invalidate(); textTimer = nil
1559
+ return setBaseline(focus, nil)
1560
+ }
1561
+ let now = recorderNowMs()
1562
+ let agent = agentGate.active()
1563
+ if var current = pending {
1564
+ current.lastChange = now
1565
+ current.agent = current.agent || agent
1566
+ pending = current
1567
+ } else {
1568
+ pending = PendingText(element: focus, context: context, el: focusEl, lastChange: now, agent: agent, latest: nil)
1569
+ }
1570
+ textTimer?.invalidate()
1571
+ textTimer = makeTimer(Self.textDebounce) { [self] in settleText() }
1572
+ if focusSampling && sampleTimer == nil {
1573
+ sampleTimer = makeTimer(Self.sampleDelay) { [self] in sampleText() }
1574
+ }
1575
+ }
1576
+
1577
+ /// 小输入框边打边采样(≤4 次/秒):抓住「发送后立刻被清空」之前的那一版。
1578
+ private func sampleText() {
1579
+ sampleTimer = nil
1580
+ // 两头都要:打字时的情境,以及现在的情境(之间若刷新成了无痕 / 排除站点,refreshWindow 已丢掉待发)。
1581
+ guard var current = pending, let focus = focusElement, CFEqual(current.element, focus), let live = context,
1582
+ wants(.text, current.context), wants(.text, live)
1583
+ else { return }
1584
+ // 整次读(角色 / 子角色复核 + 值)超过 50ms 的 App、或者文本变大了:退回只在停手时读,不给目标 App 添负担。
1585
+ // 停手那次照常复核(flushText 走同一个 readField,报字前再 fieldSecure 一次),与采不采样无关。
1586
+ let (read, keepSampling) = sampleField(focus, withCount: false)
1587
+ if !keepSampling { focusSampling = false }
1588
+ let value: String
1589
+ switch read {
1590
+ case .secure: return dropSecureField(focus)
1591
+ case .unreadable, .big: return
1592
+ case .value(let text): value = text
1593
+ }
1594
+ if value.isEmpty, let latest = current.latest, !latest.isEmpty {
1595
+ // 输入框被清空(聊天框发送的典型形态):清空前那版立刻当作一段输入报出去,基线归零。
1596
+ // 报之前先确认情境没过时(这段输入期间页面可能已同标题换到排除站点);刷新若换了情境,待发已被报掉或丢掉。
1597
+ refreshIfStale()
1598
+ guard pending != nil, let still = focusElement, CFEqual(still, focus) else { return }
1599
+ if fieldSecure(focus) { return dropSecureField(focus) }
1600
+ if let base = baseline(focus), let diff = recorderSettleText(baseline: base, latest: latest, final: value), !diff.inserted.isEmpty {
1601
+ emit(.text, context: current.context, t: current.lastChange, fields: recorderTextFields(diff).merging(["el": current.el]) { $1 }, agent: current.agent)
1602
+ }
1603
+ setBaseline(focus, value)
1604
+ pending = nil
1605
+ textTimer?.invalidate(); textTimer = nil
1606
+ return
1607
+ }
1608
+ current.latest = value
1609
+ pending = current
1610
+ }
1611
+
1612
+ /// 停手(去抖到期):先确认情境没过时 —— 同一个输入框还在,所在页面却可能已同标题换到排除站点(单页应用常见),
1613
+ /// 段首那次 refreshIfStale 管不到段中的导航。刷新若判定情境变了,refreshWindow 已按规则报掉或丢掉待发,
1614
+ /// 这里的 flushText 就是空操作。
1615
+ private func settleText() {
1616
+ textTimer = nil
1617
+ refreshIfStale()
1618
+ flushText()
1619
+ }
1620
+
1621
+ private func flushText() {
1622
+ textTimer?.invalidate(); textTimer = nil
1623
+ sampleTimer?.invalidate(); sampleTimer = nil
1624
+ guard let current = pending else { return }
1625
+ pending = nil
1626
+ let element = current.element
1627
+ if recorderSecureInputActive() { return dropSecureField(element) }
1628
+ guard wants(.text, current.context), let live = context, wants(.text, live) else {
1629
+ return setBaseline(element, nil)
1630
+ }
1631
+ let final: String
1632
+ switch readField(element, withCount: true) {
1633
+ case .secure: return dropSecureField(element)
1634
+ case .big:
1635
+ setBaseline(element, nil)
1636
+ return emit(.text, context: current.context, t: current.lastChange, fields: ["el": current.el, "bigEdit": true], agent: current.agent)
1637
+ case .unreadable:
1638
+ // 元素已销毁 / 没有角色 / 值读不到:用最后一次采样兜底(有的聊天 App 发送后直接换掉输入框)。采样时它还确认是普通框
1639
+ // (每次采样前都复核过角色与子角色)。查询超时不会走到这里 —— 那是 .secure,待发已丢。
1640
+ guard let latest = current.latest else { return }
1641
+ final = latest
1642
+ case .value(let text): final = text
1643
+ }
1644
+ // 没有基线(聚焦时读不到 / 刚从大文本缩回来):这次只建基线,不猜哪些是新打的。
1645
+ guard let base = baseline(element), let diff = recorderSettleText(baseline: base, latest: current.latest, final: final) else {
1646
+ return setBaseline(element, final)
1647
+ }
1648
+ if fieldSecure(element) { return dropSecureField(element) }
1649
+ setBaseline(element, final)
1650
+ emit(.text, context: current.context, t: current.lastChange, fields: recorderTextFields(diff).merging(["el": current.el]) { $1 }, agent: current.agent)
1651
+ }
1652
+
1653
+ /// 丢掉待发输入(不报),并作废相关基线 —— 只丢待发、不清基线的话,下一次差分会把丢掉的字又带出来。
1654
+ private func dropText() {
1655
+ textTimer?.invalidate(); textTimer = nil
1656
+ sampleTimer?.invalidate(); sampleTimer = nil
1657
+ if let element = pending?.element { setBaseline(element, nil) }
1658
+ pending = nil
1659
+ if let focusElement { setBaseline(focusElement, nil) }
1660
+ }
1661
+
1662
+ // MARK: 点击 / 快捷键 / 系统
1663
+
1664
+ fileprivate func clicked(at point: CGPoint, agent: Bool) {
1665
+ guard running, !suspended, let before = context, observedApp != nil, wants(.click, before) else { return }
1666
+ // 点的可能是刚打开的无痕窗口 / 刚切到的排除站点:先把情境对齐。
1667
+ refreshIfStale()
1668
+ guard let context, let appElement = observedApp, wants(.click, context) else { return }
1669
+ // 点到别的 App 会先切前台;那一下不记。
1670
+ guard NSWorkspace.shared.frontmostApplication?.processIdentifier == observedPid else { return }
1671
+ // 命中测试以被观察的 App 为根,返回的总是它自己的控件 —— 哪怕那一点上压着的是 Spotlight、Raycast、通知横幅、
1672
+ // Dock、菜单栏图标这类不抢前台的窗口。所以先按窗口列表确认那一点最上面的窗口确实是它的,不是就不记。
1673
+ let windows = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]] ?? []
1674
+ guard recorderWindowOwner(at: point, windows: windows, skipPid: getpid()) == observedPid else { return }
1675
+ if pending != nil { flushText() }
1676
+ var hit: AXUIElement?
1677
+ guard AXUIElementCopyElementAtPosition(appElement, Float(point.x), Float(point.y), &hit) == .success, let hit else { return }
1678
+ var pid: pid_t = 0
1679
+ guard AXUIElementGetPid(hit, &pid) == .success, pid == observedPid else { return }
1680
+ AXUIElementSetMessagingTimeout(hit, Self.axTimeout)
1681
+ let values = axMulti(hit, recorderClickAttributes)
1682
+ let role = values[0] as? String ?? ""
1683
+ guard !role.isEmpty else { return }
1684
+ var el: [String: Any] = ["role": role]
1685
+ if recorderClickLabelAllowed(role: role, subrole: values[1] as? String ?? ""),
1686
+ let label = recorderLabel(recorderFirstNonEmpty(values[2] as? String, values[3] as? String)) {
1687
+ el["label"] = label
1688
+ }
1689
+ emit(.click, context: context, fields: ["el": el], agent: agent)
1690
+ }
1691
+
1692
+ /// target = 事件自带的目标 pid(没有为 0)。有就以它为准:不是被观察的 App → 不记。没有就只能认前台 App ——
1693
+ /// 那时 Spotlight 这类不抢前台的面板拿着键盘焦点,组合键仍会记到前台 App 名下(只有组合键,不含内容)。
1694
+ fileprivate func keyed(_ combo: String, target: pid_t, agent: Bool) {
1695
+ guard running, !suspended, let before = context, !recorderSecureInputActive(), wants(.key, before) else { return }
1696
+ guard NSWorkspace.shared.frontmostApplication?.processIdentifier == observedPid else { return }
1697
+ if target > 0 && target != getpid() && target != observedPid { return }
1698
+ refreshIfStale()
1699
+ guard let context, wants(.key, context) else { return }
1700
+ if pending != nil { flushText() }
1701
+ emit(.key, context: context, fields: ["keys": combo], agent: agent)
1702
+ }
1703
+
1704
+ fileprivate func systemEvent(_ state: String) {
1705
+ guard running else { return }
1706
+ let wasSuspended = suspended
1707
+ switch state {
1708
+ case "locked": locked = true
1709
+ case "unlocked": locked = false
1710
+ case "sleep": asleep = true
1711
+ case "wake": asleep = false
1712
+ default: break
1713
+ }
1714
+ if suspended && !wasSuspended { flushText() }
1715
+ emit(.system, context: nil, fields: ["state": state])
1716
+ if wasSuspended && !suspended {
1717
+ // 回来了:重新拍一次前台情境,时间线从这里接上。
1718
+ // 只清去重键;「没记」的标记留着 —— 锁屏前在无痕窗口里,解锁后回到普通页面那条照样带 resumed。
1719
+ for subscriber in subscriberSnapshot() { subscriber.delivery.lastKey = nil }
1720
+ observedPid = 0
1721
+ activated(NSWorkspace.shared.frontmostApplication)
1722
+ }
1723
+ }
1724
+
1725
+ // MARK: 主线程观察者
1726
+
1727
+ private func installMainObservers() {
1728
+ guard mainObservers.isEmpty else { return }
1729
+ func on(_ center: NotificationCenter, _ name: Notification.Name, _ body: @escaping (Notification) -> Void) {
1730
+ mainObservers.append((center, center.addObserver(forName: name, object: nil, queue: nil) { [self] note in
1731
+ debugMainDeliveries += 1
1732
+ body(note)
1733
+ }))
1734
+ }
1735
+ let workspace = NSWorkspace.shared.notificationCenter
1736
+ on(workspace, NSWorkspace.didActivateApplicationNotification) { [self] note in
1737
+ let app = note.userInfo?[NSWorkspace.applicationUserInfoKey] as? NSRunningApplication
1738
+ perform { [self] in activated(app) }
1739
+ }
1740
+ on(workspace, NSWorkspace.willSleepNotification) { [self] _ in perform { [self] in systemEvent("sleep") } }
1741
+ on(workspace, NSWorkspace.didWakeNotification) { [self] _ in perform { [self] in systemEvent("wake") } }
1742
+ // 快速切换用户:会话让出 = 人不在这台屏幕前,按锁屏处理。
1743
+ on(workspace, NSWorkspace.sessionDidResignActiveNotification) { [self] _ in perform { [self] in systemEvent("locked") } }
1744
+ on(workspace, NSWorkspace.sessionDidBecomeActiveNotification) { [self] _ in perform { [self] in systemEvent("unlocked") } }
1745
+ let distributed = DistributedNotificationCenter.default()
1746
+ on(distributed, Notification.Name("com.apple.screenIsLocked")) { [self] _ in perform { [self] in systemEvent("locked") } }
1747
+ on(distributed, Notification.Name("com.apple.screenIsUnlocked")) { [self] _ in perform { [self] in systemEvent("unlocked") } }
1748
+ mouseMonitor = NSEvent.addGlobalMonitorForEvents(matching: .leftMouseDown) { [self] event in
1749
+ guard let point = event.cgEvent?.location else { return }
1750
+ let agent = agentGate.active()
1751
+ perform { [self] in clicked(at: point, agent: agent) }
1752
+ }
1753
+ keyMonitor = NSEvent.addGlobalMonitorForEvents(matching: .keyDown) { [self] event in
1754
+ let flags = event.modifierFlags
1755
+ // 只看修饰键:没按 ⌘ / ⌃ 的普通按键到这里就结束,键值碰都不碰。
1756
+ guard flags.contains(.command) || flags.contains(.control) else { return }
1757
+ guard let key = recorderKeyName(keyCode: event.keyCode, characters: event.charactersIgnoringModifiers),
1758
+ let combo = recorderKeyCombo(command: flags.contains(.command), control: flags.contains(.control), option: flags.contains(.option), shift: flags.contains(.shift), key: key)
1759
+ else { return }
1760
+ let target = event.cgEvent.map { pid_t(truncatingIfNeeded: $0.getIntegerValueField(.eventTargetUnixProcessID)) } ?? 0
1761
+ let agent = agentGate.active()
1762
+ perform { [self] in keyed(combo, target: target, agent: agent) }
1763
+ }
1764
+ }
1765
+
1766
+ private func removeMainObservers() {
1767
+ for (center, token) in mainObservers { center.removeObserver(token) }
1768
+ mainObservers = []
1769
+ if let mouseMonitor { NSEvent.removeMonitor(mouseMonitor) }
1770
+ if let keyMonitor { NSEvent.removeMonitor(keyMonitor) }
1771
+ mouseMonitor = nil
1772
+ keyMonitor = nil
1773
+ }
1774
+
1775
+ // MARK: AX 小工具
1776
+
1777
+ private func axCopy(_ element: AXUIElement, _ attribute: String) -> AnyObject? {
1778
+ var value: AnyObject?
1779
+ guard AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success else { return nil }
1780
+ return value
1781
+ }
1782
+
1783
+ private func axElement(_ value: AnyObject?) -> AXUIElement? {
1784
+ guard let value, CFGetTypeID(value as CFTypeRef) == AXUIElementGetTypeID() else { return nil }
1785
+ return unsafeBitCast(value, to: AXUIElement.self)
1786
+ }
1787
+
1788
+ private func axChildren(_ element: AXUIElement) -> [AXUIElement] {
1789
+ AXUIElementSetMessagingTimeout(element, Self.axTimeout)
1790
+ return (axCopy(element, kAXChildrenAttribute) as? [AnyObject] ?? []).compactMap(axElement)
1791
+ }
1792
+
1793
+ /// 一次 IPC 取多个属性;缺的属性(AXValue 包着的错误)按 nil 返回。
1794
+ private func axMulti(_ element: AXUIElement, _ attributes: [String]) -> [AnyObject?] {
1795
+ var values: CFArray?
1796
+ let status = AXUIElementCopyMultipleAttributeValues(element, attributes as CFArray, AXCopyMultipleAttributeOptions(rawValue: 0), &values)
1797
+ guard status == .success, let array = values as? [AnyObject], array.count == attributes.count else {
1798
+ return attributes.map { _ in nil }
1799
+ }
1800
+ return array.map { value -> AnyObject? in
1801
+ let ref = value as CFTypeRef
1802
+ if CFGetTypeID(ref) == AXValueGetTypeID(), AXValueGetType(unsafeBitCast(ref, to: AXValue.self)) == .axError { return nil }
1803
+ return value
1804
+ }
1805
+ }
1806
+ }