@lattices/cli 0.3.0 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. package/README.md +85 -9
  2. package/app/Info.plist +30 -0
  3. package/app/Lattices.app/Contents/Info.plist +8 -2
  4. package/app/Lattices.app/Contents/MacOS/Lattices +0 -0
  5. package/app/Lattices.app/Contents/Resources/AppIcon.icns +0 -0
  6. package/app/Lattices.app/Contents/Resources/tap.wav +0 -0
  7. package/app/Lattices.app/Contents/_CodeSignature/CodeResources +139 -0
  8. package/app/Lattices.entitlements +15 -0
  9. package/app/Package.swift +8 -1
  10. package/app/Resources/tap.wav +0 -0
  11. package/app/Sources/AdvisorLearningStore.swift +90 -0
  12. package/app/Sources/AgentSession.swift +377 -0
  13. package/app/Sources/AppDelegate.swift +45 -12
  14. package/app/Sources/AppShellView.swift +81 -8
  15. package/app/Sources/AudioProvider.swift +386 -0
  16. package/app/Sources/CheatSheetHUD.swift +261 -19
  17. package/app/Sources/DaemonProtocol.swift +13 -0
  18. package/app/Sources/DaemonServer.swift +8 -0
  19. package/app/Sources/DesktopModel.swift +189 -6
  20. package/app/Sources/DesktopModelTypes.swift +2 -0
  21. package/app/Sources/DiagnosticLog.swift +104 -2
  22. package/app/Sources/EventBus.swift +1 -0
  23. package/app/Sources/HUDBottomBar.swift +279 -0
  24. package/app/Sources/HUDController.swift +1158 -0
  25. package/app/Sources/HUDLeftBar.swift +849 -0
  26. package/app/Sources/HUDMinimap.swift +179 -0
  27. package/app/Sources/HUDRightBar.swift +774 -0
  28. package/app/Sources/HUDState.swift +367 -0
  29. package/app/Sources/HUDTopBar.swift +243 -0
  30. package/app/Sources/HandsOffSession.swift +802 -0
  31. package/app/Sources/HomeDashboardView.swift +125 -0
  32. package/app/Sources/HotkeyManager.swift +2 -0
  33. package/app/Sources/HotkeyStore.swift +49 -9
  34. package/app/Sources/IntentEngine.swift +962 -0
  35. package/app/Sources/Intents/CreateLayerIntent.swift +54 -0
  36. package/app/Sources/Intents/DistributeIntent.swift +56 -0
  37. package/app/Sources/Intents/FocusIntent.swift +69 -0
  38. package/app/Sources/Intents/HelpIntent.swift +41 -0
  39. package/app/Sources/Intents/KillIntent.swift +47 -0
  40. package/app/Sources/Intents/LatticeIntent.swift +78 -0
  41. package/app/Sources/Intents/LaunchIntent.swift +67 -0
  42. package/app/Sources/Intents/ListSessionsIntent.swift +32 -0
  43. package/app/Sources/Intents/ListWindowsIntent.swift +30 -0
  44. package/app/Sources/Intents/ScanIntent.swift +52 -0
  45. package/app/Sources/Intents/SearchIntent.swift +190 -0
  46. package/app/Sources/Intents/SwitchLayerIntent.swift +50 -0
  47. package/app/Sources/Intents/TileIntent.swift +61 -0
  48. package/app/Sources/LatticesApi.swift +1275 -30
  49. package/app/Sources/LauncherHUD.swift +348 -0
  50. package/app/Sources/MainView.swift +147 -44
  51. package/app/Sources/MouseFinder.swift +222 -0
  52. package/app/Sources/OcrModel.swift +34 -1
  53. package/app/Sources/OmniSearchState.swift +99 -102
  54. package/app/Sources/OnboardingView.swift +457 -0
  55. package/app/Sources/PermissionChecker.swift +2 -12
  56. package/app/Sources/PiChatDock.swift +454 -0
  57. package/app/Sources/PiChatSession.swift +815 -0
  58. package/app/Sources/PiWorkspaceView.swift +364 -0
  59. package/app/Sources/PlacementSpec.swift +195 -0
  60. package/app/Sources/Preferences.swift +59 -0
  61. package/app/Sources/ProjectScanner.swift +58 -45
  62. package/app/Sources/ScreenMapState.swift +701 -55
  63. package/app/Sources/ScreenMapView.swift +843 -103
  64. package/app/Sources/ScreenMapWindowController.swift +22 -0
  65. package/app/Sources/SessionLayerStore.swift +285 -0
  66. package/app/Sources/SessionManager.swift +4 -1
  67. package/app/Sources/SettingsView.swift +186 -3
  68. package/app/Sources/Theme.swift +9 -8
  69. package/app/Sources/TmuxModel.swift +7 -0
  70. package/app/Sources/TmuxQuery.swift +27 -3
  71. package/app/Sources/VoiceChatView.swift +192 -0
  72. package/app/Sources/VoiceCommandWindow.swift +1594 -0
  73. package/app/Sources/VoiceIntentResolver.swift +671 -0
  74. package/app/Sources/VoxClient.swift +454 -0
  75. package/app/Sources/WindowTiler.swift +348 -87
  76. package/app/Sources/WorkspaceManager.swift +127 -18
  77. package/app/Tests/StageDragTests.swift +333 -0
  78. package/app/Tests/StageJoinTests.swift +313 -0
  79. package/app/Tests/StageManagerTests.swift +280 -0
  80. package/app/Tests/StageTileTests.swift +353 -0
  81. package/assets/AppIcon.icns +0 -0
  82. package/bin/client.ts +16 -0
  83. package/bin/{daemon-client.js → daemon-client.ts} +49 -30
  84. package/bin/handsoff-infer.ts +280 -0
  85. package/bin/handsoff-worker.ts +740 -0
  86. package/bin/lattices-app.ts +338 -0
  87. package/bin/lattices-dev +208 -0
  88. package/bin/{lattices.js → lattices.ts} +777 -140
  89. package/bin/project-twin.ts +645 -0
  90. package/docs/agent-execution-plan.md +562 -0
  91. package/docs/agent-layer-guide.md +207 -0
  92. package/docs/agents.md +142 -0
  93. package/docs/api.md +153 -34
  94. package/docs/app.md +29 -1
  95. package/docs/config.md +5 -1
  96. package/docs/handsoff-test-scenarios.md +84 -0
  97. package/docs/layers.md +20 -20
  98. package/docs/ocr.md +14 -5
  99. package/docs/overview.md +5 -1
  100. package/docs/presentation-execution-review.md +491 -0
  101. package/docs/prompts/hands-off-system.md +374 -0
  102. package/docs/prompts/hands-off-turn.md +30 -0
  103. package/docs/prompts/voice-advisor.md +31 -0
  104. package/docs/prompts/voice-fallback.md +23 -0
  105. package/docs/tiling-reference.md +167 -0
  106. package/docs/twins.md +138 -0
  107. package/docs/voice-command-protocol.md +278 -0
  108. package/docs/voice.md +219 -0
  109. package/package.json +29 -11
  110. package/bin/client.js +0 -4
  111. package/bin/lattices-app.js +0 -221
@@ -29,7 +29,7 @@ final class CheatSheetHUD {
29
29
  let hosting = NSHostingView(rootView: view)
30
30
 
31
31
  let p = NSPanel(
32
- contentRect: NSRect(x: 0, y: 0, width: 520, height: 420),
32
+ contentRect: NSRect(x: 0, y: 0, width: 520, height: 480),
33
33
  styleMask: [.borderless, .nonactivatingPanel],
34
34
  backing: .buffered,
35
35
  defer: false
@@ -48,7 +48,7 @@ final class CheatSheetHUD {
48
48
  let screen = NSScreen.screens.first(where: { $0.frame.contains(mouseLocation) }) ?? NSScreen.main ?? NSScreen.screens.first!
49
49
  let screenFrame = screen.visibleFrame
50
50
  let x = screenFrame.midX - 260
51
- let y = screenFrame.midY - 210
51
+ let y = screenFrame.midY - 240
52
52
  p.setFrameOrigin(NSPoint(x: x, y: y))
53
53
 
54
54
  p.alphaValue = 0
@@ -66,6 +66,7 @@ final class CheatSheetHUD {
66
66
  func dismiss() {
67
67
  guard let p = panel else { return }
68
68
  removeMonitors()
69
+ TileZoneOverlay.shared.dismiss()
69
70
 
70
71
  NSAnimationContext.runAnimationGroup({ ctx in
71
72
  ctx.duration = 0.2
@@ -79,10 +80,17 @@ final class CheatSheetHUD {
79
80
  // MARK: - Event monitors
80
81
 
81
82
  private func installMonitors() {
82
- // Escape key dismisses (global — panel is non-activating so keys go to frontmost app)
83
+ // Key handling (global — panel is non-activating so keys go to frontmost app)
83
84
  localMonitor = NSEvent.addGlobalMonitorForEvents(matching: .keyDown) { [weak self] event in
84
85
  if event.keyCode == 53 { // Escape
85
86
  self?.dismiss()
87
+ } else if event.keyCode == 49 { // Space — toggle voice command
88
+ let audio = AudioLayer.shared
89
+ if audio.isListening {
90
+ audio.stopVoiceCommand()
91
+ } else {
92
+ audio.startVoiceCommand()
93
+ }
86
94
  }
87
95
  }
88
96
 
@@ -103,6 +111,8 @@ final class CheatSheetHUD {
103
111
 
104
112
  struct CheatSheetView: View {
105
113
  @ObservedObject private var hotkeyStore = HotkeyStore.shared
114
+ @ObservedObject private var audioLayer = AudioLayer.shared
115
+ @State private var hoveredAction: HotkeyAction?
106
116
 
107
117
  var body: some View {
108
118
  VStack(spacing: 0) {
@@ -140,19 +150,37 @@ struct CheatSheetView: View {
140
150
 
141
151
  Spacer(minLength: 0)
142
152
 
153
+ // Voice feedback strip
154
+ if audioLayer.isListening || audioLayer.lastTranscript != nil || audioLayer.executionResult != nil {
155
+ Rectangle().fill(Palette.border).frame(height: 0.5)
156
+ voiceFeedback
157
+ }
158
+
143
159
  Rectangle().fill(Palette.border).frame(height: 0.5)
144
160
 
145
161
  // Footer
146
- HStack {
162
+ HStack(spacing: 20) {
147
163
  Spacer()
148
- Text("Press ESC to dismiss")
149
- .font(Typo.caption(10))
150
- .foregroundColor(Palette.textMuted)
164
+ HStack(spacing: 6) {
165
+ keyBadge("Space")
166
+ Image(systemName: audioLayer.isListening ? "mic.fill" : "mic")
167
+ .font(.system(size: 11))
168
+ .foregroundColor(audioLayer.isListening ? Palette.running : Palette.text)
169
+ Text(audioLayer.isListening ? "Listening..." : "Voice")
170
+ .font(Typo.geistMono(11))
171
+ .foregroundColor(audioLayer.isListening ? Palette.running : Palette.text)
172
+ }
173
+ HStack(spacing: 6) {
174
+ keyBadge("ESC")
175
+ Text("Dismiss")
176
+ .font(Typo.geistMono(11))
177
+ .foregroundColor(Palette.textMuted)
178
+ }
151
179
  Spacer()
152
180
  }
153
- .padding(.vertical, 8)
181
+ .padding(.vertical, 10)
154
182
  }
155
- .frame(width: 520, height: 420)
183
+ .frame(width: 520, height: 480)
156
184
  .background(
157
185
  RoundedRectangle(cornerRadius: 12)
158
186
  .fill(Palette.bg)
@@ -170,6 +198,15 @@ struct CheatSheetView: View {
170
198
  VStack(alignment: .leading, spacing: 10) {
171
199
  columnHeader("Tiling")
172
200
 
201
+ // Modifier prefix
202
+ HStack(spacing: 3) {
203
+ keyBadge("Ctrl")
204
+ keyBadge("Option")
205
+ Text("+")
206
+ .font(Typo.caption(11))
207
+ .foregroundColor(Palette.textMuted)
208
+ }
209
+
173
210
  // 3x3 grid
174
211
  VStack(spacing: 2) {
175
212
  HStack(spacing: 2) {
@@ -217,8 +254,107 @@ struct CheatSheetView: View {
217
254
  // Center + Distribute
218
255
  shortcutRow(action: .tileCenter)
219
256
  shortcutRow(action: .tileDistribute)
257
+
258
+ // Hovered shortcut detail
259
+ if let hovered = hoveredAction, let binding = hotkeyStore.bindings[hovered] {
260
+ HStack(spacing: 3) {
261
+ ForEach(binding.displayParts, id: \.self) { part in
262
+ keyBadge(part)
263
+ }
264
+ Text("→ \(hovered.label)")
265
+ .font(Typo.caption(11))
266
+ .foregroundColor(Palette.textDim)
267
+ }
268
+ .transition(.opacity)
269
+ .animation(.easeInOut(duration: 0.1), value: hoveredAction)
270
+ }
220
271
  }
221
272
  .frame(maxWidth: .infinity, alignment: .leading)
273
+ .onHover { over in
274
+ if !over {
275
+ hoveredAction = nil
276
+ TileZoneOverlay.shared.dismiss()
277
+ }
278
+ }
279
+ .onDisappear {
280
+ hoveredAction = nil
281
+ TileZoneOverlay.shared.dismiss()
282
+ }
283
+ }
284
+
285
+ // MARK: - Voice Feedback
286
+
287
+ private var voiceFeedback: some View {
288
+ VStack(alignment: .leading, spacing: 6) {
289
+ if audioLayer.isListening {
290
+ HStack(spacing: 8) {
291
+ Circle()
292
+ .fill(Palette.running)
293
+ .frame(width: 8, height: 8)
294
+ Text("Listening...")
295
+ .font(Typo.geistMono(12))
296
+ .foregroundColor(Palette.running)
297
+ Spacer()
298
+ Text("Press Space to stop")
299
+ .font(Typo.caption(10))
300
+ .foregroundColor(Palette.textMuted)
301
+ }
302
+ } else if let transcript = audioLayer.lastTranscript {
303
+ // Show what was heard
304
+ HStack(spacing: 6) {
305
+ Image(systemName: "quote.opening")
306
+ .font(.system(size: 10))
307
+ .foregroundColor(Palette.textMuted)
308
+ Text(transcript)
309
+ .font(Typo.geistMono(12))
310
+ .foregroundColor(Palette.text)
311
+ .lineLimit(1)
312
+ }
313
+
314
+ // Show matched intent + result
315
+ if let intent = audioLayer.matchedIntent {
316
+ HStack(spacing: 6) {
317
+ Image(systemName: "arrow.right")
318
+ .font(.system(size: 9))
319
+ .foregroundColor(Palette.textMuted)
320
+ Text(intent.replacingOccurrences(of: "_", with: " "))
321
+ .font(Typo.geistMonoBold(11))
322
+ .foregroundColor(Palette.text)
323
+
324
+ if !audioLayer.matchedSlots.isEmpty {
325
+ let slotText = audioLayer.matchedSlots
326
+ .map { "\($0.key): \($0.value)" }
327
+ .joined(separator: ", ")
328
+ Text(slotText)
329
+ .font(Typo.caption(10))
330
+ .foregroundColor(Palette.textDim)
331
+ }
332
+
333
+ Spacer()
334
+
335
+ if let result = audioLayer.executionResult {
336
+ Text(result == "ok" ? "Done" : result)
337
+ .font(Typo.caption(10))
338
+ .foregroundColor(result == "ok" ? Palette.running : Palette.kill)
339
+ }
340
+ }
341
+ } else if let result = audioLayer.executionResult {
342
+ HStack(spacing: 6) {
343
+ Image(systemName: "questionmark.circle")
344
+ .font(.system(size: 10))
345
+ .foregroundColor(Palette.textMuted)
346
+ Text(result)
347
+ .font(Typo.caption(10))
348
+ .foregroundColor(Palette.textMuted)
349
+ }
350
+ }
351
+ }
352
+ }
353
+ .padding(.horizontal, 20)
354
+ .padding(.vertical, 10)
355
+ .background(Color.black.opacity(0.15))
356
+ .animation(.easeInOut(duration: 0.15), value: audioLayer.isListening)
357
+ .animation(.easeInOut(duration: 0.15), value: audioLayer.lastTranscript)
222
358
  }
223
359
 
224
360
  // MARK: - App Column
@@ -228,10 +364,11 @@ struct CheatSheetView: View {
228
364
  columnHeader("App")
229
365
 
230
366
  shortcutRow(action: .palette)
231
- shortcutRow(action: .screenMap)
367
+ shortcutRow(action: .unifiedWindow)
232
368
  shortcutRow(action: .bezel)
369
+ shortcutRow(action: .hud)
370
+ shortcutRow(action: .voiceCommand)
233
371
  shortcutRow(action: .cheatSheet)
234
- shortcutRow(action: .desktopInventory)
235
372
  }
236
373
  }
237
374
 
@@ -262,25 +399,34 @@ struct CheatSheetView: View {
262
399
  private func tileCell(action: HotkeyAction, label: String) -> some View {
263
400
  let binding = hotkeyStore.bindings[action]
264
401
  let badgeText = binding?.displayParts.last ?? ""
402
+ let isHovered = hoveredAction == action
265
403
 
266
- return VStack(spacing: 3) {
267
- Text(label)
268
- .font(Typo.caption(9))
269
- .foregroundColor(Palette.textDim)
404
+ return VStack(spacing: 2) {
270
405
  Text(badgeText)
271
- .font(Typo.geistMonoBold(9))
272
- .foregroundColor(Palette.text)
406
+ .font(Typo.geistMonoBold(12))
407
+ .foregroundColor(isHovered ? Color.blue : Palette.text)
408
+ Text(label)
409
+ .font(Typo.caption(8))
410
+ .foregroundColor(isHovered ? Palette.text : Palette.textMuted)
273
411
  }
274
412
  .frame(maxWidth: .infinity)
275
413
  .frame(height: 38)
276
414
  .background(
277
415
  RoundedRectangle(cornerRadius: 4)
278
- .fill(Palette.surface)
416
+ .fill(isHovered ? Color.blue.opacity(0.15) : Palette.surface)
279
417
  .overlay(
280
418
  RoundedRectangle(cornerRadius: 4)
281
- .strokeBorder(Palette.border, lineWidth: 0.5)
419
+ .strokeBorder(isHovered ? Color.blue.opacity(0.5) : Palette.border, lineWidth: 0.5)
282
420
  )
283
421
  )
422
+ .onHover { over in
423
+ if over {
424
+ hoveredAction = action
425
+ if let pos = action.tilePosition {
426
+ TileZoneOverlay.shared.show(position: pos)
427
+ }
428
+ }
429
+ }
284
430
  }
285
431
 
286
432
  private func shortcutRow(action: HotkeyAction) -> some View {
@@ -330,3 +476,99 @@ struct CheatSheetView: View {
330
476
  )
331
477
  }
332
478
  }
479
+
480
+ // MARK: - HotkeyAction → TilePosition mapping
481
+
482
+ extension HotkeyAction {
483
+ var tilePosition: TilePosition? {
484
+ switch self {
485
+ case .tileLeft: return .left
486
+ case .tileRight: return .right
487
+ case .tileTop: return .top
488
+ case .tileBottom: return .bottom
489
+ case .tileTopLeft: return .topLeft
490
+ case .tileTopRight: return .topRight
491
+ case .tileBottomLeft: return .bottomLeft
492
+ case .tileBottomRight: return .bottomRight
493
+ case .tileMaximize: return .maximize
494
+ case .tileCenter: return .center
495
+ case .tileLeftThird: return .leftThird
496
+ case .tileCenterThird: return .centerThird
497
+ case .tileRightThird: return .rightThird
498
+ default: return nil
499
+ }
500
+ }
501
+ }
502
+
503
+ // MARK: - TileZoneOverlay
504
+
505
+ final class TileZoneOverlay {
506
+ static let shared = TileZoneOverlay()
507
+
508
+ private var panel: NSPanel?
509
+
510
+ func show(position: TilePosition) {
511
+ // Instant teardown (no animation) when switching between cells
512
+ if let p = panel {
513
+ p.orderOut(nil)
514
+ self.panel = nil
515
+ }
516
+
517
+ // Use the screen where the mouse is (same as CheatSheetHUD)
518
+ let mouseLocation = NSEvent.mouseLocation
519
+ let screen = NSScreen.screens.first(where: { $0.frame.contains(mouseLocation) }) ?? NSScreen.main ?? NSScreen.screens.first!
520
+ let visible = screen.visibleFrame
521
+
522
+ let (fx, fy, fw, fh) = position.rect
523
+ // visibleFrame origin is bottom-left in AppKit coordinates
524
+ let zoneRect = NSRect(
525
+ x: visible.origin.x + visible.width * fx,
526
+ y: visible.origin.y + visible.height * (1 - fy - fh),
527
+ width: visible.width * fw,
528
+ height: visible.height * fh
529
+ )
530
+
531
+ let p = NSPanel(
532
+ contentRect: zoneRect,
533
+ styleMask: [.borderless, .nonactivatingPanel],
534
+ backing: .buffered,
535
+ defer: false
536
+ )
537
+ p.isOpaque = false
538
+ p.backgroundColor = .clear
539
+ p.level = .floating
540
+ p.hasShadow = false
541
+ p.hidesOnDeactivate = false
542
+ p.isReleasedWhenClosed = false
543
+ p.ignoresMouseEvents = true
544
+
545
+ let overlay = NSView(frame: NSRect(origin: .zero, size: zoneRect.size))
546
+ overlay.wantsLayer = true
547
+ overlay.layer?.backgroundColor = NSColor.systemBlue.withAlphaComponent(0.08).cgColor
548
+ overlay.layer?.borderColor = NSColor.systemBlue.withAlphaComponent(0.4).cgColor
549
+ overlay.layer?.borderWidth = 2
550
+ overlay.layer?.cornerRadius = 8
551
+ p.contentView = overlay
552
+
553
+ p.alphaValue = 0
554
+ p.orderFrontRegardless()
555
+
556
+ NSAnimationContext.runAnimationGroup { ctx in
557
+ ctx.duration = 0.12
558
+ p.animator().alphaValue = 1.0
559
+ }
560
+
561
+ self.panel = p
562
+ }
563
+
564
+ func dismiss() {
565
+ guard let p = panel else { return }
566
+ NSAnimationContext.runAnimationGroup({ ctx in
567
+ ctx.duration = 0.1
568
+ p.animator().alphaValue = 0
569
+ }) { [weak self] in
570
+ p.orderOut(nil)
571
+ self?.panel = nil
572
+ }
573
+ }
574
+ }
@@ -98,4 +98,17 @@ enum JSON: Codable, Equatable {
98
98
  guard case .bool(let b) = self else { return nil }
99
99
  return b
100
100
  }
101
+
102
+ var arrayValue: [JSON]? {
103
+ guard case .array(let a) = self else { return nil }
104
+ return a
105
+ }
106
+
107
+ var numericDouble: Double? {
108
+ switch self {
109
+ case .double(let d): return d
110
+ case .int(let i): return Double(i)
111
+ default: return nil
112
+ }
113
+ }
101
114
  }
@@ -394,6 +394,14 @@ final class DaemonServer: ObservableObject {
394
394
  "totalBlocks": .int(totalBlocks)
395
395
  ])
396
396
  )
397
+ case .voiceCommand(let text, let confidence):
398
+ daemonEvent = DaemonEvent(
399
+ event: "voice.command",
400
+ data: .object([
401
+ "text": .string(text),
402
+ "confidence": .double(confidence)
403
+ ])
404
+ )
397
405
  }
398
406
  broadcast(daemonEvent)
399
407
  }
@@ -1,13 +1,67 @@
1
1
  import AppKit
2
+ import ApplicationServices
2
3
  import CoreGraphics
3
4
 
4
5
  final class DesktopModel: ObservableObject {
5
6
  static let shared = DesktopModel()
6
7
 
8
+ /// System helper processes that should never appear in search results or window lists.
9
+ /// These are XPC services, agents, and background helpers — not user-facing apps.
10
+ private static let systemHelperProcesses: Set<String> = [
11
+ // Apple system helpers
12
+ "CredentialsProviderExtensionHost",
13
+ "AuthenticationServicesAgent",
14
+ "SafariPasswordExtension",
15
+ "com.apple.WebKit.WebAuthn",
16
+ "SharedWebCredentialRunner",
17
+ "ViewBridgeAuxiliary",
18
+ "universalaccessd",
19
+ "CoreServicesUIAgent",
20
+ "UserNotificationCenter",
21
+ "AutoFillPanelService",
22
+ "AutoFill",
23
+ "CoreLocationAgent",
24
+ "SecurityAgent",
25
+ "coreautha",
26
+ "coreauth",
27
+ "talagent",
28
+ "CommCenter",
29
+ "AXVisualSupportAgent",
30
+ "SystemUIServer",
31
+ "Dock",
32
+ "Window Server",
33
+ "WindowManager",
34
+ "NotificationCenter",
35
+ "ControlCenter",
36
+ "Spotlight",
37
+ "Keychain Access",
38
+ "loginwindow",
39
+ "ScreenSaverEngine",
40
+ "SoftwareUpdateNotificationManager",
41
+ "WiFiAgent",
42
+ "pboard",
43
+ "storeuid",
44
+ // Third-party helpers
45
+ "CursorUIViewService",
46
+ "Electron Helper",
47
+ "Google Chrome Helper",
48
+ ]
49
+
50
+ /// Suffixes that indicate a helper/service process, not a user-facing app
51
+ private static let helperSuffixes = ["Service", "Agent", "Helper", "Extension", "Daemon", "XPCService"]
52
+
53
+ /// Real apps that happen to match helper suffixes — don't filter these
54
+ private static let knownRealApps: Set<String> = [
55
+ "Finder",
56
+ "Activity Monitor",
57
+ ]
58
+
7
59
  @Published private(set) var windows: [UInt32: WindowEntry] = [:]
8
- /// In-memory layer tags: wid layer id (e.g. "lattices", "talkie", "hudson")
60
+ @Published private(set) var interactionDates: [UInt32: Date] = [:]
61
+ /// In-memory layer tags: wid → layer id (e.g. "lattices", "vox", "hudson")
9
62
  private(set) var windowLayerTags: [UInt32: String] = [:]
10
63
  private var timer: Timer?
64
+ private var lastFrontmostWid: UInt32?
11
65
 
12
66
  func start(interval: TimeInterval = 1.5) {
13
67
  guard timer == nil else { return }
@@ -24,7 +78,27 @@ final class DesktopModel: ObservableObject {
24
78
  }
25
79
 
26
80
  func allWindows() -> [WindowEntry] {
27
- Array(windows.values).sorted { $0.wid < $1.wid }
81
+ Array(windows.values).sorted { $0.zIndex < $1.zIndex }
82
+ }
83
+
84
+ func lastInteractionDate(for wid: UInt32) -> Date? {
85
+ interactionDates[wid]
86
+ }
87
+
88
+ func markInteraction(wid: UInt32, at date: Date = Date()) {
89
+ DispatchQueue.main.async {
90
+ self.interactionDates[wid] = date
91
+ }
92
+ }
93
+
94
+ func markInteraction(wids: [UInt32], at date: Date = Date()) {
95
+ guard !wids.isEmpty else { return }
96
+ let unique = Set(wids)
97
+ DispatchQueue.main.async {
98
+ for wid in unique {
99
+ self.interactionDates[wid] = date
100
+ }
101
+ }
28
102
  }
29
103
 
30
104
  func windowForSession(_ session: String) -> WindowEntry? {
@@ -60,13 +134,31 @@ final class DesktopModel: ObservableObject {
60
134
 
61
135
  // MARK: - Polling
62
136
 
137
+ private var lastPollTime: Date = .distantPast
138
+ private static let minPollInterval: TimeInterval = 1.0
139
+
140
+ /// Poll only if stale. Call `forcePoll()` to bypass the freshness check.
63
141
  func poll() {
142
+ let now = Date()
143
+ guard now.timeIntervalSince(lastPollTime) >= Self.minPollInterval else { return }
144
+ lastPollTime = now
145
+ performPoll()
146
+ }
147
+
148
+ /// Force a poll regardless of freshness — use sparingly.
149
+ func forcePoll() {
150
+ lastPollTime = Date()
151
+ performPoll()
152
+ }
153
+
154
+ private func performPoll() {
64
155
  guard let list = CGWindowListCopyWindowInfo(
65
- [.optionOnScreenOnly, .excludeDesktopElements],
156
+ [.optionAll, .excludeDesktopElements],
66
157
  kCGNullWindowID
67
158
  ) as? [[String: Any]] else { return }
68
159
 
69
160
  var fresh: [UInt32: WindowEntry] = [:]
161
+ var zCounter = 0
70
162
 
71
163
  for info in list {
72
164
  guard let wid = info[kCGWindowNumber as String] as? UInt32,
@@ -82,11 +174,24 @@ final class DesktopModel: ObservableObject {
82
174
 
83
175
  let title = info[kCGWindowName as String] as? String ?? ""
84
176
  let layer = info[kCGWindowLayer as String] as? Int ?? 0
85
- let isOnScreen = info[kCGWindowIsOnscreen as String] as? Bool ?? true
177
+ let isOnScreen = info[kCGWindowIsOnscreen as String] as? Bool ?? false
86
178
 
87
179
  // Skip non-standard layers (menus, overlays)
88
180
  guard layer == 0 else { continue }
89
181
 
182
+ // Skip system helper processes (autofill, credential providers, etc.)
183
+ if Self.systemHelperProcesses.contains(ownerName) { continue }
184
+
185
+ // Skip processes whose name ends with common helper suffixes
186
+ // (e.g. "CursorUIViewService", "AutoFillPanelService", "SecurityAgent")
187
+ // but not known real apps that happen to have these words
188
+ let isHelperByName = Self.helperSuffixes.contains(where: { ownerName.hasSuffix($0) })
189
+ && !Self.knownRealApps.contains(ownerName)
190
+ if isHelperByName { continue }
191
+
192
+ // Skip windows with no title from processes containing "com.apple."
193
+ if ownerName.hasPrefix("com.apple.") && title.isEmpty { continue }
194
+
90
195
  let frame = WindowFrame(
91
196
  x: Double(rect.origin.x),
92
197
  y: Double(rect.origin.y),
@@ -103,7 +208,7 @@ final class DesktopModel: ObservableObject {
103
208
  latticesSession = String(match.dropFirst(9).dropLast(1)) // drop "[lattices:" and "]"
104
209
  }
105
210
 
106
- fresh[wid] = WindowEntry(
211
+ var entry = WindowEntry(
107
212
  wid: wid,
108
213
  app: ownerName,
109
214
  pid: pid,
@@ -113,8 +218,14 @@ final class DesktopModel: ObservableObject {
113
218
  isOnScreen: isOnScreen,
114
219
  latticesSession: latticesSession
115
220
  )
221
+ entry.zIndex = zCounter
222
+ zCounter += 1
223
+ fresh[wid] = entry
116
224
  }
117
225
 
226
+ // AX reconciliation: check which CG windows actually exist in Accessibility
227
+ reconcileWithAX(&fresh)
228
+
118
229
  // Diff
119
230
  let oldKeys = Set(windows.keys)
120
231
  let newKeys = Set(fresh.keys)
@@ -122,9 +233,21 @@ final class DesktopModel: ObservableObject {
122
233
  let removed = Array(oldKeys.subtracting(newKeys))
123
234
 
124
235
  let changed = added.count > 0 || removed.count > 0 || windowsContentChanged(old: windows, new: fresh)
236
+ let frontmostWid = fresh.values.min(by: { $0.zIndex < $1.zIndex })?.wid
237
+ let markFrontmost = frontmostWid != nil && frontmostWid != lastFrontmostWid
238
+ let interactionTime = Date()
125
239
 
126
240
  DispatchQueue.main.async {
127
- self.windows = fresh
241
+ var interactions = self.interactionDates.filter { fresh[$0.key] != nil }
242
+ if markFrontmost, let frontmostWid {
243
+ interactions[frontmostWid] = interactionTime
244
+ }
245
+ // Only publish if something actually changed — avoids unnecessary SwiftUI re-renders
246
+ if changed || markFrontmost {
247
+ self.windows = fresh
248
+ self.interactionDates = interactions
249
+ }
250
+ self.lastFrontmostWid = frontmostWid
128
251
  }
129
252
 
130
253
  if changed {
@@ -136,6 +259,66 @@ final class DesktopModel: ObservableObject {
136
259
  }
137
260
  }
138
261
 
262
+ private func reconcileWithAX(_ fresh: inout [UInt32: WindowEntry]) {
263
+ // Get currently active Space IDs — AX only returns windows on these
264
+ let currentSpaceIds = Set(WindowTiler.getDisplaySpaces().map(\.currentSpaceId))
265
+ guard !currentSpaceIds.isEmpty else { return }
266
+
267
+ // Group CG windows by PID — only titled windows on current Spaces
268
+ var byPid: [Int32: [UInt32]] = [:]
269
+ for (wid, entry) in fresh where !entry.title.isEmpty {
270
+ let onCurrentSpace = entry.spaceIds.contains { currentSpaceIds.contains($0) }
271
+ if onCurrentSpace {
272
+ byPid[entry.pid, default: []].append(wid)
273
+ }
274
+ }
275
+
276
+ for (pid, wids) in byPid {
277
+ let axApp = AXUIElementCreateApplication(pid)
278
+
279
+ // Set a timeout so unresponsive apps (video calls, etc.) don't block the poll
280
+ AXUIElementSetMessagingTimeout(axApp, 0.3)
281
+
282
+ var axWindowsRef: CFTypeRef?
283
+ guard AXUIElementCopyAttributeValue(axApp, kAXWindowsAttribute as CFString, &axWindowsRef) == .success,
284
+ let axWindows = axWindowsRef as? [AXUIElement] else { continue }
285
+
286
+ // Collect AX window titles
287
+ var axTitles: [String] = []
288
+ for axWin in axWindows {
289
+ var titleRef: CFTypeRef?
290
+ AXUIElementCopyAttributeValue(axWin, kAXTitleAttribute as CFString, &titleRef)
291
+ if let title = titleRef as? String, !title.isEmpty {
292
+ axTitles.append(title)
293
+ }
294
+ }
295
+
296
+ // Mark CG windows that have no matching AX title.
297
+ // AX titles often have suffixes like " - Google Chrome - Profile"
298
+ // so check if any AX title starts with the CG title (stripped of emoji).
299
+ for wid in wids {
300
+ guard let entry = fresh[wid], !entry.title.isEmpty else { continue }
301
+ let cgClean = stripForMatch(entry.title)
302
+ let matched = axTitles.contains { axTitle in
303
+ let axClean = stripForMatch(axTitle)
304
+ return axClean.hasPrefix(cgClean) || axClean.contains(cgClean) || cgClean.hasPrefix(axClean)
305
+ }
306
+ if !matched {
307
+ fresh[wid]?.axVerified = false
308
+ }
309
+ }
310
+ }
311
+ }
312
+
313
+ private func stripForMatch(_ text: String) -> String {
314
+ // Remove emoji and non-ASCII symbols, lowercase, collapse whitespace
315
+ let scalar = text.unicodeScalars.filter { scalar in
316
+ scalar.isASCII || CharacterSet.letters.contains(scalar)
317
+ }
318
+ return String(scalar).lowercased()
319
+ .split(separator: " ").joined(separator: " ")
320
+ }
321
+
139
322
  private func windowsContentChanged(old: [UInt32: WindowEntry], new: [UInt32: WindowEntry]) -> Bool {
140
323
  // Quick check: if titles or frames changed for any existing window
141
324
  for (wid, newEntry) in new {
@@ -9,6 +9,8 @@ struct WindowEntry: Codable, Identifiable {
9
9
  let spaceIds: [Int]
10
10
  let isOnScreen: Bool
11
11
  let latticesSession: String?
12
+ var axVerified: Bool = true
13
+ var zIndex: Int = 0 // 0 = frontmost, from CGWindowList order
12
14
 
13
15
  var id: UInt32 { wid }
14
16
  }