@lattices/cli 0.3.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +85 -9
- package/app/Info.plist +30 -0
- package/app/Lattices.app/Contents/Info.plist +8 -2
- package/app/Lattices.app/Contents/MacOS/Lattices +0 -0
- package/app/Lattices.app/Contents/Resources/AppIcon.icns +0 -0
- package/app/Lattices.app/Contents/Resources/tap.wav +0 -0
- package/app/Lattices.app/Contents/_CodeSignature/CodeResources +139 -0
- package/app/Lattices.entitlements +15 -0
- package/app/Package.swift +8 -1
- package/app/Resources/tap.wav +0 -0
- package/app/Sources/AdvisorLearningStore.swift +90 -0
- package/app/Sources/AgentSession.swift +377 -0
- package/app/Sources/AppDelegate.swift +45 -12
- package/app/Sources/AppShellView.swift +81 -8
- package/app/Sources/AudioProvider.swift +386 -0
- package/app/Sources/CheatSheetHUD.swift +261 -19
- package/app/Sources/DaemonProtocol.swift +13 -0
- package/app/Sources/DaemonServer.swift +8 -0
- package/app/Sources/DesktopModel.swift +189 -6
- package/app/Sources/DesktopModelTypes.swift +2 -0
- package/app/Sources/DiagnosticLog.swift +104 -2
- package/app/Sources/EventBus.swift +1 -0
- package/app/Sources/HUDBottomBar.swift +279 -0
- package/app/Sources/HUDController.swift +1158 -0
- package/app/Sources/HUDLeftBar.swift +849 -0
- package/app/Sources/HUDMinimap.swift +179 -0
- package/app/Sources/HUDRightBar.swift +774 -0
- package/app/Sources/HUDState.swift +367 -0
- package/app/Sources/HUDTopBar.swift +243 -0
- package/app/Sources/HandsOffSession.swift +802 -0
- package/app/Sources/HomeDashboardView.swift +125 -0
- package/app/Sources/HotkeyManager.swift +2 -0
- package/app/Sources/HotkeyStore.swift +49 -9
- package/app/Sources/IntentEngine.swift +962 -0
- package/app/Sources/Intents/CreateLayerIntent.swift +54 -0
- package/app/Sources/Intents/DistributeIntent.swift +56 -0
- package/app/Sources/Intents/FocusIntent.swift +69 -0
- package/app/Sources/Intents/HelpIntent.swift +41 -0
- package/app/Sources/Intents/KillIntent.swift +47 -0
- package/app/Sources/Intents/LatticeIntent.swift +78 -0
- package/app/Sources/Intents/LaunchIntent.swift +67 -0
- package/app/Sources/Intents/ListSessionsIntent.swift +32 -0
- package/app/Sources/Intents/ListWindowsIntent.swift +30 -0
- package/app/Sources/Intents/ScanIntent.swift +52 -0
- package/app/Sources/Intents/SearchIntent.swift +190 -0
- package/app/Sources/Intents/SwitchLayerIntent.swift +50 -0
- package/app/Sources/Intents/TileIntent.swift +61 -0
- package/app/Sources/LatticesApi.swift +1275 -30
- package/app/Sources/LauncherHUD.swift +348 -0
- package/app/Sources/MainView.swift +147 -44
- package/app/Sources/MouseFinder.swift +222 -0
- package/app/Sources/OcrModel.swift +34 -1
- package/app/Sources/OmniSearchState.swift +99 -102
- package/app/Sources/OnboardingView.swift +457 -0
- package/app/Sources/PermissionChecker.swift +2 -12
- package/app/Sources/PiChatDock.swift +454 -0
- package/app/Sources/PiChatSession.swift +815 -0
- package/app/Sources/PiWorkspaceView.swift +364 -0
- package/app/Sources/PlacementSpec.swift +195 -0
- package/app/Sources/Preferences.swift +59 -0
- package/app/Sources/ProjectScanner.swift +58 -45
- package/app/Sources/ScreenMapState.swift +701 -55
- package/app/Sources/ScreenMapView.swift +843 -103
- package/app/Sources/ScreenMapWindowController.swift +22 -0
- package/app/Sources/SessionLayerStore.swift +285 -0
- package/app/Sources/SessionManager.swift +4 -1
- package/app/Sources/SettingsView.swift +186 -3
- package/app/Sources/Theme.swift +9 -8
- package/app/Sources/TmuxModel.swift +7 -0
- package/app/Sources/TmuxQuery.swift +27 -3
- package/app/Sources/VoiceChatView.swift +192 -0
- package/app/Sources/VoiceCommandWindow.swift +1594 -0
- package/app/Sources/VoiceIntentResolver.swift +671 -0
- package/app/Sources/VoxClient.swift +454 -0
- package/app/Sources/WindowTiler.swift +348 -87
- package/app/Sources/WorkspaceManager.swift +127 -18
- package/app/Tests/StageDragTests.swift +333 -0
- package/app/Tests/StageJoinTests.swift +313 -0
- package/app/Tests/StageManagerTests.swift +280 -0
- package/app/Tests/StageTileTests.swift +353 -0
- package/assets/AppIcon.icns +0 -0
- package/bin/client.ts +16 -0
- package/bin/{daemon-client.js → daemon-client.ts} +49 -30
- package/bin/handsoff-infer.ts +280 -0
- package/bin/handsoff-worker.ts +740 -0
- package/bin/lattices-app.ts +338 -0
- package/bin/lattices-dev +208 -0
- package/bin/{lattices.js → lattices.ts} +777 -140
- package/bin/project-twin.ts +645 -0
- package/docs/agent-execution-plan.md +562 -0
- package/docs/agent-layer-guide.md +207 -0
- package/docs/agents.md +142 -0
- package/docs/api.md +153 -34
- package/docs/app.md +29 -1
- package/docs/config.md +5 -1
- package/docs/handsoff-test-scenarios.md +84 -0
- package/docs/layers.md +20 -20
- package/docs/ocr.md +14 -5
- package/docs/overview.md +5 -1
- package/docs/presentation-execution-review.md +491 -0
- package/docs/prompts/hands-off-system.md +374 -0
- package/docs/prompts/hands-off-turn.md +30 -0
- package/docs/prompts/voice-advisor.md +31 -0
- package/docs/prompts/voice-fallback.md +23 -0
- package/docs/tiling-reference.md +167 -0
- package/docs/twins.md +138 -0
- package/docs/voice-command-protocol.md +278 -0
- package/docs/voice.md +219 -0
- package/package.json +29 -11
- package/bin/client.js +0 -4
- package/bin/lattices-app.js +0 -221
|
@@ -29,7 +29,7 @@ final class CheatSheetHUD {
|
|
|
29
29
|
let hosting = NSHostingView(rootView: view)
|
|
30
30
|
|
|
31
31
|
let p = NSPanel(
|
|
32
|
-
contentRect: NSRect(x: 0, y: 0, width: 520, height:
|
|
32
|
+
contentRect: NSRect(x: 0, y: 0, width: 520, height: 480),
|
|
33
33
|
styleMask: [.borderless, .nonactivatingPanel],
|
|
34
34
|
backing: .buffered,
|
|
35
35
|
defer: false
|
|
@@ -48,7 +48,7 @@ final class CheatSheetHUD {
|
|
|
48
48
|
let screen = NSScreen.screens.first(where: { $0.frame.contains(mouseLocation) }) ?? NSScreen.main ?? NSScreen.screens.first!
|
|
49
49
|
let screenFrame = screen.visibleFrame
|
|
50
50
|
let x = screenFrame.midX - 260
|
|
51
|
-
let y = screenFrame.midY -
|
|
51
|
+
let y = screenFrame.midY - 240
|
|
52
52
|
p.setFrameOrigin(NSPoint(x: x, y: y))
|
|
53
53
|
|
|
54
54
|
p.alphaValue = 0
|
|
@@ -66,6 +66,7 @@ final class CheatSheetHUD {
|
|
|
66
66
|
func dismiss() {
|
|
67
67
|
guard let p = panel else { return }
|
|
68
68
|
removeMonitors()
|
|
69
|
+
TileZoneOverlay.shared.dismiss()
|
|
69
70
|
|
|
70
71
|
NSAnimationContext.runAnimationGroup({ ctx in
|
|
71
72
|
ctx.duration = 0.2
|
|
@@ -79,10 +80,17 @@ final class CheatSheetHUD {
|
|
|
79
80
|
// MARK: - Event monitors
|
|
80
81
|
|
|
81
82
|
private func installMonitors() {
|
|
82
|
-
//
|
|
83
|
+
// Key handling (global — panel is non-activating so keys go to frontmost app)
|
|
83
84
|
localMonitor = NSEvent.addGlobalMonitorForEvents(matching: .keyDown) { [weak self] event in
|
|
84
85
|
if event.keyCode == 53 { // Escape
|
|
85
86
|
self?.dismiss()
|
|
87
|
+
} else if event.keyCode == 49 { // Space — toggle voice command
|
|
88
|
+
let audio = AudioLayer.shared
|
|
89
|
+
if audio.isListening {
|
|
90
|
+
audio.stopVoiceCommand()
|
|
91
|
+
} else {
|
|
92
|
+
audio.startVoiceCommand()
|
|
93
|
+
}
|
|
86
94
|
}
|
|
87
95
|
}
|
|
88
96
|
|
|
@@ -103,6 +111,8 @@ final class CheatSheetHUD {
|
|
|
103
111
|
|
|
104
112
|
struct CheatSheetView: View {
|
|
105
113
|
@ObservedObject private var hotkeyStore = HotkeyStore.shared
|
|
114
|
+
@ObservedObject private var audioLayer = AudioLayer.shared
|
|
115
|
+
@State private var hoveredAction: HotkeyAction?
|
|
106
116
|
|
|
107
117
|
var body: some View {
|
|
108
118
|
VStack(spacing: 0) {
|
|
@@ -140,19 +150,37 @@ struct CheatSheetView: View {
|
|
|
140
150
|
|
|
141
151
|
Spacer(minLength: 0)
|
|
142
152
|
|
|
153
|
+
// Voice feedback strip
|
|
154
|
+
if audioLayer.isListening || audioLayer.lastTranscript != nil || audioLayer.executionResult != nil {
|
|
155
|
+
Rectangle().fill(Palette.border).frame(height: 0.5)
|
|
156
|
+
voiceFeedback
|
|
157
|
+
}
|
|
158
|
+
|
|
143
159
|
Rectangle().fill(Palette.border).frame(height: 0.5)
|
|
144
160
|
|
|
145
161
|
// Footer
|
|
146
|
-
HStack {
|
|
162
|
+
HStack(spacing: 20) {
|
|
147
163
|
Spacer()
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
164
|
+
HStack(spacing: 6) {
|
|
165
|
+
keyBadge("Space")
|
|
166
|
+
Image(systemName: audioLayer.isListening ? "mic.fill" : "mic")
|
|
167
|
+
.font(.system(size: 11))
|
|
168
|
+
.foregroundColor(audioLayer.isListening ? Palette.running : Palette.text)
|
|
169
|
+
Text(audioLayer.isListening ? "Listening..." : "Voice")
|
|
170
|
+
.font(Typo.geistMono(11))
|
|
171
|
+
.foregroundColor(audioLayer.isListening ? Palette.running : Palette.text)
|
|
172
|
+
}
|
|
173
|
+
HStack(spacing: 6) {
|
|
174
|
+
keyBadge("ESC")
|
|
175
|
+
Text("Dismiss")
|
|
176
|
+
.font(Typo.geistMono(11))
|
|
177
|
+
.foregroundColor(Palette.textMuted)
|
|
178
|
+
}
|
|
151
179
|
Spacer()
|
|
152
180
|
}
|
|
153
|
-
.padding(.vertical,
|
|
181
|
+
.padding(.vertical, 10)
|
|
154
182
|
}
|
|
155
|
-
.frame(width: 520, height:
|
|
183
|
+
.frame(width: 520, height: 480)
|
|
156
184
|
.background(
|
|
157
185
|
RoundedRectangle(cornerRadius: 12)
|
|
158
186
|
.fill(Palette.bg)
|
|
@@ -170,6 +198,15 @@ struct CheatSheetView: View {
|
|
|
170
198
|
VStack(alignment: .leading, spacing: 10) {
|
|
171
199
|
columnHeader("Tiling")
|
|
172
200
|
|
|
201
|
+
// Modifier prefix
|
|
202
|
+
HStack(spacing: 3) {
|
|
203
|
+
keyBadge("Ctrl")
|
|
204
|
+
keyBadge("Option")
|
|
205
|
+
Text("+")
|
|
206
|
+
.font(Typo.caption(11))
|
|
207
|
+
.foregroundColor(Palette.textMuted)
|
|
208
|
+
}
|
|
209
|
+
|
|
173
210
|
// 3x3 grid
|
|
174
211
|
VStack(spacing: 2) {
|
|
175
212
|
HStack(spacing: 2) {
|
|
@@ -217,8 +254,107 @@ struct CheatSheetView: View {
|
|
|
217
254
|
// Center + Distribute
|
|
218
255
|
shortcutRow(action: .tileCenter)
|
|
219
256
|
shortcutRow(action: .tileDistribute)
|
|
257
|
+
|
|
258
|
+
// Hovered shortcut detail
|
|
259
|
+
if let hovered = hoveredAction, let binding = hotkeyStore.bindings[hovered] {
|
|
260
|
+
HStack(spacing: 3) {
|
|
261
|
+
ForEach(binding.displayParts, id: \.self) { part in
|
|
262
|
+
keyBadge(part)
|
|
263
|
+
}
|
|
264
|
+
Text("→ \(hovered.label)")
|
|
265
|
+
.font(Typo.caption(11))
|
|
266
|
+
.foregroundColor(Palette.textDim)
|
|
267
|
+
}
|
|
268
|
+
.transition(.opacity)
|
|
269
|
+
.animation(.easeInOut(duration: 0.1), value: hoveredAction)
|
|
270
|
+
}
|
|
220
271
|
}
|
|
221
272
|
.frame(maxWidth: .infinity, alignment: .leading)
|
|
273
|
+
.onHover { over in
|
|
274
|
+
if !over {
|
|
275
|
+
hoveredAction = nil
|
|
276
|
+
TileZoneOverlay.shared.dismiss()
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
.onDisappear {
|
|
280
|
+
hoveredAction = nil
|
|
281
|
+
TileZoneOverlay.shared.dismiss()
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
// MARK: - Voice Feedback
|
|
286
|
+
|
|
287
|
+
private var voiceFeedback: some View {
|
|
288
|
+
VStack(alignment: .leading, spacing: 6) {
|
|
289
|
+
if audioLayer.isListening {
|
|
290
|
+
HStack(spacing: 8) {
|
|
291
|
+
Circle()
|
|
292
|
+
.fill(Palette.running)
|
|
293
|
+
.frame(width: 8, height: 8)
|
|
294
|
+
Text("Listening...")
|
|
295
|
+
.font(Typo.geistMono(12))
|
|
296
|
+
.foregroundColor(Palette.running)
|
|
297
|
+
Spacer()
|
|
298
|
+
Text("Press Space to stop")
|
|
299
|
+
.font(Typo.caption(10))
|
|
300
|
+
.foregroundColor(Palette.textMuted)
|
|
301
|
+
}
|
|
302
|
+
} else if let transcript = audioLayer.lastTranscript {
|
|
303
|
+
// Show what was heard
|
|
304
|
+
HStack(spacing: 6) {
|
|
305
|
+
Image(systemName: "quote.opening")
|
|
306
|
+
.font(.system(size: 10))
|
|
307
|
+
.foregroundColor(Palette.textMuted)
|
|
308
|
+
Text(transcript)
|
|
309
|
+
.font(Typo.geistMono(12))
|
|
310
|
+
.foregroundColor(Palette.text)
|
|
311
|
+
.lineLimit(1)
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
// Show matched intent + result
|
|
315
|
+
if let intent = audioLayer.matchedIntent {
|
|
316
|
+
HStack(spacing: 6) {
|
|
317
|
+
Image(systemName: "arrow.right")
|
|
318
|
+
.font(.system(size: 9))
|
|
319
|
+
.foregroundColor(Palette.textMuted)
|
|
320
|
+
Text(intent.replacingOccurrences(of: "_", with: " "))
|
|
321
|
+
.font(Typo.geistMonoBold(11))
|
|
322
|
+
.foregroundColor(Palette.text)
|
|
323
|
+
|
|
324
|
+
if !audioLayer.matchedSlots.isEmpty {
|
|
325
|
+
let slotText = audioLayer.matchedSlots
|
|
326
|
+
.map { "\($0.key): \($0.value)" }
|
|
327
|
+
.joined(separator: ", ")
|
|
328
|
+
Text(slotText)
|
|
329
|
+
.font(Typo.caption(10))
|
|
330
|
+
.foregroundColor(Palette.textDim)
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
Spacer()
|
|
334
|
+
|
|
335
|
+
if let result = audioLayer.executionResult {
|
|
336
|
+
Text(result == "ok" ? "Done" : result)
|
|
337
|
+
.font(Typo.caption(10))
|
|
338
|
+
.foregroundColor(result == "ok" ? Palette.running : Palette.kill)
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
} else if let result = audioLayer.executionResult {
|
|
342
|
+
HStack(spacing: 6) {
|
|
343
|
+
Image(systemName: "questionmark.circle")
|
|
344
|
+
.font(.system(size: 10))
|
|
345
|
+
.foregroundColor(Palette.textMuted)
|
|
346
|
+
Text(result)
|
|
347
|
+
.font(Typo.caption(10))
|
|
348
|
+
.foregroundColor(Palette.textMuted)
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
.padding(.horizontal, 20)
|
|
354
|
+
.padding(.vertical, 10)
|
|
355
|
+
.background(Color.black.opacity(0.15))
|
|
356
|
+
.animation(.easeInOut(duration: 0.15), value: audioLayer.isListening)
|
|
357
|
+
.animation(.easeInOut(duration: 0.15), value: audioLayer.lastTranscript)
|
|
222
358
|
}
|
|
223
359
|
|
|
224
360
|
// MARK: - App Column
|
|
@@ -228,10 +364,11 @@ struct CheatSheetView: View {
|
|
|
228
364
|
columnHeader("App")
|
|
229
365
|
|
|
230
366
|
shortcutRow(action: .palette)
|
|
231
|
-
shortcutRow(action: .
|
|
367
|
+
shortcutRow(action: .unifiedWindow)
|
|
232
368
|
shortcutRow(action: .bezel)
|
|
369
|
+
shortcutRow(action: .hud)
|
|
370
|
+
shortcutRow(action: .voiceCommand)
|
|
233
371
|
shortcutRow(action: .cheatSheet)
|
|
234
|
-
shortcutRow(action: .desktopInventory)
|
|
235
372
|
}
|
|
236
373
|
}
|
|
237
374
|
|
|
@@ -262,25 +399,34 @@ struct CheatSheetView: View {
|
|
|
262
399
|
private func tileCell(action: HotkeyAction, label: String) -> some View {
|
|
263
400
|
let binding = hotkeyStore.bindings[action]
|
|
264
401
|
let badgeText = binding?.displayParts.last ?? ""
|
|
402
|
+
let isHovered = hoveredAction == action
|
|
265
403
|
|
|
266
|
-
return VStack(spacing:
|
|
267
|
-
Text(label)
|
|
268
|
-
.font(Typo.caption(9))
|
|
269
|
-
.foregroundColor(Palette.textDim)
|
|
404
|
+
return VStack(spacing: 2) {
|
|
270
405
|
Text(badgeText)
|
|
271
|
-
.font(Typo.geistMonoBold(
|
|
272
|
-
.foregroundColor(Palette.text)
|
|
406
|
+
.font(Typo.geistMonoBold(12))
|
|
407
|
+
.foregroundColor(isHovered ? Color.blue : Palette.text)
|
|
408
|
+
Text(label)
|
|
409
|
+
.font(Typo.caption(8))
|
|
410
|
+
.foregroundColor(isHovered ? Palette.text : Palette.textMuted)
|
|
273
411
|
}
|
|
274
412
|
.frame(maxWidth: .infinity)
|
|
275
413
|
.frame(height: 38)
|
|
276
414
|
.background(
|
|
277
415
|
RoundedRectangle(cornerRadius: 4)
|
|
278
|
-
.fill(Palette.surface)
|
|
416
|
+
.fill(isHovered ? Color.blue.opacity(0.15) : Palette.surface)
|
|
279
417
|
.overlay(
|
|
280
418
|
RoundedRectangle(cornerRadius: 4)
|
|
281
|
-
.strokeBorder(Palette.border, lineWidth: 0.5)
|
|
419
|
+
.strokeBorder(isHovered ? Color.blue.opacity(0.5) : Palette.border, lineWidth: 0.5)
|
|
282
420
|
)
|
|
283
421
|
)
|
|
422
|
+
.onHover { over in
|
|
423
|
+
if over {
|
|
424
|
+
hoveredAction = action
|
|
425
|
+
if let pos = action.tilePosition {
|
|
426
|
+
TileZoneOverlay.shared.show(position: pos)
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
}
|
|
284
430
|
}
|
|
285
431
|
|
|
286
432
|
private func shortcutRow(action: HotkeyAction) -> some View {
|
|
@@ -330,3 +476,99 @@ struct CheatSheetView: View {
|
|
|
330
476
|
)
|
|
331
477
|
}
|
|
332
478
|
}
|
|
479
|
+
|
|
480
|
+
// MARK: - HotkeyAction → TilePosition mapping
|
|
481
|
+
|
|
482
|
+
extension HotkeyAction {
|
|
483
|
+
var tilePosition: TilePosition? {
|
|
484
|
+
switch self {
|
|
485
|
+
case .tileLeft: return .left
|
|
486
|
+
case .tileRight: return .right
|
|
487
|
+
case .tileTop: return .top
|
|
488
|
+
case .tileBottom: return .bottom
|
|
489
|
+
case .tileTopLeft: return .topLeft
|
|
490
|
+
case .tileTopRight: return .topRight
|
|
491
|
+
case .tileBottomLeft: return .bottomLeft
|
|
492
|
+
case .tileBottomRight: return .bottomRight
|
|
493
|
+
case .tileMaximize: return .maximize
|
|
494
|
+
case .tileCenter: return .center
|
|
495
|
+
case .tileLeftThird: return .leftThird
|
|
496
|
+
case .tileCenterThird: return .centerThird
|
|
497
|
+
case .tileRightThird: return .rightThird
|
|
498
|
+
default: return nil
|
|
499
|
+
}
|
|
500
|
+
}
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
// MARK: - TileZoneOverlay
|
|
504
|
+
|
|
505
|
+
final class TileZoneOverlay {
|
|
506
|
+
static let shared = TileZoneOverlay()
|
|
507
|
+
|
|
508
|
+
private var panel: NSPanel?
|
|
509
|
+
|
|
510
|
+
func show(position: TilePosition) {
|
|
511
|
+
// Instant teardown (no animation) when switching between cells
|
|
512
|
+
if let p = panel {
|
|
513
|
+
p.orderOut(nil)
|
|
514
|
+
self.panel = nil
|
|
515
|
+
}
|
|
516
|
+
|
|
517
|
+
// Use the screen where the mouse is (same as CheatSheetHUD)
|
|
518
|
+
let mouseLocation = NSEvent.mouseLocation
|
|
519
|
+
let screen = NSScreen.screens.first(where: { $0.frame.contains(mouseLocation) }) ?? NSScreen.main ?? NSScreen.screens.first!
|
|
520
|
+
let visible = screen.visibleFrame
|
|
521
|
+
|
|
522
|
+
let (fx, fy, fw, fh) = position.rect
|
|
523
|
+
// visibleFrame origin is bottom-left in AppKit coordinates
|
|
524
|
+
let zoneRect = NSRect(
|
|
525
|
+
x: visible.origin.x + visible.width * fx,
|
|
526
|
+
y: visible.origin.y + visible.height * (1 - fy - fh),
|
|
527
|
+
width: visible.width * fw,
|
|
528
|
+
height: visible.height * fh
|
|
529
|
+
)
|
|
530
|
+
|
|
531
|
+
let p = NSPanel(
|
|
532
|
+
contentRect: zoneRect,
|
|
533
|
+
styleMask: [.borderless, .nonactivatingPanel],
|
|
534
|
+
backing: .buffered,
|
|
535
|
+
defer: false
|
|
536
|
+
)
|
|
537
|
+
p.isOpaque = false
|
|
538
|
+
p.backgroundColor = .clear
|
|
539
|
+
p.level = .floating
|
|
540
|
+
p.hasShadow = false
|
|
541
|
+
p.hidesOnDeactivate = false
|
|
542
|
+
p.isReleasedWhenClosed = false
|
|
543
|
+
p.ignoresMouseEvents = true
|
|
544
|
+
|
|
545
|
+
let overlay = NSView(frame: NSRect(origin: .zero, size: zoneRect.size))
|
|
546
|
+
overlay.wantsLayer = true
|
|
547
|
+
overlay.layer?.backgroundColor = NSColor.systemBlue.withAlphaComponent(0.08).cgColor
|
|
548
|
+
overlay.layer?.borderColor = NSColor.systemBlue.withAlphaComponent(0.4).cgColor
|
|
549
|
+
overlay.layer?.borderWidth = 2
|
|
550
|
+
overlay.layer?.cornerRadius = 8
|
|
551
|
+
p.contentView = overlay
|
|
552
|
+
|
|
553
|
+
p.alphaValue = 0
|
|
554
|
+
p.orderFrontRegardless()
|
|
555
|
+
|
|
556
|
+
NSAnimationContext.runAnimationGroup { ctx in
|
|
557
|
+
ctx.duration = 0.12
|
|
558
|
+
p.animator().alphaValue = 1.0
|
|
559
|
+
}
|
|
560
|
+
|
|
561
|
+
self.panel = p
|
|
562
|
+
}
|
|
563
|
+
|
|
564
|
+
func dismiss() {
|
|
565
|
+
guard let p = panel else { return }
|
|
566
|
+
NSAnimationContext.runAnimationGroup({ ctx in
|
|
567
|
+
ctx.duration = 0.1
|
|
568
|
+
p.animator().alphaValue = 0
|
|
569
|
+
}) { [weak self] in
|
|
570
|
+
p.orderOut(nil)
|
|
571
|
+
self?.panel = nil
|
|
572
|
+
}
|
|
573
|
+
}
|
|
574
|
+
}
|
|
@@ -98,4 +98,17 @@ enum JSON: Codable, Equatable {
|
|
|
98
98
|
guard case .bool(let b) = self else { return nil }
|
|
99
99
|
return b
|
|
100
100
|
}
|
|
101
|
+
|
|
102
|
+
var arrayValue: [JSON]? {
|
|
103
|
+
guard case .array(let a) = self else { return nil }
|
|
104
|
+
return a
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
var numericDouble: Double? {
|
|
108
|
+
switch self {
|
|
109
|
+
case .double(let d): return d
|
|
110
|
+
case .int(let i): return Double(i)
|
|
111
|
+
default: return nil
|
|
112
|
+
}
|
|
113
|
+
}
|
|
101
114
|
}
|
|
@@ -394,6 +394,14 @@ final class DaemonServer: ObservableObject {
|
|
|
394
394
|
"totalBlocks": .int(totalBlocks)
|
|
395
395
|
])
|
|
396
396
|
)
|
|
397
|
+
case .voiceCommand(let text, let confidence):
|
|
398
|
+
daemonEvent = DaemonEvent(
|
|
399
|
+
event: "voice.command",
|
|
400
|
+
data: .object([
|
|
401
|
+
"text": .string(text),
|
|
402
|
+
"confidence": .double(confidence)
|
|
403
|
+
])
|
|
404
|
+
)
|
|
397
405
|
}
|
|
398
406
|
broadcast(daemonEvent)
|
|
399
407
|
}
|
|
@@ -1,13 +1,67 @@
|
|
|
1
1
|
import AppKit
|
|
2
|
+
import ApplicationServices
|
|
2
3
|
import CoreGraphics
|
|
3
4
|
|
|
4
5
|
final class DesktopModel: ObservableObject {
|
|
5
6
|
static let shared = DesktopModel()
|
|
6
7
|
|
|
8
|
+
/// System helper processes that should never appear in search results or window lists.
|
|
9
|
+
/// These are XPC services, agents, and background helpers — not user-facing apps.
|
|
10
|
+
private static let systemHelperProcesses: Set<String> = [
|
|
11
|
+
// Apple system helpers
|
|
12
|
+
"CredentialsProviderExtensionHost",
|
|
13
|
+
"AuthenticationServicesAgent",
|
|
14
|
+
"SafariPasswordExtension",
|
|
15
|
+
"com.apple.WebKit.WebAuthn",
|
|
16
|
+
"SharedWebCredentialRunner",
|
|
17
|
+
"ViewBridgeAuxiliary",
|
|
18
|
+
"universalaccessd",
|
|
19
|
+
"CoreServicesUIAgent",
|
|
20
|
+
"UserNotificationCenter",
|
|
21
|
+
"AutoFillPanelService",
|
|
22
|
+
"AutoFill",
|
|
23
|
+
"CoreLocationAgent",
|
|
24
|
+
"SecurityAgent",
|
|
25
|
+
"coreautha",
|
|
26
|
+
"coreauth",
|
|
27
|
+
"talagent",
|
|
28
|
+
"CommCenter",
|
|
29
|
+
"AXVisualSupportAgent",
|
|
30
|
+
"SystemUIServer",
|
|
31
|
+
"Dock",
|
|
32
|
+
"Window Server",
|
|
33
|
+
"WindowManager",
|
|
34
|
+
"NotificationCenter",
|
|
35
|
+
"ControlCenter",
|
|
36
|
+
"Spotlight",
|
|
37
|
+
"Keychain Access",
|
|
38
|
+
"loginwindow",
|
|
39
|
+
"ScreenSaverEngine",
|
|
40
|
+
"SoftwareUpdateNotificationManager",
|
|
41
|
+
"WiFiAgent",
|
|
42
|
+
"pboard",
|
|
43
|
+
"storeuid",
|
|
44
|
+
// Third-party helpers
|
|
45
|
+
"CursorUIViewService",
|
|
46
|
+
"Electron Helper",
|
|
47
|
+
"Google Chrome Helper",
|
|
48
|
+
]
|
|
49
|
+
|
|
50
|
+
/// Suffixes that indicate a helper/service process, not a user-facing app
|
|
51
|
+
private static let helperSuffixes = ["Service", "Agent", "Helper", "Extension", "Daemon", "XPCService"]
|
|
52
|
+
|
|
53
|
+
/// Real apps that happen to match helper suffixes — don't filter these
|
|
54
|
+
private static let knownRealApps: Set<String> = [
|
|
55
|
+
"Finder",
|
|
56
|
+
"Activity Monitor",
|
|
57
|
+
]
|
|
58
|
+
|
|
7
59
|
@Published private(set) var windows: [UInt32: WindowEntry] = [:]
|
|
8
|
-
|
|
60
|
+
@Published private(set) var interactionDates: [UInt32: Date] = [:]
|
|
61
|
+
/// In-memory layer tags: wid → layer id (e.g. "lattices", "vox", "hudson")
|
|
9
62
|
private(set) var windowLayerTags: [UInt32: String] = [:]
|
|
10
63
|
private var timer: Timer?
|
|
64
|
+
private var lastFrontmostWid: UInt32?
|
|
11
65
|
|
|
12
66
|
func start(interval: TimeInterval = 1.5) {
|
|
13
67
|
guard timer == nil else { return }
|
|
@@ -24,7 +78,27 @@ final class DesktopModel: ObservableObject {
|
|
|
24
78
|
}
|
|
25
79
|
|
|
26
80
|
func allWindows() -> [WindowEntry] {
|
|
27
|
-
Array(windows.values).sorted { $0.
|
|
81
|
+
Array(windows.values).sorted { $0.zIndex < $1.zIndex }
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
func lastInteractionDate(for wid: UInt32) -> Date? {
|
|
85
|
+
interactionDates[wid]
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
func markInteraction(wid: UInt32, at date: Date = Date()) {
|
|
89
|
+
DispatchQueue.main.async {
|
|
90
|
+
self.interactionDates[wid] = date
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
func markInteraction(wids: [UInt32], at date: Date = Date()) {
|
|
95
|
+
guard !wids.isEmpty else { return }
|
|
96
|
+
let unique = Set(wids)
|
|
97
|
+
DispatchQueue.main.async {
|
|
98
|
+
for wid in unique {
|
|
99
|
+
self.interactionDates[wid] = date
|
|
100
|
+
}
|
|
101
|
+
}
|
|
28
102
|
}
|
|
29
103
|
|
|
30
104
|
func windowForSession(_ session: String) -> WindowEntry? {
|
|
@@ -60,13 +134,31 @@ final class DesktopModel: ObservableObject {
|
|
|
60
134
|
|
|
61
135
|
// MARK: - Polling
|
|
62
136
|
|
|
137
|
+
private var lastPollTime: Date = .distantPast
|
|
138
|
+
private static let minPollInterval: TimeInterval = 1.0
|
|
139
|
+
|
|
140
|
+
/// Poll only if stale. Call `forcePoll()` to bypass the freshness check.
|
|
63
141
|
func poll() {
|
|
142
|
+
let now = Date()
|
|
143
|
+
guard now.timeIntervalSince(lastPollTime) >= Self.minPollInterval else { return }
|
|
144
|
+
lastPollTime = now
|
|
145
|
+
performPoll()
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/// Force a poll regardless of freshness — use sparingly.
|
|
149
|
+
func forcePoll() {
|
|
150
|
+
lastPollTime = Date()
|
|
151
|
+
performPoll()
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
private func performPoll() {
|
|
64
155
|
guard let list = CGWindowListCopyWindowInfo(
|
|
65
|
-
[.
|
|
156
|
+
[.optionAll, .excludeDesktopElements],
|
|
66
157
|
kCGNullWindowID
|
|
67
158
|
) as? [[String: Any]] else { return }
|
|
68
159
|
|
|
69
160
|
var fresh: [UInt32: WindowEntry] = [:]
|
|
161
|
+
var zCounter = 0
|
|
70
162
|
|
|
71
163
|
for info in list {
|
|
72
164
|
guard let wid = info[kCGWindowNumber as String] as? UInt32,
|
|
@@ -82,11 +174,24 @@ final class DesktopModel: ObservableObject {
|
|
|
82
174
|
|
|
83
175
|
let title = info[kCGWindowName as String] as? String ?? ""
|
|
84
176
|
let layer = info[kCGWindowLayer as String] as? Int ?? 0
|
|
85
|
-
let isOnScreen = info[kCGWindowIsOnscreen as String] as? Bool ??
|
|
177
|
+
let isOnScreen = info[kCGWindowIsOnscreen as String] as? Bool ?? false
|
|
86
178
|
|
|
87
179
|
// Skip non-standard layers (menus, overlays)
|
|
88
180
|
guard layer == 0 else { continue }
|
|
89
181
|
|
|
182
|
+
// Skip system helper processes (autofill, credential providers, etc.)
|
|
183
|
+
if Self.systemHelperProcesses.contains(ownerName) { continue }
|
|
184
|
+
|
|
185
|
+
// Skip processes whose name ends with common helper suffixes
|
|
186
|
+
// (e.g. "CursorUIViewService", "AutoFillPanelService", "SecurityAgent")
|
|
187
|
+
// but not known real apps that happen to have these words
|
|
188
|
+
let isHelperByName = Self.helperSuffixes.contains(where: { ownerName.hasSuffix($0) })
|
|
189
|
+
&& !Self.knownRealApps.contains(ownerName)
|
|
190
|
+
if isHelperByName { continue }
|
|
191
|
+
|
|
192
|
+
// Skip windows with no title from processes containing "com.apple."
|
|
193
|
+
if ownerName.hasPrefix("com.apple.") && title.isEmpty { continue }
|
|
194
|
+
|
|
90
195
|
let frame = WindowFrame(
|
|
91
196
|
x: Double(rect.origin.x),
|
|
92
197
|
y: Double(rect.origin.y),
|
|
@@ -103,7 +208,7 @@ final class DesktopModel: ObservableObject {
|
|
|
103
208
|
latticesSession = String(match.dropFirst(9).dropLast(1)) // drop "[lattices:" and "]"
|
|
104
209
|
}
|
|
105
210
|
|
|
106
|
-
|
|
211
|
+
var entry = WindowEntry(
|
|
107
212
|
wid: wid,
|
|
108
213
|
app: ownerName,
|
|
109
214
|
pid: pid,
|
|
@@ -113,8 +218,14 @@ final class DesktopModel: ObservableObject {
|
|
|
113
218
|
isOnScreen: isOnScreen,
|
|
114
219
|
latticesSession: latticesSession
|
|
115
220
|
)
|
|
221
|
+
entry.zIndex = zCounter
|
|
222
|
+
zCounter += 1
|
|
223
|
+
fresh[wid] = entry
|
|
116
224
|
}
|
|
117
225
|
|
|
226
|
+
// AX reconciliation: check which CG windows actually exist in Accessibility
|
|
227
|
+
reconcileWithAX(&fresh)
|
|
228
|
+
|
|
118
229
|
// Diff
|
|
119
230
|
let oldKeys = Set(windows.keys)
|
|
120
231
|
let newKeys = Set(fresh.keys)
|
|
@@ -122,9 +233,21 @@ final class DesktopModel: ObservableObject {
|
|
|
122
233
|
let removed = Array(oldKeys.subtracting(newKeys))
|
|
123
234
|
|
|
124
235
|
let changed = added.count > 0 || removed.count > 0 || windowsContentChanged(old: windows, new: fresh)
|
|
236
|
+
let frontmostWid = fresh.values.min(by: { $0.zIndex < $1.zIndex })?.wid
|
|
237
|
+
let markFrontmost = frontmostWid != nil && frontmostWid != lastFrontmostWid
|
|
238
|
+
let interactionTime = Date()
|
|
125
239
|
|
|
126
240
|
DispatchQueue.main.async {
|
|
127
|
-
self.
|
|
241
|
+
var interactions = self.interactionDates.filter { fresh[$0.key] != nil }
|
|
242
|
+
if markFrontmost, let frontmostWid {
|
|
243
|
+
interactions[frontmostWid] = interactionTime
|
|
244
|
+
}
|
|
245
|
+
// Only publish if something actually changed — avoids unnecessary SwiftUI re-renders
|
|
246
|
+
if changed || markFrontmost {
|
|
247
|
+
self.windows = fresh
|
|
248
|
+
self.interactionDates = interactions
|
|
249
|
+
}
|
|
250
|
+
self.lastFrontmostWid = frontmostWid
|
|
128
251
|
}
|
|
129
252
|
|
|
130
253
|
if changed {
|
|
@@ -136,6 +259,66 @@ final class DesktopModel: ObservableObject {
|
|
|
136
259
|
}
|
|
137
260
|
}
|
|
138
261
|
|
|
262
|
+
private func reconcileWithAX(_ fresh: inout [UInt32: WindowEntry]) {
|
|
263
|
+
// Get currently active Space IDs — AX only returns windows on these
|
|
264
|
+
let currentSpaceIds = Set(WindowTiler.getDisplaySpaces().map(\.currentSpaceId))
|
|
265
|
+
guard !currentSpaceIds.isEmpty else { return }
|
|
266
|
+
|
|
267
|
+
// Group CG windows by PID — only titled windows on current Spaces
|
|
268
|
+
var byPid: [Int32: [UInt32]] = [:]
|
|
269
|
+
for (wid, entry) in fresh where !entry.title.isEmpty {
|
|
270
|
+
let onCurrentSpace = entry.spaceIds.contains { currentSpaceIds.contains($0) }
|
|
271
|
+
if onCurrentSpace {
|
|
272
|
+
byPid[entry.pid, default: []].append(wid)
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
for (pid, wids) in byPid {
|
|
277
|
+
let axApp = AXUIElementCreateApplication(pid)
|
|
278
|
+
|
|
279
|
+
// Set a timeout so unresponsive apps (video calls, etc.) don't block the poll
|
|
280
|
+
AXUIElementSetMessagingTimeout(axApp, 0.3)
|
|
281
|
+
|
|
282
|
+
var axWindowsRef: CFTypeRef?
|
|
283
|
+
guard AXUIElementCopyAttributeValue(axApp, kAXWindowsAttribute as CFString, &axWindowsRef) == .success,
|
|
284
|
+
let axWindows = axWindowsRef as? [AXUIElement] else { continue }
|
|
285
|
+
|
|
286
|
+
// Collect AX window titles
|
|
287
|
+
var axTitles: [String] = []
|
|
288
|
+
for axWin in axWindows {
|
|
289
|
+
var titleRef: CFTypeRef?
|
|
290
|
+
AXUIElementCopyAttributeValue(axWin, kAXTitleAttribute as CFString, &titleRef)
|
|
291
|
+
if let title = titleRef as? String, !title.isEmpty {
|
|
292
|
+
axTitles.append(title)
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
// Mark CG windows that have no matching AX title.
|
|
297
|
+
// AX titles often have suffixes like " - Google Chrome - Profile"
|
|
298
|
+
// so check if any AX title starts with the CG title (stripped of emoji).
|
|
299
|
+
for wid in wids {
|
|
300
|
+
guard let entry = fresh[wid], !entry.title.isEmpty else { continue }
|
|
301
|
+
let cgClean = stripForMatch(entry.title)
|
|
302
|
+
let matched = axTitles.contains { axTitle in
|
|
303
|
+
let axClean = stripForMatch(axTitle)
|
|
304
|
+
return axClean.hasPrefix(cgClean) || axClean.contains(cgClean) || cgClean.hasPrefix(axClean)
|
|
305
|
+
}
|
|
306
|
+
if !matched {
|
|
307
|
+
fresh[wid]?.axVerified = false
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
private func stripForMatch(_ text: String) -> String {
|
|
314
|
+
// Remove emoji and non-ASCII symbols, lowercase, collapse whitespace
|
|
315
|
+
let scalar = text.unicodeScalars.filter { scalar in
|
|
316
|
+
scalar.isASCII || CharacterSet.letters.contains(scalar)
|
|
317
|
+
}
|
|
318
|
+
return String(scalar).lowercased()
|
|
319
|
+
.split(separator: " ").joined(separator: " ")
|
|
320
|
+
}
|
|
321
|
+
|
|
139
322
|
private func windowsContentChanged(old: [UInt32: WindowEntry], new: [UInt32: WindowEntry]) -> Bool {
|
|
140
323
|
// Quick check: if titles or frames changed for any existing window
|
|
141
324
|
for (wid, newEntry) in new {
|