@lattices/cli 0.3.0 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. package/README.md +85 -9
  2. package/app/Info.plist +30 -0
  3. package/app/Lattices.app/Contents/Info.plist +8 -2
  4. package/app/Lattices.app/Contents/MacOS/Lattices +0 -0
  5. package/app/Lattices.app/Contents/Resources/AppIcon.icns +0 -0
  6. package/app/Lattices.app/Contents/Resources/tap.wav +0 -0
  7. package/app/Lattices.app/Contents/_CodeSignature/CodeResources +139 -0
  8. package/app/Lattices.entitlements +15 -0
  9. package/app/Package.swift +8 -1
  10. package/app/Resources/tap.wav +0 -0
  11. package/app/Sources/AdvisorLearningStore.swift +90 -0
  12. package/app/Sources/AgentSession.swift +377 -0
  13. package/app/Sources/AppDelegate.swift +45 -12
  14. package/app/Sources/AppShellView.swift +81 -8
  15. package/app/Sources/AudioProvider.swift +386 -0
  16. package/app/Sources/CheatSheetHUD.swift +261 -19
  17. package/app/Sources/DaemonProtocol.swift +13 -0
  18. package/app/Sources/DaemonServer.swift +8 -0
  19. package/app/Sources/DesktopModel.swift +189 -6
  20. package/app/Sources/DesktopModelTypes.swift +2 -0
  21. package/app/Sources/DiagnosticLog.swift +104 -2
  22. package/app/Sources/EventBus.swift +1 -0
  23. package/app/Sources/HUDBottomBar.swift +279 -0
  24. package/app/Sources/HUDController.swift +1158 -0
  25. package/app/Sources/HUDLeftBar.swift +849 -0
  26. package/app/Sources/HUDMinimap.swift +179 -0
  27. package/app/Sources/HUDRightBar.swift +774 -0
  28. package/app/Sources/HUDState.swift +367 -0
  29. package/app/Sources/HUDTopBar.swift +243 -0
  30. package/app/Sources/HandsOffSession.swift +802 -0
  31. package/app/Sources/HomeDashboardView.swift +125 -0
  32. package/app/Sources/HotkeyManager.swift +2 -0
  33. package/app/Sources/HotkeyStore.swift +49 -9
  34. package/app/Sources/IntentEngine.swift +962 -0
  35. package/app/Sources/Intents/CreateLayerIntent.swift +54 -0
  36. package/app/Sources/Intents/DistributeIntent.swift +56 -0
  37. package/app/Sources/Intents/FocusIntent.swift +69 -0
  38. package/app/Sources/Intents/HelpIntent.swift +41 -0
  39. package/app/Sources/Intents/KillIntent.swift +47 -0
  40. package/app/Sources/Intents/LatticeIntent.swift +78 -0
  41. package/app/Sources/Intents/LaunchIntent.swift +67 -0
  42. package/app/Sources/Intents/ListSessionsIntent.swift +32 -0
  43. package/app/Sources/Intents/ListWindowsIntent.swift +30 -0
  44. package/app/Sources/Intents/ScanIntent.swift +52 -0
  45. package/app/Sources/Intents/SearchIntent.swift +190 -0
  46. package/app/Sources/Intents/SwitchLayerIntent.swift +50 -0
  47. package/app/Sources/Intents/TileIntent.swift +61 -0
  48. package/app/Sources/LatticesApi.swift +1275 -30
  49. package/app/Sources/LauncherHUD.swift +348 -0
  50. package/app/Sources/MainView.swift +147 -44
  51. package/app/Sources/MouseFinder.swift +222 -0
  52. package/app/Sources/OcrModel.swift +34 -1
  53. package/app/Sources/OmniSearchState.swift +99 -102
  54. package/app/Sources/OnboardingView.swift +457 -0
  55. package/app/Sources/PermissionChecker.swift +2 -12
  56. package/app/Sources/PiChatDock.swift +454 -0
  57. package/app/Sources/PiChatSession.swift +815 -0
  58. package/app/Sources/PiWorkspaceView.swift +364 -0
  59. package/app/Sources/PlacementSpec.swift +195 -0
  60. package/app/Sources/Preferences.swift +59 -0
  61. package/app/Sources/ProjectScanner.swift +58 -45
  62. package/app/Sources/ScreenMapState.swift +701 -55
  63. package/app/Sources/ScreenMapView.swift +843 -103
  64. package/app/Sources/ScreenMapWindowController.swift +22 -0
  65. package/app/Sources/SessionLayerStore.swift +285 -0
  66. package/app/Sources/SessionManager.swift +4 -1
  67. package/app/Sources/SettingsView.swift +186 -3
  68. package/app/Sources/Theme.swift +9 -8
  69. package/app/Sources/TmuxModel.swift +7 -0
  70. package/app/Sources/TmuxQuery.swift +27 -3
  71. package/app/Sources/VoiceChatView.swift +192 -0
  72. package/app/Sources/VoiceCommandWindow.swift +1594 -0
  73. package/app/Sources/VoiceIntentResolver.swift +671 -0
  74. package/app/Sources/VoxClient.swift +454 -0
  75. package/app/Sources/WindowTiler.swift +348 -87
  76. package/app/Sources/WorkspaceManager.swift +127 -18
  77. package/app/Tests/StageDragTests.swift +333 -0
  78. package/app/Tests/StageJoinTests.swift +313 -0
  79. package/app/Tests/StageManagerTests.swift +280 -0
  80. package/app/Tests/StageTileTests.swift +353 -0
  81. package/assets/AppIcon.icns +0 -0
  82. package/bin/client.ts +16 -0
  83. package/bin/{daemon-client.js → daemon-client.ts} +49 -30
  84. package/bin/handsoff-infer.ts +280 -0
  85. package/bin/handsoff-worker.ts +740 -0
  86. package/bin/lattices-app.ts +338 -0
  87. package/bin/lattices-dev +208 -0
  88. package/bin/{lattices.js → lattices.ts} +777 -140
  89. package/bin/project-twin.ts +645 -0
  90. package/docs/agent-execution-plan.md +562 -0
  91. package/docs/agent-layer-guide.md +207 -0
  92. package/docs/agents.md +142 -0
  93. package/docs/api.md +153 -34
  94. package/docs/app.md +29 -1
  95. package/docs/config.md +5 -1
  96. package/docs/handsoff-test-scenarios.md +84 -0
  97. package/docs/layers.md +20 -20
  98. package/docs/ocr.md +14 -5
  99. package/docs/overview.md +5 -1
  100. package/docs/presentation-execution-review.md +491 -0
  101. package/docs/prompts/hands-off-system.md +374 -0
  102. package/docs/prompts/hands-off-turn.md +30 -0
  103. package/docs/prompts/voice-advisor.md +31 -0
  104. package/docs/prompts/voice-fallback.md +23 -0
  105. package/docs/tiling-reference.md +167 -0
  106. package/docs/twins.md +138 -0
  107. package/docs/voice-command-protocol.md +278 -0
  108. package/docs/voice.md +219 -0
  109. package/package.json +29 -11
  110. package/bin/client.js +0 -4
  111. package/bin/lattices-app.js +0 -221
@@ -0,0 +1,962 @@
1
+ import AppKit
2
+
3
+ // MARK: - Intent Definition
4
+
5
+ struct IntentDef {
6
+ let name: String
7
+ let description: String
8
+ let examples: [String] // Example phrases that map to this intent
9
+ let slots: [IntentSlot] // Named parameters extracted from the utterance
10
+ let handler: (IntentRequest) throws -> JSON
11
+ }
12
+
13
+ struct IntentSlot {
14
+ let name: String
15
+ let type: String // "string", "int", "position", "query"
16
+ let required: Bool
17
+ let description: String
18
+ let enumValues: [String]? // For constrained slots like tile positions
19
+ }
20
+
21
+ struct IntentRequest {
22
+ let intent: String
23
+ let slots: [String: JSON]
24
+ let rawText: String? // Original transcription, for fallback matching
25
+ let confidence: Double? // Transcription confidence from voice service
26
+ let source: String? // "vox", "siri", "cli", etc.
27
+ }
28
+
29
+ // MARK: - Intent Engine
30
+
31
+ final class IntentEngine {
32
+ static let shared = IntentEngine()
33
+
34
+ private var intents: [String: IntentDef] = [:]
35
+ private var intentOrder: [String] = []
36
+
37
+ private init() {
38
+ registerBuiltins()
39
+ }
40
+
41
+ func register(_ intent: IntentDef) {
42
+ intents[intent.name] = intent
43
+ if !intentOrder.contains(intent.name) {
44
+ intentOrder.append(intent.name)
45
+ }
46
+ }
47
+
48
+ func definitions() -> [IntentDef] {
49
+ intentOrder.compactMap { intents[$0] }
50
+ }
51
+
52
+ // MARK: - Execution
53
+
54
+ func execute(_ request: IntentRequest) throws -> JSON {
55
+ // 1. Direct match by intent name
56
+ if let def = intents[request.intent] {
57
+ return try def.handler(request)
58
+ }
59
+
60
+ // 2. Fuzzy match by intent name (handle voice transcription typos)
61
+ let normalized = request.intent.lowercased().replacingOccurrences(of: " ", with: "_")
62
+ .replacingOccurrences(of: "-", with: "_")
63
+ if let def = intents[normalized] {
64
+ return try def.handler(request)
65
+ }
66
+
67
+ // 3. No match
68
+ throw IntentError.unknownIntent(request.intent, available: Array(intents.keys).sorted())
69
+ }
70
+
71
+ // MARK: - Discovery
72
+
73
+ func catalog() -> JSON {
74
+ .array(intentOrder.compactMap { name in
75
+ guard let def = intents[name] else { return nil }
76
+ return .object([
77
+ "intent": .string(def.name),
78
+ "description": .string(def.description),
79
+ "examples": .array(def.examples.map { .string($0) }),
80
+ "slots": .array(def.slots.map { slot in
81
+ var obj: [String: JSON] = [
82
+ "name": .string(slot.name),
83
+ "type": .string(slot.type),
84
+ "required": .bool(slot.required),
85
+ "description": .string(slot.description),
86
+ ]
87
+ if let vals = slot.enumValues {
88
+ obj["values"] = .array(vals.map { .string($0) })
89
+ }
90
+ return .object(obj)
91
+ })
92
+ ])
93
+ })
94
+ }
95
+
96
+ // MARK: - Built-in Intents
97
+
98
+ /// Track recently tiled wids so batch operations (e.g. "tile iTerm left, iTerm right")
99
+ /// don't pick the same window twice. Resets after 2 seconds.
100
+ private static var recentlyTiledWids: Set<UInt32> = []
101
+ private static var recentlyTiledTimer: Timer?
102
+
103
+ private static func markTiled(_ wid: UInt32) {
104
+ recentlyTiledWids.insert(wid)
105
+ recentlyTiledTimer?.invalidate()
106
+ recentlyTiledTimer = Timer.scheduledTimer(withTimeInterval: 2.0, repeats: false) { _ in
107
+ recentlyTiledWids.removeAll()
108
+ }
109
+ }
110
+
111
+ private func registerBuiltins() {
112
+
113
+ // ── Window Tiling ───────────────────────────────────────
114
+
115
+ register(IntentDef(
116
+ name: "tile_window",
117
+ description: "Tile a window to a screen position",
118
+ examples: [
119
+ "tile this left",
120
+ "snap to the right half",
121
+ "maximize the window",
122
+ "put it in the top left corner",
123
+ "center the window",
124
+ "make it full screen"
125
+ ],
126
+ slots: [
127
+ IntentSlot(name: "position", type: "position", required: true,
128
+ description: "Target tile position. Named positions or grid:CxR:C,R syntax.",
129
+ enumValues: TilePosition.allCases.map(\.rawValue)),
130
+ IntentSlot(name: "app", type: "string", required: false,
131
+ description: "Target app name (defaults to frontmost)", enumValues: nil),
132
+ IntentSlot(name: "wid", type: "int", required: false,
133
+ description: "Target window ID", enumValues: nil),
134
+ IntentSlot(name: "session", type: "string", required: false,
135
+ description: "Target session name", enumValues: nil),
136
+ ],
137
+ handler: { req in
138
+ guard let posStr = req.slots["position"]?.stringValue else {
139
+ throw IntentError.missingSlot("position")
140
+ }
141
+ guard let placement = PlacementSpec(string: posStr) else {
142
+ throw IntentError.invalidSlot("Unknown position: \(posStr)")
143
+ }
144
+
145
+ // Resolve target: explicit session, wid, app name, or frontmost
146
+ if let session = req.slots["session"]?.stringValue {
147
+ return try LatticesApi.shared.dispatch(
148
+ method: "window.place",
149
+ params: .object(["session": .string(session), "placement": placement.jsonValue])
150
+ )
151
+ }
152
+
153
+ // For wid/app/frontmost: use WindowTiler directly
154
+ func tileEntry(_ entry: WindowEntry) {
155
+ IntentEngine.markTiled(entry.wid)
156
+ DispatchQueue.main.async {
157
+ WindowTiler.tileWindowById(wid: entry.wid, pid: entry.pid, to: placement)
158
+ }
159
+ }
160
+
161
+ if let wid = req.slots["wid"]?.uint32Value,
162
+ let entry = DesktopModel.shared.windows[wid] {
163
+ tileEntry(entry)
164
+ return .object(["ok": .bool(true), "wid": .int(Int(wid)), "position": .string(posStr)])
165
+ }
166
+
167
+ if let app = req.slots["app"]?.stringValue {
168
+ // Skip windows already tiled in this batch (e.g. two iTerm windows side by side)
169
+ let alreadyTiled = IntentEngine.recentlyTiledWids
170
+ if let entry = DesktopModel.shared.windows.values.first(where: {
171
+ $0.app.localizedCaseInsensitiveContains(app) && !alreadyTiled.contains($0.wid)
172
+ }) {
173
+ tileEntry(entry)
174
+ return .object(["ok": .bool(true), "app": .string(entry.app), "wid": .int(Int(entry.wid)), "position": .string(posStr)])
175
+ }
176
+ throw IntentError.targetNotFound("No window found for app '\(app)'")
177
+ }
178
+
179
+ // Default: tile frontmost window
180
+ DispatchQueue.main.async {
181
+ WindowTiler.tileFrontmostViaAX(to: placement)
182
+ }
183
+ return .object(["ok": .bool(true), "target": .string("frontmost"), "position": .string(posStr)])
184
+ }
185
+ ))
186
+
187
+ // ── Focus Window / App ──────────────────────────────────
188
+
189
+ register(IntentDef(
190
+ name: "focus",
191
+ description: "Focus a window, app, or session",
192
+ examples: [
193
+ "switch to Chrome",
194
+ "focus the terminal",
195
+ "go to my frontend project",
196
+ "show Slack"
197
+ ],
198
+ slots: [
199
+ IntentSlot(name: "app", type: "string", required: false,
200
+ description: "App name to focus", enumValues: nil),
201
+ IntentSlot(name: "session", type: "string", required: false,
202
+ description: "Session name to focus", enumValues: nil),
203
+ IntentSlot(name: "wid", type: "int", required: false,
204
+ description: "Window ID to focus", enumValues: nil),
205
+ ],
206
+ handler: { req in
207
+ if let session = req.slots["session"]?.stringValue {
208
+ return try LatticesApi.shared.dispatch(
209
+ method: "window.focus",
210
+ params: .object(["session": .string(session)])
211
+ )
212
+ }
213
+ if let wid = req.slots["wid"]?.intValue {
214
+ return try LatticesApi.shared.dispatch(
215
+ method: "window.focus",
216
+ params: .object(["wid": .int(wid)])
217
+ )
218
+ }
219
+ if let app = req.slots["app"]?.stringValue {
220
+ if let entry = DesktopModel.shared.windows.values.first(where: {
221
+ $0.app.localizedCaseInsensitiveContains(app)
222
+ }) {
223
+ return try LatticesApi.shared.dispatch(
224
+ method: "window.focus",
225
+ params: .object(["wid": .int(Int(entry.wid))])
226
+ )
227
+ }
228
+ // Try launching the app
229
+ NSWorkspace.shared.launchApplication(app)
230
+ return .object(["ok": .bool(true), "launched": .string(app)])
231
+ }
232
+ throw IntentError.missingSlot("app, session, or wid")
233
+ }
234
+ ))
235
+
236
+ // ── Launch Session ──────────────────────────────────────
237
+
238
+ register(IntentDef(
239
+ name: "launch",
240
+ description: "Launch a project session",
241
+ examples: [
242
+ "open my frontend project",
243
+ "launch the API",
244
+ "start working on lattices",
245
+ "open the backend"
246
+ ],
247
+ slots: [
248
+ IntentSlot(name: "project", type: "string", required: true,
249
+ description: "Project name or path", enumValues: nil),
250
+ ],
251
+ handler: { req in
252
+ guard let project = req.slots["project"]?.stringValue else {
253
+ throw IntentError.missingSlot("project")
254
+ }
255
+
256
+ // Try matching by name against discovered projects
257
+ let projects = try LatticesApi.shared.dispatch(method: "projects.list", params: nil)
258
+ if case .array(let list) = projects {
259
+ for p in list {
260
+ let name = p["name"]?.stringValue ?? ""
261
+ let path = p["path"]?.stringValue ?? ""
262
+ if name.localizedCaseInsensitiveContains(project) ||
263
+ path.localizedCaseInsensitiveContains(project) {
264
+ return try LatticesApi.shared.dispatch(
265
+ method: "session.launch",
266
+ params: .object(["path": .string(path)])
267
+ )
268
+ }
269
+ }
270
+ }
271
+ throw IntentError.targetNotFound("No project matching '\(project)'")
272
+ }
273
+ ))
274
+
275
+ // ── Switch Layer ────────────────────────────────────────
276
+
277
+ register(IntentDef(
278
+ name: "switch_layer",
279
+ description: "Switch to a workspace layer",
280
+ examples: [
281
+ "switch to the web layer",
282
+ "go to mobile",
283
+ "layer 2",
284
+ "switch to review"
285
+ ],
286
+ slots: [
287
+ IntentSlot(name: "layer", type: "string", required: true,
288
+ description: "Layer name or index", enumValues: nil),
289
+ ],
290
+ handler: { req in
291
+ guard let layer = req.slots["layer"]?.stringValue else {
292
+ throw IntentError.missingSlot("layer")
293
+ }
294
+
295
+ // Try as index first
296
+ if let index = Int(layer) {
297
+ // Try session layers first, then config layers
298
+ let session = SessionLayerStore.shared
299
+ if !session.layers.isEmpty && index < session.layers.count {
300
+ DispatchQueue.main.async { session.switchTo(index: index) }
301
+ return .object(["ok": .bool(true), "type": .string("session"), "index": .int(index)])
302
+ }
303
+ return try LatticesApi.shared.dispatch(
304
+ method: "layer.switch",
305
+ params: .object(["index": .int(index)])
306
+ )
307
+ }
308
+
309
+ // Try as name — session layers first
310
+ let session = SessionLayerStore.shared
311
+ if let idx = session.layers.firstIndex(where: {
312
+ $0.name.localizedCaseInsensitiveContains(layer)
313
+ }) {
314
+ DispatchQueue.main.async { session.switchTo(index: idx) }
315
+ return .object(["ok": .bool(true), "type": .string("session"), "name": .string(session.layers[idx].name)])
316
+ }
317
+
318
+ // Then config layers
319
+ return try LatticesApi.shared.dispatch(
320
+ method: "layer.switch",
321
+ params: .object(["name": .string(layer)])
322
+ )
323
+ }
324
+ ))
325
+
326
+ // ── Search Windows ─────────────────────────────────────
327
+
328
+ register(IntentDef(
329
+ name: "search",
330
+ description: "Search for windows by app name, title, session, or screen text",
331
+ examples: [
332
+ "find the error message",
333
+ "search for TODO",
334
+ "find all terminal windows",
335
+ "find chrome",
336
+ "where does it say build failed",
337
+ "look for port 3000"
338
+ ],
339
+ slots: [
340
+ IntentSlot(name: "query", type: "query", required: true,
341
+ description: "Text to search for", enumValues: nil),
342
+ ],
343
+ handler: { req in
344
+ return try SearchIntent().perform(slots: req.slots)
345
+ }
346
+ ))
347
+
348
+ // ── List Windows ────────────────────────────────────────
349
+
350
+ register(IntentDef(
351
+ name: "list_windows",
352
+ description: "List all visible windows",
353
+ examples: [
354
+ "what windows are open",
355
+ "show me all windows",
356
+ "what's on screen"
357
+ ],
358
+ slots: [],
359
+ handler: { _ in
360
+ try LatticesApi.shared.dispatch(method: "windows.list", params: nil)
361
+ }
362
+ ))
363
+
364
+ // ── List Sessions ───────────────────────────────────────
365
+
366
+ register(IntentDef(
367
+ name: "list_sessions",
368
+ description: "List active terminal sessions",
369
+ examples: [
370
+ "what sessions are running",
371
+ "show my projects",
372
+ "list sessions"
373
+ ],
374
+ slots: [],
375
+ handler: { _ in
376
+ try LatticesApi.shared.dispatch(method: "tmux.sessions", params: nil)
377
+ }
378
+ ))
379
+
380
+ // ── Distribute Windows ──────────────────────────────────
381
+
382
+ register(IntentDef(
383
+ name: "distribute",
384
+ description: "Distribute windows evenly in a grid, optionally filtered by app and constrained to a screen region",
385
+ examples: [
386
+ "spread out the windows",
387
+ "distribute everything",
388
+ "organize the windows",
389
+ "clean up the layout",
390
+ "grid the terminals on the right",
391
+ "tile all iTerm windows on the left half",
392
+ "arrange my chrome windows in the bottom"
393
+ ],
394
+ slots: [
395
+ IntentSlot(name: "app", type: "string", required: false,
396
+ description: "Filter to windows of this app (e.g. 'iTerm2', 'Google Chrome')", enumValues: nil),
397
+ IntentSlot(name: "region", type: "position", required: false,
398
+ description: "Constrain the grid to a screen region. Uses tile position names.",
399
+ enumValues: ["left", "right", "top", "bottom", "top-left", "top-right", "bottom-left", "bottom-right",
400
+ "left-third", "center-third", "right-third"]),
401
+ ],
402
+ handler: { req in
403
+ var params: [String: JSON] = [:]
404
+ if let app = req.slots["app"]?.stringValue {
405
+ params["app"] = .string(app)
406
+ }
407
+ if let region = req.slots["region"]?.stringValue {
408
+ params["region"] = .string(region)
409
+ }
410
+ return try LatticesApi.shared.dispatch(
411
+ method: "layout.distribute",
412
+ params: params.isEmpty ? nil : .object(params)
413
+ )
414
+ }
415
+ ))
416
+
417
+ // ── Create Layer ────────────────────────────────────────
418
+
419
+ register(IntentDef(
420
+ name: "create_layer",
421
+ description: "Create a new session layer from current windows",
422
+ examples: [
423
+ "save this layout as review",
424
+ "create a layer called deploy",
425
+ "make a new layer"
426
+ ],
427
+ slots: [
428
+ IntentSlot(name: "name", type: "string", required: true,
429
+ description: "Name for the new layer", enumValues: nil),
430
+ IntentSlot(name: "capture_visible", type: "bool", required: false,
431
+ description: "Auto-capture visible windows into the layer", enumValues: nil),
432
+ ],
433
+ handler: { req in
434
+ guard let name = req.slots["name"]?.stringValue else {
435
+ throw IntentError.missingSlot("name")
436
+ }
437
+
438
+ var windowIds: [JSON] = []
439
+ if req.slots["capture_visible"]?.boolValue == true {
440
+ for entry in DesktopModel.shared.windows.values where entry.isOnScreen {
441
+ windowIds.append(.int(Int(entry.wid)))
442
+ }
443
+ }
444
+
445
+ return try LatticesApi.shared.dispatch(
446
+ method: "session.layers.create",
447
+ params: .object([
448
+ "name": .string(name),
449
+ "windowIds": .array(windowIds)
450
+ ])
451
+ )
452
+ }
453
+ ))
454
+
455
+ // ── Kill Session ────────────────────────────────────────
456
+
457
+ register(IntentDef(
458
+ name: "kill",
459
+ description: "Kill a terminal session",
460
+ examples: [
461
+ "stop the frontend session",
462
+ "kill the API",
463
+ "shut down that project"
464
+ ],
465
+ slots: [
466
+ IntentSlot(name: "session", type: "string", required: true,
467
+ description: "Session name or project name", enumValues: nil),
468
+ ],
469
+ handler: { req in
470
+ guard let session = req.slots["session"]?.stringValue else {
471
+ throw IntentError.missingSlot("session")
472
+ }
473
+
474
+ // Try direct name first
475
+ let sessions = try LatticesApi.shared.dispatch(method: "tmux.sessions", params: nil)
476
+ if case .array(let list) = sessions {
477
+ for s in list {
478
+ let name = s["name"]?.stringValue ?? ""
479
+ if name.localizedCaseInsensitiveContains(session) {
480
+ return try LatticesApi.shared.dispatch(
481
+ method: "session.kill",
482
+ params: .object(["name": .string(name)])
483
+ )
484
+ }
485
+ }
486
+ }
487
+ throw IntentError.targetNotFound("No session matching '\(session)'")
488
+ }
489
+ ))
490
+
491
+ // ── Scan (trigger OCR) ──────────────────────────────────
492
+
493
+ register(IntentDef(
494
+ name: "scan",
495
+ description: "Trigger an immediate screen text scan",
496
+ examples: [
497
+ "scan the screen",
498
+ "read what's on screen",
499
+ "update OCR"
500
+ ],
501
+ slots: [],
502
+ handler: { _ in
503
+ try LatticesApi.shared.dispatch(method: "ocr.scan", params: nil)
504
+ }
505
+ ))
506
+
507
+ // ── Swap Windows ───────────────────────────────────────
508
+
509
+ register(IntentDef(
510
+ name: "swap",
511
+ description: "Swap the positions of two windows",
512
+ examples: [
513
+ "swap Chrome and iTerm",
514
+ "switch those two",
515
+ "swap the left and right windows"
516
+ ],
517
+ slots: [
518
+ IntentSlot(name: "wid_a", type: "int", required: true,
519
+ description: "Window ID of the first window", enumValues: nil),
520
+ IntentSlot(name: "wid_b", type: "int", required: true,
521
+ description: "Window ID of the second window", enumValues: nil),
522
+ ],
523
+ handler: { req in
524
+ guard let widA = req.slots["wid_a"]?.uint32Value,
525
+ let widB = req.slots["wid_b"]?.uint32Value else {
526
+ throw IntentError.missingSlot("wid_a and wid_b")
527
+ }
528
+ guard let entryA = DesktopModel.shared.windows[widA],
529
+ let entryB = DesktopModel.shared.windows[widB] else {
530
+ throw IntentError.targetNotFound("One or both windows not found")
531
+ }
532
+
533
+ // Read current CG frames (top-left origin) directly from CGWindowList
534
+ guard let windowList = CGWindowListCopyWindowInfo([.optionAll, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]] else {
535
+ throw IntentError.targetNotFound("Couldn't read window list")
536
+ }
537
+ var cgFrames: [UInt32: CGRect] = [:]
538
+ for info in windowList {
539
+ guard let num = info[kCGWindowNumber as String] as? UInt32,
540
+ (num == widA || num == widB),
541
+ let dict = info[kCGWindowBounds as String] as? NSDictionary else { continue }
542
+ var rect = CGRect.zero
543
+ if CGRectMakeWithDictionaryRepresentation(dict, &rect) {
544
+ cgFrames[num] = rect
545
+ }
546
+ }
547
+ guard let frameA = cgFrames[widA], let frameB = cgFrames[widB] else {
548
+ throw IntentError.targetNotFound("Couldn't read window frames")
549
+ }
550
+
551
+ // Swap: move A to B's frame, B to A's frame
552
+ let moves: [(wid: UInt32, pid: Int32, frame: CGRect)] = [
553
+ (wid: widA, pid: entryA.pid, frame: frameB),
554
+ (wid: widB, pid: entryB.pid, frame: frameA),
555
+ ]
556
+ DispatchQueue.main.async {
557
+ WindowTiler.batchMoveAndRaiseWindows(moves)
558
+ }
559
+ return .object([
560
+ "ok": .bool(true),
561
+ "swapped": .array([.int(Int(widA)), .int(Int(widB))]),
562
+ ])
563
+ }
564
+ ))
565
+
566
+ // ── Hide / Minimize ────────────────────────────────────
567
+
568
+ register(IntentDef(
569
+ name: "hide",
570
+ description: "Hide or minimize a window or app",
571
+ examples: [
572
+ "hide Slack",
573
+ "minimize that",
574
+ "put away Messages",
575
+ "hide the browser"
576
+ ],
577
+ slots: [
578
+ IntentSlot(name: "app", type: "string", required: false,
579
+ description: "App name to hide", enumValues: nil),
580
+ IntentSlot(name: "wid", type: "int", required: false,
581
+ description: "Window ID to minimize", enumValues: nil),
582
+ ],
583
+ handler: { req in
584
+ // Hide by wid — minimize just that window via AX
585
+ if let wid = req.slots["wid"]?.uint32Value,
586
+ let entry = DesktopModel.shared.windows[wid] {
587
+ let appRef = AXUIElementCreateApplication(entry.pid)
588
+ var windowsRef: CFTypeRef?
589
+ if AXUIElementCopyAttributeValue(appRef, kAXWindowsAttribute as CFString, &windowsRef) == .success,
590
+ let axWindows = windowsRef as? [AXUIElement] {
591
+ for axWin in axWindows {
592
+ var windowId: CGWindowID = 0
593
+ if _AXUIElementGetWindow(axWin, &windowId) == .success, windowId == wid {
594
+ AXUIElementSetAttributeValue(axWin, kAXMinimizedAttribute as CFString, kCFBooleanTrue)
595
+ return .object(["ok": .bool(true), "action": .string("minimized"), "wid": .int(Int(wid))])
596
+ }
597
+ }
598
+ }
599
+ throw IntentError.targetNotFound("Couldn't find AX window for wid \(wid)")
600
+ }
601
+
602
+ // Hide by app name — hide the entire app
603
+ if let appName = req.slots["app"]?.stringValue {
604
+ let apps = NSWorkspace.shared.runningApplications
605
+ if let app = apps.first(where: {
606
+ ($0.localizedName ?? "").localizedCaseInsensitiveContains(appName)
607
+ }) {
608
+ app.hide()
609
+ return .object(["ok": .bool(true), "action": .string("hidden"), "app": .string(app.localizedName ?? appName)])
610
+ }
611
+ throw IntentError.targetNotFound("No running app matching '\(appName)'")
612
+ }
613
+
614
+ throw IntentError.missingSlot("app or wid")
615
+ }
616
+ ))
617
+
618
+ // ── Highlight ──────────────────────────────────────────
619
+
620
+ register(IntentDef(
621
+ name: "highlight",
622
+ description: "Flash a window's border to identify it visually",
623
+ examples: [
624
+ "which one is the lattices terminal",
625
+ "highlight Chrome",
626
+ "show me that window",
627
+ "flash the iTerm window"
628
+ ],
629
+ slots: [
630
+ IntentSlot(name: "wid", type: "int", required: false,
631
+ description: "Window ID to highlight", enumValues: nil),
632
+ IntentSlot(name: "app", type: "string", required: false,
633
+ description: "App name to highlight", enumValues: nil),
634
+ ],
635
+ handler: { req in
636
+ if let wid = req.slots["wid"]?.uint32Value {
637
+ DispatchQueue.main.async {
638
+ WindowTiler.highlightWindowById(wid: wid)
639
+ }
640
+ return .object(["ok": .bool(true), "wid": .int(Int(wid))])
641
+ }
642
+
643
+ if let appName = req.slots["app"]?.stringValue {
644
+ if let entry = DesktopModel.shared.windows.values.first(where: {
645
+ $0.app.localizedCaseInsensitiveContains(appName)
646
+ }) {
647
+ DispatchQueue.main.async {
648
+ WindowTiler.highlightWindowById(wid: entry.wid)
649
+ }
650
+ return .object(["ok": .bool(true), "wid": .int(Int(entry.wid)), "app": .string(entry.app)])
651
+ }
652
+ throw IntentError.targetNotFound("No window found for app '\(appName)'")
653
+ }
654
+
655
+ throw IntentError.missingSlot("wid or app")
656
+ }
657
+ ))
658
+
659
+ // ── Move to Display ────────────────────────────────────
660
+
661
+ register(IntentDef(
662
+ name: "move_to_display",
663
+ description: "Move a window to another monitor/display, optionally positioning it",
664
+ examples: [
665
+ "put this on the vertical monitor",
666
+ "move Chrome to the second display",
667
+ "send iTerm to the other screen",
668
+ "move that to my main monitor"
669
+ ],
670
+ slots: [
671
+ IntentSlot(name: "wid", type: "int", required: false,
672
+ description: "Window ID to move", enumValues: nil),
673
+ IntentSlot(name: "app", type: "string", required: false,
674
+ description: "App name to move", enumValues: nil),
675
+ IntentSlot(name: "display", type: "int", required: true,
676
+ description: "Target display index (0 = main, 1 = second, etc.)", enumValues: nil),
677
+ IntentSlot(name: "position", type: "position", required: false,
678
+ description: "Tile position on the target display (e.g. 'left', 'maximize')",
679
+ enumValues: ["left", "right", "top", "bottom", "maximize", "center",
680
+ "top-left", "top-right", "bottom-left", "bottom-right"]),
681
+ ],
682
+ handler: { req in
683
+ guard let display = req.slots["display"]?.intValue else {
684
+ throw IntentError.missingSlot("display")
685
+ }
686
+
687
+ // Resolve window target
688
+ let wid: UInt32
689
+ if let w = req.slots["wid"]?.uint32Value {
690
+ wid = w
691
+ } else if let appName = req.slots["app"]?.stringValue,
692
+ let entry = DesktopModel.shared.windows.values.first(where: {
693
+ $0.app.localizedCaseInsensitiveContains(appName)
694
+ }) {
695
+ wid = entry.wid
696
+ } else {
697
+ // Frontmost window
698
+ guard let frontApp = NSWorkspace.shared.frontmostApplication,
699
+ frontApp.bundleIdentifier != "com.arach.lattices" else {
700
+ throw IntentError.targetNotFound("No frontmost window")
701
+ }
702
+ let appRef = AXUIElementCreateApplication(frontApp.processIdentifier)
703
+ var focusedRef: CFTypeRef?
704
+ guard AXUIElementCopyAttributeValue(appRef, kAXFocusedWindowAttribute as CFString, &focusedRef) == .success else {
705
+ throw IntentError.targetNotFound("No focused window")
706
+ }
707
+ var frontWid: CGWindowID = 0
708
+ guard _AXUIElementGetWindow(focusedRef as! AXUIElement, &frontWid) == .success else {
709
+ throw IntentError.targetNotFound("Couldn't get frontmost window ID")
710
+ }
711
+ wid = frontWid
712
+ }
713
+
714
+ // Use window.present with display + optional position
715
+ var params: [String: JSON] = [
716
+ "wid": .int(Int(wid)),
717
+ "display": .int(display),
718
+ ]
719
+ if let pos = req.slots["position"]?.stringValue {
720
+ params["position"] = .string(pos)
721
+ }
722
+ return try LatticesApi.shared.dispatch(method: "window.present", params: .object(params))
723
+ }
724
+ ))
725
+
726
+ // ── Find / Summon Mouse ────────────────────────────────
727
+
728
+ register(IntentDef(
729
+ name: "find_mouse",
730
+ description: "Show a sonar pulse at the current mouse cursor position",
731
+ examples: [
732
+ "where's my mouse",
733
+ "find the cursor",
734
+ "I lost my mouse",
735
+ "find mouse",
736
+ "show cursor"
737
+ ],
738
+ slots: [],
739
+ handler: { _ in
740
+ DispatchQueue.main.async { MouseFinder.shared.find() }
741
+ let pos = NSEvent.mouseLocation
742
+ return .object(["ok": .bool(true), "x": .int(Int(pos.x)), "y": .int(Int(pos.y))])
743
+ }
744
+ ))
745
+
746
+ register(IntentDef(
747
+ name: "summon_mouse",
748
+ description: "Warp the mouse cursor to the center of the screen",
749
+ examples: [
750
+ "summon mouse",
751
+ "bring the cursor here",
752
+ "center the mouse",
753
+ "mouse come here",
754
+ "bring mouse back"
755
+ ],
756
+ slots: [],
757
+ handler: { _ in
758
+ DispatchQueue.main.async { MouseFinder.shared.summon() }
759
+ return .object(["ok": .bool(true)])
760
+ }
761
+ ))
762
+
763
+ // ── Undo / Restore ─────────────────────────────────────
764
+
765
+ register(IntentDef(
766
+ name: "undo",
767
+ description: "Undo the last window move — restore windows to their previous positions",
768
+ examples: [
769
+ "put it back",
770
+ "undo that",
771
+ "restore the windows",
772
+ "that was wrong, undo"
773
+ ],
774
+ slots: [],
775
+ handler: { _ in
776
+ let history = HandsOffSession.shared.frameHistory
777
+ guard !history.isEmpty else {
778
+ throw IntentError.targetNotFound("No window moves to undo")
779
+ }
780
+
781
+ let restores = history.map { (wid: $0.wid, pid: $0.pid, frame: $0.frame) }
782
+ DispatchQueue.main.async {
783
+ WindowTiler.batchRestoreWindows(restores)
784
+ }
785
+ HandsOffSession.shared.clearFrameHistory()
786
+ return .object([
787
+ "ok": .bool(true),
788
+ "restored": .int(restores.count),
789
+ ])
790
+ }
791
+ ))
792
+ }
793
+ }
794
+
795
+ // MARK: - Errors
796
+
797
+ enum IntentError: LocalizedError {
798
+ case unknownIntent(String, available: [String])
799
+ case missingSlot(String)
800
+ case invalidSlot(String)
801
+ case targetNotFound(String)
802
+
803
+ var errorDescription: String? {
804
+ switch self {
805
+ case .unknownIntent(let name, let available):
806
+ return "Unknown intent '\(name)'. Available: \(available.joined(separator: ", "))"
807
+ case .missingSlot(let name):
808
+ return "Missing required slot: \(name)"
809
+ case .invalidSlot(let detail):
810
+ return detail
811
+ case .targetNotFound(let detail):
812
+ return detail
813
+ }
814
+ }
815
+ }
816
+
817
+ // MARK: - Claude CLI Fallback
818
+
819
+ struct ClaudeResolvedIntent {
820
+ let intent: String
821
+ let slots: [String: JSON]
822
+ }
823
+
824
+ struct ClaudeAgentPlan {
825
+ let steps: [ClaudeResolvedIntent]
826
+ let reasoning: String
827
+ }
828
+
829
+ enum ClaudeFallback {
830
+
831
+ private static var claudePath: String? { Preferences.resolveClaudePath() }
832
+
833
+ /// Shell out to Claude CLI to resolve a voice command transcript into an intent + slots.
834
+ /// Runs synchronously — call from a background thread.
835
+ static func resolve(
836
+ transcript: String,
837
+ windows: [WindowEntry],
838
+ intentCatalog: JSON
839
+ ) -> ClaudeResolvedIntent? {
840
+
841
+ let timer = DiagnosticLog.shared.startTimed("Claude fallback")
842
+
843
+ // Build window context (compact)
844
+ // Compact window list — just app and title, max 20
845
+ let windowList = windows.prefix(20).map { "\($0.app): \($0.title)" }.joined(separator: "\n")
846
+
847
+ // Compact intent list — just name and slot names
848
+ var intentList = ""
849
+ if case .array(let intents) = intentCatalog {
850
+ for intent in intents {
851
+ let name = intent["intent"]?.stringValue ?? ""
852
+ var slotNames: [String] = []
853
+ if case .array(let slots) = intent["slots"] {
854
+ slotNames = slots.compactMap { $0["name"]?.stringValue }
855
+ }
856
+ let s = slotNames.isEmpty ? "" : "(\(slotNames.joined(separator: ",")))"
857
+ intentList += "\(name)\(s), "
858
+ }
859
+ }
860
+
861
+ let prompt = """
862
+ Voice command resolver. Whisper transcript (may have typos): "\(transcript)"
863
+ Intents: \(intentList.trimmingCharacters(in: .init(charactersIn: ", ")))
864
+ Windows: \(windowList)
865
+ Return ONLY a JSON object like {"intent":"search","slots":{"query":"dewey"},"reasoning":"user wants to find dewey windows"}. For search, extract the key term. Use window names from the list. If unclear, use intent "unknown".
866
+ """
867
+
868
+ guard let path = claudePath else {
869
+ DiagnosticLog.shared.warn("ClaudeFallback: claude CLI not found")
870
+ DiagnosticLog.shared.finish(timer)
871
+ return nil
872
+ }
873
+
874
+ let proc = Process()
875
+ proc.executableURL = URL(fileURLWithPath: path)
876
+
877
+ proc.arguments = [
878
+ "-p", prompt,
879
+ "--model", "haiku",
880
+ "--output-format", "text",
881
+ "--no-session-persistence",
882
+ "--max-budget-usd", "0.50",
883
+ ]
884
+
885
+ // Clear CLAUDECODE env var to allow nested invocation
886
+ var env = ProcessInfo.processInfo.environment
887
+ env.removeValue(forKey: "CLAUDECODE")
888
+ proc.environment = env
889
+
890
+ let pipe = Pipe()
891
+ let errPipe = Pipe()
892
+ proc.standardOutput = pipe
893
+ proc.standardError = errPipe
894
+
895
+ do {
896
+ try proc.run()
897
+ } catch {
898
+ DiagnosticLog.shared.warn("ClaudeFallback: failed to launch claude CLI — \(error)")
899
+ return nil
900
+ }
901
+
902
+ proc.waitUntilExit()
903
+ let exitCode = proc.terminationStatus
904
+ DiagnosticLog.shared.finish(timer)
905
+ DiagnosticLog.shared.info("ClaudeFallback: exit code \(exitCode)")
906
+
907
+ let output = String(data: pipe.fileHandleForReading.readDataToEndOfFile(), encoding: .utf8)?
908
+ .trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
909
+ let errOutput = String(data: errPipe.fileHandleForReading.readDataToEndOfFile(), encoding: .utf8)?
910
+ .trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
911
+
912
+ if !errOutput.isEmpty {
913
+ DiagnosticLog.shared.warn("ClaudeFallback: stderr → \(errOutput.prefix(200))")
914
+ }
915
+ DiagnosticLog.shared.info("ClaudeFallback: raw output → \(output.prefix(300))")
916
+
917
+ // Parse JSON from text output
918
+ guard let jsonStr = extractJSON(from: output),
919
+ let jsonData = jsonStr.data(using: .utf8),
920
+ let json = try? JSONSerialization.jsonObject(with: jsonData) as? [String: Any],
921
+ let intent = json["intent"] as? String,
922
+ intent != "unknown" else {
923
+ DiagnosticLog.shared.info("ClaudeFallback: couldn't parse response")
924
+ return nil
925
+ }
926
+
927
+ if let reasoning = json["reasoning"] as? String {
928
+ DiagnosticLog.shared.info("ClaudeFallback: reasoning → \(reasoning)")
929
+ }
930
+
931
+ // Convert slots
932
+ var slots: [String: JSON] = [:]
933
+ if let rawSlots = json["slots"] as? [String: Any] {
934
+ for (key, value) in rawSlots {
935
+ if let s = value as? String {
936
+ slots[key] = .string(s)
937
+ } else if let n = value as? Int {
938
+ slots[key] = .int(n)
939
+ } else if let b = value as? Bool {
940
+ slots[key] = .bool(b)
941
+ }
942
+ }
943
+ }
944
+
945
+ return ClaudeResolvedIntent(intent: intent, slots: slots)
946
+ }
947
+
948
+ private static func extractJSON(from text: String) -> String? {
949
+ // Try to find JSON object in the response
950
+ // Claude might return it directly, or wrapped in ```json ... ```
951
+ let cleaned = text
952
+ .replacingOccurrences(of: "```json", with: "")
953
+ .replacingOccurrences(of: "```", with: "")
954
+ .trimmingCharacters(in: .whitespacesAndNewlines)
955
+
956
+ // Find first { and last }
957
+ guard let start = cleaned.firstIndex(of: "{"),
958
+ let end = cleaned.lastIndex(of: "}") else { return nil }
959
+
960
+ return String(cleaned[start...end])
961
+ }
962
+ }