@lattices/cli 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. package/README.md +85 -9
  2. package/app/Package.swift +8 -1
  3. package/app/Sources/AdvisorLearningStore.swift +90 -0
  4. package/app/Sources/AgentSession.swift +377 -0
  5. package/app/Sources/AppDelegate.swift +44 -12
  6. package/app/Sources/AppShellView.swift +81 -8
  7. package/app/Sources/AudioProvider.swift +386 -0
  8. package/app/Sources/CheatSheetHUD.swift +261 -19
  9. package/app/Sources/DaemonProtocol.swift +13 -0
  10. package/app/Sources/DaemonServer.swift +8 -0
  11. package/app/Sources/DesktopModel.swift +164 -5
  12. package/app/Sources/DesktopModelTypes.swift +2 -0
  13. package/app/Sources/DiagnosticLog.swift +104 -2
  14. package/app/Sources/EventBus.swift +1 -0
  15. package/app/Sources/HUDBottomBar.swift +279 -0
  16. package/app/Sources/HUDController.swift +1158 -0
  17. package/app/Sources/HUDLeftBar.swift +849 -0
  18. package/app/Sources/HUDMinimap.swift +179 -0
  19. package/app/Sources/HUDRightBar.swift +774 -0
  20. package/app/Sources/HUDState.swift +367 -0
  21. package/app/Sources/HUDTopBar.swift +243 -0
  22. package/app/Sources/HandsOffSession.swift +733 -0
  23. package/app/Sources/HomeDashboardView.swift +125 -0
  24. package/app/Sources/HotkeyManager.swift +2 -0
  25. package/app/Sources/HotkeyStore.swift +45 -9
  26. package/app/Sources/IntentEngine.swift +925 -0
  27. package/app/Sources/Intents/CreateLayerIntent.swift +54 -0
  28. package/app/Sources/Intents/DistributeIntent.swift +56 -0
  29. package/app/Sources/Intents/FocusIntent.swift +69 -0
  30. package/app/Sources/Intents/HelpIntent.swift +41 -0
  31. package/app/Sources/Intents/KillIntent.swift +47 -0
  32. package/app/Sources/Intents/LatticeIntent.swift +78 -0
  33. package/app/Sources/Intents/LaunchIntent.swift +67 -0
  34. package/app/Sources/Intents/ListSessionsIntent.swift +32 -0
  35. package/app/Sources/Intents/ListWindowsIntent.swift +30 -0
  36. package/app/Sources/Intents/ScanIntent.swift +52 -0
  37. package/app/Sources/Intents/SearchIntent.swift +190 -0
  38. package/app/Sources/Intents/SwitchLayerIntent.swift +50 -0
  39. package/app/Sources/Intents/TileIntent.swift +61 -0
  40. package/app/Sources/LatticesApi.swift +1235 -30
  41. package/app/Sources/LauncherHUD.swift +348 -0
  42. package/app/Sources/MainView.swift +147 -44
  43. package/app/Sources/OcrModel.swift +34 -1
  44. package/app/Sources/OmniSearchState.swift +99 -102
  45. package/app/Sources/OnboardingView.swift +457 -0
  46. package/app/Sources/PermissionChecker.swift +2 -12
  47. package/app/Sources/PiChatDock.swift +454 -0
  48. package/app/Sources/PiChatSession.swift +815 -0
  49. package/app/Sources/PiWorkspaceView.swift +364 -0
  50. package/app/Sources/PlacementSpec.swift +195 -0
  51. package/app/Sources/Preferences.swift +59 -0
  52. package/app/Sources/ProjectScanner.swift +1 -1
  53. package/app/Sources/ScreenMapState.swift +701 -55
  54. package/app/Sources/ScreenMapView.swift +843 -103
  55. package/app/Sources/ScreenMapWindowController.swift +22 -0
  56. package/app/Sources/SessionLayerStore.swift +285 -0
  57. package/app/Sources/SessionManager.swift +4 -1
  58. package/app/Sources/SettingsView.swift +186 -3
  59. package/app/Sources/Theme.swift +9 -8
  60. package/app/Sources/TmuxModel.swift +7 -0
  61. package/app/Sources/TmuxQuery.swift +27 -3
  62. package/app/Sources/VoiceChatView.swift +192 -0
  63. package/app/Sources/VoiceCommandWindow.swift +1594 -0
  64. package/app/Sources/VoiceIntentResolver.swift +671 -0
  65. package/app/Sources/VoxClient.swift +454 -0
  66. package/app/Sources/WindowTiler.swift +348 -87
  67. package/app/Sources/WorkspaceManager.swift +127 -18
  68. package/bin/client.ts +16 -0
  69. package/bin/{daemon-client.js → daemon-client.ts} +49 -30
  70. package/bin/handsoff-infer.ts +280 -0
  71. package/bin/handsoff-worker.ts +731 -0
  72. package/bin/{lattices-app.js → lattices-app.ts} +67 -32
  73. package/bin/lattices-dev +160 -0
  74. package/bin/{lattices.js → lattices.ts} +600 -137
  75. package/bin/project-twin.ts +645 -0
  76. package/docs/agent-execution-plan.md +562 -0
  77. package/docs/agents.md +142 -0
  78. package/docs/api.md +153 -34
  79. package/docs/app.md +29 -1
  80. package/docs/config.md +5 -1
  81. package/docs/handsoff-test-scenarios.md +84 -0
  82. package/docs/layers.md +20 -20
  83. package/docs/ocr.md +14 -5
  84. package/docs/overview.md +5 -1
  85. package/docs/presentation-execution-review.md +491 -0
  86. package/docs/prompts/hands-off-system.md +374 -0
  87. package/docs/prompts/hands-off-turn.md +30 -0
  88. package/docs/prompts/voice-advisor.md +31 -0
  89. package/docs/prompts/voice-fallback.md +23 -0
  90. package/docs/tiling-reference.md +167 -0
  91. package/docs/twins.md +138 -0
  92. package/docs/voice-command-protocol.md +278 -0
  93. package/docs/voice.md +219 -0
  94. package/package.json +21 -10
  95. package/bin/client.js +0 -4
@@ -0,0 +1,454 @@
1
+ import AppKit
2
+
3
+ /// WebSocket JSON-RPC client for the Vox transcription runtime.
4
+ ///
5
+ /// Vox is a local-first transcription daemon (voxd) that runs on a configurable port
6
+ /// (default 42137). Service discovery is file-based via ~/.vox/runtime.json.
7
+ ///
8
+ /// Key differences from the old Talkie integration:
9
+ /// - Discovery: ~/.vox/runtime.json (not ~/.talkie/services.json)
10
+ /// - Port: 42137 (not 19823)
11
+ /// - No distributed notifications — poll runtime.json or check on demand
12
+ /// - API: transcribe.startSession/stopSession (not startDictation/stopDictation)
13
+ /// - No register call — pass clientId per request
14
+ /// - All session events flow on the startSession call ID
15
+ final class VoxClient: ObservableObject {
16
+ static let shared = VoxClient()
17
+
18
+ enum ConnectionState: Equatable {
19
+ case disconnected
20
+ case connecting
21
+ case connected
22
+ case unavailable(reason: String)
23
+ }
24
+
25
+ @Published var connectionState: ConnectionState = .disconnected
26
+
27
+ static let clientId = "lattices"
28
+
29
+ private var pendingCalls: [String: PendingCall] = [:]
30
+ private var eventHandler: ((String, [String: Any]) -> Void)?
31
+ private var reconnectDelay: TimeInterval = 0.5
32
+ private var reconnectTimer: DispatchSourceTimer?
33
+ private var heartbeatTimer: DispatchSourceTimer?
34
+ private var intentionalDisconnect = false
35
+ private let queue = DispatchQueue(label: "com.lattices.vox-client")
36
+
37
+ private struct PendingCall {
38
+ let completion: (Result<[String: Any], VoxError>) -> Void
39
+ let onProgress: ((String, [String: Any]) -> Void)?
40
+ let timer: DispatchSourceTimer?
41
+ }
42
+
43
+ enum VoxError: LocalizedError {
44
+ case notConnected
45
+ case callFailed(String)
46
+ case timeout(String)
47
+ case sessionBusy
48
+ case connectionDropped
49
+ case daemonNotRunning
50
+
51
+ var errorDescription: String? {
52
+ switch self {
53
+ case .notConnected: return "Not connected to Vox"
54
+ case .callFailed(let msg): return msg
55
+ case .timeout(let method): return "Call to '\(method)' timed out"
56
+ case .sessionBusy: return "A live session is already active"
57
+ case .connectionDropped: return "Connection to Vox dropped"
58
+ case .daemonNotRunning: return "Vox daemon not running — start with 'vox daemon start'"
59
+ }
60
+ }
61
+ }
62
+
63
+ // MARK: - Service Discovery (file-based via ~/.vox/runtime.json)
64
+
65
+ private static let defaultPort: UInt16 = 42137
66
+ private static let runtimePath = NSHomeDirectory() + "/.vox/runtime.json"
67
+
68
+ struct RuntimeInfo {
69
+ let port: UInt16
70
+ let pid: Int
71
+ let version: String
72
+ }
73
+
74
+ /// Read ~/.vox/runtime.json and check if the daemon is alive.
75
+ func discoverDaemon() -> RuntimeInfo? {
76
+ guard let data = FileManager.default.contents(atPath: Self.runtimePath),
77
+ let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any],
78
+ let port = json["port"] as? Int,
79
+ let pid = json["pid"] as? Int else {
80
+ return nil
81
+ }
82
+
83
+ // Verify the PID is still alive
84
+ let alive = kill(Int32(pid), 0) == 0
85
+ guard alive else {
86
+ DiagnosticLog.shared.warn("VoxClient: stale runtime.json — pid \(pid) not running")
87
+ return nil
88
+ }
89
+
90
+ let version = json["version"] as? String ?? "unknown"
91
+ return RuntimeInfo(port: UInt16(port), pid: pid, version: version)
92
+ }
93
+
94
+ // MARK: - Connection
95
+
96
+ func connect() {
97
+ if connectionState == .connected || connectionState == .connecting { return }
98
+
99
+ intentionalDisconnect = false
100
+
101
+ guard let runtime = discoverDaemon() else {
102
+ DiagnosticLog.shared.warn("VoxClient: daemon not found — check ~/.vox/runtime.json")
103
+ DispatchQueue.main.async {
104
+ self.connectionState = .unavailable(reason: "Vox daemon not running")
105
+ }
106
+ return
107
+ }
108
+
109
+ DiagnosticLog.shared.info("VoxClient: discovered daemon v\(runtime.version) on port \(runtime.port) (pid \(runtime.pid))")
110
+ connectToPort(runtime.port)
111
+ }
112
+
113
+ func disconnect() {
114
+ intentionalDisconnect = true
115
+ reconnectTimer?.cancel()
116
+ reconnectTimer = nil
117
+ heartbeatTimer?.cancel()
118
+ heartbeatTimer = nil
119
+ wsTask?.cancel(with: .goingAway, reason: nil)
120
+ wsTask = nil
121
+ wsSession?.invalidateAndCancel()
122
+ wsSession = nil
123
+ pendingCalls.removeAll()
124
+ DispatchQueue.main.async {
125
+ self.connectionState = .disconnected
126
+ }
127
+ }
128
+
129
+ /// Force a full disconnect + reconnect cycle.
130
+ func reconnect() {
131
+ DiagnosticLog.shared.info("VoxClient: forced reconnect requested")
132
+ disconnect()
133
+ DispatchQueue.main.asyncAfter(deadline: .now() + 0.3) { [weak self] in
134
+ self?.connect()
135
+ }
136
+ }
137
+
138
+ private var wsTask: URLSessionWebSocketTask?
139
+ private var wsSession: URLSession?
140
+
141
+ private func connectToPort(_ port: UInt16) {
142
+ DispatchQueue.main.async {
143
+ self.connectionState = .connecting
144
+ }
145
+
146
+ let connectStart = Date()
147
+ DiagnosticLog.shared.info("VoxClient: connecting to ws://127.0.0.1:\(port)")
148
+
149
+ let url = URL(string: "ws://127.0.0.1:\(port)")!
150
+ let config = URLSessionConfiguration.default
151
+ config.connectionProxyDictionary = [:]
152
+ let session = URLSession(configuration: config)
153
+ let task = session.webSocketTask(with: url)
154
+
155
+ self.wsSession = session
156
+ self.wsTask = task
157
+ task.resume()
158
+
159
+ // Verify with a health check instead of raw ping
160
+ task.sendPing { [weak self] error in
161
+ guard let self else { return }
162
+ let ms = Int(Date().timeIntervalSince(connectStart) * 1000)
163
+ if let error {
164
+ DiagnosticLog.shared.warn("VoxClient: WebSocket ping failed (\(ms)ms) — \(error)")
165
+ self.handleDisconnect()
166
+ } else {
167
+ self.reconnectDelay = 0.5
168
+ DiagnosticLog.shared.info("VoxClient: connected on port \(port) (\(ms)ms)")
169
+ self.receiveLoop()
170
+ self.startHeartbeat()
171
+ DispatchQueue.main.async {
172
+ self.connectionState = .connected
173
+ }
174
+ // Verify with a health RPC
175
+ self.call(method: "health") { result in
176
+ switch result {
177
+ case .success(let data):
178
+ let svc = data["serviceName"] as? String ?? "?"
179
+ let ver = data["version"] as? String ?? "?"
180
+ DiagnosticLog.shared.info("VoxClient: health OK — \(svc) v\(ver)")
181
+ case .failure(let error):
182
+ DiagnosticLog.shared.warn("VoxClient: health check failed — \(error.localizedDescription)")
183
+ }
184
+ }
185
+ }
186
+ }
187
+ }
188
+
189
+ /// Periodic WebSocket ping every 30s to detect dead connections early.
190
+ private func startHeartbeat() {
191
+ heartbeatTimer?.cancel()
192
+ let timer = DispatchSource.makeTimerSource(queue: queue)
193
+ timer.schedule(deadline: .now() + 30, repeating: 30)
194
+ timer.setEventHandler { [weak self] in
195
+ guard let self, let task = self.wsTask else { return }
196
+ task.sendPing { error in
197
+ if let error {
198
+ DiagnosticLog.shared.warn("VoxClient: heartbeat failed — \(error)")
199
+ self.handleDisconnect()
200
+ }
201
+ }
202
+ }
203
+ timer.resume()
204
+ heartbeatTimer = timer
205
+ }
206
+
207
+ private func handleDisconnect() {
208
+ heartbeatTimer?.cancel()
209
+ heartbeatTimer = nil
210
+ wsTask?.cancel(with: .goingAway, reason: nil)
211
+ wsTask = nil
212
+
213
+ for (_, pending) in pendingCalls {
214
+ pending.timer?.cancel()
215
+ pending.completion(.failure(.connectionDropped))
216
+ }
217
+ pendingCalls.removeAll()
218
+
219
+ DispatchQueue.main.async {
220
+ self.connectionState = .disconnected
221
+ }
222
+
223
+ guard !intentionalDisconnect else { return }
224
+
225
+ let delay = reconnectDelay
226
+ reconnectDelay = min(reconnectDelay * 2, 10)
227
+ DiagnosticLog.shared.info("VoxClient: reconnecting in \(delay)s")
228
+
229
+ let timer = DispatchSource.makeTimerSource(queue: queue)
230
+ timer.schedule(deadline: .now() + delay)
231
+ timer.setEventHandler { [weak self] in
232
+ self?.connect()
233
+ }
234
+ timer.resume()
235
+ reconnectTimer = timer
236
+ }
237
+
238
+ // MARK: - WebSocket I/O
239
+
240
+ private func receiveLoop() {
241
+ guard let task = wsTask else { return }
242
+
243
+ task.receive { [weak self] result in
244
+ guard let self else { return }
245
+ switch result {
246
+ case .success(let message):
247
+ switch message {
248
+ case .string(let text): self.handleMessage(text)
249
+ case .data(let data):
250
+ if let text = String(data: data, encoding: .utf8) { self.handleMessage(text) }
251
+ @unknown default: break
252
+ }
253
+ self.receiveLoop()
254
+ case .failure(let error):
255
+ DiagnosticLog.shared.warn("VoxClient: receive error — \(error)")
256
+ self.handleDisconnect()
257
+ }
258
+ }
259
+ }
260
+
261
+ private func handleMessage(_ text: String) {
262
+ guard let data = text.data(using: .utf8),
263
+ let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { return }
264
+
265
+ // Match by request ID
266
+ if let id = json["id"] as? String, let pending = pendingCalls.removeValue(forKey: id) {
267
+ pending.timer?.cancel()
268
+
269
+ // Streaming event — has "event" key alongside "id"
270
+ if let event = json["event"] as? String {
271
+ let eventData = json["data"] as? [String: Any] ?? [:]
272
+ pending.onProgress?(event, eventData)
273
+ // Re-add — still pending until final result/error
274
+ pendingCalls[id] = pending
275
+ return
276
+ }
277
+
278
+ // Final result or error
279
+ if let errorStr = json["error"] as? String {
280
+ if errorStr == "live_session_busy" {
281
+ pending.completion(.failure(.sessionBusy))
282
+ } else {
283
+ pending.completion(.failure(.callFailed(errorStr)))
284
+ }
285
+ } else {
286
+ let result = json["result"] as? [String: Any] ?? [:]
287
+ pending.completion(.success(result))
288
+ }
289
+ return
290
+ }
291
+
292
+ // Push event (no matching ID)
293
+ if let event = json["event"] as? String {
294
+ let eventData = json["data"] as? [String: Any] ?? [:]
295
+ DispatchQueue.main.async {
296
+ self.eventHandler?(event, eventData)
297
+ }
298
+ }
299
+ }
300
+
301
+ private func sendJSON(_ dict: [String: Any]) {
302
+ guard let task = wsTask,
303
+ let data = try? JSONSerialization.data(withJSONObject: dict),
304
+ let text = String(data: data, encoding: .utf8) else { return }
305
+
306
+ task.send(.string(text)) { error in
307
+ if let error {
308
+ DiagnosticLog.shared.warn("VoxClient: send error — \(error)")
309
+ }
310
+ }
311
+ }
312
+
313
+ // MARK: - RPC (fire-and-forget and request-response)
314
+
315
+ func call(method: String, params: [String: Any]? = nil, timeout: TimeInterval = 30,
316
+ completion: @escaping (Result<[String: Any], VoxError>) -> Void) {
317
+ guard wsTask != nil, connectionState == .connected else {
318
+ completion(.failure(.notConnected))
319
+ return
320
+ }
321
+
322
+ let id = UUID().uuidString
323
+ var payload: [String: Any] = ["id": id, "method": method]
324
+ if var p = params {
325
+ // Inject clientId into all calls
326
+ if p["clientId"] == nil { p["clientId"] = Self.clientId }
327
+ payload["params"] = p
328
+ } else {
329
+ payload["params"] = ["clientId": Self.clientId]
330
+ }
331
+
332
+ let timer = DispatchSource.makeTimerSource(queue: queue)
333
+ timer.schedule(deadline: .now() + timeout)
334
+ timer.setEventHandler { [weak self] in
335
+ if let pending = self?.pendingCalls.removeValue(forKey: id) {
336
+ pending.completion(.failure(.timeout(method)))
337
+ }
338
+ }
339
+ timer.resume()
340
+
341
+ pendingCalls[id] = PendingCall(completion: completion, onProgress: nil, timer: timer)
342
+ sendJSON(payload)
343
+ }
344
+
345
+ /// Streaming RPC — receives progress events before the final result.
346
+ /// Used for transcribe.startSession where events flow on the start call ID.
347
+ func callStreaming(method: String, params: [String: Any]? = nil, timeout: TimeInterval = 120,
348
+ onProgress: @escaping (String, [String: Any]) -> Void,
349
+ completion: @escaping (Result<[String: Any], VoxError>) -> Void) {
350
+ guard wsTask != nil, connectionState == .connected else {
351
+ completion(.failure(.notConnected))
352
+ return
353
+ }
354
+
355
+ let id = UUID().uuidString
356
+ var payload: [String: Any] = ["id": id, "method": method]
357
+ if var p = params {
358
+ if p["clientId"] == nil { p["clientId"] = Self.clientId }
359
+ payload["params"] = p
360
+ } else {
361
+ payload["params"] = ["clientId": Self.clientId]
362
+ }
363
+
364
+ let timer = DispatchSource.makeTimerSource(queue: queue)
365
+ timer.schedule(deadline: .now() + timeout)
366
+ timer.setEventHandler { [weak self] in
367
+ if let pending = self?.pendingCalls.removeValue(forKey: id) {
368
+ pending.completion(.failure(.timeout(method)))
369
+ }
370
+ }
371
+ timer.resume()
372
+
373
+ pendingCalls[id] = PendingCall(completion: completion, onProgress: { event, data in
374
+ timer.schedule(deadline: .now() + timeout) // Reset timeout on activity
375
+ onProgress(event, data)
376
+ }, timer: timer)
377
+
378
+ sendJSON(payload)
379
+ }
380
+
381
+ func onServiceEvent(_ handler: @escaping (String, [String: Any]) -> Void) {
382
+ eventHandler = handler
383
+ }
384
+
385
+ // MARK: - High-level session helpers
386
+
387
+ /// Current active session ID, if any.
388
+ @Published var activeSessionId: String?
389
+
390
+ /// Start a live transcription session. Vox records from the mic and transcribes on stop.
391
+ ///
392
+ /// Events arrive on this call's ID:
393
+ /// - session.state: {state: "starting"|"recording"|"processing"|"done", sessionId, previous}
394
+ /// - session.final: {sessionId, text, words[], elapsedMs, metrics}
395
+ func startSession(
396
+ modelId: String = "parakeet:v3",
397
+ onProgress: @escaping (String, [String: Any]) -> Void,
398
+ completion: @escaping (Result<[String: Any], VoxError>) -> Void
399
+ ) {
400
+ callStreaming(
401
+ method: "transcribe.startSession",
402
+ params: ["modelId": modelId],
403
+ onProgress: { [weak self] event, data in
404
+ if event == "session.state", let sid = data["sessionId"] as? String {
405
+ DispatchQueue.main.async { self?.activeSessionId = sid }
406
+ }
407
+ onProgress(event, data)
408
+ },
409
+ completion: { [weak self] result in
410
+ DispatchQueue.main.async { self?.activeSessionId = nil }
411
+ completion(result)
412
+ }
413
+ )
414
+ }
415
+
416
+ /// Stop the current live session. The final transcript arrives via the startSession callback.
417
+ func stopSession(completion: ((Result<[String: Any], VoxError>) -> Void)? = nil) {
418
+ guard let sessionId = activeSessionId else {
419
+ completion?(.failure(.callFailed("No active session")))
420
+ return
421
+ }
422
+ call(method: "transcribe.stopSession", params: ["sessionId": sessionId]) { result in
423
+ completion?(result)
424
+ }
425
+ }
426
+
427
+ /// Cancel the current session without waiting for transcription.
428
+ func cancelSession(completion: ((Result<[String: Any], VoxError>) -> Void)? = nil) {
429
+ guard let sessionId = activeSessionId else {
430
+ completion?(.failure(.callFailed("No active session")))
431
+ return
432
+ }
433
+ call(method: "transcribe.cancelSession", params: ["sessionId": sessionId]) { result in
434
+ completion?(result)
435
+ }
436
+ }
437
+
438
+ /// Request model warm-up so first transcription is fast.
439
+ func warmup(modelId: String = "parakeet:v3") {
440
+ call(method: "warmup.start", params: ["modelId": modelId]) { result in
441
+ switch result {
442
+ case .success: DiagnosticLog.shared.info("VoxClient: warmup started")
443
+ case .failure(let e): DiagnosticLog.shared.warn("VoxClient: warmup failed — \(e.localizedDescription)")
444
+ }
445
+ }
446
+ }
447
+
448
+ // MARK: - Init
449
+
450
+ private init() {
451
+ // No distributed notifications for Vox — discovery is file-based.
452
+ // We connect on demand when voice mode activates.
453
+ }
454
+ }