@lattices/cli 0.3.0 → 0.4.0

package/app/Sources/AppDelegate.swift

@@ -81,19 +81,47 @@ class AppDelegate: NSObject, NSApplicationDelegate, NSPopoverDelegate {
 
         let store = HotkeyStore.shared
         store.register(action: .palette) { CommandPaletteWindow.shared.toggle() }
-        store.register(action: .screenMap) { ScreenMapWindowController.shared.toggle() }
+        store.register(action: .unifiedWindow) { ScreenMapWindowController.shared.toggle() }
         store.register(action: .bezel) { WindowBezel.showBezelForFrontmostWindow() }
         store.register(action: .cheatSheet) { CheatSheetHUD.shared.toggle() }
-        store.register(action: .desktopInventory) { CommandModeWindow.shared.toggle() }
+        store.register(action: .voiceCommand) {
+            DiagnosticLog.shared.info("Hotkey: voiceCommand triggered")
+            VoiceCommandWindow.shared.toggle()
+        }
+        store.register(action: .handsOff) {
+            DiagnosticLog.shared.info("Hotkey: handsOff triggered")
+            HandsOffSession.shared.toggle()
+            // Show voice bar when starting, hide when stopping
+            if HandsOffSession.shared.state != .idle {
+                HUDController.shared.showVoiceBar()
+            } else {
+                HUDController.shared.hideVoiceBar()
+            }
+        }
+        store.register(action: .hud) { HUDController.shared.toggle() }
+
+        // Pre-render HUD panels off-screen for instant first open
+        DispatchQueue.main.async { HUDController.shared.warmUp() }
         store.register(action: .omniSearch) { OmniSearchWindow.shared.toggle() }
 
-        // Layer-switching hotkeys
+        // Session layer cycling
+        store.register(action: .layerNext) { SessionLayerStore.shared.cycleNext() }
+        store.register(action: .layerPrev) { SessionLayerStore.shared.cyclePrev() }
+        store.register(action: .layerTag)  { SessionLayerStore.shared.tagFrontmostWindow() }
+
+        // Layer-switching hotkeys (1-9): session layers take priority
         let workspace = WorkspaceManager.shared
-        let layerCount = (workspace.config?.layers ?? []).count
-        for (i, action) in HotkeyAction.layerActions.prefix(layerCount).enumerated() {
+        let configLayerCount = (workspace.config?.layers ?? []).count
+        let maxLayers = max(configLayerCount, 9)
+        for (i, action) in HotkeyAction.layerActions.prefix(maxLayers).enumerated() {
             let index = i
             store.register(action: action) {
-                workspace.tileLayer(index: index, launch: true, force: true)
+                let session = SessionLayerStore.shared
+                if !session.layers.isEmpty && index < session.layers.count {
+                    session.switchTo(index: index)
+                } else {
+                    workspace.focusLayer(index: index)
+                }
                 EventBus.shared.post(.layerSwitched(index: index))
             }
         }
@@ -113,8 +141,10 @@ class AppDelegate: NSObject, NSApplicationDelegate, NSPopoverDelegate {
         }
         store.register(action: .tileDistribute) { WindowTiler.distributeVisible() }
 
-        // Check macOS permissions (Accessibility, Screen Recording)
-        PermissionChecker.shared.check()
+        // Onboarding on first launch; otherwise just check permissions
+        if !OnboardingWindowController.shared.showIfNeeded() {
+            PermissionChecker.shared.check()
+        }
 
         // Start daemon services
         let diag = DiagnosticLog.shared
@@ -126,6 +156,8 @@ class AppDelegate: NSObject, NSApplicationDelegate, NSPopoverDelegate {
         ProcessModel.shared.start()
         LatticesApi.setup()
         DaemonServer.shared.start()
+        AgentPool.shared.start()
+        HandsOffSession.shared.start()
         diag.finish(tBoot)
 
         // --diagnostics flag: auto-open diagnostics panel on launch
@@ -175,7 +207,7 @@ class AppDelegate: NSObject, NSApplicationDelegate, NSPopoverDelegate {
         let p = NSPopover()
         p.contentViewController = NSHostingController(rootView: MainView(scanner: ProjectScanner.shared))
         p.behavior = .transient
-        p.contentSize = NSSize(width: 380, height: 520)
+        p.contentSize = NSSize(width: 380, height: 560)
         p.appearance = NSAppearance(named: .darkAqua)
         p.delegate = self
         popover = p
@@ -196,8 +228,8 @@ class AppDelegate: NSObject, NSApplicationDelegate, NSPopoverDelegate {
 
         let actions: [(String, String, Selector)] = [
             ("Command Palette", "⌘⇧M", #selector(menuCommandPalette)),
-            ("Screen Map", "", #selector(menuScreenMap)),
-            ("Desktop Inventory", "", #selector(menuDesktopInventory)),
+            ("Unified Window", "", #selector(menuScreenMap)),
+            ("HUD", "", #selector(menuHUD)),
             ("Window Bezel", "", #selector(menuWindowBezel)),
             ("Cheat Sheet", "", #selector(menuCheatSheet)),
             ("Omni Search", "", #selector(menuOmniSearch)),
@@ -232,7 +264,7 @@ class AppDelegate: NSObject, NSApplicationDelegate, NSPopoverDelegate {
 
     @objc private func menuCommandPalette() { CommandPaletteWindow.shared.toggle() }
     @objc private func menuScreenMap() { ScreenMapWindowController.shared.toggle() }
-    @objc private func menuDesktopInventory() { CommandModeWindow.shared.toggle() }
+    @objc private func menuHUD() { HUDController.shared.toggle() }
     @objc private func menuWindowBezel() { WindowBezel.showBezelForFrontmostWindow() }
     @objc private func menuCheatSheet() { CheatSheetHUD.shared.toggle() }
     @objc private func menuOmniSearch() { OmniSearchWindow.shared.toggle() }

package/app/Sources/AppShellView.swift

@@ -3,25 +3,37 @@ import SwiftUI
 // MARK: - Navigation Pages
 
 enum AppPage: String, CaseIterable {
+    case home
     case screenMap
+    case desktopInventory
+    case pi
     case settings
     case docs
 
     var label: String {
         switch self {
-        case .screenMap: return "Screen Map"
-        case .settings:  return "Settings"
-        case .docs:      return "Docs"
+        case .home:             return "Home"
+        case .screenMap:        return "Screen Map"
+        case .desktopInventory: return "Desktop Inventory"
+        case .pi:               return "Pi"
+        case .settings:         return "Settings"
+        case .docs:             return "Docs"
         }
     }
 
     var icon: String {
         switch self {
-        case .screenMap: return "rectangle.3.group"
-        case .settings:  return "gearshape"
-        case .docs:      return "book"
+        case .home:             return "house"
+        case .screenMap:        return "rectangle.3.group"
+        case .desktopInventory: return "macwindow.on.rectangle"
+        case .pi:               return "terminal"
+        case .settings:         return "gearshape"
+        case .docs:             return "book"
         }
     }
+
+    /// Pages shown as primary tabs in the unified window
+    static var primaryTabs: [AppPage] { [.home, .screenMap, .desktopInventory, .pi] }
 }
 
 // MARK: - App Shell View
@@ -29,10 +41,61 @@ enum AppPage: String, CaseIterable {
 struct AppShellView: View {
     @ObservedObject var controller: ScreenMapController
     @ObservedObject var windowController = ScreenMapWindowController.shared
+    @StateObject private var commandState = CommandModeState()
 
     var body: some View {
-        contentArea
-            .background(Palette.bg)
+        VStack(spacing: 0) {
+            // Tab bar (only on primary pages)
+            if AppPage.primaryTabs.contains(windowController.activePage) {
+                tabBar
+                Rectangle().fill(Palette.border).frame(height: 0.5)
+            }
+
+            contentArea
+        }
+        .background(Palette.bg)
+        .onAppear {
+            commandState.onDismiss = { windowController.activePage = .home }
+        }
+    }
+
+    // MARK: - Tab Bar
+
+    private var tabBar: some View {
+        HStack(spacing: 0) {
+            ForEach(AppPage.primaryTabs, id: \.rawValue) { tab in
+                tabButton(tab)
+            }
+            Spacer()
+        }
+        .padding(.horizontal, 12)
+        .padding(.top, 8)
+        .padding(.bottom, 4)
+    }
+
+    private func tabButton(_ tab: AppPage) -> some View {
+        let isActive = windowController.activePage == tab
+
+        return Button {
+            windowController.activePage = tab
+            if tab == .screenMap { controller.enter() }
+            if tab == .desktopInventory { commandState.enter() }
+        } label: {
+            HStack(spacing: 5) {
+                Image(systemName: tab.icon)
+                    .font(.system(size: 10))
+                Text(tab.label)
+                    .font(Typo.monoBold(11))
+            }
+            .foregroundColor(isActive ? Palette.text : Palette.textMuted)
+            .padding(.horizontal, 12)
+            .padding(.vertical, 6)
+            .background(
+                RoundedRectangle(cornerRadius: 6)
+                    .fill(isActive ? Palette.surfaceHov : Color.clear)
+            )
+        }
+        .buttonStyle(.plain)
     }
 
     // MARK: - Content Area
@@ -40,10 +103,20 @@ struct AppShellView: View {
     @ViewBuilder
     private var contentArea: some View {
         switch windowController.activePage {
+        case .home:
+            HomeDashboardView(onNavigate: { page in
+                windowController.activePage = page
+                if page == .screenMap { controller.enter() }
+                if page == .desktopInventory { commandState.enter() }
+            })
         case .screenMap:
             ScreenMapView(controller: controller, onNavigate: { page in
                 windowController.activePage = page
             })
+        case .desktopInventory:
+            CommandModeView(state: commandState)
+        case .pi:
+            PiWorkspaceView()
         case .settings:
             SettingsContentView(
                 prefs: Preferences.shared,

package/app/Sources/AudioProvider.swift

@@ -0,0 +1,386 @@
+import AppKit
+
+// MARK: - Audio Provider Protocol
+
+/// A provider that can capture audio and return transcriptions.
+/// Lattices doesn't do transcription itself — it delegates to an external
+/// service (Vox, Whisper, etc.) and maps the result to intents.
+protocol AudioProvider: AnyObject {
+    var isAvailable: Bool { get }
+    var isListening: Bool { get }
+
+    /// Start listening. Transcription arrives via the callback.
+    func startListening(onTranscript: @escaping (Transcription) -> Void)
+
+    /// Stop listening and return the final transcription.
+    func stopListening(completion: @escaping (Transcription?) -> Void)
+
+    /// Check if the provider service is reachable.
+    func checkHealth(completion: @escaping (Bool) -> Void)
+}
+
+struct Transcription {
+    let text: String
+    let confidence: Double
+    let source: String           // "vox", "whisper", etc.
+    let isPartial: Bool          // true for streaming partial results
+    let durationMs: Int?
+}
+
+// MARK: - Audio Layer (coordinates provider + intent engine)
+
+final class AudioLayer: ObservableObject {
+    static let shared = AudioLayer()
+
+    @Published var isListening = false
+    @Published var lastTranscript: String?
+    @Published var matchedIntent: String?
+    @Published var matchedSlots: [String: String] = [:]
+    @Published var matchConfidence: Double = 0
+    @Published var executionResult: String?   // "ok" or error message
+    @Published var executionData: JSON?       // Full result data from intent execution
+    @Published var provider: (any AudioProvider)?
+    @Published var providerName: String = "none"
+    @Published var agentResponse: AgentResponse?
+
+
+    private init() {
+        let vox = VoxAudioProvider()
+        provider = vox
+        providerName = "vox"
+        // Connection is managed by VoiceCommandWindow — not here.
+        // Connecting here would race with (and destroy) the existing WebSocket.
+    }
+
+    /// Start a voice command capture. Transcription is piped to the intent engine.
+    func startVoiceCommand() {
+        guard !isListening else { return }
+
+        // Clear previous state
+        lastTranscript = nil
+        matchedIntent = nil
+        matchedSlots = [:]
+        matchConfidence = 0
+        executionResult = nil
+        didExecuteIntent = false
+
+        guard let provider = provider else {
+            executionResult = "No voice provider — install Vox"
+            return
+        }
+
+        isListening = true
+
+        provider.startListening { [weak self] transcription in
+            DispatchQueue.main.async {
+                guard let self = self else { return }
+
+                if transcription.isPartial {
+                    self.lastTranscript = transcription.text
+                    return
+                }
+
+                // Final transcript (e.g. from streaming providers)
+                self.lastTranscript = transcription.text
+                self.isListening = false
+
+                // Empty transcript = transcription failed, don't try to execute
+                guard !transcription.text.isEmpty else {
+                    if self.executionResult == nil || self.executionResult == "Transcribing..." {
+                        self.executionResult = "No speech detected"
+                    }
+                    return
+                }
+
+                EventBus.shared.post(.voiceCommand(text: transcription.text, confidence: transcription.confidence))
+                self.executeVoiceIntent(transcription)
+            }
+        }
+    }
+
+    /// Track whether we already executed for this recording session.
+    private var didExecuteIntent = false
+
+    func stopVoiceCommand() {
+        guard let provider = provider, isListening else { return }
+
+        isListening = false
+        executionResult = "Transcribing..."
+
+        provider.stopListening { [weak self] transcription in
+            DispatchQueue.main.async {
+                guard let self = self else { return }
+                if let t = transcription {
+                    self.lastTranscript = t.text
+                    // Skip if the streaming callback already executed the intent
+                    guard !self.didExecuteIntent else { return }
+                    EventBus.shared.post(.voiceCommand(text: t.text, confidence: t.confidence))
+                    self.executeVoiceIntent(t)
+                } else if !self.didExecuteIntent {
+                    self.executionResult = "No speech detected"
+                }
+            }
+        }
+    }
+
+    private func executeVoiceIntent(_ transcription: Transcription) {
+        didExecuteIntent = true
+        let matcher = PhraseMatcher.shared
+
+        // Clear previous agent response
+        agentResponse = nil
+
+        if let match = matcher.match(text: transcription.text) {
+            matchedIntent = match.intentName
+            matchConfidence = match.confidence
+            matchedSlots = match.slots.reduce(into: [:]) { dict, pair in
+                dict[pair.key] = pair.value.stringValue ?? "\(pair.value)"
+            }
+            DiagnosticLog.shared.info("AudioLayer: matched '\(match.intentName)' via '\(match.matchedPhrase)' slots=\(matchedSlots)")
+
+            do {
+                let result = try matcher.execute(match)
+                executionResult = "ok"
+                executionData = result
+                DiagnosticLog.shared.info("AudioLayer: executed '\(match.intentName)' → ok")
+            } catch {
+                DiagnosticLog.shared.info("AudioLayer: intent error — \(error.localizedDescription), falling back to Claude")
+                executionResult = "thinking..."
+                executionData = nil
+                claudeFallback(transcription: transcription)
+            }
+
+            // Fire parallel Haiku advisor for 5+ word utterances
+            fireAdvisor(transcript: transcription.text, matched: "\(match.intentName)(\(matchedSlots))")
+
+        } else {
+            // No local match — Claude fallback
+            DiagnosticLog.shared.info("AudioLayer: no phrase match for '\(transcription.text)', falling back to Claude")
+            matchedIntent = nil
+            matchedSlots = [:]
+            executionResult = "thinking..."
+            executionData = nil
+            claudeFallback(transcription: transcription)
+        }
+    }
+
+    /// Fire the Haiku advisor in parallel — non-blocking, result arrives later.
+    private func fireAdvisor(transcript: String, matched: String) {
+        let haiku = AgentPool.shared.haiku
+        guard haiku.isReady else {
+            DiagnosticLog.shared.info("AudioLayer: advisor skipped (haiku not ready)")
+            return
+        }
+
+        let message = "Transcript: \"\(transcript)\"\nMatched: \(matched)"
+        DiagnosticLog.shared.info("AudioLayer: firing haiku advisor")
+
+        haiku.send(message: message) { [weak self] response in
+            guard let self = self, let response = response else { return }
+            self.agentResponse = response
+            if let commentary = response.commentary {
+                DiagnosticLog.shared.info("AudioLayer: haiku says — \(commentary)")
+            }
+            if let suggestion = response.suggestion {
+                DiagnosticLog.shared.info("AudioLayer: haiku suggests — \(suggestion.label) → \(suggestion.intent)")
+            }
+        }
+    }
+
+    private func claudeFallback(transcription: Transcription) {
+        DispatchQueue.global(qos: .userInitiated).async { [weak self] in
+            guard let self else { return }
+
+            let result = ClaudeFallback.resolve(
+                transcript: transcription.text,
+                windows: DesktopModel.shared.windows.values.map { $0 },
+                intentCatalog: PhraseMatcher.shared.catalog()
+            )
+
+            DispatchQueue.main.async {
+                guard let resolved = result else {
+                    self.executionResult = "Claude couldn't resolve intent"
+                    DiagnosticLog.shared.info("AudioLayer: Claude fallback returned nil")
+                    return
+                }
+
+                DiagnosticLog.shared.info("AudioLayer: Claude resolved → \(resolved.intent) \(resolved.slots)")
+                self.matchedIntent = resolved.intent
+                self.matchedSlots = resolved.slots.reduce(into: [:]) { dict, pair in
+                    dict[pair.key] = pair.value.stringValue ?? "\(pair.value)"
+                }
+
+                let intentMatch = IntentMatch(
+                    intentName: resolved.intent,
+                    slots: resolved.slots,
+                    confidence: 0.8,
+                    matchedPhrase: "claude-fallback"
+                )
+
+                do {
+                    let execResult = try PhraseMatcher.shared.execute(intentMatch)
+                    self.executionResult = "ok"
+                    self.executionData = execResult
+                    DiagnosticLog.shared.info("AudioLayer: Claude-resolved executed → \(execResult)")
+                } catch {
+                    self.executionResult = error.localizedDescription
+                    self.executionData = nil
+                    DiagnosticLog.shared.info("AudioLayer: Claude-resolved execution error — \(error.localizedDescription)")
+                }
+            }
+        }
+    }
+}
+
+// Old IntentExtractor removed — PhraseMatcher handles all intent matching now.
+// See app/Sources/Intents/LatticeIntent.swift
+
+
+// MARK: - Vox Audio Provider (WebSocket JSON-RPC via VoxClient)
+//
+// Delegates recording and transcription entirely to the Vox daemon (voxd).
+// Lattices never touches the mic — Vox owns the mic, recording, and
+// transcription. We call transcribe.startSession to begin recording
+// and transcribe.stopSession to stop and get the transcript.
+//
+// Session events flow on the startSession call ID:
+//   session.state: {state, sessionId, previous}
+//   session.final: {sessionId, text, words[], elapsedMs, metrics}
+
+final class VoxAudioProvider: AudioProvider {
+    private var onTranscript: ((Transcription) -> Void)?
+    private var stopCompletion: ((Transcription?) -> Void)?
+    private var _isListening = false
+    private var startTime: Date?
+
+    var isAvailable: Bool {
+        VoxClient.shared.connectionState == .connected
+    }
+
+    var isListening: Bool { _isListening }
+
+    func checkHealth(completion: @escaping (Bool) -> Void) {
+        let client = VoxClient.shared
+        if client.connectionState == .connected {
+            client.call(method: "health") { result in
+                switch result {
+                case .success: DispatchQueue.main.async { completion(true) }
+                case .failure: DispatchQueue.main.async { completion(false) }
+                }
+            }
+        } else {
+            completion(false)
+        }
+    }
+
+    func startListening(onTranscript: @escaping (Transcription) -> Void) {
+        let client = VoxClient.shared
+        guard client.connectionState == .connected else {
+            DiagnosticLog.shared.warn("VoxAudioProvider: not connected to Vox")
+            onTranscript(Transcription(text: "", confidence: 0, source: "vox", isPartial: false, durationMs: nil))
+            return
+        }
+
+        self.onTranscript = onTranscript
+        _isListening = true
+        startTime = Date()
+
+        DiagnosticLog.shared.info("VoxAudioProvider: starting session via Vox")
+
+        // transcribe.startSession — Vox records from mic, emits session events on this call ID
+        client.startSession(
+            onProgress: { [weak self] event, data in
+                guard let self else { return }
+                DispatchQueue.main.async {
+                    switch event {
+                    case "session.state":
+                        let state = data["state"] as? String ?? ""
+                        DiagnosticLog.shared.info("VoxAudioProvider: session → \(state)")
+
+                    case "session.final":
+                        // Final transcript arrived — deliver it
+                        if let text = data["text"] as? String, !text.isEmpty {
+                            let elapsed = data["elapsedMs"] as? Int
+                            let t = Transcription(
+                                text: text, confidence: 0.95, source: "vox",
+                                isPartial: false, durationMs: elapsed
+                            )
+                            DiagnosticLog.shared.info("VoxAudioProvider: transcribed → '\(text)' (\(elapsed ?? 0)ms)")
+                            self.onTranscript?(t)
+                            self.stopCompletion?(t)
+                            self.stopCompletion = nil
+                        }
+
+                    default:
+                        break
+                    }
+                }
+            },
+            completion: { [weak self] result in
+                guard let self else { return }
+                DispatchQueue.main.async {
+                    self._isListening = false
+
+                    switch result {
+                    case .success(let data):
+                        // Final result also comes here (same data as session.final)
+                        if let text = data["text"] as? String, !text.isEmpty,
+                           self.stopCompletion != nil {
+                            // Only deliver if session.final didn't already
+                            let elapsed = data["elapsedMs"] as? Int
+                            let t = Transcription(
+                                text: text, confidence: 0.95, source: "vox",
+                                isPartial: false, durationMs: elapsed
+                            )
+                            self.onTranscript?(t)
+                            self.stopCompletion?(t)
+                            self.stopCompletion = nil
+                        } else if self.stopCompletion != nil {
+                            self.stopCompletion?(nil)
+                            self.stopCompletion = nil
+                        }
+
+                    case .failure(let error):
+                        DiagnosticLog.shared.warn("VoxAudioProvider: session error — \(error.localizedDescription)")
+                        if case .sessionBusy = error {
+                            AudioLayer.shared.executionResult = "Session already active"
+                        } else {
+                            AudioLayer.shared.executionResult = "Transcription failed"
+                        }
+                        self.onTranscript?(Transcription(
+                            text: "", confidence: 0, source: "vox",
+                            isPartial: false, durationMs: nil
+                        ))
+                        self.stopCompletion?(nil)
+                        self.stopCompletion = nil
+                    }
+                }
+            }
+        )
+    }
+
+    func stopListening(completion: @escaping (Transcription?) -> Void) {
+        _isListening = false
+
+        let client = VoxClient.shared
+        guard client.connectionState == .connected else {
+            completion(nil)
+            return
+        }
+
+        DiagnosticLog.shared.info("VoxAudioProvider: stopping session")
+
+        // Store completion — the startSession's session.final event delivers the transcript
+        self.stopCompletion = completion
+
+        client.stopSession { result in
+            if case .failure(let error) = result {
+                DiagnosticLog.shared.warn("VoxAudioProvider: stopSession error — \(error.localizedDescription)")
+                DispatchQueue.main.async {
+                    self.stopCompletion?(nil)
+                    self.stopCompletion = nil
+                }
+            }
+        }
+    }
+}