@lattices/cli 0.4.9 → 0.4.11

package/{app → apps/mac}/Sources/Core/Voice/AudioProvider.swift

@@ -43,13 +43,15 @@ final class AudioLayer: ObservableObject {
     @Published var providerName: String = "none"
     @Published var agentResponse: AgentResponse?
 
+    private var pendingVoiceStart = false
+    private var voiceConnectionRetry: DispatchWorkItem?
 
     private init() {
         let vox = VoxAudioProvider()
         provider = vox
         providerName = "vox"
-        // Connection is managed by VoiceCommandWindow — not here.
-        // Connecting here would race with (and destroy) the existing WebSocket.
+        // Voice entry points can arrive from the desktop UI, daemon, or iOS
+        // bridge, so connection setup is handled lazily when capture starts.
     }
 
     /// Start a voice command capture. Transcription is piped to the intent engine.
@@ -69,6 +71,47 @@ final class AudioLayer: ObservableObject {
             return
         }
 
+        pendingVoiceStart = true
+        voiceConnectionRetry?.cancel()
+        startVoiceCommandWhenReady(provider: provider, attempt: 0)
+    }
+
+    private func startVoiceCommandWhenReady(provider: any AudioProvider, attempt: Int) {
+        guard pendingVoiceStart, !isListening else { return }
+
+        if provider.isAvailable {
+            pendingVoiceStart = false
+            beginVoiceCommand(provider: provider)
+            return
+        }
+
+        let client = VoxClient.shared
+        if attempt == 0 {
+            let launched = launchVoxIfNeeded()
+            executionResult = launched ? "Starting Vox..." : "Connecting to Vox..."
+            client.connect()
+        } else if case .disconnected = client.connectionState {
+            client.connect()
+        } else if case .unavailable = client.connectionState {
+            client.connect()
+        }
+
+        guard attempt < 40 else {
+            pendingVoiceStart = false
+            executionResult = "Vox unavailable — open Vox and try again"
+            DiagnosticLog.shared.warn("AudioLayer: Vox connection failed before voice start")
+            return
+        }
+
+        let retry = DispatchWorkItem { [weak self] in
+            guard let self else { return }
+            self.startVoiceCommandWhenReady(provider: provider, attempt: attempt + 1)
+        }
+        voiceConnectionRetry = retry
+        DispatchQueue.main.asyncAfter(deadline: .now() + 0.25, execute: retry)
+    }
+
+    private func beginVoiceCommand(provider: any AudioProvider) {
         isListening = true
 
         provider.startListening { [weak self] transcription in
@@ -85,7 +128,7 @@ final class AudioLayer: ObservableObject {
                 self.isListening = false
 
                 // Empty transcript = transcription failed, don't try to execute
-                guard !transcription.text.isEmpty else {
+                guard !transcription.text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
                     if self.executionResult == nil || self.executionResult == "Transcribing..." {
                         self.executionResult = "No speech detected"
                     }
@@ -98,19 +141,54 @@ final class AudioLayer: ObservableObject {
         }
     }
 
+    private func launchVoxIfNeeded() -> Bool {
+        guard VoxClient.shared.discoverDaemon() == nil else { return false }
+
+        let candidates = [
+            "/Applications/Vox.app",
+            NSHomeDirectory() + "/Applications/Vox.app",
+        ]
+
+        guard let path = candidates.first(where: { FileManager.default.fileExists(atPath: $0) }) else {
+            DiagnosticLog.shared.warn("AudioLayer: Vox daemon unavailable and Vox.app was not found")
+            return false
+        }
+
+        let configuration = NSWorkspace.OpenConfiguration()
+        configuration.activates = false
+        NSWorkspace.shared.openApplication(at: URL(fileURLWithPath: path), configuration: configuration) { _, error in
+            if let error {
+                DiagnosticLog.shared.warn("AudioLayer: failed to open Vox — \(error.localizedDescription)")
+            } else {
+                DiagnosticLog.shared.info("AudioLayer: opened Vox for voice command")
+            }
+        }
+        return true
+    }
+
     /// Track whether we already executed for this recording session.
     private var didExecuteIntent = false
 
     func stopVoiceCommand() {
+        if pendingVoiceStart, !isListening {
+            pendingVoiceStart = false
+            voiceConnectionRetry?.cancel()
+            voiceConnectionRetry = nil
+            executionResult = "Voice cancelled"
+            return
+        }
+
         guard let provider = provider, isListening else { return }
 
+        pendingVoiceStart = false
         isListening = false
         executionResult = "Transcribing..."
 
         provider.stopListening { [weak self] transcription in
             DispatchQueue.main.async {
                 guard let self = self else { return }
-                if let t = transcription {
+                if let t = transcription,
+                   !t.text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
                     self.lastTranscript = t.text
                     // Skip if the streaming callback already executed the intent
                     guard !self.didExecuteIntent else { return }
@@ -130,6 +208,16 @@ final class AudioLayer: ObservableObject {
         // Clear previous agent response
         agentResponse = nil
 
+        if shouldAnswerWithAssistant(transcription.text) {
+            DiagnosticLog.shared.info("AudioLayer: question-like voice request, asking Assistant provider")
+            matchedIntent = nil
+            matchedSlots = [:]
+            executionResult = "thinking..."
+            executionData = nil
+            assistantQuestion(transcription: transcription)
+            return
+        }
+
         if let match = matcher.match(text: transcription.text) {
             matchedIntent = match.intentName
             matchConfidence = match.confidence
@@ -140,100 +228,181 @@ final class AudioLayer: ObservableObject {
 
             do {
                 let result = try matcher.execute(match)
-                executionResult = "ok"
+                executionResult = voiceResultSummary(for: match, result: result)
                 executionData = result
-                DiagnosticLog.shared.info("AudioLayer: executed '\(match.intentName)' → ok")
+                DiagnosticLog.shared.info("AudioLayer: executed '\(match.intentName)' → \(executionResult ?? "ok")")
             } catch {
-                DiagnosticLog.shared.info("AudioLayer: intent error — \(error.localizedDescription), falling back to Claude")
+                DiagnosticLog.shared.info("AudioLayer: intent error — \(error.localizedDescription), asking Assistant provider")
                 executionResult = "thinking..."
                 executionData = nil
-                claudeFallback(transcription: transcription)
+                assistantFallback(transcription: transcription)
             }
 
-            // Fire parallel Haiku advisor for 5+ word utterances
+            // Fire parallel provider-backed advisor for 5+ word utterances.
             fireAdvisor(transcript: transcription.text, matched: "\(match.intentName)(\(matchedSlots))")
 
         } else {
-            // No local match — Claude fallback
-            DiagnosticLog.shared.info("AudioLayer: no phrase match for '\(transcription.text)', falling back to Claude")
+            // No local match — ask the selected Assistant provider.
+            DiagnosticLog.shared.info("AudioLayer: no phrase match for '\(transcription.text)', asking Assistant provider")
             matchedIntent = nil
             matchedSlots = [:]
             executionResult = "thinking..."
             executionData = nil
-            claudeFallback(transcription: transcription)
+            assistantFallback(transcription: transcription)
         }
     }
 
-    /// Fire the Haiku advisor in parallel — non-blocking, result arrives later.
+    /// Fire the selected Assistant provider in parallel — non-blocking, result arrives later.
     private func fireAdvisor(transcript: String, matched: String) {
-        let haiku = AgentPool.shared.haiku
-        guard haiku.isReady else {
-            DiagnosticLog.shared.info("AudioLayer: advisor skipped (haiku not ready)")
+        let assistant = PiChatSession.shared
+        guard assistant.isProviderInferenceReady else {
+            DiagnosticLog.shared.info("AudioLayer: advisor skipped (Assistant provider not ready)")
             return
         }
 
-        let message = "Transcript: \"\(transcript)\"\nMatched: \(matched)"
-        DiagnosticLog.shared.info("AudioLayer: firing haiku advisor")
+        DiagnosticLog.shared.info("AudioLayer: firing Assistant advisor via \(assistant.currentProvider.name)")
 
-        haiku.send(message: message) { [weak self] response in
+        assistant.askVoiceAdvisor(transcript: transcript, matched: matched) { [weak self] response in
             guard let self = self, let response = response else { return }
             self.agentResponse = response
             if let commentary = response.commentary {
-                DiagnosticLog.shared.info("AudioLayer: haiku says — \(commentary)")
+                DiagnosticLog.shared.info("AudioLayer: Assistant advisor says — \(commentary)")
             }
             if let suggestion = response.suggestion {
-                DiagnosticLog.shared.info("AudioLayer: haiku suggests — \(suggestion.label) → \(suggestion.intent)")
+                DiagnosticLog.shared.info("AudioLayer: Assistant advisor suggests — \(suggestion.label) → \(suggestion.intent)")
             }
         }
     }
 
-    private func claudeFallback(transcription: Transcription) {
-        DispatchQueue.global(qos: .userInitiated).async { [weak self] in
+    private func assistantFallback(transcription: Transcription) {
+        let assistant = PiChatSession.shared
+        guard assistant.isProviderInferenceReady else {
+            executionResult = "Connect an Assistant provider in Settings"
+            DiagnosticLog.shared.info("AudioLayer: Assistant provider not ready")
+            return
+        }
+
+        assistant.resolveVoiceIntent(transcript: transcription.text) { [weak self] resolved in
             guard let self else { return }
+            guard let resolved else {
+                self.executionResult = "Assistant couldn't resolve intent"
+                DiagnosticLog.shared.info("AudioLayer: Assistant provider returned no intent")
+                return
+            }
 
-            let result = ClaudeFallback.resolve(
-                transcript: transcription.text,
-                windows: DesktopModel.shared.windows.values.map { $0 },
-                intentCatalog: PhraseMatcher.shared.catalog()
+            DiagnosticLog.shared.info("AudioLayer: Assistant resolved → \(resolved.intent) \(resolved.slots)")
+            self.matchedIntent = resolved.intent
+            self.matchedSlots = resolved.slots.reduce(into: [:]) { dict, pair in
+                dict[pair.key] = pair.value.stringValue ?? "\(pair.value)"
+            }
+
+            let intentMatch = IntentMatch(
+                intentName: resolved.intent,
+                slots: resolved.slots,
+                confidence: 0.8,
+                matchedPhrase: "assistant-provider"
             )
 
-            DispatchQueue.main.async {
-                guard let resolved = result else {
-                    self.executionResult = "Claude couldn't resolve intent"
-                    DiagnosticLog.shared.info("AudioLayer: Claude fallback returned nil")
-                    return
-                }
+            do {
+                let execResult = try PhraseMatcher.shared.execute(intentMatch)
+                self.executionResult = self.voiceResultSummary(for: intentMatch, result: execResult)
+                self.executionData = execResult
+                DiagnosticLog.shared.info("AudioLayer: Assistant-resolved executed → \(self.executionResult ?? "\(execResult)")")
+            } catch {
+                self.executionResult = error.localizedDescription
+                self.executionData = nil
+                DiagnosticLog.shared.info("AudioLayer: Assistant-resolved execution error — \(error.localizedDescription)")
+            }
+        }
+    }
 
-                DiagnosticLog.shared.info("AudioLayer: Claude resolved → \(resolved.intent) \(resolved.slots)")
-                self.matchedIntent = resolved.intent
-                self.matchedSlots = resolved.slots.reduce(into: [:]) { dict, pair in
-                    dict[pair.key] = pair.value.stringValue ?? "\(pair.value)"
-                }
+    private func assistantQuestion(transcription: Transcription) {
+        let assistant = PiChatSession.shared
+        guard assistant.isProviderInferenceReady else {
+            executionResult = "Connect an Assistant provider in Settings"
+            DiagnosticLog.shared.info("AudioLayer: Assistant provider not ready for question")
+            return
+        }
 
-                let intentMatch = IntentMatch(
-                    intentName: resolved.intent,
-                    slots: resolved.slots,
-                    confidence: 0.8,
-                    matchedPhrase: "claude-fallback"
-                )
-
-                do {
-                    let execResult = try PhraseMatcher.shared.execute(intentMatch)
-                    self.executionResult = "ok"
-                    self.executionData = execResult
-                    DiagnosticLog.shared.info("AudioLayer: Claude-resolved executed → \(execResult)")
-                } catch {
-                    self.executionResult = error.localizedDescription
-                    self.executionData = nil
-                    DiagnosticLog.shared.info("AudioLayer: Claude-resolved execution error — \(error.localizedDescription)")
-                }
+        assistant.answerVoiceQuestion(transcription.text) { [weak self] response in
+            guard let self else { return }
+            guard let response else {
+                self.executionResult = "Assistant couldn't answer"
+                DiagnosticLog.shared.info("AudioLayer: Assistant provider returned no answer")
+                return
+            }
+            self.agentResponse = response
+            self.executionResult = "ok"
+            self.executionData = nil
+            if let commentary = response.commentary {
+                DiagnosticLog.shared.info("AudioLayer: Assistant answered — \(commentary.prefix(160))")
             }
         }
     }
+
+    private func shouldAnswerWithAssistant(_ text: String) -> Bool {
+        let lower = text.lowercased()
+        let questionStarters = [
+            "what", "how", "why", "where", "when", "who",
+            "can you tell", "tell me about", "explain", "summarize", "describe"
+        ]
+        let asksQuestion = text.contains("?") || questionStarters.contains(where: lower.hasPrefix)
+        guard asksQuestion else { return false }
+
+        let assistantTopics = [
+            "setting", "settings", "configured", "enabled", "disabled",
+            "mouse", "shortcut", "shortcuts", "gesture", "gestures",
+            "ocr", "terminal", "scan root", "assistant", "provider",
+            "lattices", "workspace"
+        ]
+        return assistantTopics.contains(where: lower.contains)
+    }
+
+    private func voiceResultSummary(for match: IntentMatch, result: JSON) -> String {
+        if let summary = result["summary"]?.stringValue, !summary.isEmpty {
+            return summary
+        }
+        if let message = result["message"]?.stringValue, !message.isEmpty {
+            return message
+        }
+        if result["ok"]?.boolValue == false {
+            return result["reason"]?.stringValue ?? "Voice command did not complete"
+        }
+
+        switch match.intentName {
+        case "tile_window":
+            let position = match.slots["position"]?.stringValue
+                ?? result["position"]?.stringValue
+                ?? "requested position"
+            return "Moved window to \(position)"
+
+        case "focus":
+            let target = result["focused"]?.stringValue
+                ?? match.slots["app"]?.stringValue
+                ?? "target"
+            return "Focused \(target)"
+
+        case "launch":
+            if let launched = result["launched"]?.stringValue {
+                return "Launched \(launched)"
+            }
+            let target = match.slots["project"]?.stringValue ?? "requested target"
+            return "Opened \(target)"
+
+        case "kill":
+            let target = match.slots["session"]?.stringValue
+                ?? match.slots["app"]?.stringValue
+                ?? "target"
+            return "Closed \(target)"
+
+        default:
+            return "ok"
+        }
+    }
 }
 
 // Old IntentExtractor removed — PhraseMatcher handles all intent matching now.
-// See app/Sources/Intents/LatticeIntent.swift
+// See apps/mac/Sources/Intents/LatticeIntent.swift
 
 
 // MARK: - Vox Audio Provider (WebSocket JSON-RPC via VoxClient)

package/bin/handsoff-infer.ts

@@ -4,7 +4,7 @@
  *
  * Usage: echo '{"transcript":"tile chrome left","snapshot":{...}}' | bun run bin/handsoff-infer.ts
  *
- * Reads JSON from stdin, calls Groq via lib/infer.ts, prints JSON result to stdout.
+ * Reads JSON from stdin, calls the configured voice inference provider, prints JSON result to stdout.
  * All logging goes to stderr so it doesn't pollute the JSON output.
  */
 
@@ -14,7 +14,9 @@ import {
   normalizeAssistantPlan,
   tryLocalAssistantPlan,
 } from "./assistant-intelligence.ts";
-import { inferJSON } from "../lib/infer.ts";
+import { inferJSON, resolveVoiceInferenceOptions } from "./infer.ts";
+
+const INFER_TIMEOUT_MS = 15_000;
 
 // ── Read input from stdin ──────────────────────────────────────────
 
@@ -36,6 +38,7 @@ const req = JSON.parse(input) as {
 const transcript = req.transcript ?? "";
 const systemPrompt = buildAssistantSystemPrompt();
 const userMessage = buildAssistantContextMessage(transcript, req.snapshot ?? {});
+const voiceInference = resolveVoiceInferenceOptions();
 
 const localPlan = tryLocalAssistantPlan(transcript, req.snapshot ?? {});
 if (localPlan) {
@@ -50,14 +53,18 @@ const messages = (req.history ?? []).map((h) => ({
   content: h.content,
 }));
 
+const controller = new AbortController();
+const timer = setTimeout(() => controller.abort(), INFER_TIMEOUT_MS);
+
 try {
   const { data, raw } = await inferJSON(userMessage, {
-    provider: "groq",
-    model: "llama-3.3-70b-versatile",
+    provider: voiceInference.provider,
+    model: voiceInference.model,
     system: systemPrompt,
     messages,
     temperature: 0.2,
     maxTokens: 512,
+    abortSignal: controller.signal,
     tag: "hands-off",
   });
 
@@ -83,5 +90,7 @@ try {
       _meta: { error: err.message },
     })
   );
-  process.exit(1);
+  process.exitCode = 1;
+} finally {
+  clearTimeout(timer);
 }

package/bin/handsoff-worker.ts

@@ -3,7 +3,7 @@
  * Hands-off worker — long-running process that handles both inference and TTS.
  *
  * Reads newline-delimited JSON commands from stdin, writes JSON responses to stdout.
- * Keeps SpeakEasy and inference warm — no cold starts.
+ * Keeps TTS and inference warm — no cold starts.
  *
  * Commands:
  *   {"cmd":"infer","transcript":"...","snapshot":{...},"history":[...]}
@@ -23,9 +23,10 @@ import {
   normalizeAssistantPlan,
   tryLocalAssistantPlan,
 } from "./assistant-intelligence.ts";
-import { infer } from "../lib/infer.ts";
+import { infer, resolveVoiceInferenceOptions } from "./infer.ts";
 
 const INFER_TIMEOUT_MS = 15_000;
+const voiceInference = resolveVoiceInferenceOptions();
 
 /** Call infer and parse JSON if possible, otherwise treat as spoken-only response */
 async function inferSmart(prompt: string, options: any): Promise<{ data: any; raw: any }> {
@@ -291,12 +292,15 @@ log("worker started, streaming TTS ready");
 
 const systemPrompt = buildAssistantSystemPrompt();
 log("system prompt loaded");
+log(`voice inference: ${voiceInference.provider}/${voiceInference.model}`);
 
 // ── Auto-restart on file changes ───────────────────────────────────
 
 const watchFiles = [
   assistantPromptPath,
   join(import.meta.dir, "assistant-intelligence.ts"),
+  join(import.meta.dir, "..", ".env"),
+  join(import.meta.dir, "..", ".env.local"),
   import.meta.path, // this script itself
 ];
 
@@ -376,8 +380,8 @@ async function processLine(line: string) {
         }));
 
         const { data, raw } = await inferSmart(userMessage, {
-          provider: "xai",
-          model: "grok-4.20-beta-0309-non-reasoning",
+          provider: voiceInference.provider,
+          model: voiceInference.model,
           system: systemPrompt,
           messages,
           temperature: 0.2,
@@ -416,7 +420,7 @@ async function processLine(line: string) {
       //
       // Timeline:
       //   t=0  ──┬── ack TTS (fire & forget)
-      //          └── Groq inference
+      //          └── model inference
       //   t=~600ms ─┬── narrate TTS (what we're doing)
       //             └── execute actions (in parallel with narrate)
       //   t=done ── respond with results
@@ -445,8 +449,8 @@ async function processLine(line: string) {
         const userMessage = buildAssistantContextMessage(transcript, snap);
         try {
           const { data, raw } = await inferSmart(userMessage, {
-            provider: "xai",
-            model: "grok-4.20-beta-0309-non-reasoning",
+            provider: voiceInference.provider,
+            model: voiceInference.model,
             system: systemPrompt,
             messages,
             temperature: 0.2,