agentix-cli 0.58.0 → 0.60.0

package/apps/mac-voice/Sources/AgentXVoice/SileroVAD.swift

@@ -0,0 +1,92 @@
+import CoreML
+import Foundation
+
+/// Silero VAD on Core ML: how likely each 32 ms of 16 kHz audio is speech.
+///
+/// Core ML rather than ONNX Runtime: the model is 0.9 MB, macOS already
+/// ships the runtime, and the app keeps building with plain swiftc and no
+/// package dependency. The weights come from ModelStore.vad on first use.
+///
+/// Streaming: the model is an LSTM, so each call carries the previous
+/// call's state and the last 64 samples as context, exactly as Silero's
+/// own ONNX wrapper does. One `SileroVAD` per recording; the loaded model
+/// is shared.
+final class SileroVAD {
+    static let chunk = 512
+    static let context = 64
+    static let sampleRate = 16_000.0
+    static var frameDuration: TimeInterval { Double(chunk) / sampleRate }
+
+    private static let lock = NSLock()
+    nonisolated(unsafe) private static var cached: MLModel?
+
+    /// The shared model, loaded once. Nil when it is not installed or will
+    /// not load; the caller then falls back to the volume threshold.
+    static func loadModel() throws -> MLModel? {
+        lock.lock(); defer { lock.unlock() }
+        if let cached { return cached }
+        guard ModelStore.vad.isInstalled else { return nil }
+        let config = MLModelConfiguration()
+        // Tiny and called every 32 ms: the CPU answers faster than a trip
+        // to the Neural Engine.
+        config.computeUnits = .cpuOnly
+        let url = ModelStore.vad.url("silero-vad-unified-v6.0.0.mlmodelc")
+        let model = try MLModel(contentsOf: url, configuration: config)
+        cached = model
+        return model
+    }
+
+    private let model: MLModel
+    private let audio: MLMultiArray
+    private var hidden: MLMultiArray
+    private var cell: MLMultiArray
+    private var pending: [Float] = []
+    private var tail = [Float](repeating: 0, count: SileroVAD.context)
+
+    init(model: MLModel) throws {
+        self.model = model
+        audio = try MLMultiArray(shape: [1, NSNumber(value: Self.context + Self.chunk)], dataType: .float32)
+        hidden = try Self.zeros()
+        cell = try Self.zeros()
+    }
+
+    private static func zeros() throws -> MLMultiArray {
+        let a = try MLMultiArray(shape: [1, 128], dataType: .float32)
+        for i in 0..<a.count { a[i] = 0 }
+        return a
+    }
+
+    /// Feeds samples (16 kHz, -1…1) and returns one probability per
+    /// complete 512-sample frame; leftovers wait for the next call.
+    func process(_ samples: [Float]) throws -> [Float] {
+        pending.append(contentsOf: samples)
+        var out: [Float] = []
+        var start = 0
+        while pending.count - start >= Self.chunk {
+            let frame = pending[start..<start + Self.chunk]
+            out.append(try score(frame))
+            start += Self.chunk
+        }
+        pending.removeFirst(start)
+        return out
+    }
+
+    private func score(_ frame: ArraySlice<Float>) throws -> Float {
+        let ptr = audio.dataPointer.bindMemory(to: Float.self, capacity: audio.count)
+        for i in 0..<Self.context { ptr[i] = tail[i] }
+        var i = Self.context
+        for s in frame { ptr[i] = s; i += 1 }
+        tail = Array(frame.suffix(Self.context))
+
+        let input = try MLDictionaryFeatureProvider(dictionary: [
+            "audio_input": MLFeatureValue(multiArray: audio),
+            "hidden_state": MLFeatureValue(multiArray: hidden),
+            "cell_state": MLFeatureValue(multiArray: cell),
+        ])
+        let result = try model.prediction(from: input)
+        if let h = result.featureValue(for: "new_hidden_state")?.multiArrayValue { hidden = h }
+        if let c = result.featureValue(for: "new_cell_state")?.multiArrayValue { cell = c }
+        guard let p = result.featureValue(for: "vad_output")?.multiArrayValue, p.count > 0 else { return 0 }
+        return p[0].floatValue
+    }
+}

package/apps/mac-voice/Sources/AgentXVoice/Speech.swift

@@ -38,16 +38,18 @@ enum Speech {
     // MARK: Speech to text
 
     /// `engine` is agentx.json's `voice.stt`: "auto" and "elevenlabs" use
-    /// ElevenLabs when there is a key, falling back to mlx-whisper; "local"
-    /// never sends audio off the Mac.
-    static func transcribe(wav: Data, engine: String = "auto") async throws -> String {
+    /// ElevenLabs when there is a key, falling back to the local engine;
+    /// "local" never sends audio off the Mac. `local` is `voice.localStt`:
+    /// "mlx-whisper" or "parakeet".
+    static func transcribe(wav: Data, engine: String = "auto", local: String = "mlx-whisper") async throws -> String {
         if engine == "elevenlabs" && Config.elevenLabsKey == nil {
-            Log.warn("speech to text is set to ElevenLabs but no key is set; using mlx-whisper")
+            Log.warn("speech to text is set to ElevenLabs but no key is set; using \(local)")
         }
         if engine != "local", let key = Config.elevenLabsKey {
             do { return try await elevenLabsSTT(wav: wav, key: key) }
-            catch { Log.warn("ElevenLabs STT failed (\(error.localizedDescription)); falling back to mlx-whisper") }
+            catch { Log.warn("ElevenLabs STT failed (\(error.localizedDescription)); falling back to \(local)") }
         }
+        if local == "parakeet", let text = await LocalSTT.parakeet(wav: wav) { return text }
         return try await mlxWhisper(wav: wav)
     }
 

package/apps/mac-voice/Sources/AgentXVoice/StatusMenu.swift

@@ -23,6 +23,8 @@ final class StatusMenu: NSObject, NSMenuDelegate {
     /// The daemon's speaking queue, polled while anything is in flight.
     private var queue: AgentClient.QueueState?
     private var queuePoll: Timer?
+    /// The last three voice exchanges, for a quick replay.
+    private let recent = RecentExchanges()
 
     /// Questions this widget has in flight, per agent. Set by the app.
     var thinking: [String: Int] = [:] {
@@ -68,6 +70,15 @@ final class StatusMenu: NSObject, NSMenuDelegate {
         return NSColor(srgbRed: c.r, green: c.g, blue: c.b, alpha: 1)
     }
 
+    /// The agent's orb palette, five colours deep to light, or nil when
+    /// the daemon sent none.
+    func palette(of id: String) -> [NSColor]? {
+        guard case .loaded(let agents) = roster,
+              let hexes = agents.first(where: { $0.id == id })?.palette?.colors, hexes.count == 5 else { return nil }
+        let colors = hexes.compactMap(OrbMath.parseHex).map { NSColor(srgbRed: $0.r, green: $0.g, blue: $0.b, alpha: 1) }
+        return colors.count == 5 ? colors : nil
+    }
+
     /// Open the menu from the keyboard.
     func open() { item.button?.performClick(nil) }
 
@@ -76,6 +87,7 @@ final class StatusMenu: NSObject, NSMenuDelegate {
             let agents = await AgentClient.agents()
             roster = agents.map { .loaded($0) } ?? .down
             queue = await AgentClient.queueState()
+            await recent.refresh()
             rebuild()
             showBadge()
             if reopenAfterRefresh { reopenAfterRefresh = false; open() }
@@ -175,6 +187,7 @@ final class StatusMenu: NSObject, NSMenuDelegate {
                 menu.addItem(row)
             }
         }
+        recent.addItems(to: menu, name: name(of:))
 
         menu.addItem(.separator())
         menu.addItem(action("Stop speaking", #selector(stop), key: ".", modifiers: [.command, .option]))
@@ -251,7 +264,7 @@ final class StatusMenu: NSObject, NSMenuDelegate {
 
     @objc private func openSettings() { onSettings?() }
     @objc private func openDashboardHome() { openDashboard("/admin") }
-    @objc private func openHistory() { openDashboard("/activity") }
+    @objc private func openHistory() { HistoryWindow.shared.show() }
 
     private func openDashboard(_ path: String) {
         guard let url = URL(string: Config.dashboardURL + path) else { return }

package/apps/mac-voice/Sources/AgentXVoice/Surface.swift

@@ -0,0 +1,105 @@
+import AppKit
+
+/// The pill's background, drawn the way macOS draws its own floating
+/// surfaces: popover vibrancy, a continuous-corner shape, a hairline
+/// border, and the window's soft shadow around it.
+///
+/// It follows light and dark, Increase Contrast and Reduce Transparency as
+/// they change, since layer colours are fixed values that do not update
+/// themselves.
+final class Surface: NSVisualEffectView {
+    /// An opaque-ish wash over the blur, and it is not a style preference.
+    ///
+    /// `blendingMode = .behindWindow` composites whatever is behind the
+    /// pill, so an editor's panel dividers came through it as a faint
+    /// rectangle that read as a border the widget was drawing. The wash
+    /// keeps enough blur to feel native while making the pill read as one
+    /// solid object wherever it is parked.
+    private let wash = NSView()
+
+    /// Lays out the pill's row and its answer on every size change. Set by
+    /// the panel: the row stays put while the widget grows around it, which
+    /// autoresizing masks cannot say.
+    var layoutContent: ((NSRect) -> Void)?
+
+    override init(frame: NSRect) {
+        super.init(frame: frame)
+        material = .popover
+        blendingMode = .behindWindow
+        state = .active
+        wantsLayer = true
+        layer?.cornerRadius = Brand.Radius.lg
+        layer?.cornerCurve = .continuous
+        layer?.masksToBounds = true
+        // The layer's corners clip the wash and the answer, but not the
+        // blur: the window server draws that behind the whole rectangle
+        // unless the view has a mask image of the same shape.
+        maskImage = Self.mask(radius: Brand.Radius.lg)
+
+        wash.wantsLayer = true
+        wash.frame = bounds
+        wash.autoresizingMask = [.width, .height]
+        addSubview(wash)
+
+        NSWorkspace.shared.notificationCenter.addObserver(
+            self, selector: #selector(displayOptionsChanged),
+            name: NSWorkspace.accessibilityDisplayOptionsDidChangeNotification, object: nil)
+        restyle()
+    }
+
+    @available(*, unavailable)
+    required init?(coder: NSCoder) { fatalError("not from a nib") }
+
+    override func resizeSubviews(withOldSize oldSize: NSSize) {
+        super.resizeSubviews(withOldSize: oldSize)
+        layoutContent?(bounds)
+    }
+
+    override func viewDidChangeEffectiveAppearance() {
+        super.viewDidChangeEffectiveAppearance()
+        restyle()
+    }
+
+    override func viewDidChangeBackingProperties() {
+        super.viewDidChangeBackingProperties()
+        restyle()
+    }
+
+    @objc private func displayOptionsChanged() { restyle() }
+
+    /// A stretchable continuous-corner shape: the corners stay, the middle
+    /// stretches to any size.
+    private static func mask(radius: CGFloat) -> NSImage {
+        // A continuous corner starts bending before `radius` from the edge.
+        let cap = ceil(radius * 1.6)
+        let side = cap * 2 + 1
+        let image = NSImage(size: NSSize(width: side, height: side), flipped: false) { rect in
+            guard let ctx = NSGraphicsContext.current?.cgContext else { return false }
+            let shape = CALayer()
+            shape.frame = rect
+            shape.backgroundColor = NSColor.black.cgColor
+            shape.cornerRadius = radius
+            shape.cornerCurve = .continuous
+            shape.render(in: ctx)
+            return true
+        }
+        image.capInsets = NSEdgeInsets(top: cap, left: cap, bottom: cap, right: cap)
+        image.resizingMode = .stretch
+        return image
+    }
+
+    /// Colours for the current appearance and accessibility settings.
+    private func restyle() {
+        let ws = NSWorkspace.shared
+        let contrast = ws.accessibilityDisplayShouldIncreaseContrast
+        let solid = ws.accessibilityDisplayShouldReduceTransparency
+        effectiveAppearance.performAsCurrentDrawingAppearance {
+            wash.layer?.backgroundColor = NSColor.windowBackgroundColor
+                .withAlphaComponent(solid || contrast ? 0.96 : 0.72).cgColor
+            // One device pixel: a hairline, as system popovers have. With
+            // Increase Contrast it becomes a clear line in the text colour.
+            layer?.borderColor = (contrast ? NSColor.labelColor.withAlphaComponent(0.6) : NSColor.separatorColor).cgColor
+            layer?.borderWidth = (contrast ? 1.5 : 1) / (window?.backingScaleFactor ?? 2)
+        }
+    }
+}

package/apps/mac-voice/Sources/AgentXVoice/TurnEnd.swift

@@ -0,0 +1,110 @@
+import Foundation
+
+/// When a hands-free turn has ended: the person spoke, then stopped for
+/// long enough that the sentence is over.
+///
+/// It is fed one audio frame at a time with how likely that frame is to be
+/// speech (0…1) and how long it lasted. Two sources feed it:
+///
+/// - Silero VAD, 32 ms frames with a real probability. A voice model
+///   tells a word from a fan, a keyboard or a café, so the noise of a room
+///   neither holds the microphone open nor counts as talking.
+/// - The old volume check when the VAD model is not available: 1 above
+///   `speechLevel`, 0 below, per audio buffer.
+///
+/// Foundation only and time passed in, so the tests can drive it with
+/// synthetic frames.
+struct TurnEnd {
+    struct Tuning: Equatable {
+        /// A frame at or above this starts speech (Silero's own default).
+        var start: Float
+        /// Once talking, a frame at or above this still counts as speech.
+        /// Lower than `start`: the soft end of a word is not a pause.
+        var keep: Float
+        /// Speech must last this long in one go before the turn counts as
+        /// started, so a cough, a door or a click does not open it.
+        var minSpeech: TimeInterval
+        /// A burst this short inside a pause does not restart the pause:
+        /// one noisy frame must not hold the microphone open.
+        var resume: TimeInterval
+        /// A pause this long after speech ends the turn. People pause
+        /// inside sentences; this is longer than those pauses.
+        var endSilence: TimeInterval
+
+        /// For Silero VAD probabilities.
+        static let vad = Tuning(start: 0.5, keep: 0.35, minSpeech: 0.25, resume: 0.1, endSilence: 1.2)
+
+        /// The volume threshold as it always worked: any buffer above the
+        /// level is speech, and 1.2 s below it ends the turn.
+        static let level = Tuning(start: 0.5, keep: 0.5, minSpeech: 0, resume: 0, endSilence: 1.2)
+    }
+
+    enum State: Equatable {
+        /// Nothing said yet.
+        case waiting
+        /// Speech heard; the turn is open (possibly in a short pause).
+        case speaking
+        /// Speech, then `endSilence` of quiet: send it.
+        case ended
+    }
+
+    /// RMS above which the microphone is hearing a voice rather than a
+    /// room. The fallback's whole notion of speech.
+    static let speechLevel: Float = 0.02
+
+    let tuning: Tuning
+    private(set) var state: State = .waiting
+    /// How long the current run of speech frames has lasted.
+    private var speechRun: TimeInterval = 0
+    /// How long the current pause has lasted (a short burst does not
+    /// reset it).
+    private var silence: TimeInterval = 0
+
+    init(_ tuning: Tuning) { self.tuning = tuning }
+
+    /// One frame: `probability` that it is speech, `duration` it covers.
+    @discardableResult
+    mutating func feed(probability: Float, duration: TimeInterval) -> State {
+        guard state != .ended else { return state }
+        let threshold = state == .speaking && silence == 0 ? tuning.keep : tuning.start
+        let speech = probability >= threshold
+
+        switch state {
+        case .waiting:
+            speechRun = speech ? speechRun + duration : 0
+            if speech && speechRun >= tuning.minSpeech {
+                state = .speaking
+                silence = 0
+            }
+        case .speaking:
+            if speech {
+                speechRun += duration
+                if silence == 0 || speechRun >= tuning.resume {
+                    silence = 0
+                } else {
+                    silence += duration
+                }
+            } else {
+                speechRun = 0
+                silence += duration
+            }
+            if silence >= tuning.endSilence { state = .ended }
+        case .ended:
+            break
+        }
+        return state
+    }
+
+    /// The fallback's frame: the smoothed microphone level.
+    @discardableResult
+    mutating func feed(level: Float, duration: TimeInterval) -> State {
+        feed(probability: level > Self.speechLevel ? 1 : 0, duration: duration)
+    }
+
+    /// The microphone level after one buffer of loudness `rms`. Attack
+    /// fast, release slow: a level that drops instantly makes a pause
+    /// between words look like the end of a sentence.
+    static func smooth(level: Float, rms: Float) -> Float {
+        rms > level ? rms : level * 0.82 + rms * 0.18
+    }
+}

package/apps/mac-voice/Sources/AgentXVoice/VoiceSettings.swift

@@ -8,11 +8,43 @@ struct VoiceSettings: Codable, Equatable {
         var stop: String
         var paste: String
     }
+    /// The answer shown in the pill: seconds open once spoken (0: until
+    /// closed), and its tallest height in points.
+    struct Card: Codable, Equatable {
+        var timeout: Double
+        var maxHeight: Double
+        static let standard = Card(timeout: 30, maxHeight: 320)
+    }
     struct General: Codable, Equatable {
         var provider: String
         var fallback: String
         var stt: String
+        /// The on-device engine: "mlx-whisper" or "parakeet".
+        var localStt: String
+        /// How a hands-free turn's end is heard: "vad" or "volume".
+        var endOfTurn: String
         var hotkeys: Hotkeys
+        /// Nil from a daemon older than the card settings.
+        var card: Card?
+
+        init(provider: String, fallback: String, stt: String, localStt: String = "mlx-whisper",
+             endOfTurn: String = "vad", hotkeys: Hotkeys, card: Card? = nil) {
+            self.provider = provider; self.fallback = fallback; self.stt = stt
+            self.localStt = localStt; self.endOfTurn = endOfTurn; self.hotkeys = hotkeys
+            self.card = card
+        }
+
+        /// A daemon from before these fields leaves them out: the defaults.
+        init(from decoder: Decoder) throws {
+            let c = try decoder.container(keyedBy: CodingKeys.self)
+            provider = try c.decode(String.self, forKey: .provider)
+            fallback = try c.decode(String.self, forKey: .fallback)
+            stt = try c.decode(String.self, forKey: .stt)
+            localStt = try c.decodeIfPresent(String.self, forKey: .localStt) ?? "mlx-whisper"
+            endOfTurn = try c.decodeIfPresent(String.self, forKey: .endOfTurn) ?? "vad"
+            hotkeys = try c.decode(Hotkeys.self, forKey: .hotkeys)
+            card = try c.decodeIfPresent(Card.self, forKey: .card)
+        }
     }
     struct Voice: Codable, Equatable {
         var provider: String?
@@ -33,6 +65,10 @@ struct VoiceSettings: Codable, Equatable {
         let name: String
         var color: String
         var colorSet: Bool
+        /// The chosen orb palette; nil follows the colour.
+        var palette: String?
+        /// The palette the colour gives when none is chosen.
+        let paletteDefault: String?
         var voice: Voice
         let speaks: Speaks
     }
@@ -42,9 +78,18 @@ struct VoiceSettings: Codable, Equatable {
         let locale: String
     }
 
+    /// One of the orb's nature palettes: five colours, deep to light.
+    struct Palette: Codable, Equatable, Identifiable {
+        let id: String
+        let label: String
+        let colors: [String]
+    }
+
     var general: General
     var agents: [Agent]
     let systemVoices: [SystemVoice]
+    /// Nil from a daemon older than the palettes.
+    let palettes: [Palette]?
     let menuHotkey: String
 
     /// The change from `old` to this, as POST /voice/settings takes it:
@@ -55,11 +100,19 @@ struct VoiceSettings: Codable, Equatable {
         var general: [String: Any] = [:]
         if self.general.provider != old.general.provider { general["provider"] = self.general.provider }
         if self.general.stt != old.general.stt { general["stt"] = self.general.stt }
+        if self.general.localStt != old.general.localStt { general["localStt"] = self.general.localStt }
+        if self.general.endOfTurn != old.general.endOfTurn { general["endOfTurn"] = self.general.endOfTurn }
         var keys: [String: Any] = [:]
         if self.general.hotkeys.talk != old.general.hotkeys.talk { keys["talk"] = self.general.hotkeys.talk }
         if self.general.hotkeys.stop != old.general.hotkeys.stop { keys["stop"] = self.general.hotkeys.stop }
         if self.general.hotkeys.paste != old.general.hotkeys.paste { keys["paste"] = self.general.hotkeys.paste }
         if !keys.isEmpty { general["hotkeys"] = keys }
+        if let card = self.general.card {
+            var c: [String: Any] = [:]
+            if card.timeout != old.general.card?.timeout { c["timeout"] = card.timeout }
+            if card.maxHeight != old.general.card?.maxHeight { c["maxHeight"] = card.maxHeight }
+            if !c.isEmpty { general["card"] = c }
+        }
         if !general.isEmpty { out["general"] = general }
 
         var agents: [String: Any] = [:]
@@ -77,6 +130,7 @@ struct VoiceSettings: Codable, Equatable {
             field("narrate", agent.voice.narrate, before.voice.narrate)
             field("priority", agent.voice.priority, before.voice.priority)
             field("hotkey", agent.voice.hotkey, before.voice.hotkey)
+            field("palette", agent.palette, before.palette)
             if agent.colorSet != before.colorSet || (agent.colorSet && agent.color != before.color) {
                 a["color"] = agent.colorSet ? agent.color as Any : NSNull()
             }

package/apps/mac-voice/Sources/AgentXVoice/main.swift

@@ -18,7 +18,6 @@ import Carbon.HIToolbox
 final class App: NSObject, NSApplicationDelegate {
     private let panel = Panel()
     private let statusMenu = StatusMenu()
-    private let card = ResultCard()
     /// The agent whose by-name answer is on screen now, while no turn runs.
     private var asideSpeaker: String?
     private let recorder = Recorder()
@@ -155,6 +154,9 @@ final class App: NSObject, NSApplicationDelegate {
     /// Settings from the daemon: shortcuts now, the rest on next use.
     private func apply(_ saved: VoiceSettings) {
         settings = saved
+        let card = saved.general.card ?? .standard
+        panel.cardTimeout = card.timeout
+        panel.cardMaxHeight = card.maxHeight
         if settingsWindow.model.recording == nil { registerHotkeys() }
         // Colours may have changed.
         statusMenu.refresh()
@@ -170,6 +172,14 @@ final class App: NSObject, NSApplicationDelegate {
         return busy && !turnAgent.isEmpty ? turnAgent : Config.effectiveAgentID
     }
 
+    /// The answer's "Open in chat": the agent's page in the dashboard, with
+    /// its chat open.
+    private func openChat() {
+        let who = shownAgent
+        guard !who.isEmpty, let url = URL(string: "\(Config.dashboardURL)/admin/agents/\(who)#chat") else { return }
+        NSWorkspace.shared.open(url)
+    }
+
     /// Close, Esc or "Hide pill": the pill goes and the voice stops, the
     /// same stop as ⌘⌥. An open microphone closes without sending. The
     /// next talk key brings the pill back.
@@ -246,8 +256,11 @@ final class App: NSObject, NSApplicationDelegate {
         panel.orb.setAnimated(Config.animatedOrb)
         panel.orb.levelSource = { [weak self] in self?.recorder.level ?? 0 }
         panel.onDismiss = { [weak self] in self?.dismissPill() }
-        panel.onMove = { [weak self] in self?.card.follow() }
-        card.anchor = { [weak self] in self?.panel.frame ?? .zero }
+        panel.agentPalette = { [weak self] in
+            guard let self else { return nil }
+            return self.statusMenu.palette(of: self.shownAgent)
+        }
+        panel.answer.onOpenChat = { [weak self] in self?.openChat() }
         statusMenu.pillVisible = { [weak self] in self?.panel.isVisible ?? false }
         statusMenu.onHidePill = { [weak self] in self?.dismissPill() }
         statusMenu.onResetPosition = { [weak self] in self?.panel.resetPosition() }
@@ -321,20 +334,14 @@ final class App: NSObject, NSApplicationDelegate {
     // Ending on SILENCE rather than a timer is the whole trick. A fixed
     // window either cuts people off mid-sentence or leaves the mic open
     // staring at them; the only honest signal that a sentence has finished
-    // is the room going quiet.
+    // is the voice stopping. The recorder judges that (TurnEnd.swift).
 
-    /// RMS above which the microphone is hearing a voice rather than a room.
-    private let speechLevel: Float = 0.02
-    /// Quiet this long after speech ends the turn.
-    private let endSilence: TimeInterval = 1.2
     /// How long an unprompted follow-up window waits before giving up.
     private let followUpPatience: TimeInterval = 4.0
     /// How long a clicked session waits for you to start talking.
     private let clickPatience: TimeInterval = 8.0
 
     private var listenPoll: Timer?
-    private var heardSpeech = false
-    private var quietSince: Date?
     private var openedAt = Date()
     private var patience: TimeInterval = 8.0
     /// True when nothing was said and the window should close in silence.
@@ -343,14 +350,13 @@ final class App: NSObject, NSApplicationDelegate {
     /// Open the microphone with no key held.
     private func listenHandsFree(followUp: Bool) {
         guard !busy, !recorder.isRecording else { return }
+        recorder.endOfTurn = settings?.general.endOfTurn ?? "vad"
         do {
             try recorder.start()
         } catch {
             panel.render(.error(error.localizedDescription))
             return
         }
-        heardSpeech = false
-        quietSince = nil
         openedAt = Date()
         patience = followUp ? followUpPatience : clickPatience
         silentClose = followUp
@@ -369,23 +375,13 @@ final class App: NSObject, NSApplicationDelegate {
 
     private func pollLevel() {
         guard recorder.isRecording else { stopPolling(); return }
-        let level = recorder.level
-
-        if level > speechLevel {
-            heardSpeech = true
-            quietSince = nil
-            return
-        }
-        if heardSpeech {
-            // Speech has stopped. Give it a beat before deciding the
-            // sentence is over — people pause inside sentences.
-            let since = quietSince ?? Date()
-            quietSince = since
-            if Date().timeIntervalSince(since) >= endSilence {
-                stopPolling()
-                stopAndSend()
-            }
+        switch recorder.turnState {
+        case .speaking: return
+        case .ended:
+            stopPolling()
+            stopAndSend()
             return
+        case .waiting: break
         }
         // Nothing said yet. Close quietly rather than making the person
         // dismiss a window they did not ask for.
@@ -444,6 +440,8 @@ final class App: NSObject, NSApplicationDelegate {
         lastSpokeAt = Date()
         talkCheck = Task { await AgentClient.hush() }
         Log.info("door: opened\(busy ? " (a turn is running)" : "")")
+        // Push-to-talk ends on key release; no end-of-turn detection needed.
+        recorder.endOfTurn = "hold"
         do {
             try recorder.start()
             asideSpeaker = nil
@@ -474,7 +472,8 @@ final class App: NSObject, NSApplicationDelegate {
 
         Task { @MainActor in
             var heard = ""
-            do { heard = try await Speech.transcribe(wav: wav, engine: settings?.general.stt ?? "auto") }
+            do { heard = try await Speech.transcribe(wav: wav, engine: settings?.general.stt ?? "auto",
+                                                          local: settings?.general.localStt ?? "mlx-whisper") }
             catch { Log.warn("transcription failed: \(error.localizedDescription)") }
             guard !heard.isEmpty else {
                 _ = await door.value
@@ -566,10 +565,10 @@ final class App: NSObject, NSApplicationDelegate {
             switch result {
             case .success(let answer):
                 Log.info("aside answer (\(answer.agentID ?? agent)): \(answer.text)")
-                if ResultCard.isWorthShowing(spoken: answer.text, written: answer.written,
+                if AnswerView.isWorthShowing(spoken: answer.text, written: answer.written,
                                              buttons: answer.buttons, imageURL: answer.imageURL) {
-                    card.show(spoken: answer.text, written: answer.written,
-                              buttons: answer.buttons, imageURL: answer.imageURL)
+                    panel.showAnswer(spoken: answer.text, written: answer.written,
+                                     buttons: answer.buttons, imageURL: answer.imageURL)
                 }
                 // Nothing else on screen: the pill shows this answer, its
                 // orb in this agent's colour, while it is spoken.
@@ -631,12 +630,12 @@ final class App: NSObject, NSApplicationDelegate {
             // the speech cannot deliver — a link, an image, or more
             // text than was read aloud. A card that opens on every
             // "Ok." teaches you to ignore it.
-            if ResultCard.isWorthShowing(spoken: answer.text, written: answer.written,
+            if AnswerView.isWorthShowing(spoken: answer.text, written: answer.written,
                                          buttons: answer.buttons, imageURL: answer.imageURL) {
-                card.show(spoken: answer.text, written: answer.written,
-                          buttons: answer.buttons, imageURL: answer.imageURL)
+                panel.showAnswer(spoken: answer.text, written: answer.written,
+                                 buttons: answer.buttons, imageURL: answer.imageURL)
             } else {
-                card.orderOut(nil)
+                panel.collapse()
             }
             // Stopped while it was thinking: the answer is not spoken.
             if silenced { silenced = false; panel.render(.idle); busy = false; return }

package/apps/mac-voice/build.sh

@@ -18,9 +18,18 @@ rm -rf build && mkdir -p "$BIN" "$APP/Contents/Resources"
 swiftc -O \
   -o "$BIN/AgentXVoice" \
   Sources/AgentXVoice/*.swift \
-  -framework AppKit -framework AVFoundation -framework Carbon -framework SwiftUI \
+  -framework AppKit -framework AVFoundation -framework Carbon -framework CoreML -framework SwiftUI \
   -target arm64-apple-macosx14.0
 
+# The on-device models from a terminal: fetch them, transcribe a file, see
+# where a turn would end. Same sources as the app; the models themselves
+# are downloaded on first use into ~/.agentx/models, never bundled.
+swiftc -O \
+  -o "$BIN/agentx-voice-local" \
+  Sources/AgentXVoice/ModelStore.swift Sources/AgentXVoice/SileroVAD.swift \
+  Sources/AgentXVoice/Parakeet.swift Sources/AgentXVoice/TurnEnd.swift Tools/LocalSTT/main.swift \
+  -framework CoreML -target arm64-apple-macosx14.0
+
 cp Resources/Info.plist "$APP/Contents/Info.plist"
 
 # Ad-hoc signature. Unsigned bundles get a fresh TCC identity on every