agentix-cli 0.57.0 → 0.59.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (123) hide show
  1. package/apps/mac-voice/Sources/AgentXVoice/Config.swift +19 -5
  2. package/apps/mac-voice/Sources/AgentXVoice/HistoryClient.swift +55 -0
  3. package/apps/mac-voice/Sources/AgentXVoice/HistoryModel.swift +153 -0
  4. package/apps/mac-voice/Sources/AgentXVoice/HistoryWindow.swift +301 -0
  5. package/apps/mac-voice/Sources/AgentXVoice/LocalSTT.swift +79 -0
  6. package/apps/mac-voice/Sources/AgentXVoice/ModelStore.swift +184 -0
  7. package/apps/mac-voice/Sources/AgentXVoice/Orb.swift +131 -169
  8. package/apps/mac-voice/Sources/AgentXVoice/Panel.swift +174 -35
  9. package/apps/mac-voice/Sources/AgentXVoice/Parakeet.swift +264 -0
  10. package/apps/mac-voice/Sources/AgentXVoice/PillPlacement.swift +70 -0
  11. package/apps/mac-voice/Sources/AgentXVoice/Recorder.swift +81 -3
  12. package/apps/mac-voice/Sources/AgentXVoice/ResultCard.swift +26 -22
  13. package/apps/mac-voice/Sources/AgentXVoice/SettingsWindow.swift +10 -2
  14. package/apps/mac-voice/Sources/AgentXVoice/SileroVAD.swift +92 -0
  15. package/apps/mac-voice/Sources/AgentXVoice/Speech.swift +7 -5
  16. package/apps/mac-voice/Sources/AgentXVoice/StatusMenu.swift +23 -6
  17. package/apps/mac-voice/Sources/AgentXVoice/TurnEnd.swift +110 -0
  18. package/apps/mac-voice/Sources/AgentXVoice/VoiceSettings.swift +23 -0
  19. package/apps/mac-voice/Sources/AgentXVoice/main.swift +47 -70
  20. package/apps/mac-voice/build.sh +10 -1
  21. package/dist/{App-Z5VOAUXB.js → App-QEJGXZTY.js} +2 -2
  22. package/dist/{ChatApp-BE7ONQKS.js → ChatApp-VKWQN5GM.js} +2 -2
  23. package/dist/{agent-Q74EISKP.js → agent-MJKLI2L5.js} +2 -2
  24. package/dist/{board-dashboard-A7PQOI6C.js → board-dashboard-2KVT72J7.js} +221 -138
  25. package/dist/board-dashboard-2KVT72J7.js.map +1 -0
  26. package/dist/{chunk-I2LYULR4.js → chunk-2XZGHIVP.js} +5 -5
  27. package/dist/{chunk-5LLGIX5M.js → chunk-3EVCHRJF.js} +2 -2
  28. package/dist/{chunk-6QNPCVRN.js → chunk-4ZTV6JV7.js} +3 -3
  29. package/dist/chunk-4ZTV6JV7.js.map +1 -0
  30. package/dist/{chunk-VI7GBDKV.js → chunk-5OF3EGCZ.js} +2 -2
  31. package/dist/{chunk-RXEAACVH.js → chunk-6DUTIPTY.js} +2 -2
  32. package/dist/{chunk-CQUVUVXR.js → chunk-6ZV2QFXE.js} +2 -2
  33. package/dist/{chunk-2MIOMYT2.js → chunk-DAAWUFO4.js} +2 -2
  34. package/dist/{chunk-547G3HYQ.js → chunk-EJQOKSKP.js} +2 -2
  35. package/dist/{chunk-POL7RAJQ.js → chunk-ILXW3ERG.js} +117 -117
  36. package/dist/chunk-ILXW3ERG.js.map +1 -0
  37. package/dist/{chunk-ERL6AS3O.js → chunk-IW4XIZAZ.js} +8 -8
  38. package/dist/{chunk-ERL6AS3O.js.map → chunk-IW4XIZAZ.js.map} +1 -1
  39. package/dist/{chunk-Q6SMLSRO.js → chunk-JUY77L7C.js} +2 -2
  40. package/dist/{chunk-KW7TIOS3.js → chunk-K62U5EJJ.js} +2 -2
  41. package/dist/{chunk-KKGFZEXS.js → chunk-L5OCF33B.js} +37 -36
  42. package/dist/chunk-L5OCF33B.js.map +1 -0
  43. package/dist/{chunk-XTD3SZIP.js → chunk-N5GOBLUI.js} +2 -2
  44. package/dist/{chunk-VVBGPT6W.js → chunk-OIDMPPV5.js} +2 -2
  45. package/dist/{chunk-OZZPJ2YG.js → chunk-SDBU6747.js} +190 -177
  46. package/dist/chunk-SDBU6747.js.map +1 -0
  47. package/dist/{chunk-NKMFHEGV.js → chunk-WFPRVZBA.js} +2 -2
  48. package/dist/{chunk-6OABTCRA.js → chunk-WI5R64WO.js} +2 -2
  49. package/dist/{chunk-5W7NJHCS.js → chunk-YCIZ2WHX.js} +2 -2
  50. package/dist/{chunk-U5YBQJ5Q.js → chunk-YQU74AOL.js} +2 -2
  51. package/dist/{chunk-V7ZK6ZIO.js → chunk-YRB75GMN.js} +2 -2
  52. package/dist/cli.js +1 -1
  53. package/dist/client-X2PTYKVX.js +2 -0
  54. package/dist/{config-P6YXC6XZ.js → config-XB2ORKWS.js} +2 -2
  55. package/dist/config-mutator-KMXR3XDG.js +2 -0
  56. package/dist/doctor-6BIMNEGU.js +2 -0
  57. package/dist/index.d.ts +17 -0
  58. package/dist/index.js +1 -1
  59. package/dist/program-DHDC6XYB.js +2 -0
  60. package/dist/{promote-KVPLDSKT.js → promote-GBR6YSQW.js} +2 -2
  61. package/dist/query-672RGMXO.js +2 -0
  62. package/dist/{registry-AUOI4TW2.js → registry-P535FNLB.js} +2 -2
  63. package/dist/{request-planner-LMZ5ZAXC.js → request-planner-KHTCHF7N.js} +2 -2
  64. package/dist/{review-candidates-WCBXLLDS.js → review-candidates-Y5KCRUNU.js} +2 -2
  65. package/dist/{routing-FBASYKUL.js → routing-RWCCDJ4D.js} +2 -2
  66. package/dist/schedule-tool-H46AK53H.js +2 -0
  67. package/dist/scheduler-3XLWBCX7.js +2 -0
  68. package/dist/{seat-DBFW3Z5P.js → seat-PJPKLH6L.js} +2 -2
  69. package/dist/{select-6CNVVFA5.js → select-VMTJAX5X.js} +2 -2
  70. package/dist/{set-enabled-AYZH3KSB.js → set-enabled-LPXH3ZQP.js} +2 -2
  71. package/dist/{tool-PKS4NBZ6.js → tool-MWXG44OT.js} +2 -2
  72. package/dist/wiki-QHMHVFWU.js +2 -0
  73. package/package.json +1 -1
  74. package/dist/board-dashboard-A7PQOI6C.js.map +0 -1
  75. package/dist/chunk-6QNPCVRN.js.map +0 -1
  76. package/dist/chunk-KKGFZEXS.js.map +0 -1
  77. package/dist/chunk-OZZPJ2YG.js.map +0 -1
  78. package/dist/chunk-POL7RAJQ.js.map +0 -1
  79. package/dist/client-EFP2PKA3.js +0 -2
  80. package/dist/config-mutator-3GKXLTHH.js +0 -2
  81. package/dist/doctor-JEWMHXYH.js +0 -2
  82. package/dist/program-7DUQJDSQ.js +0 -2
  83. package/dist/query-AGOMCYTT.js +0 -2
  84. package/dist/schedule-tool-RJRJ63SS.js +0 -2
  85. package/dist/scheduler-ATA7Z44S.js +0 -2
  86. package/dist/wiki-QW6BDNEQ.js +0 -2
  87. /package/dist/{App-Z5VOAUXB.js.map → App-QEJGXZTY.js.map} +0 -0
  88. /package/dist/{ChatApp-BE7ONQKS.js.map → ChatApp-VKWQN5GM.js.map} +0 -0
  89. /package/dist/{agent-Q74EISKP.js.map → agent-MJKLI2L5.js.map} +0 -0
  90. /package/dist/{chunk-I2LYULR4.js.map → chunk-2XZGHIVP.js.map} +0 -0
  91. /package/dist/{chunk-5LLGIX5M.js.map → chunk-3EVCHRJF.js.map} +0 -0
  92. /package/dist/{chunk-VI7GBDKV.js.map → chunk-5OF3EGCZ.js.map} +0 -0
  93. /package/dist/{chunk-RXEAACVH.js.map → chunk-6DUTIPTY.js.map} +0 -0
  94. /package/dist/{chunk-CQUVUVXR.js.map → chunk-6ZV2QFXE.js.map} +0 -0
  95. /package/dist/{chunk-2MIOMYT2.js.map → chunk-DAAWUFO4.js.map} +0 -0
  96. /package/dist/{chunk-547G3HYQ.js.map → chunk-EJQOKSKP.js.map} +0 -0
  97. /package/dist/{chunk-Q6SMLSRO.js.map → chunk-JUY77L7C.js.map} +0 -0
  98. /package/dist/{chunk-KW7TIOS3.js.map → chunk-K62U5EJJ.js.map} +0 -0
  99. /package/dist/{chunk-XTD3SZIP.js.map → chunk-N5GOBLUI.js.map} +0 -0
  100. /package/dist/{chunk-VVBGPT6W.js.map → chunk-OIDMPPV5.js.map} +0 -0
  101. /package/dist/{chunk-NKMFHEGV.js.map → chunk-WFPRVZBA.js.map} +0 -0
  102. /package/dist/{chunk-6OABTCRA.js.map → chunk-WI5R64WO.js.map} +0 -0
  103. /package/dist/{chunk-5W7NJHCS.js.map → chunk-YCIZ2WHX.js.map} +0 -0
  104. /package/dist/{chunk-U5YBQJ5Q.js.map → chunk-YQU74AOL.js.map} +0 -0
  105. /package/dist/{chunk-V7ZK6ZIO.js.map → chunk-YRB75GMN.js.map} +0 -0
  106. /package/dist/{client-EFP2PKA3.js.map → client-X2PTYKVX.js.map} +0 -0
  107. /package/dist/{config-P6YXC6XZ.js.map → config-XB2ORKWS.js.map} +0 -0
  108. /package/dist/{config-mutator-3GKXLTHH.js.map → config-mutator-KMXR3XDG.js.map} +0 -0
  109. /package/dist/{doctor-JEWMHXYH.js.map → doctor-6BIMNEGU.js.map} +0 -0
  110. /package/dist/{program-7DUQJDSQ.js.map → program-DHDC6XYB.js.map} +0 -0
  111. /package/dist/{promote-KVPLDSKT.js.map → promote-GBR6YSQW.js.map} +0 -0
  112. /package/dist/{query-AGOMCYTT.js.map → query-672RGMXO.js.map} +0 -0
  113. /package/dist/{registry-AUOI4TW2.js.map → registry-P535FNLB.js.map} +0 -0
  114. /package/dist/{request-planner-LMZ5ZAXC.js.map → request-planner-KHTCHF7N.js.map} +0 -0
  115. /package/dist/{review-candidates-WCBXLLDS.js.map → review-candidates-Y5KCRUNU.js.map} +0 -0
  116. /package/dist/{routing-FBASYKUL.js.map → routing-RWCCDJ4D.js.map} +0 -0
  117. /package/dist/{schedule-tool-RJRJ63SS.js.map → schedule-tool-H46AK53H.js.map} +0 -0
  118. /package/dist/{scheduler-ATA7Z44S.js.map → scheduler-3XLWBCX7.js.map} +0 -0
  119. /package/dist/{seat-DBFW3Z5P.js.map → seat-PJPKLH6L.js.map} +0 -0
  120. /package/dist/{select-6CNVVFA5.js.map → select-VMTJAX5X.js.map} +0 -0
  121. /package/dist/{set-enabled-AYZH3KSB.js.map → set-enabled-LPXH3ZQP.js.map} +0 -0
  122. /package/dist/{tool-PKS4NBZ6.js.map → tool-MWXG44OT.js.map} +0 -0
  123. /package/dist/{wiki-QW6BDNEQ.js.map → wiki-QHMHVFWU.js.map} +0 -0
@@ -29,6 +29,28 @@ final class Recorder {
29
29
  /// between two adjacent 100ms windows — which changes nothing.
30
30
  private(set) var level: Float = 0
31
31
 
32
+ /// How a hands-free turn's end is heard: "vad" (Silero VAD, once its
33
+ /// model is downloaded; the volume threshold until then) or "volume",
34
+ /// from agentx.json's `voice.endOfTurn`. Push-to-talk sets "hold": its
35
+ /// turn ends on key release, so no detector runs. Set before `start()`.
36
+ var endOfTurn = "vad"
37
+ /// True while this recording's turn is judged by Silero VAD.
38
+ private(set) var usingVAD = false
39
+ private var vad: SileroVAD?
40
+ private var turn = TurnEnd(.level)
41
+ private let turnLock = NSLock()
42
+ /// Bumped by every start, so VAD work queued for an earlier recording
43
+ /// never lands in this one.
44
+ private var generation = 0
45
+ private let vadQueue = DispatchQueue(label: "tn.acme.agentx.voice.vad")
46
+
47
+ /// Where the hands-free turn stands: nothing said, talking (a pause
48
+ /// included), or over. Read from the main thread.
49
+ var turnState: TurnEnd.State {
50
+ turnLock.lock(); defer { turnLock.unlock() }
51
+ return turn.state
52
+ }
53
+
32
54
  /// 16 kHz mono int16 — the format both ElevenLabs and whisper prefer,
33
55
  /// and small enough that a 30-second utterance is under 1 MB.
34
56
  private let target = AVAudioFormat(
@@ -58,6 +80,7 @@ final class Recorder {
58
80
  level = 0
59
81
  guard !isRecording else { return }
60
82
  lock.lock(); pcm.removeAll(keepingCapacity: true); lock.unlock()
83
+ prepareTurn()
61
84
 
62
85
  let input = engine.inputNode
63
86
  let inputFormat = input.inputFormat(forBus: 0)
@@ -110,9 +133,8 @@ final class Recorder {
110
133
  sum += sample * sample
111
134
  }
112
135
  let rms = (sum / Float(max(Int(out.frameLength), 1))).squareRoot()
113
- // Attack fast, release slow: a level that drops instantly makes a
114
- // pause between words look like the end of a sentence.
115
- level = rms > level ? rms : level * 0.82 + rms * 0.18
136
+ level = TurnEnd.smooth(level: level, rms: rms)
137
+ hearTurn(channel, count: Int(out.frameLength))
116
138
 
117
139
  let bytes = Int(out.frameLength) * MemoryLayout<Int16>.size
118
140
  lock.lock()
@@ -122,6 +144,62 @@ final class Recorder {
122
144
  lock.unlock()
123
145
  }
124
146
 
147
+ /// A fresh turn for a new recording, judged by Silero VAD when its
148
+ /// model is installed and loads; otherwise by the volume threshold,
149
+ /// with the model downloading in the background for next time.
150
+ private func prepareTurn() {
151
+ var detector: SileroVAD?
152
+ // Only a hands-free turn listens for its own end; push-to-talk ("hold")
153
+ // ends on key release, so it never loads or runs the model.
154
+ if endOfTurn == "vad" {
155
+ do {
156
+ if let model = try SileroVAD.loadModel() {
157
+ detector = try SileroVAD(model: model)
158
+ } else {
159
+ ModelStore.installInBackground(ModelStore.vad) { Log.info($0) }
160
+ }
161
+ } catch {
162
+ Log.warn("Silero VAD would not load (\(error.localizedDescription)); using the volume threshold")
163
+ }
164
+ }
165
+ turnLock.lock()
166
+ generation += 1
167
+ vad = detector
168
+ usingVAD = detector != nil
169
+ turn = TurnEnd(detector != nil ? .vad : .level)
170
+ turnLock.unlock()
171
+ }
172
+
173
+ /// One converted buffer: to the VAD off the audio thread, or straight
174
+ /// to the volume check.
175
+ private func hearTurn(_ channel: UnsafePointer<Int16>, count: Int) {
176
+ turnLock.lock()
177
+ let detector = vad, gen = generation
178
+ if detector == nil {
179
+ turn.feed(level: level, duration: Double(count) / target.sampleRate)
180
+ }
181
+ turnLock.unlock()
182
+ guard let detector else { return }
183
+ var samples = [Float](repeating: 0, count: count)
184
+ for i in 0..<count { samples[i] = Float(channel[i]) / 32768.0 }
185
+ vadQueue.async { [weak self] in
186
+ guard let self else { return }
187
+ let probabilities: [Float]
188
+ do { probabilities = try detector.process(samples) } catch {
189
+ Log.warn("Silero VAD failed (\(error.localizedDescription)); using the volume threshold")
190
+ self.turnLock.lock()
191
+ if self.generation == gen { self.vad = nil; self.usingVAD = false; self.turn = TurnEnd(.level) }
192
+ self.turnLock.unlock()
193
+ return
194
+ }
195
+ self.turnLock.lock()
196
+ if self.generation == gen {
197
+ for p in probabilities { self.turn.feed(probability: p, duration: SileroVAD.frameDuration) }
198
+ }
199
+ self.turnLock.unlock()
200
+ }
201
+ }
202
+
125
203
  /// Minimal RIFF/WAVE header for 16-bit mono PCM.
126
204
  private func wav(_ body: Data) -> Data {
127
205
  let rate = UInt32(target.sampleRate), channels: UInt16 = 1, bits: UInt16 = 16
@@ -22,9 +22,8 @@ final class ResultCard: NSPanel {
22
22
  private let links = NSStackView()
23
23
  private let thumb = NSImageView()
24
24
 
25
- /// The orb's frame on screen while it shows, for the card to slide in
26
- /// under it. Nil: the card sits above the pill. Set by the app.
27
- var anchor: (() -> NSRect?)?
25
+ /// The pill's frame on screen, which the card attaches to. Set by the app.
26
+ var anchor: (() -> NSRect)?
28
27
 
29
28
  init() {
30
29
  super.init(contentRect: NSRect(x: 0, y: 0, width: 380, height: 260),
@@ -129,19 +128,32 @@ final class ResultCard: NSPanel {
129
128
  }
130
129
 
131
130
  if let c = contentView { layout(c) }
132
- if let orb = anchor?() {
133
- slideIn(under: orb)
134
- } else {
135
- positionAboveWidget()
136
- orderFrontRegardless()
137
- }
131
+ slideIn()
132
+ }
133
+
134
+ /// Where the card goes against the pill: above it when there is at
135
+ /// least as much room above as below, else below; right edges aligned.
136
+ @MainActor
137
+ private func placement() -> PillPlacement.Card? {
138
+ guard let pill = anchor?() else { return nil }
139
+ let screen = NSScreen.screens.first { $0.frame.intersects(pill) } ?? NSScreen.main
140
+ guard let visible = screen?.visibleFrame else { return nil }
141
+ return PillPlacement.card(size: frame.size, pill: pill, visible: visible)
138
142
  }
139
143
 
140
- /// Under the orb, right edges aligned, sliding down into place. With
141
- /// Reduce Motion it simply appears there.
144
+ /// The pill moved: the open card moves with it, without animation.
142
145
  @MainActor
143
- private func slideIn(under orb: NSRect) {
144
- let target = NSPoint(x: orb.maxX - frame.width, y: orb.minY - frame.height - 8)
146
+ func follow() {
147
+ guard isVisible, let spot = placement() else { return }
148
+ setFrameOrigin(spot.origin)
149
+ }
150
+
151
+ /// Slides out of the pill into place: up when it opens above, down
152
+ /// when below. With Reduce Motion it simply appears there.
153
+ @MainActor
154
+ private func slideIn() {
155
+ guard let spot = placement() else { alphaValue = 1; orderFrontRegardless(); return }
156
+ let target = spot.origin
145
157
  let wasVisible = isVisible
146
158
  guard !wasVisible, !NSWorkspace.shared.accessibilityDisplayShouldReduceMotion else {
147
159
  setFrameOrigin(target)
@@ -149,7 +161,7 @@ final class ResultCard: NSPanel {
149
161
  orderFrontRegardless()
150
162
  return
151
163
  }
152
- setFrameOrigin(NSPoint(x: target.x, y: target.y + 18))
164
+ setFrameOrigin(NSPoint(x: target.x, y: target.y + (spot.above ? -18 : 18)))
153
165
  alphaValue = 0
154
166
  orderFrontRegardless()
155
167
  NSAnimationContext.runAnimationGroup { ctx in
@@ -165,14 +177,6 @@ final class ResultCard: NSPanel {
165
177
  NSWorkspace.shared.open(u)
166
178
  }
167
179
 
168
- /// Sits just above the pill so the two read as one thing.
169
- private func positionAboveWidget() {
170
- guard let screen = NSScreen.main else { return }
171
- let v = screen.visibleFrame
172
- setFrameOrigin(NSPoint(x: v.maxX - frame.width - 24, y: v.minY + 24 + 54 + 10))
173
- alphaValue = 1
174
- }
175
-
176
180
  override var canBecomeKey: Bool { true }
177
181
  }
178
182
 
@@ -365,7 +365,15 @@ private struct GeneralTab: View {
365
365
  Picker("Speech to text", selection: binding(\.stt)) {
366
366
  Text("Automatic").tag("auto")
367
367
  Text("ElevenLabs").tag("elevenlabs")
368
- Text("On this Mac (Whisper)").tag("local")
368
+ Text("On this Mac").tag("local")
369
+ }
370
+ Picker("On-this-Mac engine", selection: binding(\.localStt)) {
371
+ Text("Whisper (mlx-whisper)").tag("mlx-whisper")
372
+ Text("Parakeet (downloads 483 MB)").tag("parakeet")
373
+ }
374
+ Picker("End of a hands-free turn", selection: binding(\.endOfTurn)) {
375
+ Text("Voice detection (Silero)").tag("vad")
376
+ Text("Volume").tag("volume")
369
377
  }
370
378
  Picker("Default voice provider", selection: binding(\.provider)) {
371
379
  Text("Mac voices (free)").tag("system")
@@ -374,7 +382,7 @@ private struct GeneralTab: View {
374
382
  } header: {
375
383
  Text("Speech")
376
384
  } footer: {
377
- Text("Automatic uses ElevenLabs when a key is set, and Whisper on this Mac otherwise. The default provider is for agents that don't choose their own. Both apply to the next question.")
385
+ Text("Automatic uses ElevenLabs when a key is set, and the engine on this Mac otherwise. Parakeet has no Arabic; until its model has downloaded, Whisper answers instead. Voice detection ends a turn when you stop talking, not when the room goes quiet. The default provider is for agents that don't choose their own. All apply to the next question.")
378
386
  .font(.caption).foregroundStyle(.secondary)
379
387
  }
380
388
  Section {
@@ -0,0 +1,92 @@
1
+ import CoreML
2
+ import Foundation
3
+
4
+ /// Silero VAD on Core ML: how likely each 32 ms of 16 kHz audio is speech.
5
+ ///
6
+ /// Core ML rather than ONNX Runtime: the model is 0.9 MB, macOS already
7
+ /// ships the runtime, and the app keeps building with plain swiftc and no
8
+ /// package dependency. The weights come from ModelStore.vad on first use.
9
+ ///
10
+ /// Streaming: the model is an LSTM, so each call carries the previous
11
+ /// call's state and the last 64 samples as context, exactly as Silero's
12
+ /// own ONNX wrapper does. One `SileroVAD` per recording; the loaded model
13
+ /// is shared.
14
+ final class SileroVAD {
15
+ static let chunk = 512
16
+ static let context = 64
17
+ static let sampleRate = 16_000.0
18
+ static var frameDuration: TimeInterval { Double(chunk) / sampleRate }
19
+
20
+ private static let lock = NSLock()
21
+ nonisolated(unsafe) private static var cached: MLModel?
22
+
23
+ /// The shared model, loaded once. Nil when it is not installed or will
24
+ /// not load; the caller then falls back to the volume threshold.
25
+ static func loadModel() throws -> MLModel? {
26
+ lock.lock(); defer { lock.unlock() }
27
+ if let cached { return cached }
28
+ guard ModelStore.vad.isInstalled else { return nil }
29
+ let config = MLModelConfiguration()
30
+ // Tiny and called every 32 ms: the CPU answers faster than a trip
31
+ // to the Neural Engine.
32
+ config.computeUnits = .cpuOnly
33
+ let url = ModelStore.vad.url("silero-vad-unified-v6.0.0.mlmodelc")
34
+ let model = try MLModel(contentsOf: url, configuration: config)
35
+ cached = model
36
+ return model
37
+ }
38
+
39
+ private let model: MLModel
40
+ private let audio: MLMultiArray
41
+ private var hidden: MLMultiArray
42
+ private var cell: MLMultiArray
43
+ private var pending: [Float] = []
44
+ private var tail = [Float](repeating: 0, count: SileroVAD.context)
45
+
46
+ init(model: MLModel) throws {
47
+ self.model = model
48
+ audio = try MLMultiArray(shape: [1, NSNumber(value: Self.context + Self.chunk)], dataType: .float32)
49
+ hidden = try Self.zeros()
50
+ cell = try Self.zeros()
51
+ }
52
+
53
+ private static func zeros() throws -> MLMultiArray {
54
+ let a = try MLMultiArray(shape: [1, 128], dataType: .float32)
55
+ for i in 0..<a.count { a[i] = 0 }
56
+ return a
57
+ }
58
+
59
+ /// Feeds samples (16 kHz, -1…1) and returns one probability per
60
+ /// complete 512-sample frame; leftovers wait for the next call.
61
+ func process(_ samples: [Float]) throws -> [Float] {
62
+ pending.append(contentsOf: samples)
63
+ var out: [Float] = []
64
+ var start = 0
65
+ while pending.count - start >= Self.chunk {
66
+ let frame = pending[start..<start + Self.chunk]
67
+ out.append(try score(frame))
68
+ start += Self.chunk
69
+ }
70
+ pending.removeFirst(start)
71
+ return out
72
+ }
73
+
74
+ private func score(_ frame: ArraySlice<Float>) throws -> Float {
75
+ let ptr = audio.dataPointer.bindMemory(to: Float.self, capacity: audio.count)
76
+ for i in 0..<Self.context { ptr[i] = tail[i] }
77
+ var i = Self.context
78
+ for s in frame { ptr[i] = s; i += 1 }
79
+ tail = Array(frame.suffix(Self.context))
80
+
81
+ let input = try MLDictionaryFeatureProvider(dictionary: [
82
+ "audio_input": MLFeatureValue(multiArray: audio),
83
+ "hidden_state": MLFeatureValue(multiArray: hidden),
84
+ "cell_state": MLFeatureValue(multiArray: cell),
85
+ ])
86
+ let result = try model.prediction(from: input)
87
+ if let h = result.featureValue(for: "new_hidden_state")?.multiArrayValue { hidden = h }
88
+ if let c = result.featureValue(for: "new_cell_state")?.multiArrayValue { cell = c }
89
+ guard let p = result.featureValue(for: "vad_output")?.multiArrayValue, p.count > 0 else { return 0 }
90
+ return p[0].floatValue
91
+ }
92
+ }
@@ -38,16 +38,18 @@ enum Speech {
38
38
  // MARK: Speech to text
39
39
 
40
40
  /// `engine` is agentx.json's `voice.stt`: "auto" and "elevenlabs" use
41
- /// ElevenLabs when there is a key, falling back to mlx-whisper; "local"
42
- /// never sends audio off the Mac.
43
- static func transcribe(wav: Data, engine: String = "auto") async throws -> String {
41
+ /// ElevenLabs when there is a key, falling back to the local engine;
42
+ /// "local" never sends audio off the Mac. `local` is `voice.localStt`:
43
+ /// "mlx-whisper" or "parakeet".
44
+ static func transcribe(wav: Data, engine: String = "auto", local: String = "mlx-whisper") async throws -> String {
44
45
  if engine == "elevenlabs" && Config.elevenLabsKey == nil {
45
- Log.warn("speech to text is set to ElevenLabs but no key is set; using mlx-whisper")
46
+ Log.warn("speech to text is set to ElevenLabs but no key is set; using \(local)")
46
47
  }
47
48
  if engine != "local", let key = Config.elevenLabsKey {
48
49
  do { return try await elevenLabsSTT(wav: wav, key: key) }
49
- catch { Log.warn("ElevenLabs STT failed (\(error.localizedDescription)); falling back to mlx-whisper") }
50
+ catch { Log.warn("ElevenLabs STT failed (\(error.localizedDescription)); falling back to \(local)") }
50
51
  }
52
+ if local == "parakeet", let text = await LocalSTT.parakeet(wav: wav) { return text }
51
53
  return try await mlxWhisper(wav: wav)
52
54
  }
53
55
 
@@ -23,6 +23,8 @@ final class StatusMenu: NSObject, NSMenuDelegate {
23
23
  /// The daemon's speaking queue, polled while anything is in flight.
24
24
  private var queue: AgentClient.QueueState?
25
25
  private var queuePoll: Timer?
26
+ /// The last three voice exchanges, for a quick replay.
27
+ private let recent = RecentExchanges()
26
28
 
27
29
  /// Questions this widget has in flight, per agent. Set by the app.
28
30
  var thinking: [String: Int] = [:] {
@@ -34,7 +36,13 @@ final class StatusMenu: NSObject, NSMenuDelegate {
34
36
  var onHoldChanged: ((Bool) -> Void)?
35
37
  var onTargetChanged: ((String) -> Void)?
36
38
  var onPillChanged: ((Bool) -> Void)?
37
- var onOrbChanged: ((Bool) -> Void)?
39
+ var onAnimatedOrbChanged: ((Bool) -> Void)?
40
+ /// Hide the pill and stop speech, as its close button does.
41
+ var onHidePill: (() -> Void)?
42
+ /// Put the pill back in the bottom-right corner.
43
+ var onResetPosition: (() -> Void)?
44
+ /// Whether the pill is on screen, for "Hide pill".
45
+ var pillVisible: () -> Bool = { false }
38
46
  var onSettings: (() -> Void)?
39
47
 
40
48
  override init() {
@@ -70,6 +78,7 @@ final class StatusMenu: NSObject, NSMenuDelegate {
70
78
  let agents = await AgentClient.agents()
71
79
  roster = agents.map { .loaded($0) } ?? .down
72
80
  queue = await AgentClient.queueState()
81
+ await recent.refresh()
73
82
  rebuild()
74
83
  showBadge()
75
84
  if reopenAfterRefresh { reopenAfterRefresh = false; open() }
@@ -169,6 +178,7 @@ final class StatusMenu: NSObject, NSMenuDelegate {
169
178
  menu.addItem(row)
170
179
  }
171
180
  }
181
+ recent.addItems(to: menu, name: name(of:))
172
182
 
173
183
  menu.addItem(.separator())
174
184
  menu.addItem(action("Stop speaking", #selector(stop), key: ".", modifiers: [.command, .option]))
@@ -178,9 +188,13 @@ final class StatusMenu: NSObject, NSMenuDelegate {
178
188
  let pill = action("Show floating pill", #selector(togglePill), key: "")
179
189
  pill.state = Config.showPill ? .on : .off
180
190
  menu.addItem(pill)
181
- let orb = action("Show orb", #selector(toggleOrb), key: "")
182
- orb.state = Config.showOrb ? .on : .off
191
+ let orb = action("Animated orb", #selector(toggleOrb), key: "")
192
+ orb.state = Config.animatedOrb ? .on : .off
183
193
  menu.addItem(orb)
194
+ let hide = action("Hide pill", #selector(hidePill), key: "")
195
+ hide.isEnabled = pillVisible()
196
+ menu.addItem(hide)
197
+ menu.addItem(action("Reset position", #selector(resetPosition), key: ""))
184
198
 
185
199
  menu.addItem(.separator())
186
200
  menu.addItem(action("Settings…", #selector(openSettings), key: ","))
@@ -232,13 +246,16 @@ final class StatusMenu: NSObject, NSMenuDelegate {
232
246
  }
233
247
 
234
248
  @objc private func toggleOrb() {
235
- Config.showOrb.toggle()
236
- onOrbChanged?(Config.showOrb)
249
+ Config.animatedOrb.toggle()
250
+ onAnimatedOrbChanged?(Config.animatedOrb)
237
251
  }
238
252
 
253
+ @objc private func hidePill() { onHidePill?() }
254
+ @objc private func resetPosition() { onResetPosition?() }
255
+
239
256
  @objc private func openSettings() { onSettings?() }
240
257
  @objc private func openDashboardHome() { openDashboard("/admin") }
241
- @objc private func openHistory() { openDashboard("/activity") }
258
+ @objc private func openHistory() { HistoryWindow.shared.show() }
242
259
 
243
260
  private func openDashboard(_ path: String) {
244
261
  guard let url = URL(string: Config.dashboardURL + path) else { return }
@@ -0,0 +1,110 @@
1
+ import Foundation
2
+
3
+ /// When a hands-free turn has ended: the person spoke, then stopped for
4
+ /// long enough that the sentence is over.
5
+ ///
6
+ /// It is fed one audio frame at a time with how likely that frame is to be
7
+ /// speech (0…1) and how long it lasted. Two sources feed it:
8
+ ///
9
+ /// - Silero VAD, 32 ms frames with a real probability. A voice model
10
+ /// tells a word from a fan, a keyboard or a café, so the noise of a room
11
+ /// neither holds the microphone open nor counts as talking.
12
+ /// - The old volume check when the VAD model is not available: 1 above
13
+ /// `speechLevel`, 0 below, per audio buffer.
14
+ ///
15
+ /// Foundation only and time passed in, so the tests can drive it with
16
+ /// synthetic frames.
17
+ struct TurnEnd {
18
+ struct Tuning: Equatable {
19
+ /// A frame at or above this starts speech (Silero's own default).
20
+ var start: Float
21
+ /// Once talking, a frame at or above this still counts as speech.
22
+ /// Lower than `start`: the soft end of a word is not a pause.
23
+ var keep: Float
24
+ /// Speech must last this long in one go before the turn counts as
25
+ /// started, so a cough, a door or a click does not open it.
26
+ var minSpeech: TimeInterval
27
+ /// A burst this short inside a pause does not restart the pause:
28
+ /// one noisy frame must not hold the microphone open.
29
+ var resume: TimeInterval
30
+ /// A pause this long after speech ends the turn. People pause
31
+ /// inside sentences; this is longer than those pauses.
32
+ var endSilence: TimeInterval
33
+
34
+ /// For Silero VAD probabilities.
35
+ static let vad = Tuning(start: 0.5, keep: 0.35, minSpeech: 0.25, resume: 0.1, endSilence: 1.2)
36
+
37
+ /// The volume threshold as it always worked: any buffer above the
38
+ /// level is speech, and 1.2 s below it ends the turn.
39
+ static let level = Tuning(start: 0.5, keep: 0.5, minSpeech: 0, resume: 0, endSilence: 1.2)
40
+ }
41
+
42
+ enum State: Equatable {
43
+ /// Nothing said yet.
44
+ case waiting
45
+ /// Speech heard; the turn is open (possibly in a short pause).
46
+ case speaking
47
+ /// Speech, then `endSilence` of quiet: send it.
48
+ case ended
49
+ }
50
+
51
+ /// RMS above which the microphone is hearing a voice rather than a
52
+ /// room. The fallback's whole notion of speech.
53
+ static let speechLevel: Float = 0.02
54
+
55
+ let tuning: Tuning
56
+ private(set) var state: State = .waiting
57
+ /// How long the current run of speech frames has lasted.
58
+ private var speechRun: TimeInterval = 0
59
+ /// How long the current pause has lasted (a short burst does not
60
+ /// reset it).
61
+ private var silence: TimeInterval = 0
62
+
63
+ init(_ tuning: Tuning) { self.tuning = tuning }
64
+
65
+ /// One frame: `probability` that it is speech, `duration` it covers.
66
+ @discardableResult
67
+ mutating func feed(probability: Float, duration: TimeInterval) -> State {
68
+ guard state != .ended else { return state }
69
+ let threshold = state == .speaking && silence == 0 ? tuning.keep : tuning.start
70
+ let speech = probability >= threshold
71
+
72
+ switch state {
73
+ case .waiting:
74
+ speechRun = speech ? speechRun + duration : 0
75
+ if speech && speechRun >= tuning.minSpeech {
76
+ state = .speaking
77
+ silence = 0
78
+ }
79
+ case .speaking:
80
+ if speech {
81
+ speechRun += duration
82
+ if silence == 0 || speechRun >= tuning.resume {
83
+ silence = 0
84
+ } else {
85
+ silence += duration
86
+ }
87
+ } else {
88
+ speechRun = 0
89
+ silence += duration
90
+ }
91
+ if silence >= tuning.endSilence { state = .ended }
92
+ case .ended:
93
+ break
94
+ }
95
+ return state
96
+ }
97
+
98
+ /// The fallback's frame: the smoothed microphone level.
99
+ @discardableResult
100
+ mutating func feed(level: Float, duration: TimeInterval) -> State {
101
+ feed(probability: level > Self.speechLevel ? 1 : 0, duration: duration)
102
+ }
103
+
104
+ /// The microphone level after one buffer of loudness `rms`. Attack
105
+ /// fast, release slow: a level that drops instantly makes a pause
106
+ /// between words look like the end of a sentence.
107
+ static func smooth(level: Float, rms: Float) -> Float {
108
+ rms > level ? rms : level * 0.82 + rms * 0.18
109
+ }
110
+ }
@@ -12,7 +12,28 @@ struct VoiceSettings: Codable, Equatable {
12
12
  var provider: String
13
13
  var fallback: String
14
14
  var stt: String
15
+ /// The on-device engine: "mlx-whisper" or "parakeet".
16
+ var localStt: String
17
+ /// How a hands-free turn's end is heard: "vad" or "volume".
18
+ var endOfTurn: String
15
19
  var hotkeys: Hotkeys
20
+
21
+ init(provider: String, fallback: String, stt: String, localStt: String = "mlx-whisper",
22
+ endOfTurn: String = "vad", hotkeys: Hotkeys) {
23
+ self.provider = provider; self.fallback = fallback; self.stt = stt
24
+ self.localStt = localStt; self.endOfTurn = endOfTurn; self.hotkeys = hotkeys
25
+ }
26
+
27
+ /// A daemon from before these fields leaves them out: the defaults.
28
+ init(from decoder: Decoder) throws {
29
+ let c = try decoder.container(keyedBy: CodingKeys.self)
30
+ provider = try c.decode(String.self, forKey: .provider)
31
+ fallback = try c.decode(String.self, forKey: .fallback)
32
+ stt = try c.decode(String.self, forKey: .stt)
33
+ localStt = try c.decodeIfPresent(String.self, forKey: .localStt) ?? "mlx-whisper"
34
+ endOfTurn = try c.decodeIfPresent(String.self, forKey: .endOfTurn) ?? "vad"
35
+ hotkeys = try c.decode(Hotkeys.self, forKey: .hotkeys)
36
+ }
16
37
  }
17
38
  struct Voice: Codable, Equatable {
18
39
  var provider: String?
@@ -55,6 +76,8 @@ struct VoiceSettings: Codable, Equatable {
55
76
  var general: [String: Any] = [:]
56
77
  if self.general.provider != old.general.provider { general["provider"] = self.general.provider }
57
78
  if self.general.stt != old.general.stt { general["stt"] = self.general.stt }
79
+ if self.general.localStt != old.general.localStt { general["localStt"] = self.general.localStt }
80
+ if self.general.endOfTurn != old.general.endOfTurn { general["endOfTurn"] = self.general.endOfTurn }
58
81
  var keys: [String: Any] = [:]
59
82
  if self.general.hotkeys.talk != old.general.hotkeys.talk { keys["talk"] = self.general.hotkeys.talk }
60
83
  if self.general.hotkeys.stop != old.general.hotkeys.stop { keys["stop"] = self.general.hotkeys.stop }