agentix-cli 0.57.0 → 0.59.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/apps/mac-voice/Sources/AgentXVoice/Config.swift +19 -5
- package/apps/mac-voice/Sources/AgentXVoice/HistoryClient.swift +55 -0
- package/apps/mac-voice/Sources/AgentXVoice/HistoryModel.swift +153 -0
- package/apps/mac-voice/Sources/AgentXVoice/HistoryWindow.swift +301 -0
- package/apps/mac-voice/Sources/AgentXVoice/LocalSTT.swift +79 -0
- package/apps/mac-voice/Sources/AgentXVoice/ModelStore.swift +184 -0
- package/apps/mac-voice/Sources/AgentXVoice/Orb.swift +131 -169
- package/apps/mac-voice/Sources/AgentXVoice/Panel.swift +174 -35
- package/apps/mac-voice/Sources/AgentXVoice/Parakeet.swift +264 -0
- package/apps/mac-voice/Sources/AgentXVoice/PillPlacement.swift +70 -0
- package/apps/mac-voice/Sources/AgentXVoice/Recorder.swift +81 -3
- package/apps/mac-voice/Sources/AgentXVoice/ResultCard.swift +26 -22
- package/apps/mac-voice/Sources/AgentXVoice/SettingsWindow.swift +10 -2
- package/apps/mac-voice/Sources/AgentXVoice/SileroVAD.swift +92 -0
- package/apps/mac-voice/Sources/AgentXVoice/Speech.swift +7 -5
- package/apps/mac-voice/Sources/AgentXVoice/StatusMenu.swift +23 -6
- package/apps/mac-voice/Sources/AgentXVoice/TurnEnd.swift +110 -0
- package/apps/mac-voice/Sources/AgentXVoice/VoiceSettings.swift +23 -0
- package/apps/mac-voice/Sources/AgentXVoice/main.swift +47 -70
- package/apps/mac-voice/build.sh +10 -1
- package/dist/{App-Z5VOAUXB.js → App-QEJGXZTY.js} +2 -2
- package/dist/{ChatApp-BE7ONQKS.js → ChatApp-VKWQN5GM.js} +2 -2
- package/dist/{agent-Q74EISKP.js → agent-MJKLI2L5.js} +2 -2
- package/dist/{board-dashboard-A7PQOI6C.js → board-dashboard-2KVT72J7.js} +221 -138
- package/dist/board-dashboard-2KVT72J7.js.map +1 -0
- package/dist/{chunk-I2LYULR4.js → chunk-2XZGHIVP.js} +5 -5
- package/dist/{chunk-5LLGIX5M.js → chunk-3EVCHRJF.js} +2 -2
- package/dist/{chunk-6QNPCVRN.js → chunk-4ZTV6JV7.js} +3 -3
- package/dist/chunk-4ZTV6JV7.js.map +1 -0
- package/dist/{chunk-VI7GBDKV.js → chunk-5OF3EGCZ.js} +2 -2
- package/dist/{chunk-RXEAACVH.js → chunk-6DUTIPTY.js} +2 -2
- package/dist/{chunk-CQUVUVXR.js → chunk-6ZV2QFXE.js} +2 -2
- package/dist/{chunk-2MIOMYT2.js → chunk-DAAWUFO4.js} +2 -2
- package/dist/{chunk-547G3HYQ.js → chunk-EJQOKSKP.js} +2 -2
- package/dist/{chunk-POL7RAJQ.js → chunk-ILXW3ERG.js} +117 -117
- package/dist/chunk-ILXW3ERG.js.map +1 -0
- package/dist/{chunk-ERL6AS3O.js → chunk-IW4XIZAZ.js} +8 -8
- package/dist/{chunk-ERL6AS3O.js.map → chunk-IW4XIZAZ.js.map} +1 -1
- package/dist/{chunk-Q6SMLSRO.js → chunk-JUY77L7C.js} +2 -2
- package/dist/{chunk-KW7TIOS3.js → chunk-K62U5EJJ.js} +2 -2
- package/dist/{chunk-KKGFZEXS.js → chunk-L5OCF33B.js} +37 -36
- package/dist/chunk-L5OCF33B.js.map +1 -0
- package/dist/{chunk-XTD3SZIP.js → chunk-N5GOBLUI.js} +2 -2
- package/dist/{chunk-VVBGPT6W.js → chunk-OIDMPPV5.js} +2 -2
- package/dist/{chunk-OZZPJ2YG.js → chunk-SDBU6747.js} +190 -177
- package/dist/chunk-SDBU6747.js.map +1 -0
- package/dist/{chunk-NKMFHEGV.js → chunk-WFPRVZBA.js} +2 -2
- package/dist/{chunk-6OABTCRA.js → chunk-WI5R64WO.js} +2 -2
- package/dist/{chunk-5W7NJHCS.js → chunk-YCIZ2WHX.js} +2 -2
- package/dist/{chunk-U5YBQJ5Q.js → chunk-YQU74AOL.js} +2 -2
- package/dist/{chunk-V7ZK6ZIO.js → chunk-YRB75GMN.js} +2 -2
- package/dist/cli.js +1 -1
- package/dist/client-X2PTYKVX.js +2 -0
- package/dist/{config-P6YXC6XZ.js → config-XB2ORKWS.js} +2 -2
- package/dist/config-mutator-KMXR3XDG.js +2 -0
- package/dist/doctor-6BIMNEGU.js +2 -0
- package/dist/index.d.ts +17 -0
- package/dist/index.js +1 -1
- package/dist/program-DHDC6XYB.js +2 -0
- package/dist/{promote-KVPLDSKT.js → promote-GBR6YSQW.js} +2 -2
- package/dist/query-672RGMXO.js +2 -0
- package/dist/{registry-AUOI4TW2.js → registry-P535FNLB.js} +2 -2
- package/dist/{request-planner-LMZ5ZAXC.js → request-planner-KHTCHF7N.js} +2 -2
- package/dist/{review-candidates-WCBXLLDS.js → review-candidates-Y5KCRUNU.js} +2 -2
- package/dist/{routing-FBASYKUL.js → routing-RWCCDJ4D.js} +2 -2
- package/dist/schedule-tool-H46AK53H.js +2 -0
- package/dist/scheduler-3XLWBCX7.js +2 -0
- package/dist/{seat-DBFW3Z5P.js → seat-PJPKLH6L.js} +2 -2
- package/dist/{select-6CNVVFA5.js → select-VMTJAX5X.js} +2 -2
- package/dist/{set-enabled-AYZH3KSB.js → set-enabled-LPXH3ZQP.js} +2 -2
- package/dist/{tool-PKS4NBZ6.js → tool-MWXG44OT.js} +2 -2
- package/dist/wiki-QHMHVFWU.js +2 -0
- package/package.json +1 -1
- package/dist/board-dashboard-A7PQOI6C.js.map +0 -1
- package/dist/chunk-6QNPCVRN.js.map +0 -1
- package/dist/chunk-KKGFZEXS.js.map +0 -1
- package/dist/chunk-OZZPJ2YG.js.map +0 -1
- package/dist/chunk-POL7RAJQ.js.map +0 -1
- package/dist/client-EFP2PKA3.js +0 -2
- package/dist/config-mutator-3GKXLTHH.js +0 -2
- package/dist/doctor-JEWMHXYH.js +0 -2
- package/dist/program-7DUQJDSQ.js +0 -2
- package/dist/query-AGOMCYTT.js +0 -2
- package/dist/schedule-tool-RJRJ63SS.js +0 -2
- package/dist/scheduler-ATA7Z44S.js +0 -2
- package/dist/wiki-QW6BDNEQ.js +0 -2
- /package/dist/{App-Z5VOAUXB.js.map → App-QEJGXZTY.js.map} +0 -0
- /package/dist/{ChatApp-BE7ONQKS.js.map → ChatApp-VKWQN5GM.js.map} +0 -0
- /package/dist/{agent-Q74EISKP.js.map → agent-MJKLI2L5.js.map} +0 -0
- /package/dist/{chunk-I2LYULR4.js.map → chunk-2XZGHIVP.js.map} +0 -0
- /package/dist/{chunk-5LLGIX5M.js.map → chunk-3EVCHRJF.js.map} +0 -0
- /package/dist/{chunk-VI7GBDKV.js.map → chunk-5OF3EGCZ.js.map} +0 -0
- /package/dist/{chunk-RXEAACVH.js.map → chunk-6DUTIPTY.js.map} +0 -0
- /package/dist/{chunk-CQUVUVXR.js.map → chunk-6ZV2QFXE.js.map} +0 -0
- /package/dist/{chunk-2MIOMYT2.js.map → chunk-DAAWUFO4.js.map} +0 -0
- /package/dist/{chunk-547G3HYQ.js.map → chunk-EJQOKSKP.js.map} +0 -0
- /package/dist/{chunk-Q6SMLSRO.js.map → chunk-JUY77L7C.js.map} +0 -0
- /package/dist/{chunk-KW7TIOS3.js.map → chunk-K62U5EJJ.js.map} +0 -0
- /package/dist/{chunk-XTD3SZIP.js.map → chunk-N5GOBLUI.js.map} +0 -0
- /package/dist/{chunk-VVBGPT6W.js.map → chunk-OIDMPPV5.js.map} +0 -0
- /package/dist/{chunk-NKMFHEGV.js.map → chunk-WFPRVZBA.js.map} +0 -0
- /package/dist/{chunk-6OABTCRA.js.map → chunk-WI5R64WO.js.map} +0 -0
- /package/dist/{chunk-5W7NJHCS.js.map → chunk-YCIZ2WHX.js.map} +0 -0
- /package/dist/{chunk-U5YBQJ5Q.js.map → chunk-YQU74AOL.js.map} +0 -0
- /package/dist/{chunk-V7ZK6ZIO.js.map → chunk-YRB75GMN.js.map} +0 -0
- /package/dist/{client-EFP2PKA3.js.map → client-X2PTYKVX.js.map} +0 -0
- /package/dist/{config-P6YXC6XZ.js.map → config-XB2ORKWS.js.map} +0 -0
- /package/dist/{config-mutator-3GKXLTHH.js.map → config-mutator-KMXR3XDG.js.map} +0 -0
- /package/dist/{doctor-JEWMHXYH.js.map → doctor-6BIMNEGU.js.map} +0 -0
- /package/dist/{program-7DUQJDSQ.js.map → program-DHDC6XYB.js.map} +0 -0
- /package/dist/{promote-KVPLDSKT.js.map → promote-GBR6YSQW.js.map} +0 -0
- /package/dist/{query-AGOMCYTT.js.map → query-672RGMXO.js.map} +0 -0
- /package/dist/{registry-AUOI4TW2.js.map → registry-P535FNLB.js.map} +0 -0
- /package/dist/{request-planner-LMZ5ZAXC.js.map → request-planner-KHTCHF7N.js.map} +0 -0
- /package/dist/{review-candidates-WCBXLLDS.js.map → review-candidates-Y5KCRUNU.js.map} +0 -0
- /package/dist/{routing-FBASYKUL.js.map → routing-RWCCDJ4D.js.map} +0 -0
- /package/dist/{schedule-tool-RJRJ63SS.js.map → schedule-tool-H46AK53H.js.map} +0 -0
- /package/dist/{scheduler-ATA7Z44S.js.map → scheduler-3XLWBCX7.js.map} +0 -0
- /package/dist/{seat-DBFW3Z5P.js.map → seat-PJPKLH6L.js.map} +0 -0
- /package/dist/{select-6CNVVFA5.js.map → select-VMTJAX5X.js.map} +0 -0
- /package/dist/{set-enabled-AYZH3KSB.js.map → set-enabled-LPXH3ZQP.js.map} +0 -0
- /package/dist/{tool-PKS4NBZ6.js.map → tool-MWXG44OT.js.map} +0 -0
- /package/dist/{wiki-QW6BDNEQ.js.map → wiki-QHMHVFWU.js.map} +0 -0
|
@@ -29,6 +29,28 @@ final class Recorder {
|
|
|
29
29
|
/// between two adjacent 100ms windows — which changes nothing.
|
|
30
30
|
private(set) var level: Float = 0
|
|
31
31
|
|
|
32
|
+
/// How a hands-free turn's end is heard: "vad" (Silero VAD, once its
|
|
33
|
+
/// model is downloaded; the volume threshold until then) or "volume",
|
|
34
|
+
/// from agentx.json's `voice.endOfTurn`. Push-to-talk sets "hold": its
|
|
35
|
+
/// turn ends on key release, so no detector runs. Set before `start()`.
|
|
36
|
+
var endOfTurn = "vad"
|
|
37
|
+
/// True while this recording's turn is judged by Silero VAD.
|
|
38
|
+
private(set) var usingVAD = false
|
|
39
|
+
private var vad: SileroVAD?
|
|
40
|
+
private var turn = TurnEnd(.level)
|
|
41
|
+
private let turnLock = NSLock()
|
|
42
|
+
/// Bumped by every start, so VAD work queued for an earlier recording
|
|
43
|
+
/// never lands in this one.
|
|
44
|
+
private var generation = 0
|
|
45
|
+
private let vadQueue = DispatchQueue(label: "tn.acme.agentx.voice.vad")
|
|
46
|
+
|
|
47
|
+
/// Where the hands-free turn stands: nothing said, talking (a pause
|
|
48
|
+
/// included), or over. Read from the main thread.
|
|
49
|
+
var turnState: TurnEnd.State {
|
|
50
|
+
turnLock.lock(); defer { turnLock.unlock() }
|
|
51
|
+
return turn.state
|
|
52
|
+
}
|
|
53
|
+
|
|
32
54
|
/// 16 kHz mono int16 — the format both ElevenLabs and whisper prefer,
|
|
33
55
|
/// and small enough that a 30-second utterance is under 1 MB.
|
|
34
56
|
private let target = AVAudioFormat(
|
|
@@ -58,6 +80,7 @@ final class Recorder {
|
|
|
58
80
|
level = 0
|
|
59
81
|
guard !isRecording else { return }
|
|
60
82
|
lock.lock(); pcm.removeAll(keepingCapacity: true); lock.unlock()
|
|
83
|
+
prepareTurn()
|
|
61
84
|
|
|
62
85
|
let input = engine.inputNode
|
|
63
86
|
let inputFormat = input.inputFormat(forBus: 0)
|
|
@@ -110,9 +133,8 @@ final class Recorder {
|
|
|
110
133
|
sum += sample * sample
|
|
111
134
|
}
|
|
112
135
|
let rms = (sum / Float(max(Int(out.frameLength), 1))).squareRoot()
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
level = rms > level ? rms : level * 0.82 + rms * 0.18
|
|
136
|
+
level = TurnEnd.smooth(level: level, rms: rms)
|
|
137
|
+
hearTurn(channel, count: Int(out.frameLength))
|
|
116
138
|
|
|
117
139
|
let bytes = Int(out.frameLength) * MemoryLayout<Int16>.size
|
|
118
140
|
lock.lock()
|
|
@@ -122,6 +144,62 @@ final class Recorder {
|
|
|
122
144
|
lock.unlock()
|
|
123
145
|
}
|
|
124
146
|
|
|
147
|
+
/// A fresh turn for a new recording, judged by Silero VAD when its
|
|
148
|
+
/// model is installed and loads; otherwise by the volume threshold,
|
|
149
|
+
/// with the model downloading in the background for next time.
|
|
150
|
+
private func prepareTurn() {
|
|
151
|
+
var detector: SileroVAD?
|
|
152
|
+
// Only a hands-free turn listens for its own end; push-to-talk ("hold")
|
|
153
|
+
// ends on key release, so it never loads or runs the model.
|
|
154
|
+
if endOfTurn == "vad" {
|
|
155
|
+
do {
|
|
156
|
+
if let model = try SileroVAD.loadModel() {
|
|
157
|
+
detector = try SileroVAD(model: model)
|
|
158
|
+
} else {
|
|
159
|
+
ModelStore.installInBackground(ModelStore.vad) { Log.info($0) }
|
|
160
|
+
}
|
|
161
|
+
} catch {
|
|
162
|
+
Log.warn("Silero VAD would not load (\(error.localizedDescription)); using the volume threshold")
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
turnLock.lock()
|
|
166
|
+
generation += 1
|
|
167
|
+
vad = detector
|
|
168
|
+
usingVAD = detector != nil
|
|
169
|
+
turn = TurnEnd(detector != nil ? .vad : .level)
|
|
170
|
+
turnLock.unlock()
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
/// One converted buffer: to the VAD off the audio thread, or straight
|
|
174
|
+
/// to the volume check.
|
|
175
|
+
private func hearTurn(_ channel: UnsafePointer<Int16>, count: Int) {
|
|
176
|
+
turnLock.lock()
|
|
177
|
+
let detector = vad, gen = generation
|
|
178
|
+
if detector == nil {
|
|
179
|
+
turn.feed(level: level, duration: Double(count) / target.sampleRate)
|
|
180
|
+
}
|
|
181
|
+
turnLock.unlock()
|
|
182
|
+
guard let detector else { return }
|
|
183
|
+
var samples = [Float](repeating: 0, count: count)
|
|
184
|
+
for i in 0..<count { samples[i] = Float(channel[i]) / 32768.0 }
|
|
185
|
+
vadQueue.async { [weak self] in
|
|
186
|
+
guard let self else { return }
|
|
187
|
+
let probabilities: [Float]
|
|
188
|
+
do { probabilities = try detector.process(samples) } catch {
|
|
189
|
+
Log.warn("Silero VAD failed (\(error.localizedDescription)); using the volume threshold")
|
|
190
|
+
self.turnLock.lock()
|
|
191
|
+
if self.generation == gen { self.vad = nil; self.usingVAD = false; self.turn = TurnEnd(.level) }
|
|
192
|
+
self.turnLock.unlock()
|
|
193
|
+
return
|
|
194
|
+
}
|
|
195
|
+
self.turnLock.lock()
|
|
196
|
+
if self.generation == gen {
|
|
197
|
+
for p in probabilities { self.turn.feed(probability: p, duration: SileroVAD.frameDuration) }
|
|
198
|
+
}
|
|
199
|
+
self.turnLock.unlock()
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
|
|
125
203
|
/// Minimal RIFF/WAVE header for 16-bit mono PCM.
|
|
126
204
|
private func wav(_ body: Data) -> Data {
|
|
127
205
|
let rate = UInt32(target.sampleRate), channels: UInt16 = 1, bits: UInt16 = 16
|
|
@@ -22,9 +22,8 @@ final class ResultCard: NSPanel {
|
|
|
22
22
|
private let links = NSStackView()
|
|
23
23
|
private let thumb = NSImageView()
|
|
24
24
|
|
|
25
|
-
/// The
|
|
26
|
-
|
|
27
|
-
var anchor: (() -> NSRect?)?
|
|
25
|
+
/// The pill's frame on screen, which the card attaches to. Set by the app.
|
|
26
|
+
var anchor: (() -> NSRect)?
|
|
28
27
|
|
|
29
28
|
init() {
|
|
30
29
|
super.init(contentRect: NSRect(x: 0, y: 0, width: 380, height: 260),
|
|
@@ -129,19 +128,32 @@ final class ResultCard: NSPanel {
|
|
|
129
128
|
}
|
|
130
129
|
|
|
131
130
|
if let c = contentView { layout(c) }
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
131
|
+
slideIn()
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/// Where the card goes against the pill: above it when there is at
|
|
135
|
+
/// least as much room above as below, else below; right edges aligned.
|
|
136
|
+
@MainActor
|
|
137
|
+
private func placement() -> PillPlacement.Card? {
|
|
138
|
+
guard let pill = anchor?() else { return nil }
|
|
139
|
+
let screen = NSScreen.screens.first { $0.frame.intersects(pill) } ?? NSScreen.main
|
|
140
|
+
guard let visible = screen?.visibleFrame else { return nil }
|
|
141
|
+
return PillPlacement.card(size: frame.size, pill: pill, visible: visible)
|
|
138
142
|
}
|
|
139
143
|
|
|
140
|
-
///
|
|
141
|
-
/// Reduce Motion it simply appears there.
|
|
144
|
+
/// The pill moved: the open card moves with it, without animation.
|
|
142
145
|
@MainActor
|
|
143
|
-
|
|
144
|
-
let
|
|
146
|
+
func follow() {
|
|
147
|
+
guard isVisible, let spot = placement() else { return }
|
|
148
|
+
setFrameOrigin(spot.origin)
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/// Slides out of the pill into place: up when it opens above, down
|
|
152
|
+
/// when below. With Reduce Motion it simply appears there.
|
|
153
|
+
@MainActor
|
|
154
|
+
private func slideIn() {
|
|
155
|
+
guard let spot = placement() else { alphaValue = 1; orderFrontRegardless(); return }
|
|
156
|
+
let target = spot.origin
|
|
145
157
|
let wasVisible = isVisible
|
|
146
158
|
guard !wasVisible, !NSWorkspace.shared.accessibilityDisplayShouldReduceMotion else {
|
|
147
159
|
setFrameOrigin(target)
|
|
@@ -149,7 +161,7 @@ final class ResultCard: NSPanel {
|
|
|
149
161
|
orderFrontRegardless()
|
|
150
162
|
return
|
|
151
163
|
}
|
|
152
|
-
setFrameOrigin(NSPoint(x: target.x, y: target.y + 18))
|
|
164
|
+
setFrameOrigin(NSPoint(x: target.x, y: target.y + (spot.above ? -18 : 18)))
|
|
153
165
|
alphaValue = 0
|
|
154
166
|
orderFrontRegardless()
|
|
155
167
|
NSAnimationContext.runAnimationGroup { ctx in
|
|
@@ -165,14 +177,6 @@ final class ResultCard: NSPanel {
|
|
|
165
177
|
NSWorkspace.shared.open(u)
|
|
166
178
|
}
|
|
167
179
|
|
|
168
|
-
/// Sits just above the pill so the two read as one thing.
|
|
169
|
-
private func positionAboveWidget() {
|
|
170
|
-
guard let screen = NSScreen.main else { return }
|
|
171
|
-
let v = screen.visibleFrame
|
|
172
|
-
setFrameOrigin(NSPoint(x: v.maxX - frame.width - 24, y: v.minY + 24 + 54 + 10))
|
|
173
|
-
alphaValue = 1
|
|
174
|
-
}
|
|
175
|
-
|
|
176
180
|
override var canBecomeKey: Bool { true }
|
|
177
181
|
}
|
|
178
182
|
|
|
@@ -365,7 +365,15 @@ private struct GeneralTab: View {
|
|
|
365
365
|
Picker("Speech to text", selection: binding(\.stt)) {
|
|
366
366
|
Text("Automatic").tag("auto")
|
|
367
367
|
Text("ElevenLabs").tag("elevenlabs")
|
|
368
|
-
Text("On this Mac
|
|
368
|
+
Text("On this Mac").tag("local")
|
|
369
|
+
}
|
|
370
|
+
Picker("On-this-Mac engine", selection: binding(\.localStt)) {
|
|
371
|
+
Text("Whisper (mlx-whisper)").tag("mlx-whisper")
|
|
372
|
+
Text("Parakeet (downloads 483 MB)").tag("parakeet")
|
|
373
|
+
}
|
|
374
|
+
Picker("End of a hands-free turn", selection: binding(\.endOfTurn)) {
|
|
375
|
+
Text("Voice detection (Silero)").tag("vad")
|
|
376
|
+
Text("Volume").tag("volume")
|
|
369
377
|
}
|
|
370
378
|
Picker("Default voice provider", selection: binding(\.provider)) {
|
|
371
379
|
Text("Mac voices (free)").tag("system")
|
|
@@ -374,7 +382,7 @@ private struct GeneralTab: View {
|
|
|
374
382
|
} header: {
|
|
375
383
|
Text("Speech")
|
|
376
384
|
} footer: {
|
|
377
|
-
Text("Automatic uses ElevenLabs when a key is set, and
|
|
385
|
+
Text("Automatic uses ElevenLabs when a key is set, and the engine on this Mac otherwise. Parakeet has no Arabic; until its model has downloaded, Whisper answers instead. Voice detection ends a turn when you stop talking, not when the room goes quiet. The default provider is for agents that don't choose their own. All apply to the next question.")
|
|
378
386
|
.font(.caption).foregroundStyle(.secondary)
|
|
379
387
|
}
|
|
380
388
|
Section {
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
import CoreML
|
|
2
|
+
import Foundation
|
|
3
|
+
|
|
4
|
+
/// Silero VAD on Core ML: how likely each 32 ms of 16 kHz audio is speech.
|
|
5
|
+
///
|
|
6
|
+
/// Core ML rather than ONNX Runtime: the model is 0.9 MB, macOS already
|
|
7
|
+
/// ships the runtime, and the app keeps building with plain swiftc and no
|
|
8
|
+
/// package dependency. The weights come from ModelStore.vad on first use.
|
|
9
|
+
///
|
|
10
|
+
/// Streaming: the model is an LSTM, so each call carries the previous
|
|
11
|
+
/// call's state and the last 64 samples as context, exactly as Silero's
|
|
12
|
+
/// own ONNX wrapper does. One `SileroVAD` per recording; the loaded model
|
|
13
|
+
/// is shared.
|
|
14
|
+
final class SileroVAD {
|
|
15
|
+
static let chunk = 512
|
|
16
|
+
static let context = 64
|
|
17
|
+
static let sampleRate = 16_000.0
|
|
18
|
+
static var frameDuration: TimeInterval { Double(chunk) / sampleRate }
|
|
19
|
+
|
|
20
|
+
private static let lock = NSLock()
|
|
21
|
+
nonisolated(unsafe) private static var cached: MLModel?
|
|
22
|
+
|
|
23
|
+
/// The shared model, loaded once. Nil when it is not installed or will
|
|
24
|
+
/// not load; the caller then falls back to the volume threshold.
|
|
25
|
+
static func loadModel() throws -> MLModel? {
|
|
26
|
+
lock.lock(); defer { lock.unlock() }
|
|
27
|
+
if let cached { return cached }
|
|
28
|
+
guard ModelStore.vad.isInstalled else { return nil }
|
|
29
|
+
let config = MLModelConfiguration()
|
|
30
|
+
// Tiny and called every 32 ms: the CPU answers faster than a trip
|
|
31
|
+
// to the Neural Engine.
|
|
32
|
+
config.computeUnits = .cpuOnly
|
|
33
|
+
let url = ModelStore.vad.url("silero-vad-unified-v6.0.0.mlmodelc")
|
|
34
|
+
let model = try MLModel(contentsOf: url, configuration: config)
|
|
35
|
+
cached = model
|
|
36
|
+
return model
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
private let model: MLModel
|
|
40
|
+
private let audio: MLMultiArray
|
|
41
|
+
private var hidden: MLMultiArray
|
|
42
|
+
private var cell: MLMultiArray
|
|
43
|
+
private var pending: [Float] = []
|
|
44
|
+
private var tail = [Float](repeating: 0, count: SileroVAD.context)
|
|
45
|
+
|
|
46
|
+
init(model: MLModel) throws {
|
|
47
|
+
self.model = model
|
|
48
|
+
audio = try MLMultiArray(shape: [1, NSNumber(value: Self.context + Self.chunk)], dataType: .float32)
|
|
49
|
+
hidden = try Self.zeros()
|
|
50
|
+
cell = try Self.zeros()
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
private static func zeros() throws -> MLMultiArray {
|
|
54
|
+
let a = try MLMultiArray(shape: [1, 128], dataType: .float32)
|
|
55
|
+
for i in 0..<a.count { a[i] = 0 }
|
|
56
|
+
return a
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/// Feeds samples (16 kHz, -1…1) and returns one probability per
|
|
60
|
+
/// complete 512-sample frame; leftovers wait for the next call.
|
|
61
|
+
func process(_ samples: [Float]) throws -> [Float] {
|
|
62
|
+
pending.append(contentsOf: samples)
|
|
63
|
+
var out: [Float] = []
|
|
64
|
+
var start = 0
|
|
65
|
+
while pending.count - start >= Self.chunk {
|
|
66
|
+
let frame = pending[start..<start + Self.chunk]
|
|
67
|
+
out.append(try score(frame))
|
|
68
|
+
start += Self.chunk
|
|
69
|
+
}
|
|
70
|
+
pending.removeFirst(start)
|
|
71
|
+
return out
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
private func score(_ frame: ArraySlice<Float>) throws -> Float {
|
|
75
|
+
let ptr = audio.dataPointer.bindMemory(to: Float.self, capacity: audio.count)
|
|
76
|
+
for i in 0..<Self.context { ptr[i] = tail[i] }
|
|
77
|
+
var i = Self.context
|
|
78
|
+
for s in frame { ptr[i] = s; i += 1 }
|
|
79
|
+
tail = Array(frame.suffix(Self.context))
|
|
80
|
+
|
|
81
|
+
let input = try MLDictionaryFeatureProvider(dictionary: [
|
|
82
|
+
"audio_input": MLFeatureValue(multiArray: audio),
|
|
83
|
+
"hidden_state": MLFeatureValue(multiArray: hidden),
|
|
84
|
+
"cell_state": MLFeatureValue(multiArray: cell),
|
|
85
|
+
])
|
|
86
|
+
let result = try model.prediction(from: input)
|
|
87
|
+
if let h = result.featureValue(for: "new_hidden_state")?.multiArrayValue { hidden = h }
|
|
88
|
+
if let c = result.featureValue(for: "new_cell_state")?.multiArrayValue { cell = c }
|
|
89
|
+
guard let p = result.featureValue(for: "vad_output")?.multiArrayValue, p.count > 0 else { return 0 }
|
|
90
|
+
return p[0].floatValue
|
|
91
|
+
}
|
|
92
|
+
}
|
|
@@ -38,16 +38,18 @@ enum Speech {
|
|
|
38
38
|
// MARK: Speech to text
|
|
39
39
|
|
|
40
40
|
/// `engine` is agentx.json's `voice.stt`: "auto" and "elevenlabs" use
|
|
41
|
-
/// ElevenLabs when there is a key, falling back to
|
|
42
|
-
/// never sends audio off the Mac.
|
|
43
|
-
|
|
41
|
+
/// ElevenLabs when there is a key, falling back to the local engine;
|
|
42
|
+
/// "local" never sends audio off the Mac. `local` is `voice.localStt`:
|
|
43
|
+
/// "mlx-whisper" or "parakeet".
|
|
44
|
+
static func transcribe(wav: Data, engine: String = "auto", local: String = "mlx-whisper") async throws -> String {
|
|
44
45
|
if engine == "elevenlabs" && Config.elevenLabsKey == nil {
|
|
45
|
-
Log.warn("speech to text is set to ElevenLabs but no key is set; using
|
|
46
|
+
Log.warn("speech to text is set to ElevenLabs but no key is set; using \(local)")
|
|
46
47
|
}
|
|
47
48
|
if engine != "local", let key = Config.elevenLabsKey {
|
|
48
49
|
do { return try await elevenLabsSTT(wav: wav, key: key) }
|
|
49
|
-
catch { Log.warn("ElevenLabs STT failed (\(error.localizedDescription)); falling back to
|
|
50
|
+
catch { Log.warn("ElevenLabs STT failed (\(error.localizedDescription)); falling back to \(local)") }
|
|
50
51
|
}
|
|
52
|
+
if local == "parakeet", let text = await LocalSTT.parakeet(wav: wav) { return text }
|
|
51
53
|
return try await mlxWhisper(wav: wav)
|
|
52
54
|
}
|
|
53
55
|
|
|
@@ -23,6 +23,8 @@ final class StatusMenu: NSObject, NSMenuDelegate {
|
|
|
23
23
|
/// The daemon's speaking queue, polled while anything is in flight.
|
|
24
24
|
private var queue: AgentClient.QueueState?
|
|
25
25
|
private var queuePoll: Timer?
|
|
26
|
+
/// The last three voice exchanges, for a quick replay.
|
|
27
|
+
private let recent = RecentExchanges()
|
|
26
28
|
|
|
27
29
|
/// Questions this widget has in flight, per agent. Set by the app.
|
|
28
30
|
var thinking: [String: Int] = [:] {
|
|
@@ -34,7 +36,13 @@ final class StatusMenu: NSObject, NSMenuDelegate {
|
|
|
34
36
|
var onHoldChanged: ((Bool) -> Void)?
|
|
35
37
|
var onTargetChanged: ((String) -> Void)?
|
|
36
38
|
var onPillChanged: ((Bool) -> Void)?
|
|
37
|
-
var
|
|
39
|
+
var onAnimatedOrbChanged: ((Bool) -> Void)?
|
|
40
|
+
/// Hide the pill and stop speech, as its close button does.
|
|
41
|
+
var onHidePill: (() -> Void)?
|
|
42
|
+
/// Put the pill back in the bottom-right corner.
|
|
43
|
+
var onResetPosition: (() -> Void)?
|
|
44
|
+
/// Whether the pill is on screen, for "Hide pill".
|
|
45
|
+
var pillVisible: () -> Bool = { false }
|
|
38
46
|
var onSettings: (() -> Void)?
|
|
39
47
|
|
|
40
48
|
override init() {
|
|
@@ -70,6 +78,7 @@ final class StatusMenu: NSObject, NSMenuDelegate {
|
|
|
70
78
|
let agents = await AgentClient.agents()
|
|
71
79
|
roster = agents.map { .loaded($0) } ?? .down
|
|
72
80
|
queue = await AgentClient.queueState()
|
|
81
|
+
await recent.refresh()
|
|
73
82
|
rebuild()
|
|
74
83
|
showBadge()
|
|
75
84
|
if reopenAfterRefresh { reopenAfterRefresh = false; open() }
|
|
@@ -169,6 +178,7 @@ final class StatusMenu: NSObject, NSMenuDelegate {
|
|
|
169
178
|
menu.addItem(row)
|
|
170
179
|
}
|
|
171
180
|
}
|
|
181
|
+
recent.addItems(to: menu, name: name(of:))
|
|
172
182
|
|
|
173
183
|
menu.addItem(.separator())
|
|
174
184
|
menu.addItem(action("Stop speaking", #selector(stop), key: ".", modifiers: [.command, .option]))
|
|
@@ -178,9 +188,13 @@ final class StatusMenu: NSObject, NSMenuDelegate {
|
|
|
178
188
|
let pill = action("Show floating pill", #selector(togglePill), key: "")
|
|
179
189
|
pill.state = Config.showPill ? .on : .off
|
|
180
190
|
menu.addItem(pill)
|
|
181
|
-
let orb = action("
|
|
182
|
-
orb.state = Config.
|
|
191
|
+
let orb = action("Animated orb", #selector(toggleOrb), key: "")
|
|
192
|
+
orb.state = Config.animatedOrb ? .on : .off
|
|
183
193
|
menu.addItem(orb)
|
|
194
|
+
let hide = action("Hide pill", #selector(hidePill), key: "")
|
|
195
|
+
hide.isEnabled = pillVisible()
|
|
196
|
+
menu.addItem(hide)
|
|
197
|
+
menu.addItem(action("Reset position", #selector(resetPosition), key: ""))
|
|
184
198
|
|
|
185
199
|
menu.addItem(.separator())
|
|
186
200
|
menu.addItem(action("Settings…", #selector(openSettings), key: ","))
|
|
@@ -232,13 +246,16 @@ final class StatusMenu: NSObject, NSMenuDelegate {
|
|
|
232
246
|
}
|
|
233
247
|
|
|
234
248
|
@objc private func toggleOrb() {
|
|
235
|
-
Config.
|
|
236
|
-
|
|
249
|
+
Config.animatedOrb.toggle()
|
|
250
|
+
onAnimatedOrbChanged?(Config.animatedOrb)
|
|
237
251
|
}
|
|
238
252
|
|
|
253
|
+
@objc private func hidePill() { onHidePill?() }
|
|
254
|
+
@objc private func resetPosition() { onResetPosition?() }
|
|
255
|
+
|
|
239
256
|
@objc private func openSettings() { onSettings?() }
|
|
240
257
|
@objc private func openDashboardHome() { openDashboard("/admin") }
|
|
241
|
-
@objc private func openHistory() {
|
|
258
|
+
@objc private func openHistory() { HistoryWindow.shared.show() }
|
|
242
259
|
|
|
243
260
|
private func openDashboard(_ path: String) {
|
|
244
261
|
guard let url = URL(string: Config.dashboardURL + path) else { return }
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
import Foundation
|
|
2
|
+
|
|
3
|
+
/// When a hands-free turn has ended: the person spoke, then stopped for
|
|
4
|
+
/// long enough that the sentence is over.
|
|
5
|
+
///
|
|
6
|
+
/// It is fed one audio frame at a time with how likely that frame is to be
|
|
7
|
+
/// speech (0…1) and how long it lasted. Two sources feed it:
|
|
8
|
+
///
|
|
9
|
+
/// - Silero VAD, 32 ms frames with a real probability. A voice model
|
|
10
|
+
/// tells a word from a fan, a keyboard or a café, so the noise of a room
|
|
11
|
+
/// neither holds the microphone open nor counts as talking.
|
|
12
|
+
/// - The old volume check when the VAD model is not available: 1 above
|
|
13
|
+
/// `speechLevel`, 0 below, per audio buffer.
|
|
14
|
+
///
|
|
15
|
+
/// Foundation only and time passed in, so the tests can drive it with
|
|
16
|
+
/// synthetic frames.
|
|
17
|
+
struct TurnEnd {
|
|
18
|
+
struct Tuning: Equatable {
|
|
19
|
+
/// A frame at or above this starts speech (Silero's own default).
|
|
20
|
+
var start: Float
|
|
21
|
+
/// Once talking, a frame at or above this still counts as speech.
|
|
22
|
+
/// Lower than `start`: the soft end of a word is not a pause.
|
|
23
|
+
var keep: Float
|
|
24
|
+
/// Speech must last this long in one go before the turn counts as
|
|
25
|
+
/// started, so a cough, a door or a click does not open it.
|
|
26
|
+
var minSpeech: TimeInterval
|
|
27
|
+
/// A burst this short inside a pause does not restart the pause:
|
|
28
|
+
/// one noisy frame must not hold the microphone open.
|
|
29
|
+
var resume: TimeInterval
|
|
30
|
+
/// A pause this long after speech ends the turn. People pause
|
|
31
|
+
/// inside sentences; this is longer than those pauses.
|
|
32
|
+
var endSilence: TimeInterval
|
|
33
|
+
|
|
34
|
+
/// For Silero VAD probabilities.
|
|
35
|
+
static let vad = Tuning(start: 0.5, keep: 0.35, minSpeech: 0.25, resume: 0.1, endSilence: 1.2)
|
|
36
|
+
|
|
37
|
+
/// The volume threshold as it always worked: any buffer above the
|
|
38
|
+
/// level is speech, and 1.2 s below it ends the turn.
|
|
39
|
+
static let level = Tuning(start: 0.5, keep: 0.5, minSpeech: 0, resume: 0, endSilence: 1.2)
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
enum State: Equatable {
|
|
43
|
+
/// Nothing said yet.
|
|
44
|
+
case waiting
|
|
45
|
+
/// Speech heard; the turn is open (possibly in a short pause).
|
|
46
|
+
case speaking
|
|
47
|
+
/// Speech, then `endSilence` of quiet: send it.
|
|
48
|
+
case ended
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/// RMS above which the microphone is hearing a voice rather than a
|
|
52
|
+
/// room. The fallback's whole notion of speech.
|
|
53
|
+
static let speechLevel: Float = 0.02
|
|
54
|
+
|
|
55
|
+
let tuning: Tuning
|
|
56
|
+
private(set) var state: State = .waiting
|
|
57
|
+
/// How long the current run of speech frames has lasted.
|
|
58
|
+
private var speechRun: TimeInterval = 0
|
|
59
|
+
/// How long the current pause has lasted (a short burst does not
|
|
60
|
+
/// reset it).
|
|
61
|
+
private var silence: TimeInterval = 0
|
|
62
|
+
|
|
63
|
+
init(_ tuning: Tuning) { self.tuning = tuning }
|
|
64
|
+
|
|
65
|
+
/// One frame: `probability` that it is speech, `duration` it covers.
|
|
66
|
+
@discardableResult
|
|
67
|
+
mutating func feed(probability: Float, duration: TimeInterval) -> State {
|
|
68
|
+
guard state != .ended else { return state }
|
|
69
|
+
let threshold = state == .speaking && silence == 0 ? tuning.keep : tuning.start
|
|
70
|
+
let speech = probability >= threshold
|
|
71
|
+
|
|
72
|
+
switch state {
|
|
73
|
+
case .waiting:
|
|
74
|
+
speechRun = speech ? speechRun + duration : 0
|
|
75
|
+
if speech && speechRun >= tuning.minSpeech {
|
|
76
|
+
state = .speaking
|
|
77
|
+
silence = 0
|
|
78
|
+
}
|
|
79
|
+
case .speaking:
|
|
80
|
+
if speech {
|
|
81
|
+
speechRun += duration
|
|
82
|
+
if silence == 0 || speechRun >= tuning.resume {
|
|
83
|
+
silence = 0
|
|
84
|
+
} else {
|
|
85
|
+
silence += duration
|
|
86
|
+
}
|
|
87
|
+
} else {
|
|
88
|
+
speechRun = 0
|
|
89
|
+
silence += duration
|
|
90
|
+
}
|
|
91
|
+
if silence >= tuning.endSilence { state = .ended }
|
|
92
|
+
case .ended:
|
|
93
|
+
break
|
|
94
|
+
}
|
|
95
|
+
return state
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/// The fallback's frame: the smoothed microphone level.
|
|
99
|
+
@discardableResult
|
|
100
|
+
mutating func feed(level: Float, duration: TimeInterval) -> State {
|
|
101
|
+
feed(probability: level > Self.speechLevel ? 1 : 0, duration: duration)
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/// The microphone level after one buffer of loudness `rms`. Attack
|
|
105
|
+
/// fast, release slow: a level that drops instantly makes a pause
|
|
106
|
+
/// between words look like the end of a sentence.
|
|
107
|
+
static func smooth(level: Float, rms: Float) -> Float {
|
|
108
|
+
rms > level ? rms : level * 0.82 + rms * 0.18
|
|
109
|
+
}
|
|
110
|
+
}
|
|
@@ -12,7 +12,28 @@ struct VoiceSettings: Codable, Equatable {
|
|
|
12
12
|
var provider: String
|
|
13
13
|
var fallback: String
|
|
14
14
|
var stt: String
|
|
15
|
+
/// The on-device engine: "mlx-whisper" or "parakeet".
|
|
16
|
+
var localStt: String
|
|
17
|
+
/// How a hands-free turn's end is heard: "vad" or "volume".
|
|
18
|
+
var endOfTurn: String
|
|
15
19
|
var hotkeys: Hotkeys
|
|
20
|
+
|
|
21
|
+
init(provider: String, fallback: String, stt: String, localStt: String = "mlx-whisper",
|
|
22
|
+
endOfTurn: String = "vad", hotkeys: Hotkeys) {
|
|
23
|
+
self.provider = provider; self.fallback = fallback; self.stt = stt
|
|
24
|
+
self.localStt = localStt; self.endOfTurn = endOfTurn; self.hotkeys = hotkeys
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/// A daemon from before these fields leaves them out: the defaults.
|
|
28
|
+
init(from decoder: Decoder) throws {
|
|
29
|
+
let c = try decoder.container(keyedBy: CodingKeys.self)
|
|
30
|
+
provider = try c.decode(String.self, forKey: .provider)
|
|
31
|
+
fallback = try c.decode(String.self, forKey: .fallback)
|
|
32
|
+
stt = try c.decode(String.self, forKey: .stt)
|
|
33
|
+
localStt = try c.decodeIfPresent(String.self, forKey: .localStt) ?? "mlx-whisper"
|
|
34
|
+
endOfTurn = try c.decodeIfPresent(String.self, forKey: .endOfTurn) ?? "vad"
|
|
35
|
+
hotkeys = try c.decode(Hotkeys.self, forKey: .hotkeys)
|
|
36
|
+
}
|
|
16
37
|
}
|
|
17
38
|
struct Voice: Codable, Equatable {
|
|
18
39
|
var provider: String?
|
|
@@ -55,6 +76,8 @@ struct VoiceSettings: Codable, Equatable {
|
|
|
55
76
|
var general: [String: Any] = [:]
|
|
56
77
|
if self.general.provider != old.general.provider { general["provider"] = self.general.provider }
|
|
57
78
|
if self.general.stt != old.general.stt { general["stt"] = self.general.stt }
|
|
79
|
+
if self.general.localStt != old.general.localStt { general["localStt"] = self.general.localStt }
|
|
80
|
+
if self.general.endOfTurn != old.general.endOfTurn { general["endOfTurn"] = self.general.endOfTurn }
|
|
58
81
|
var keys: [String: Any] = [:]
|
|
59
82
|
if self.general.hotkeys.talk != old.general.hotkeys.talk { keys["talk"] = self.general.hotkeys.talk }
|
|
60
83
|
if self.general.hotkeys.stop != old.general.hotkeys.stop { keys["stop"] = self.general.hotkeys.stop }
|