@livx.cc/agentx 0.96.10 → 0.96.11

package/dist/native/Info.plist

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+    <key>CFBundleIdentifier</key><string>cc.livx.mic-aec</string>
+    <key>CFBundleName</key><string>mic-aec</string>
+    <key>NSMicrophoneUsageDescription</key><string>Echo-cancelled mic capture for voice experiment</string>
+</dict>
+</plist>

package/dist/native/mic-aec.swift

@@ -0,0 +1,231 @@
+// mic-aec — full-duplex echo-cancelled audio for the voice CLI.
+// Apple's voice-processing unit (same as FaceTime) subtracts rendered output from the mic. The
+// ROOT echo fix is rendering TTS through THIS engine: VPIO then has a sample-accurate, time-aligned
+// reference for exactly what we emit (vs. guessing from the system-output loopback when a separate
+// process like ffplay plays it — cancellation quality became a per-run spectrum).
+//   stdin  (optional): framed playback — [u32le len][s16le mono 44.1kHz PCM]; len==0 = FLUSH;
+//                      len==0xFFFFFFFF = PAUSE, 0xFFFFFFFE = RESUME (overlap trail-off — exact-sample hold)
+//                      (drop queued+playing audio NOW — the barge-in primitive). In-band framing
+//                      makes flush perfectly ordered: stale frames already in the pipe are consumed
+//                      and discarded by the stop, frames after it belong to the new turn.
+//   stdout: s16le mono 16kHz echo-cancelled mic PCM.
+// No stdin writer (legacy capture-only use) is fine — the reader thread just blocks.
+// Build: swiftc -O -o mic-aec mic-aec.swift -Xlinker -sectcreate -Xlinker __TEXT -Xlinker __info_plist -Xlinker Info.plist && codesign -fs - mic-aec
+import AVFoundation
+
+// VPIO checks mic TCC explicitly (plain HAL capture may work while VPIO silently mutes) — request it
+let sem = DispatchSemaphore(value: 0)
+var granted = false
+AVCaptureDevice.requestAccess(for: .audio) { ok in granted = ok; sem.signal() }
+sem.wait()
+FileHandle.standardError.write("mic access granted: \(granted)\n".data(using: .utf8)!)
+
+let engine = AVAudioEngine()
+let input = engine.inputNode
+do { try input.setVoiceProcessingEnabled(true) } catch {
+    FileHandle.standardError.write("AEC unavailable: \(error)\n".data(using: .utf8)!)
+    exit(1)
+}
+// the VP I/O unit is full-duplex: input stays muted unless the output side renders too. The mixer→
+// output format must match the VP output's hardware format (nil/default fails kAUInitialize -10875).
+let outHw = engine.outputNode.outputFormat(forBus: 0)
+engine.connect(engine.mainMixerNode, to: engine.outputNode, format: outHw)
+engine.mainMixerNode.outputVolume = 1 // playback is audible now (was 0 when this path was only a keep-alive)
+// VP ducks all other system audio by default — disable (we ARE the audio).
+if #available(macOS 14.0, *) {
+    input.voiceProcessingOtherAudioDuckingConfiguration = .init(enableAdvancedDucking: false, duckingLevel: .min)
+}
+
+// ---- playback: framed s16le 44.1k mono on stdin → player node (the VPIO reference signal) ----
+let playFmt = AVAudioFormat(commonFormat: .pcmFormatFloat32, sampleRate: 44100, channels: 1, interleaved: false)!
+let player = AVAudioPlayerNode()
+engine.attach(player)
+engine.connect(player, to: engine.mainMixerNode, format: playFmt)
+
+// ---- residual-echo gate: VPIO suppresses our own playback ~40dB, but modern STT transcribes
+// -40dB speech just fine — so acoustic cancellation alone still leaks ghost turns. While WE are
+// audible (scheduled buffers outstanding + a short tail), gate the mic by level: the residue sits
+// near a quiet baseline (tracked adaptively), a real barging user is 10-100x louder and passes. ----
+let gateLock = NSLock()
+var audibleUntil: Double = 0 // CACurrentMediaTime deadline; extended as buffers schedule/complete
+var pausedSince: Double = 0  // overlap pause: ~300ms tail then the gate OPENS — nothing is playing,
+                             // so nothing can bleed; whatever the mic hears now IS the user. (The
+                             // earlier 'keep gating while paused' muted the very speech the pause
+                             // existed to listen for.)
+var echoBaseline: Float = 0  // adaptive EMA of the residual level while audible
+// Playback-envelope-aware gating: residue scales with how LOUD we are right now. Per-window rms of
+// the scheduled audio, playerTime-aligned, × adaptive leak factor = expected residue.
+let ENV_WIN = 2048 // samples per envelope window @44.1k (~46ms)
+var schedEnv: [Float] = [] // rms per scheduled window (playerTime timeline; cleared on FLUSH)
+var leak: Float = 0.05     // mic-residue / playback-loudness ratio (adaptive EMA, clamped)
+let gateEnabled = ProcessInfo.processInfo.environment["MIC_GATE"] != "0" // escape hatch while tuning
+// TEST HARNESS: INJECT frames (stdin, len==0xFFFFFFFD) queue s16le 16k mono "user speech" that the
+// tap MIXES into the real mic stream BEFORE the gate — the full live pipeline (gate, leak, STT,
+// engine) runs unchanged while a conversation simulator plays the human. Mirrors reality: injected
+// speech, like a real voice, is in NO echo-cancellation reference.
+var injectBuf: [Int16] = []
+
+/** playback rms around the sample being rendered NOW (±1 window — alignment is approximate) */
+func playbackEnvNow() -> Float {
+    guard let nt = player.lastRenderTime, let pt = player.playerTime(forNodeTime: nt) else { return 0 }
+    let w = Int(pt.sampleTime) / ENV_WIN
+    var m: Float = 0
+    for i in max(0, w - 1)...(w + 1) where i < schedEnv.count { m = max(m, schedEnv[i]) }
+    return m
+}
+
+// ---- capture: VPIO exposes a multi-channel (e.g. 9ch) format with IDENTICAL channels (measured —
+// processed mono replicated); AVAudioConverter's N→1 mapping yields silence — take channel 0
+// manually and decimate to 16k by averaging. ----
+let inFmt = input.outputFormat(forBus: 0)
+let decim = max(1, Int(inFmt.sampleRate / 16000))
+let out = FileHandle.standardOutput
+
+input.installTap(onBus: 0, bufferSize: 2048, format: inFmt) { buf, _ in
+    guard let ch = buf.floatChannelData else { return }
+    let n = Int(buf.frameLength)
+    var pcm = [Int16]()
+    pcm.reserveCapacity(n / decim + 1)
+    var i = 0
+    var sq: Float = 0
+    while i + decim <= n {
+        var acc: Float = 0
+        for j in 0..<decim { acc += ch[0][i + j] }
+        let v = max(-1, min(1, acc / Float(decim)))
+        sq += v * v
+        pcm.append(Int16(v * 32767))
+        i += decim
+    }
+    gateLock.lock()
+    if !injectBuf.isEmpty, !pcm.isEmpty { // mix simulated user speech (pre-gate, post-AEC — like a real voice)
+        let n = min(injectBuf.count, pcm.count)
+        for k in 0..<n { pcm[k] = Int16(clamping: Int32(pcm[k]) + Int32(injectBuf[k])) }
+        injectBuf.removeFirst(n)
+    }
+    var sq2: Float = 0
+    for v in pcm { let f = Float(v) / 32767; sq2 += f * f }
+    let rms = pcm.isEmpty ? 0 : (sq2 / Float(pcm.count)).squareRoot() * 32767
+    _ = sq
+    let nowT = CACurrentMediaTime()
+    let audible = pausedSince > 0 ? nowT < pausedSince + 0.3 : nowT < audibleUntil + 0.8
+    var threshold: Float = 0
+    if audible {
+        if echoBaseline == 0 { echoBaseline = max(rms, 8) }
+        // track the baseline only while the level looks like residue (don't learn the user's voice)
+        if rms < echoBaseline * 4 { echoBaseline = echoBaseline * 0.9 + max(rms, 8) * 0.1 }
+        let env = playbackEnvNow()
+        // learn the leak ratio ONLY from clearly-residue-shaped moments (≤2.5× current estimate) —
+        // a looser bound (6×) learned the USER'S overlapping speech as leak, inflating the threshold
+        // until speech itself was gated (3s barge latency: Soniox got fragments). Clamp low: leak
+        // beyond ~0.2 means AEC is broken, not that speech should be silenced.
+        if env > 300, rms < leak * env * 2.5 {
+            leak = min(0.2, max(0.005, leak * 0.9 + (rms / env) * 0.1))
+        }
+        // 2× (not 3×): user speech must reach the STT — Soniox is the VAD/discriminator. Residue
+        // BURSTS that slip a 2× gate are 100-200ms fragments a speech model does not tokenize
+        // (the historical full-sentence bleed was CONTINUOUS residue — that still gates).
+        threshold = max(2 * leak * env, echoBaseline * 4, 120)
+    }
+    let gate = gateEnabled && audible && rms < threshold // user speech must clearly dominate the expected residue
+    gateLock.unlock()
+    if gate { for k in 0..<pcm.count { pcm[k] = 0 } } // keep the stream's timing — send silence, not nothing
+    pcm.withUnsafeBufferPointer { p in
+        out.write(Data(buffer: p))
+    }
+}
+
+do { try engine.start() } catch {
+    FileHandle.standardError.write("engine start failed: \(error)\n".data(using: .utf8)!)
+    exit(1)
+}
+player.play()
+
+// stdin reader (background): framed playback + in-band FLUSH. EOF = parent gone → clean exit.
+DispatchQueue.global(qos: .userInteractive).async {
+    let stdin = FileHandle.standardInput
+    var pausedAt: Double = 0 // overlap pause (control frames); resume shifts the gate window
+    func readN(_ n: Int) -> Data? {
+        var d = Data()
+        while d.count < n {
+            let chunk = stdin.readData(ofLength: n - d.count)
+            if chunk.isEmpty { return nil } // EOF
+            d.append(chunk)
+        }
+        return d
+    }
+    while true {
+        guard let hdr = readN(4) else { break }
+        let len = hdr.withUnsafeBytes { $0.loadUnaligned(as: UInt32.self) } // LE host
+        if len == 0 { // FLUSH: stop drops every scheduled buffer; restart for the next turn
+            player.stop()
+            player.play()
+            pausedAt = 0
+            gateLock.lock(); pausedSince = 0; audibleUntil = CACurrentMediaTime(); schedEnv.removeAll(); gateLock.unlock() // gate tail only; envelope timeline resets with playerTime
+            continue
+        }
+        if len == UInt32.max { // PAUSE: hold playback exactly (overlap trail-off; nothing is lost)
+            if pausedAt == 0 {
+                player.pause()
+                pausedAt = CACurrentMediaTime()
+                gateLock.lock(); pausedSince = pausedAt; gateLock.unlock()
+            }
+            continue
+        }
+        if len == UInt32.max - 2 { // INJECT (test harness): queue s16le 16k mono simulated user speech
+            guard let nl = readN(4) else { break }
+            let plen = nl.withUnsafeBytes { $0.loadUnaligned(as: UInt32.self) }
+            guard plen <= 1 << 22, let payload = readN(Int(plen)) else { break }
+            payload.withUnsafeBytes { raw in
+                let v = raw.bindMemory(to: Int16.self)
+                gateLock.lock(); injectBuf.append(contentsOf: v); gateLock.unlock()
+            }
+            continue
+        }
+        if len == UInt32.max - 1 { // RESUME: continue from the precise sample; gate window shifts by the pause
+            if pausedAt > 0 {
+                gateLock.lock()
+                audibleUntil += CACurrentMediaTime() - pausedAt
+                pausedSince = 0
+                gateLock.unlock()
+                pausedAt = 0
+                player.play()
+            }
+            continue
+        }
+        guard len <= 1 << 22, let payload = readN(Int(len)) else { break } // 4MB sanity cap
+        let frames = payload.count / 2
+        guard frames > 0, let buf = AVAudioPCMBuffer(pcmFormat: playFmt, frameCapacity: AVAudioFrameCount(frames)) else { continue }
+        buf.frameLength = AVAudioFrameCount(frames)
+        payload.withUnsafeBytes { raw in
+            let s = raw.bindMemory(to: Int16.self)
+            let dst = buf.floatChannelData![0]
+            for i in 0..<frames { dst[i] = Float(s[i]) / 32768.0 }
+            // playback envelope: rms per window, appended on the playerTime timeline
+            gateLock.lock()
+            var i = 0
+            while i < frames {
+                let n = min(ENV_WIN, frames - i)
+                var acc: Float = 0
+                for j in 0..<n { let v = Float(s[i + j]) / 32768.0; acc += v * v }
+                schedEnv.append((acc / Float(n)).squareRoot() * 32767)
+                i += n
+            }
+            gateLock.unlock()
+        }
+        player.scheduleBuffer(buf, completionHandler: nil)
+        // extend the audible window by this buffer's duration (+0.8s tail covers device latency)
+        gateLock.lock()
+        let nowT = CACurrentMediaTime()
+        audibleUntil = max(audibleUntil, nowT) + Double(frames) / 44100.0
+        gateLock.unlock()
+    }
+    engine.stop()
+    exit(0)
+}
+
+// Graceful teardown: SIGKILL leaves the VPIO session wedged inside coreaudiod (repeated kills
+// degrade system-wide echo cancellation until coreaudiod restarts) — stop the engine first.
+signal(SIGTERM) { _ in engine.stop(); exit(0) }
+signal(SIGINT) { _ in engine.stop(); exit(0) }
+FileHandle.standardError.write("aec duplex audio running\n".data(using: .utf8)!)
+RunLoop.main.run()

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@livx.cc/agentx",
-  "version": "0.96.10",
+  "version": "0.96.11",
   "description": "Edge-native AI agent runtime — drives a virtual filesystem via any LLM (ai.libx.js). Same bytes run in node, browser, or edge.",
   "type": "module",
   "main": "./dist/index.js",