@mentra/bluetooth-sdk 0.1.0

package/ios/Packages/VAD/Common/VAD/VADQuality.swift

@@ -0,0 +1,41 @@
+//
+//  VADQuality.swift
+//  ios-vad
+//
+//  Created by baochuquan on 2024/11/9.
+//
+
+import Foundation
+
+enum VADQuality: String, CaseIterable {
+    case normal = "NORMAL"
+    case low_bitrate = "LOW_BITERATE"
+    case aggressive = "AGGRESSIVE"
+    case very_aggressive = "VERY_AGGRESSIVE"
+
+    var desc: String {
+        rawValue
+    }
+
+    var threshold: Float {
+        switch self {
+        case .normal: return 0.3
+        case .low_bitrate: return 0.7
+        case .aggressive: return 0.8
+        case .very_aggressive: return 0.90
+        }
+    }
+
+    var webrtcMode: Int {
+        switch self {
+        case .normal: return 0
+        case .low_bitrate: return 1
+        case .aggressive: return 2
+        case .very_aggressive: return 3
+        }
+    }
+
+    static let webrtc: [VADQuality] = [.normal, .low_bitrate, .aggressive, .very_aggressive]
+    static let silero: [VADQuality] = [.normal, .aggressive, .very_aggressive]
+    static let yamnet: [VADQuality] = [.normal]
+}

package/ios/Packages/VAD/Common/VAD/VADState.swift

@@ -0,0 +1,26 @@
+//
+//  VADState.swift
+//  ios-vad
+//
+//  Created by baochuquan on 2024/11/9.
+//
+
+import Foundation
+
+enum VADState {
+    case start // 开始说话
+    case speeching // 说话中
+    case end // 结束说话
+    case silence // 静默中
+    case error // 出错了
+
+    var desc: String {
+        switch self {
+        case .start: return "VAD Start"
+        case .speeching: return "VAD Speeching"
+        case .end: return "VAD End"
+        case .silence: return "VAD Silence"
+        case .error: return "Error"
+        }
+    }
+}

package/ios/Packages/VAD/Common/VAD/VADStrategy.swift

@@ -0,0 +1,29 @@
+//
+//  VADStrategy.swift
+//  ios-vad
+//
+//  Created by baochuquan on 2024/11/9.
+//
+
+import Foundation
+
+protocol VADStrategy {
+    /// vad setup method
+    /// - Parameters:
+    ///   - sampleRate: sample rate
+    ///   - frameSize: frame size
+    ///   - quality: vad quality level
+    ///   - silenceTriggerDurationMs: the minimum duration required to switch from speech to silence. Unit: millisecond.
+    ///   - speechTriggerDurationMs: the minimum duration required to switch from silence to speech. Unit: millisecond.
+    func setup(sampleRate: SampleRate, frameSize: FrameSize, quality: VADQuality, silenceTriggerDurationMs: Int64, speechTriggerDurationMs: Int64)
+
+    /// vad check result callback
+    /// - Parameters:
+    ///   - pcm: pcm Data
+    ///   - handler: callback.
+    func checkVAD(pcm: [Int16], handler: @escaping (VADState) -> Void)
+
+    /// current vad state
+    /// - Returns: vad state
+    func currentState() -> VADState
+}

package/ios/Packages/VAD/Common/VAD/VADType.swift

@@ -0,0 +1,14 @@
+//
+//  VADType.swift
+//  ios-vad
+//
+//  Created by baochuquan on 2024/11/9.
+//
+
+import Foundation
+
+enum VADType: String {
+    case webrtc = "WebRTC VAD"
+    case silero = "Silero VAD"
+    case yamnet = "Yamnet VAD"
+}

package/ios/Packages/VAD/Data/Configuration.swift

@@ -0,0 +1,68 @@
+//
+//  Configuration.swift
+//  ios-vad
+//
+//  Created by baochuquan on 2024/11/9.
+//
+
+import Foundation
+
+// MARK: - Sample Rate Config
+
+struct SampleRateConfiguration: Hashable {
+    var selectedOption: SampleRate
+    var options: [SampleRate]
+
+    static let webrtc = SampleRateConfiguration(selectedOption: SampleRate.webrtc[0], options: SampleRate.webrtc)
+
+    static let silero = SampleRateConfiguration(selectedOption: SampleRate.silero[0], options: SampleRate.silero)
+
+    static let yamnet = SampleRateConfiguration(selectedOption: SampleRate.yamnet[0], options: SampleRate.yamnet)
+
+    func frameSizeConfiguration(type: VADType) -> FrameSizeConfiguration {
+        let options = selectedOption.frameSizeOptions(type: type)
+        return FrameSizeConfiguration(selectedOption: options[0], options: options)
+    }
+}
+
+// MARK: - Frame Size Config
+
+struct FrameSizeConfiguration: Hashable {
+    var selectedOption: FrameSize
+    var options: [FrameSize]
+}
+
+// MARK: - Mode Config
+
+struct QualityConfiguration: Hashable {
+    var selectedOption: VADQuality
+    var options: [VADQuality]
+
+    static let webrtc = QualityConfiguration(
+        selectedOption: .very_aggressive,
+        options: [
+            .normal,
+            .low_bitrate,
+            .aggressive,
+            .very_aggressive,
+        ]
+    )
+
+    static let silero = QualityConfiguration(
+        selectedOption: .normal,
+        options: [
+            .normal,
+            .aggressive,
+            .very_aggressive,
+        ]
+    )
+
+    static let yamnet = QualityConfiguration(
+        selectedOption: .normal,
+        options: [
+            .normal,
+            .aggressive,
+            .very_aggressive,
+        ]
+    )
+}

package/ios/Packages/VAD/Data/FrameSize.swift

@@ -0,0 +1,39 @@
+//
+//  FrameSize.swift
+//  ios-vad
+//
+//  Created by baochuquan on 2024/11/9.
+//
+
+import Foundation
+
+enum FrameSize: Int, CaseIterable {
+    case size_80 = 80
+    case size_160 = 160
+    case size_240 = 240
+    case size_320 = 320
+    case size_256 = 256
+    case size_480 = 480
+    case size_512 = 512
+    case size_640 = 640
+    case size_768 = 768
+    case size_960 = 960
+    case size_1024 = 1024
+    case size_1440 = 1440
+    case size_1536 = 1536
+    case size_15600 = 15600
+
+    var desc: String {
+        "FRAME_SIZE_\(rawValue)"
+    }
+
+    static let webrtc_8k: [FrameSize] = [.size_80, .size_160, .size_240]
+    static let webrtc_16k: [FrameSize] = [.size_160, .size_320, .size_480]
+    static let webrtc_32k: [FrameSize] = [.size_320, .size_640, .size_960]
+    static let webrtc_48k: [FrameSize] = [.size_480, .size_960, .size_1440]
+
+    static let silero_8k: [FrameSize] = [.size_256, .size_512, .size_768]
+    static let silero_16k: [FrameSize] = [.size_512, .size_1024, .size_1536]
+
+    static let yamnet_15k: [FrameSize] = [.size_15600]
+}

package/ios/Packages/VAD/Data/Record.swift

@@ -0,0 +1,13 @@
+//
+//  Record.swift
+//  ios-vad
+//
+//  Created by baochuquan on 2024/11/9.
+//
+
+import Foundation
+
+enum VADRecord: Hashable {
+    case idle
+    case work
+}

package/ios/Packages/VAD/Data/Result.swift

@@ -0,0 +1,22 @@
+//
+//  Result.swift
+//  ios-vad
+//
+//  Created by baochuquan on 2024/11/9.
+//
+
+import Foundation
+
+enum VADResult {
+    case idle
+    case speech
+    case silence
+
+    var desc: String {
+        switch self {
+        case .idle: return "Press button to start VAD!"
+        case .speech: return "Speeching"
+        case .silence: return "Silence"
+        }
+    }
+}

package/ios/Packages/VAD/Data/SampleRate.swift

@@ -0,0 +1,48 @@
+//
+//  SampleRate.swift
+//  ios-vad
+//
+//  Created by baochuquan on 2024/11/9.
+//
+
+import Foundation
+
+enum SampleRate: Int, CaseIterable {
+    case rate_8k = 8000
+    case rate_15K = 15600
+    case rate_16k = 16000
+    case rate_32k = 32000
+    case rate_48k = 48000
+
+    var desc: String {
+        "SAMPLE_RATE_\(rawValue)"
+    }
+
+    static let webrtc: [SampleRate] = [.rate_8k, .rate_16k, .rate_32k, .rate_48k]
+    static let silero: [SampleRate] = [.rate_8k, .rate_16k]
+    static let yamnet: [SampleRate] = [.rate_15K]
+
+    func frameSizeOptions(type: VADType) -> [FrameSize] {
+        switch type {
+        case .webrtc:
+            switch self {
+            case .rate_8k: return FrameSize.webrtc_8k
+            case .rate_16k: return FrameSize.webrtc_16k
+            case .rate_32k: return FrameSize.webrtc_32k
+            case .rate_48k: return FrameSize.webrtc_48k
+            default: return []
+            }
+        case .silero:
+            switch self {
+            case .rate_8k: return FrameSize.silero_8k
+            case .rate_16k: return FrameSize.silero_16k
+            default: return []
+            }
+        case .yamnet:
+            switch self {
+            case .rate_15K: return FrameSize.yamnet_15k
+            default: return []
+            }
+        }
+    }
+}

package/ios/Packages/VAD/Silero/SileroVAD.swift

@@ -0,0 +1,284 @@
+//
+//  SileroVAD.swift
+//  ios-vad
+//
+//  Created by baochuquan on 2024/11/9.
+//
+
+import Foundation
+import onnxruntime_objc
+
+public protocol SileroVADDelegate: AnyObject {
+    func sileroVADDidDetectSpeechStart()
+    func sileroVADDidDetectSpeechEnd()
+    func sileroVADDidDetectSpeeching()
+    func sileroVADDidDetectSilence()
+}
+
+public class SileroVAD: NSObject {
+    private enum State {
+        case silence
+        case start
+        case speeching
+        case end
+    }
+
+    private class InternalBuffer {
+        private let size: Int
+        // Use ContiguousArray for better memory layout and to avoid copy-on-write issues
+        // when the array buffer is accessed from multiple threads
+        private var buffer: ContiguousArray<Bool>
+        private let lock = NSLock()
+
+        init(size: Int) {
+            self.size = size
+            // Pre-allocate capacity to minimize reallocations
+            buffer = ContiguousArray<Bool>()
+            buffer.reserveCapacity(size + 1)
+        }
+
+        func append(_ isSpeech: Bool) {
+            lock.lock()
+            // Perform all mutations while holding the lock
+            if buffer.count >= size {
+                // Remove oldest elements to make room
+                let removeCount = buffer.count - size + 1
+                buffer.removeFirst(removeCount)
+            }
+            buffer.append(isSpeech)
+            lock.unlock()
+        }
+
+        func isAllSpeech() -> Bool {
+            lock.lock()
+            // Copy the values we need while holding the lock
+            let count = buffer.count
+            let targetSize = size
+            let allTrue = count == targetSize && !buffer.contains(false)
+            lock.unlock()
+            return allTrue
+        }
+
+        func isAllNotSpeech() -> Bool {
+            lock.lock()
+            // Copy the values we need while holding the lock
+            let count = buffer.count
+            let targetSize = size
+            let allFalse = count == targetSize && !buffer.contains(true)
+            lock.unlock()
+            return allFalse
+        }
+    }
+
+    // 支持两种采样率，不同的采样率时，支持的 windowSizeSmaple 不一样
+    // sample rate: 8000;  sliceSize: 256/512/768
+    // sample rate: 16000; sliceSize: 512/1024/1536
+
+    static var modelPath: String {
+        // Try to find the model in the bundle containing this class (for CocoaPods)
+        let bundle = Bundle(for: SileroVAD.self)
+        if let path = bundle.path(forResource: "silero_vad", ofType: "onnx") {
+            return path
+        }
+        // Fall back to main bundle
+        return Bundle.main.path(forResource: "silero_vad", ofType: "onnx") ?? ""
+    }
+
+    public weak var delegate: SileroVADDelegate?
+    // 配置参数
+    private let sampleRate: Int64
+    private let sliceSizeSamples: Int64
+    private let threshold: Float
+    // 内部状态
+    private var state: State = .silence
+    private var silenceBuffer: InternalBuffer
+    private var speechBuffer: InternalBuffer
+
+    // 神经网络迭代状态
+    private var hidden: [[[Float]]]
+    private var cell: [[[Float]]]
+    private let hcSize: Int = 2 * 1 * 64
+
+    private var env: ORTEnv?
+    private var session: ORTSession
+
+    // Pre-allocated buffers to avoid creating new objects every frame (memory leak fix)
+    private var inputData: NSMutableData!
+    private var srData: NSMutableData!
+    private var hData: NSMutableData!
+    private var cData: NSMutableData!
+
+    /**
+     * sampleRate: 16000, 8000
+     * sliceSize:
+     *     - sampleRate: 8000; sliceSize: 256, 512, 768
+     *     - sampleRate: 16000; sliceSize: 512, 1024, 1536
+     */
+    public init(sampleRate: Int64, sliceSize: Int64, threshold: Float, silenceTriggerDurationMs: Int64, speechTriggerDurationMs: Int64, modelPath: String = "") {
+        self.sampleRate = sampleRate
+        sliceSizeSamples = sliceSize
+        self.threshold = threshold
+
+        let samplesPerMs = sampleRate / 1000
+        let silenceBufferSize = Int(ceil(Float(samplesPerMs * silenceTriggerDurationMs) / Float(sliceSize)))
+        let speechBufferSize = Int(ceil(Float(samplesPerMs * speechTriggerDurationMs) / Float(sliceSize)))
+        silenceBuffer = InternalBuffer(size: silenceBufferSize)
+        speechBuffer = InternalBuffer(size: speechBufferSize)
+
+        hidden = Array(repeating: Array(repeating: Array(repeating: Float(0.0), count: 64), count: 1), count: 2)
+        cell = Array(repeating: Array(repeating: Array(repeating: Float(0.0), count: 64), count: 1), count: 2)
+
+        do {
+            env = try? ORTEnv(loggingLevel: .warning)
+            let sessionOptions = try? ORTSessionOptions()
+            try sessionOptions?.setIntraOpNumThreads(1)
+            try sessionOptions?.setGraphOptimizationLevel(.all)
+            let path: String
+            if modelPath.isEmpty {
+                path = Self.modelPath
+            } else {
+                path = modelPath
+            }
+            let session = try? ORTSession(env: env!, modelPath: path, sessionOptions: sessionOptions!)
+            self.session = session!
+        } catch {
+            fatalError()
+        }
+
+        // Pre-allocate reusable buffers for ONNX inference (prevents memory leak)
+        inputData = NSMutableData(length: Int(sliceSize) * MemoryLayout<Float>.size)!
+        srData = NSMutableData(bytes: [sampleRate], length: MemoryLayout<Int64>.size)
+        hData = NSMutableData(length: hcSize * MemoryLayout<Float>.size)!
+        cData = NSMutableData(length: hcSize * MemoryLayout<Float>.size)!
+
+        super.init()
+        debugLog("SampleRate = \(sampleRate); sliceSize = \(sliceSize); threshold = \(threshold); silenceTriggerDurationMs = \(silenceTriggerDurationMs); speechTriggerDurationMs = \(speechTriggerDurationMs)")
+    }
+
+    public func resetState() {
+        hidden = Array(repeating: Array(repeating: Array(repeating: Float(0.0), count: 64), count: 1), count: 2)
+        cell = Array(repeating: Array(repeating: Array(repeating: Float(0.0), count: 64), count: 1), count: 2)
+    }
+
+    public func predict(data: [Float]) throws {
+        let inputShape: [NSNumber] = [1, NSNumber(value: sliceSizeSamples)]
+
+        // Copy input data into pre-allocated buffer (reuse buffer to avoid memory leak)
+        data.withUnsafeBytes { ptr in
+            inputData.replaceBytes(in: NSRange(location: 0, length: inputData.length), withBytes: ptr.baseAddress!)
+        }
+
+        // Copy hidden state into pre-allocated buffer
+        let flatHidden = hidden.flatMap { $0.flatMap { $0 } }
+        flatHidden.withUnsafeBytes { ptr in
+            hData.replaceBytes(in: NSRange(location: 0, length: hData.length), withBytes: ptr.baseAddress!)
+        }
+
+        // Copy cell state into pre-allocated buffer
+        let flatCell = cell.flatMap { $0.flatMap { $0 } }
+        flatCell.withUnsafeBytes { ptr in
+            cData.replaceBytes(in: NSRange(location: 0, length: cData.length), withBytes: ptr.baseAddress!)
+        }
+
+        // Create tensors from pre-allocated buffers (ORTValue creation is unavoidable but buffers are reused)
+        let inputTensor = try ORTValue(tensorData: inputData, elementType: .float, shape: inputShape)
+        let srTensor = try ORTValue(tensorData: srData, elementType: .int64, shape: [1])
+        let hTensor = try ORTValue(tensorData: hData, elementType: .float, shape: [2, 1, 64])
+        let cTensor = try ORTValue(tensorData: cData, elementType: .float, shape: [2, 1, 64])
+
+        let outputTensor = try session.run(
+            withInputs: ["input": inputTensor, "sr": srTensor, "h": hTensor, "c": cTensor],
+            outputNames: ["output", "hn", "cn"],
+            runOptions: nil
+        )
+        guard let outputValue = outputTensor["output"], let hiddenValue = outputTensor["hn"], let cellValue = outputTensor["cn"] else {
+            throw NSError(domain: "VadIterator", code: 1, userInfo: nil)
+        }
+
+        let outputData = try outputValue.tensorData() as Data
+        let probability = outputData.withUnsafeBytes { (buffer: UnsafeRawBufferPointer) -> Float in
+            let floatBuffer = buffer.bindMemory(to: Float.self)
+            return floatBuffer[0]
+        }
+
+        let hc_shape = (2, 1, 64)
+
+        let hiddenData = try hiddenValue.tensorData() as Data
+        hiddenData.withUnsafeBytes { (buffer: UnsafeRawBufferPointer) in
+            let floatBuffer = buffer.bindMemory(to: Float.self)
+            for i in 0 ..< hc_shape.0 {
+                for j in 0 ..< hc_shape.1 {
+                    for k in 0 ..< hc_shape.2 {
+                        hidden[i][j][k] = floatBuffer[i * hc_shape.1 * hc_shape.2 + j * hc_shape.2 + k]
+                    }
+                }
+            }
+        }
+
+        let cellData = try cellValue.tensorData() as Data
+        cellData.withUnsafeBytes { (buffer: UnsafeRawBufferPointer) in
+            let floatBuffer = buffer.bindMemory(to: Float.self)
+            for i in 0 ..< hc_shape.0 {
+                for j in 0 ..< hc_shape.1 {
+                    for k in 0 ..< hc_shape.2 {
+                        cell[i][j][k] = floatBuffer[i * hc_shape.1 * hc_shape.2 + j * hc_shape.2 + k]
+                    }
+                }
+            }
+        }
+
+        let isSpeech = probability > threshold
+        if isSpeech {
+            debugLog("\(timestamp()) prob -> \(probability), true")
+        } else {
+            debugLog("\(timestamp()) prob -> \(probability)")
+        }
+
+        // 缓存结果
+        silenceBuffer.append(isSpeech)
+        speechBuffer.append(isSpeech)
+        // 状态迁移
+        switch state {
+        case .silence:
+            if speechBuffer.isAllSpeech() {
+                state = .start
+                delegate?.sileroVADDidDetectSpeechStart()
+                state = .speeching
+                delegate?.sileroVADDidDetectSpeeching()
+            }
+        case .speeching:
+            if silenceBuffer.isAllNotSpeech() {
+                state = .end
+                delegate?.sileroVADDidDetectSpeechEnd()
+                state = .silence
+                delegate?.sileroVADDidDetectSilence()
+            }
+        default:
+            break
+        }
+    }
+
+    private func timestamp() -> String {
+        let date = Date()
+        let dateFormatter = DateFormatter()
+        dateFormatter.dateFormat = "HH:mm:ss.SSS"
+        return dateFormatter.string(from: date)
+    }
+
+    private func debugLog(_: String) {
+        #if DEBUG
+//        print("[Silero VAD]: " + content)
+        #endif
+    }
+}
+
+public extension Data {
+    /// 针对采样位数为 16 位的情况
+    func int16Array() -> [Int16] {
+        var array = [Int16](repeating: 0, count: count / MemoryLayout<Int16>.stride)
+        _ = array.withUnsafeMutableBytes {
+            self.copyBytes(to: $0, from: 0 ..< count)
+        }
+        return array
+    }
+}

package/ios/Packages/VAD/Silero/SileroVADStrategy.swift

@@ -0,0 +1,64 @@
+//
+//  SileroVADStrategy.swift
+//  ios-vad
+//
+//  Created by baochuquan on 2024/11/10.
+//
+
+import Foundation
+
+// MARK: - SileroVADStrategy
+
+class SileroVADStrategy: VADStrategy {
+    private var sileroVAD: SileroVAD?
+    private var state: VADState = .silence
+    private var handler: ((VADState) -> Void)?
+
+    func setup(sampleRate: SampleRate, frameSize: FrameSize, quality: VADQuality, silenceTriggerDurationMs: Int64, speechTriggerDurationMs: Int64) {
+        sileroVAD = SileroVAD(
+            sampleRate: Int64(sampleRate.rawValue),
+            sliceSize: Int64(frameSize.rawValue),
+            threshold: quality.threshold,
+            silenceTriggerDurationMs: silenceTriggerDurationMs,
+            speechTriggerDurationMs: speechTriggerDurationMs
+        )
+        sileroVAD?.delegate = self
+    }
+
+    func checkVAD(pcm: [Int16], handler: @escaping (VADState) -> Void) {
+        self.handler = handler
+        let data: [Float] = pcm.map { Float($0) / 32768.0 } // normalize
+
+        do {
+            try sileroVAD?.predict(data: data)
+        } catch _ {
+            fatalError()
+        }
+    }
+
+    func currentState() -> VADState {
+        return state
+    }
+}
+
+extension SileroVADStrategy: SileroVADDelegate {
+    func sileroVADDidDetectSpeechStart() {
+        state = .start
+        handler?(.start)
+    }
+
+    func sileroVADDidDetectSpeechEnd() {
+        state = .end
+        handler?(.end)
+    }
+
+    func sileroVADDidDetectSilence() {
+        state = .silence
+        handler?(.silence)
+    }
+
+    func sileroVADDidDetectSpeeching() {
+        state = .speeching
+        handler?(.speeching)
+    }
+}

package/ios/Packages/libbz2/module.modulemap

@@ -0,0 +1,5 @@
+module libbz2 [system] {
+    header "shim.h"
+    link "bz2"
+    export *
+}

package/ios/Packages/libbz2/shim.h

@@ -0,0 +1 @@
+#include <bzlib.h>