@glassly/bluetooth-sdk 0.1.0-dev.26

package/ios/Source/stt/STTTools.swift

@@ -0,0 +1,191 @@
+import Foundation
+
+class STTTools {
+    // MARK: - SherpaOnnxTranscriber / STT Model Management
+
+    /// How stale a SpeakerIdEngine match may be and still label a transcript.
+    private static let speakerMatchWindowMs: Int64 = 5000
+
+    /// Native-side join with SpeakerIdEngine: stamp the most recent
+    /// identification match onto the transcript. Fallback stays `0`/unknown
+    /// when the engine is off or nothing matched, so downstream schemas are
+    /// unchanged; the resolved name is additive.
+    private static func speakerFields() -> [String: Any] {
+        guard let match = SpeakerIdEngine.shared.recentMatch(withinMs: speakerMatchWindowMs) else {
+            return ["speakerId": 0]
+        }
+        return [
+            "speakerId": match.profileId,
+            "resolvedSpeakerId": match.profileId,
+            "resolvedSpeakerName": match.name,
+        ]
+    }
+
+    static func didReceivePartialTranscription(_ text: String) {
+        // Send partial result to server witgetConnectedBluetoothNameh proper formatting
+        let transcriptionLanguage =
+            UserDefaults.standard.string(forKey: "STTModelLanguageCode") ?? "en-US"
+        // Bridge.log("Glassly: Sending partial transcription: \(text), \(transcriptionLanguage)")
+        var transcription: [String: Any] = [
+            "type": "local_transcription",
+            "text": transcriptionLanguage == "en-US" ? text.lowercased() : text,
+            "isFinal": false,
+            "startTime": Int(Date().timeIntervalSince1970 * 1000) - 1000, // 1 second ago
+            "endTime": Int(Date().timeIntervalSince1970 * 1000),
+            "transcribeLanguage": transcriptionLanguage,
+            "provider": "sherpa-onnx",
+        ]
+        transcription.merge(speakerFields()) { _, new in new }
+
+        Bridge.sendLocalTranscription(transcription: transcription)
+    }
+
+    static func didReceiveFinalTranscription(_ text: String) {
+        // Send final result to server with proper formatting
+        let transcriptionLanguage =
+            UserDefaults.standard.string(forKey: "STTModelLanguageCode") ?? "en-US"
+        Bridge.log("Glassly: Sending final transcription: \(text), \(transcriptionLanguage)")
+        if !text.isEmpty {
+            var transcription: [String: Any] = [
+                "type": "local_transcription",
+                "text": transcriptionLanguage == "en-US" ? text.lowercased() : text,
+                "isFinal": true,
+                "startTime": Int(Date().timeIntervalSince1970 * 1000) - 2000, // 2 seconds ago
+                "endTime": Int(Date().timeIntervalSince1970 * 1000),
+                "transcribeLanguage": transcriptionLanguage,
+                "provider": "sherpa-onnx",
+            ]
+            let fields = speakerFields()
+            transcription.merge(fields) { _, new in new }
+            if let name = fields["resolvedSpeakerName"] as? String {
+                Bridge.log("SpeakerID: local final stamped as \(name)")
+            }
+
+            Bridge.sendLocalTranscription(transcription: transcription)
+        }
+    }
+
+    static func setSttModelDetails(_ path: String, _ languageCode: String) {
+        UserDefaults.standard.set(path, forKey: "STTModelPath")
+        UserDefaults.standard.set(languageCode, forKey: "STTModelLanguageCode")
+        UserDefaults.standard.synchronize()
+    }
+
+    static func getSttModelPath() -> String {
+        return UserDefaults.standard.string(forKey: "STTModelPath") ?? ""
+    }
+
+    static func checkSTTModelAvailable() -> Bool {
+        guard let modelPath = UserDefaults.standard.string(forKey: "STTModelPath") else {
+            return false
+        }
+
+        let fileManager = FileManager.default
+
+        // Check for tokens.txt (required for all models)
+        let tokensPath = (modelPath as NSString).appendingPathComponent("tokens.txt")
+        if !fileManager.fileExists(atPath: tokensPath) {
+            return false
+        }
+
+        // Check for CTC model
+        if firstExistingFile(in: modelPath, candidates: ["model.int8.onnx", "model.onnx"]) != nil {
+            return true
+        }
+
+        // Check for transducer model
+        let transducerFiles = [
+            ["encoder.onnx", "encoder.int8.onnx"],
+            ["decoder.onnx", "decoder.int8.onnx"],
+            ["joiner.onnx", "joiner.int8.onnx"],
+        ]
+        for candidates in transducerFiles {
+            if firstExistingFile(in: modelPath, candidates: candidates) == nil {
+                return false
+            }
+        }
+
+        return true
+    }
+
+    static func validateSTTModel(_ path: String) -> Bool {
+        // do {
+        let fileManager = FileManager.default
+
+        // Check for tokens.txt (required for all models)
+        let tokensPath = (path as NSString).appendingPathComponent("tokens.txt")
+        if !fileManager.fileExists(atPath: tokensPath) {
+            return false
+        }
+
+        // Check for CTC model
+        if firstExistingFile(in: path, candidates: ["model.int8.onnx", "model.onnx"]) != nil {
+            return true
+        }
+
+        // Check for transducer model
+        let transducerFiles = [
+            ["encoder.onnx", "encoder.int8.onnx"],
+            ["decoder.onnx", "decoder.int8.onnx"],
+            ["joiner.onnx", "joiner.int8.onnx"],
+        ]
+        var allTransducerFilesPresent = true
+
+        for candidates in transducerFiles {
+            if firstExistingFile(in: path, candidates: candidates) == nil {
+                allTransducerFilesPresent = false
+                break
+            }
+        }
+
+        return allTransducerFilesPresent
+        // } catch {
+        // Bridge.log("STT_ERROR: \(error.localizedDescription)")
+        // return false
+        // }
+    }
+
+    static func extractTarBz2(sourcePath: String, destinationPath: String) -> Bool {
+        do {
+            let fileManager = FileManager.default
+
+            // Create destination directory if it doesn't exist
+            try fileManager.createDirectory(
+                atPath: destinationPath,
+                withIntermediateDirectories: true,
+                attributes: nil
+            )
+
+            // Use the Swift TarBz2Extractor with SWCompression
+            var extractionError: NSError?
+            let success = TarBz2Extractor.extractTarBz2From(
+                sourcePath,
+                to: destinationPath,
+                error: &extractionError
+            )
+
+            if !success || extractionError != nil {
+                print(
+                    "EXTRACTION_ERROR: \(extractionError?.localizedDescription ?? "Failed to extract tar.bz2")"
+                )
+                return false
+            }
+
+        } catch {
+            Bridge.log("EXTRACTION_ERROR: \(error.localizedDescription)")
+            return false
+        }
+        return true
+    }
+
+    private static func firstExistingFile(in directory: String, candidates: [String]) -> String? {
+        let fileManager = FileManager.default
+        for candidate in candidates {
+            let path = (directory as NSString).appendingPathComponent(candidate)
+            if fileManager.fileExists(atPath: path) {
+                return path
+            }
+        }
+        return nil
+    }
+}

package/ios/Source/stt/SherpaOnnxTranscriber.swift

@@ -0,0 +1,466 @@
+import Foundation
+import UIKit
+
+/**
+ * SherpaOnnxTranscriber handles real-time audio transcription using Sherpa-ONNX.
+ *
+ * It works fully offline and processes PCM audio in real-time to provide partial and final ASR results.
+ * This class runs on a background thread, processes short PCM chunks, and emits transcribed text using a delegate.
+ */
+class SherpaOnnxTranscriber {
+    private static let TAG = "SherpaOnnxTranscriber"
+
+    private static let SAMPLE_RATE = 16000 // Sherpa-ONNX model's required sample rate
+    private static let QUEUE_CAPACITY = 100 // Max number of audio buffers to keep in queue
+
+    private let pcmQueue = DispatchQueue(label: "com.augmentos.sherpaonnx.pcmQueue", qos: .userInteractive)
+    private var pcmBuffers = [Data]()
+    private var isRunning = false
+    private var processingQueue: DispatchQueue?
+    private var processingTask: DispatchWorkItem?
+
+    /// The underlying Sherpa-ONNX objects
+    private var recognizer: SherpaOnnxRecognizer?
+
+    private var lastPartialResult = ""
+
+    /// Parent context
+    private weak var context: UIViewController?
+
+    /// Session start time for relative timestamps
+    private var transcriptionSessionStart: Date
+
+    /// Dynamic model path support
+    private static var customModelPath: String? {
+        guard let storedPath = UserDefaults.standard.string(forKey: "STTModelPath") else {
+            return nil
+        }
+
+        // Always resolve current Documents directory
+        let documentsURL = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask).first!
+
+        // Extract relative subpath after "Documents/"
+        // NOTE: Doing this because the application id changes between the development builds and files can't be found.
+        if let range = storedPath.range(of: "/Documents/") {
+            let relativePath = String(storedPath[range.upperBound...]) // e.g. "stt_models/..."
+            let fixedPath = documentsURL.appendingPathComponent(relativePath).path
+
+            Bridge.log("Reconstructed STTModelPath: \(fixedPath)")
+            return fixedPath
+        }
+
+        // If nothing matched, just return as-is
+        Bridge.log("STTModelPath (raw): \(storedPath)")
+        return storedPath
+    }
+
+    private static func firstExistingFile(in directory: String, candidates: [String]) -> String? {
+        let fileManager = FileManager.default
+        for candidate in candidates {
+            let path = (directory as NSString).appendingPathComponent(candidate)
+            if fileManager.fileExists(atPath: path) {
+                return path
+            }
+        }
+        return nil
+    }
+
+    /**
+     * Heuristically detect an NVIDIA Nemotron model directory.
+     *
+     * Nemotron is file-identical to a Zipformer transducer (encoder/decoder/joiner + tokens.txt),
+     * so filenames alone cannot distinguish the two. We check the on-disk directory name first —
+     * STTModelManager names the folder after the model id — and fall back to the README.md that
+     * ships inside every upstream archive.
+     *
+     * This is advisory only: sherpa-onnx does the authoritative dispatch from ONNX metadata. A
+     * false negative costs nothing but a less precise log line.
+     */
+    private static func isNemotronModelDir(_ directory: String) -> Bool {
+        if (directory as NSString).lastPathComponent.range(of: "nemotron", options: .caseInsensitive) != nil {
+            return true
+        }
+
+        let readmePath = (directory as NSString).appendingPathComponent("README.md")
+        guard let readme = try? String(contentsOfFile: readmePath, encoding: .utf8) else {
+            return false
+        }
+        // Archive READMEs are small; cap the scan so a malformed file can't stall startup.
+        return readme.prefix(8192).range(of: "nemotron", options: .caseInsensitive) != nil
+    }
+
+    /**
+     * Constructor that accepts a UIViewController to load model assets.
+     */
+    init(context: UIViewController) {
+        self.context = context
+        transcriptionSessionStart = Date()
+    }
+
+    deinit {
+        shutdown()
+    }
+
+    /**
+     * Initialize the Sherpa-ONNX recognizer.
+     * Loads models and configuration, sets up processing thread.
+     */
+    func initialize() {
+        do {
+            var tokensPath: String
+            var modelType = "unknown"
+            let fileManager = FileManager.default
+
+            // Check if we have a custom model path set
+            if let customPath = SherpaOnnxTranscriber.customModelPath {
+                // Detect model type based on available files
+                let ctcModelPath = Self.firstExistingFile(
+                    in: customPath,
+                    candidates: ["model.int8.onnx", "model.onnx"]
+                )
+                let transducerEncoderPath = Self.firstExistingFile(
+                    in: customPath,
+                    candidates: ["encoder.int8.onnx", "encoder.onnx"]
+                )
+
+                tokensPath = (customPath as NSString).appendingPathComponent("tokens.txt")
+
+                // Verify tokens file exists
+                guard fileManager.fileExists(atPath: tokensPath) else {
+                    throw NSError(domain: "SherpaOnnxTranscriber", code: 1, userInfo: [
+                        NSLocalizedDescriptionKey: "tokens.txt not found at path: \(customPath)",
+                    ])
+                }
+
+                if let ctcModelPath {
+                    // CTC model detected
+                    modelType = "ctc"
+                    Bridge.log("Detected CTC model at \(customPath)")
+
+                    // Build the config and construct the recognizer inside one
+                    // C-string scope: the config's `const char*` fields point at
+                    // buffers owned by that scope (see toCPointer in
+                    // SherpaOnnx.swift). The config must not escape it.
+                    recognizer = withSherpaOnnxCStrings {
+                        // Create CTC model config using Zipformer2Ctc
+                        let nemoCtc = sherpaOnnxOnlineNemoCtcModelConfig(
+                            model: ctcModelPath
+                        )
+
+                        // Create model config with CTC
+                        let modelConfig = sherpaOnnxOnlineModelConfig(
+                            tokens: tokensPath,
+                            numThreads: 1,
+                            nemoCtc: nemoCtc
+                        )
+
+                        // Configure recognizer
+                        let featureConfig = sherpaOnnxFeatureConfig()
+
+                        var config = sherpaOnnxOnlineRecognizerConfig(
+                            featConfig: featureConfig,
+                            modelConfig: modelConfig,
+                            enableEndpoint: true,
+                            rule1MinTrailingSilence: 1.2,
+                            rule2MinTrailingSilence: 0.8,
+                            rule3MinUtteranceLength: 10.0
+                        )
+
+                        // Create recognizer with the wrapper
+                        return SherpaOnnxRecognizer(config: &config)
+                    }
+
+                } else if let transducerEncoderPath {
+                    // Transducer model detected. NVIDIA Nemotron ships the same
+                    // encoder/decoder/joiner triplet as a Zipformer transducer and is loaded
+                    // through this very same config — sherpa-onnx sniffs the ONNX metadata and
+                    // dispatches to its NeMo transducer implementation on its own. We detect it
+                    // only to log it distinctly; there is no Nemotron-specific config struct.
+                    //
+                    // Critically, we must NOT set a modelType override for Nemotron:
+                    // "transducer" would force the generic Zipformer path and mis-decode.
+                    // Leaving it unset lets sherpa-onnx auto-detect.
+                    let isNemotron = Self.isNemotronModelDir(customPath)
+                    modelType = isNemotron ? "nemotron" : "transducer"
+                    Bridge.log("Detected \(modelType) model at \(customPath)")
+
+                    let decoderPath = Self.firstExistingFile(
+                        in: customPath,
+                        candidates: ["decoder.int8.onnx", "decoder.onnx"]
+                    )
+                    let joinerPath = Self.firstExistingFile(
+                        in: customPath,
+                        candidates: ["joiner.int8.onnx", "joiner.onnx"]
+                    )
+
+                    // Verify all transducer files exist
+                    guard let decoderPath,
+                          let joinerPath
+                    else {
+                        throw NSError(domain: "SherpaOnnxTranscriber", code: 1, userInfo: [
+                            NSLocalizedDescriptionKey: "Transducer model files incomplete at path: \(customPath)",
+                        ])
+                    }
+
+                    // Build the config and construct the recognizer inside one
+                    // C-string scope: the config's `const char*` fields point at
+                    // buffers owned by that scope (see toCPointer in
+                    // SherpaOnnx.swift). The config must not escape it.
+                    recognizer = withSherpaOnnxCStrings {
+                        // Create Sherpa-ONNX transducer model config
+                        let transducer = sherpaOnnxOnlineTransducerModelConfig(
+                            encoder: transducerEncoderPath,
+                            decoder: decoderPath,
+                            joiner: joinerPath
+                        )
+
+                        // Create model config
+                        let modelConfig = sherpaOnnxOnlineModelConfig(
+                            tokens: tokensPath,
+                            transducer: transducer,
+                            numThreads: 1
+                        )
+
+                        // Configure recognizer
+                        let featureConfig = sherpaOnnxFeatureConfig()
+
+                        var config = sherpaOnnxOnlineRecognizerConfig(
+                            featConfig: featureConfig,
+                            modelConfig: modelConfig,
+                            enableEndpoint: true,
+                            rule1MinTrailingSilence: 1.2,
+                            rule2MinTrailingSilence: 0.8,
+                            rule3MinUtteranceLength: 10.0
+                        )
+
+                        // Create recognizer with the wrapper
+                        return SherpaOnnxRecognizer(config: &config)
+                    }
+
+                } else {
+                    throw NSError(domain: "SherpaOnnxTranscriber", code: 1, userInfo: [
+                        NSLocalizedDescriptionKey: "No valid model files found at path: \(customPath)",
+                    ])
+                }
+            } else {
+                Bridge.log("No Sherpa ONNX model available. Transcription will be disabled.")
+                Bridge.log("Please download a model using the model downloader in settings.")
+                recognizer = nil
+                isRunning = false
+                return
+            }
+
+            if recognizer == nil {
+                throw NSError(domain: "SherpaOnnxTranscriber", code: 2, userInfo: [NSLocalizedDescriptionKey: "Failed to create recognizer"])
+            }
+
+            startProcessingTask()
+            isRunning = true
+
+            Bridge.log("Sherpa-ONNX ASR initialized successfully with \(modelType) model")
+
+        } catch {
+            Bridge.log("Failed to initialize Sherpa-ONNX: \(error.localizedDescription)")
+        }
+    }
+
+    /**
+     * Handle transcription results - send only to delegate
+     */
+    private func handleTranscriptionResult(text: String, isFinal: Bool) {
+        // Forward to delegate if set
+        DispatchQueue.main.async { [weak self] in
+            if isFinal {
+                STTTools.didReceiveFinalTranscription(text)
+            } else {
+                STTTools.didReceivePartialTranscription(text)
+            }
+        }
+    }
+
+    /**
+     * Feed PCM audio data (16-bit little endian) into the transcriber.
+     * This method should be called continuously with short chunks (e.g., 100-300ms).
+     *
+     * Audio is queued directly; microphone VAD gating is not applied in the SDK.
+     */
+    func acceptAudio(pcm16le: Data) {
+        guard isRunning else {
+            return
+        }
+
+        queueAudioData(pcm16le)
+    }
+
+    private func queueAudioData(_ pcm16le: Data) {
+        pcmQueue.async { [weak self] in
+            guard let self = self else { return }
+
+            let queueSizeBefore = self.pcmBuffers.count
+            self.pcmBuffers.append(pcm16le)
+
+            // Keep queue size manageable
+            if self.pcmBuffers.count > Self.QUEUE_CAPACITY {
+                let removedBuffer = self.pcmBuffers.removeFirst()
+                Bridge.log("⚠️ Audio queue overflow - dropped buffer of \(removedBuffer.count) bytes")
+            }
+        }
+    }
+
+    /**
+     * Start a background task to continuously consume audio and decode using Sherpa.
+     */
+    private func startProcessingTask() {
+        Bridge.log("🚀 Starting Sherpa-ONNX processing task...")
+
+        processingQueue = DispatchQueue(label: "com.augmentos.sherpaonnx.processor", qos: .userInitiated)
+
+        let workItem = DispatchWorkItem { [weak self] in
+            self?.runLoop()
+        }
+
+        processingTask = workItem
+        processingQueue?.async(execute: workItem)
+    }
+
+    /**
+     * Main processing loop that handles transcription in real-time.
+     * Pulls audio from queue, feeds into Sherpa, emits partial/final results.
+     */
+    private func runLoop() {
+        Bridge.log("🔄 Sherpa-ONNX processing loop started")
+
+        while isRunning {
+            // Pull data from queue
+            var audioData: Data?
+
+            pcmQueue.sync {
+                if !self.pcmBuffers.isEmpty {
+                    audioData = self.pcmBuffers.removeFirst()
+                }
+            }
+
+            if let data = audioData {
+                // Synchronize access to recognizer to prevent race conditions
+                objc_sync_enter(self)
+                defer { objc_sync_exit(self) }
+
+                guard let recognizer = recognizer else {
+                    Bridge.log("⚠️ Recognizer not available, skipping audio chunk")
+                    continue
+                }
+
+                do {
+                    // Convert PCM to float [-1.0, 1.0]
+                    let floatBuf = toFloatArray(from: data)
+
+                    // Pass audio data to the Sherpa-ONNX stream
+                    recognizer.acceptWaveform(samples: floatBuf, sampleRate: Self.SAMPLE_RATE)
+
+                    // Decode continuously while model is ready
+                    var decodeCount = 0
+                    while recognizer.isReady() {
+                        recognizer.decode()
+                        decodeCount += 1
+                    }
+
+                    // If utterance endpoint detected
+                    if recognizer.isEndpoint() {
+                        let result = recognizer.getResult()
+                        let finalText = result.text.trimmingCharacters(in: .whitespacesAndNewlines)
+
+                        if !finalText.isEmpty {
+                            handleTranscriptionResult(text: finalText, isFinal: true)
+                        }
+
+                        recognizer.reset() // Start new utterance
+                        lastPartialResult = ""
+                    } else {
+                        // Emit partial results if changed
+                        let result = recognizer.getResult()
+                        let partial = result.text.trimmingCharacters(in: .whitespacesAndNewlines)
+
+                        if partial != lastPartialResult, !partial.isEmpty {
+                            handleTranscriptionResult(text: partial, isFinal: false)
+                            lastPartialResult = partial
+                        }
+                    }
+                } catch {
+                    Bridge.log("❌ Error processing audio: \(error.localizedDescription)")
+                }
+            } else {
+                // Sleep briefly to avoid tight CPU loop if no audio is available
+                Thread.sleep(forTimeInterval: 0.01)
+            }
+        }
+
+        Bridge.log("ASR processing thread stopped")
+    }
+
+    /**
+     * Convert 16-bit PCM byte data (little-endian) to float array [-1.0, 1.0].
+     */
+    private func toFloatArray(from pcmData: Data) -> [Float] {
+        let count = pcmData.count / 2
+        var samples = [Float](repeating: 0, count: count)
+
+        pcmData.withUnsafeBytes { (bufferPointer: UnsafeRawBufferPointer) in
+            if let address = bufferPointer.baseAddress {
+                let int16Pointer = address.bindMemory(to: Int16.self, capacity: count)
+
+                for i in 0 ..< count {
+                    // Convert from little-endian if needed
+                    var sample = int16Pointer[i]
+                    if CFByteOrderGetCurrent() == CFByteOrder(CFByteOrderBigEndian.rawValue) {
+                        sample = Int16(littleEndian: sample)
+                    }
+                    samples[i] = Float(sample) / 32768.0
+                }
+            }
+        }
+
+        return samples
+    }
+
+    /**
+     * Stop transcription processing.
+     * This shuts down the processing thread and releases Sherpa-ONNX resources.
+     */
+    func shutdown() {
+        Bridge.log("🛑 Shutting down SherpaOnnxTranscriber...")
+
+        isRunning = false
+        processingTask?.cancel()
+
+        // Synchronize access to recognizer during shutdown
+        objc_sync_enter(self)
+        defer { objc_sync_exit(self) }
+
+        // The recognizer will be automatically cleaned up by ARC when set to nil
+        if recognizer != nil {
+            Bridge.log("🧹 Cleaning up Sherpa-ONNX recognizer")
+            recognizer = nil
+        }
+
+        // Clear any remaining audio buffers
+        pcmQueue.sync {
+            let remainingBuffers = self.pcmBuffers.count
+            if remainingBuffers > 0 {
+                Bridge.log("🗑️ Clearing \(remainingBuffers) remaining audio buffers")
+            }
+            self.pcmBuffers.removeAll()
+        }
+
+        Bridge.log("✅ SherpaOnnxTranscriber shutdown complete")
+    }
+
+    /**
+     * Restarts the transcriber after a model change.
+     * Shuts down existing resources, clears buffers, and reinitializes the recognizer.
+     */
+    func restart() {
+        Bridge.log("♻️ Restarting SherpaOnnxTranscriber...")
+        shutdown()
+        initialize()
+    }
+}