@mentra/bluetooth-sdk 3.2.1-dev.311 → 3.2.1

package/ios/Source/sgcs/NimoCanvasSession.swift

@@ -0,0 +1,166 @@
+import Foundation
+
+/// Serializes launch/update/exit. A business ACK means acceptance, not visible pixels.
+final class NimoCanvasSession {
+    enum Action: Equatable {
+        case send(key: Int, frame: Data, ticket: UInt64)
+        case reconnect(String)
+        case rejected(Int)
+    }
+
+    private struct Flight {
+        let key: Int
+        let frame: Data
+        let ticket: UInt64
+    }
+
+    private var ready = false
+    private var transportReady = false
+    private var active = false
+    private var locked = false
+    private var launchBlocked = false
+    private var waitingReadiness = false
+    private var observedNotReady = false
+    private var readinessRetries = 0
+    private var probeRetries = 0
+    private var exitRequested = false
+    private var desired: Data?
+    private var accepted: Data?
+    private var rejected: Data?
+    private var scope: String?
+    private var forceRevision: UInt64 = 0
+    private var sentRevision: UInt64 = 0
+    private var ticket: UInt64 = 0
+    private var flight: Flight?
+
+    func offer(_ frame: Data, scope: String, force: Bool = false) -> [Action] {
+        if self.scope != scope || force { forceRevision &+= 1 }
+        self.scope = scope
+        desired = frame
+        exitRequested = false
+        return pump()
+    }
+
+    /// Enter a blank canvas unless a newer host scene survived the connection change.
+    func activate() -> [Action] {
+        if desired == nil { desired = Data([0, 0, 1]) }
+        exitRequested = false
+        return pump()
+    }
+
+    /// The firmware validates peer readiness on Launch. Only an explicit NOT_READY
+    /// permits a bounded retry; an ambiguous ACK still requires a new transport.
+    func retryNotReady() -> [Action] {
+        guard transportReady, waitingReadiness, desired != nil, probeRetries < 3 else { return [] }
+        probeRetries += 1
+        waitingReadiness = false
+        ready = true
+        return pump()
+    }
+
+    /// Only a fresh heartbeat samples both TWS and peer Companion readiness.
+    func readiness(_ value: Bool, confirmed: Bool = false) -> [Action] {
+        transportReady = value
+        if !value { readinessRetries = 0; probeRetries = 0 }
+        if !value, flight != nil { return resetLink("Readiness lost during canvas command") }
+        if waitingReadiness {
+            if !value { observedNotReady = true }
+            else if !observedNotReady {
+                if !confirmed || readinessRetries >= 3 { return [] }
+                readinessRetries += 1
+            }
+        }
+        ready = value
+        if !value { active = false; accepted = nil }
+        if value { waitingReadiness = false }
+        return pump()
+    }
+
+    /// Keep only the latest desired scene across a new transport generation.
+    func disconnected() {
+        ready = false; transportReady = false; active = false; locked = false; launchBlocked = false
+        waitingReadiness = false; observedNotReady = false; readinessRetries = 0; probeRetries = 0
+        accepted = nil; rejected = nil; flight = nil
+        ticket &+= 1; forceRevision &+= 1
+    }
+
+    func exit() -> [Action] {
+        desired = nil; scope = nil; accepted = nil; rejected = nil; exitRequested = true
+        return pump()
+    }
+
+    /// Stock UI reports do not relinquish host ownership. Replay after the current ACK.
+    func nativeApp(_ appId: Int, entered: Bool) -> [Action] {
+        if (appId == NimoCanvasCodec.appId && !entered) || (appId != NimoCanvasCodec.appId && entered) {
+            active = false
+            accepted = nil
+            return pump()
+        }
+        return []
+    }
+
+    func acceptsResponse(key: Int, payload: Data) -> Bool {
+        guard let current = flight, current.key == key, let status = payload.first else { return false }
+        let echo = key == 4 ? Data([0, 0xFD, 0, 0, 0]) : Data([0, 0xFD])
+        if status == 0 { return payload == echo }
+        return payload.count == 1 || (payload.count == echo.count && payload.dropFirst() == echo.dropFirst())
+    }
+
+    func response(key: Int, payload: Data) -> [Action] {
+        guard acceptsResponse(key: key, payload: payload), let current = flight, let first = payload.first else { return [] }
+        let status = Int(first)
+        flight = nil
+        if status != 0 {
+            switch status {
+            case 7:
+                ready = false; waitingReadiness = true; observedNotReady = false; active = false; accepted = nil
+            case 8:
+                locked = true; active = false; accepted = nil
+            case 2:
+                return [.rejected(status)] + resetLink("Canvas device timeout is ambiguous")
+            default:
+                if key == 4 { rejected = current.frame }
+                else {
+                    if key == 1 { launchBlocked = true } else { desired = nil }
+                    active = false; exitRequested = false
+                }
+            }
+            return [.rejected(status)] + pump()
+        }
+        switch key {
+        case 1: active = true; accepted = nil
+        case 4: accepted = current.frame; rejected = nil; readinessRetries = 0; probeRetries = 0
+        case 3: active = false; accepted = nil; exitRequested = false
+        default: break
+        }
+        return pump()
+    }
+
+    func timeout(_ expectedTicket: UInt64) -> [Action] {
+        guard flight?.ticket == expectedTicket else { return [] }
+        return resetLink("Canvas ACK timeout; reconnect before reusing a command key")
+    }
+
+    private func resetLink(_ reason: String) -> [Action] {
+        ready = false; transportReady = false; active = false; accepted = nil; flight = nil
+        ticket &+= 1; forceRevision &+= 1
+        return [.reconnect(reason)]
+    }
+
+    private func pump() -> [Action] {
+        guard ready, !locked, !launchBlocked, !waitingReadiness, flight == nil else { return [] }
+        let key: Int
+        if exitRequested, active { key = 3 }
+        else if exitRequested { exitRequested = false; return [] }
+        else if desired == nil { return [] }
+        else if !active { key = 1 }
+        else if desired == rejected { return [] }
+        else if desired == accepted, sentRevision == forceRevision { return [] }
+        else { key = 4 }
+        let frame = key == 4 ? desired! : Data()
+        if key == 4 { sentRevision = forceRevision }
+        ticket &+= 1
+        flight = Flight(key: key, frame: frame, ticket: ticket)
+        return [.send(key: key, frame: frame, ticket: ticket)]
+    }
+}

package/ios/Source/sgcs/NimoDiscovery.swift

@@ -0,0 +1,49 @@
+import Foundation
+
+/// Combines system-connected classic devices with incoming CoreBluetooth discoveries.
+/// Kept independent of the radio so discovery can be exercised without hardware.
+@MainActor
+final class NimoDiscovery<Peripheral> {
+    private(set) var active = false
+    private let register: () -> Void
+    private let connected: () -> [Peripheral]
+    private let scan: () -> Void
+    private let stop: () -> Void
+    private let name: (Peripheral) -> String?
+    private let found: (Peripheral, String, Int?) -> Void
+
+    init(register: @escaping () -> Void, connected: @escaping () -> [Peripheral],
+         scan: @escaping () -> Void, stop: @escaping () -> Void,
+         name: @escaping (Peripheral) -> String?, found: @escaping (Peripheral, String, Int?) -> Void)
+    {
+        self.register = register
+        self.connected = connected
+        self.scan = scan
+        self.stop = stop
+        self.name = name
+        self.found = found
+    }
+
+    func start() {
+        active = true
+        // Register first so pairing in Settings cannot race the connected snapshot.
+        register()
+        for peripheral in connected() {
+            receive(peripheral)
+        }
+        if active { scan() }
+    }
+
+    func cancel() {
+        active = false
+        stop()
+    }
+
+    func receive(_ peripheral: Peripheral, advertisedName: String? = nil, rssi: Int? = nil) {
+        guard active, let name = advertisedName ?? name(peripheral) else { return }
+        let lower = name.lowercased()
+        // The separate ANCS peripheral does not host the Companion data service.
+        guard lower.hasPrefix("nimo"), !lower.hasSuffix("_ble") else { return }
+        found(peripheral, name, rssi)
+    }
+}

package/ios/Source/sgcs/SGCManager.swift

@@ -96,6 +96,10 @@ protocol SGCManager {
 
     // MARK: - Display Control
 
+    /// Full-frame adapters preserve the active scene instead of showing temporary overlays.
+    var showBrightnessConfirmation: Bool { get }
+    var showConnectionConfirmation: Bool { get }
+    var sceneHandoffRequiresClear: Bool { get }
     func setBrightness(_ level: Int, autoMode: Bool)
     func clearDisplay()
     func sendText(_ text: String) async
@@ -228,6 +232,10 @@ protocol SGCManager {
 
     func sendLoudnessGateSetting()
 
+    // MARK: - Auto power-off when unworn
+
+    func sendAutoPowerOffSetting()
+
     // MARK: - Mic tuning (super-mode only)
 
     func sendMicTuningSetting()
@@ -254,6 +262,18 @@ protocol SGCManager {
 /// doesn't seem to work for concurrency reasons :(
 /// we can make read-only getters for convienence though:
 extension SGCManager {
+    var showBrightnessConfirmation: Bool {
+        true
+    }
+
+    var showConnectionConfirmation: Bool {
+        true
+    }
+
+    var sceneHandoffRequiresClear: Bool {
+        true
+    }
+
     var isMicSuspendedForAudio: Bool {
         false
     }
@@ -435,6 +455,10 @@ extension SGCManager {
 
     func sendLoudnessGateSetting() {}
 
+    // MARK: - Auto power-off (default no-op — Mentra Live supports this)
+
+    func sendAutoPowerOffSetting() {}
+
     // MARK: - Mic tuning (default no-op — Mentra Live supports this)
 
     func sendMicTuningSetting() {}

package/ios/Source/sgcs/VuzixConnectionTarget.swift

@@ -0,0 +1,15 @@
+import Foundation
+
+enum VuzixConnectionTarget {
+    static func identifier(_ name: String) -> String {
+        guard let start = name.lastIndex(of: "["),
+              let end = name[start...].firstIndex(of: "]")
+        else { return name }
+        return String(name[name.index(after: start) ..< end])
+    }
+
+    static func matches(name: String?, target: String) -> Bool {
+        guard let name, !target.isEmpty else { return false }
+        return identifier(name) == identifier(target)
+    }
+}

package/ios/Source/status/DeviceStatus.swift

@@ -123,6 +123,7 @@ struct GlassesStatus: CustomStringConvertible {
             "wifiLocalIp": "",
             "signalStrength": -1,
             "signalStrengthUpdatedAt": 0,
+            "g2MissingArm": NSNull(),
         ]))
     }
 

package/ios/Source/stt/SherpaOnnxTranscriber.swift

@@ -1,5 +1,4 @@
 import Foundation
-import UIKit
 
 /**
  * SherpaOnnxTranscriber handles real-time audio transcription using Sherpa-ONNX.
@@ -11,24 +10,30 @@ class SherpaOnnxTranscriber {
     private static let TAG = "SherpaOnnxTranscriber"
 
     private static let SAMPLE_RATE = 16000 // Sherpa-ONNX model's required sample rate
-    private static let QUEUE_CAPACITY = 100 // Max number of audio buffers to keep in queue
+    private final class Session {
+        let recognizer: SherpaOnnxRecognizer
+        var lastPartialResult = ""
 
-    private let pcmQueue = DispatchQueue(label: "com.augmentos.sherpaonnx.pcmQueue", qos: .userInteractive)
-    private var pcmBuffers = [Data]()
-    private var isRunning = false
-    private var processingQueue: DispatchQueue?
-    private var processingTask: DispatchWorkItem?
-
-    /// The underlying Sherpa-ONNX objects
-    private var recognizer: SherpaOnnxRecognizer?
+        init(_ recognizer: SherpaOnnxRecognizer) {
+            self.recognizer = recognizer
+        }
+    }
 
-    private var lastPartialResult = ""
+    private lazy var worker = DemandDrivenAudioWorker<Session>(
+        makeProcessor: { Self.makeRecognizer().map(Session.init) },
+        process: { [weak self] session, data, generation in
+            self?.process(session, data: data, generation: generation)
+        }
+    )
 
-    /// Parent context
-    private weak var context: UIViewController?
+    /// Construct the worker on the manager's actor before audio can arrive.
+    init() {
+        _ = worker
+    }
 
-    /// Session start time for relative timestamps
-    private var transcriptionSessionStart: Date
+    func setActive(_ active: Bool) {
+        worker.setActive(active)
+    }
 
     /// Dynamic model path support
     private static var customModelPath: String? {
@@ -65,23 +70,9 @@ class SherpaOnnxTranscriber {
         return nil
     }
 
-    /**
-     * Constructor that accepts a UIViewController to load model assets.
-     */
-    init(context: UIViewController) {
-        self.context = context
-        transcriptionSessionStart = Date()
-    }
-
-    deinit {
-        shutdown()
-    }
-
-    /**
-     * Initialize the Sherpa-ONNX recognizer.
-     * Loads models and configuration, sets up processing thread.
-     */
-    func initialize() {
+    /// Runs exclusively on the worker queue, never on the main/audio delivery thread.
+    private static func makeRecognizer() -> SherpaOnnxRecognizer? {
+        var recognizer: SherpaOnnxRecognizer?
         do {
             var tokensPath: String
             var modelType = "unknown"
@@ -201,30 +192,26 @@ class SherpaOnnxTranscriber {
                 Bridge.log("No Sherpa ONNX model available. Transcription will be disabled.")
                 Bridge.log("Please download a model using the model downloader in settings.")
                 recognizer = nil
-                isRunning = false
-                return
+                return nil
             }
 
             if recognizer == nil {
                 throw NSError(domain: "SherpaOnnxTranscriber", code: 2, userInfo: [NSLocalizedDescriptionKey: "Failed to create recognizer"])
             }
 
-            startProcessingTask()
-            isRunning = true
-
             Bridge.log("Sherpa-ONNX ASR initialized successfully with \(modelType) model")
+            return recognizer
 
         } catch {
             Bridge.log("Failed to initialize Sherpa-ONNX: \(error.localizedDescription)")
+            return nil
         }
     }
 
-    /**
-     * Handle transcription results - send only to delegate
-     */
-    private func handleTranscriptionResult(text: String, isFinal: Bool) {
-        // Forward to delegate if set
+    private func handleTranscriptionResult(text: String, isFinal: Bool, generation: UInt64) {
         DispatchQueue.main.async { [weak self] in
+            // A decode or main-queue delivery may finish after stop/model replacement.
+            guard let self, self.worker.isCurrent(generation) else { return }
             if isFinal {
                 STTTools.didReceiveFinalTranscription(text)
             } else {
@@ -233,123 +220,30 @@ class SherpaOnnxTranscriber {
         }
     }
 
-    /**
-     * Feed PCM audio data (16-bit little endian) into the transcriber.
-     * This method should be called continuously with short chunks (e.g., 100-300ms).
-     *
-     * Audio is queued directly; microphone VAD gating is not applied in the SDK.
-     */
     func acceptAudio(pcm16le: Data) {
-        guard isRunning else {
-            return
-        }
-
-        queueAudioData(pcm16le)
-    }
-
-    private func queueAudioData(_ pcm16le: Data) {
-        pcmQueue.async { [weak self] in
-            guard let self = self else { return }
-
-            let queueSizeBefore = self.pcmBuffers.count
-            self.pcmBuffers.append(pcm16le)
-
-            // Keep queue size manageable
-            if self.pcmBuffers.count > Self.QUEUE_CAPACITY {
-                let removedBuffer = self.pcmBuffers.removeFirst()
-                Bridge.log("⚠️ Audio queue overflow - dropped buffer of \(removedBuffer.count) bytes")
-            }
-        }
+        worker.accept(pcm16le)
     }
 
-    /**
-     * Start a background task to continuously consume audio and decode using Sherpa.
-     */
-    private func startProcessingTask() {
-        Bridge.log("🚀 Starting Sherpa-ONNX processing task...")
-
-        processingQueue = DispatchQueue(label: "com.augmentos.sherpaonnx.processor", qos: .userInitiated)
-
-        let workItem = DispatchWorkItem { [weak self] in
-            self?.runLoop()
+    private func process(_ session: Session, data: Data, generation: UInt64) {
+        guard worker.isCurrent(generation) else { return }
+        let recognizer = session.recognizer
+        recognizer.acceptWaveform(samples: toFloatArray(from: data), sampleRate: Self.SAMPLE_RATE)
+        while recognizer.isReady() {
+            guard worker.isCurrent(generation) else { return }
+            recognizer.decode()
         }
-
-        processingTask = workItem
-        processingQueue?.async(execute: workItem)
-    }
-
-    /**
-     * Main processing loop that handles transcription in real-time.
-     * Pulls audio from queue, feeds into Sherpa, emits partial/final results.
-     */
-    private func runLoop() {
-        Bridge.log("🔄 Sherpa-ONNX processing loop started")
-
-        while isRunning {
-            // Pull data from queue
-            var audioData: Data?
-
-            pcmQueue.sync {
-                if !self.pcmBuffers.isEmpty {
-                    audioData = self.pcmBuffers.removeFirst()
-                }
-            }
-
-            if let data = audioData {
-                // Synchronize access to recognizer to prevent race conditions
-                objc_sync_enter(self)
-                defer { objc_sync_exit(self) }
-
-                guard let recognizer = recognizer else {
-                    Bridge.log("⚠️ Recognizer not available, skipping audio chunk")
-                    continue
-                }
-
-                do {
-                    // Convert PCM to float [-1.0, 1.0]
-                    let floatBuf = toFloatArray(from: data)
-
-                    // Pass audio data to the Sherpa-ONNX stream
-                    recognizer.acceptWaveform(samples: floatBuf, sampleRate: Self.SAMPLE_RATE)
-
-                    // Decode continuously while model is ready
-                    var decodeCount = 0
-                    while recognizer.isReady() {
-                        recognizer.decode()
-                        decodeCount += 1
-                    }
-
-                    // If utterance endpoint detected
-                    if recognizer.isEndpoint() {
-                        let result = recognizer.getResult()
-                        let finalText = result.text.trimmingCharacters(in: .whitespacesAndNewlines)
-
-                        if !finalText.isEmpty {
-                            handleTranscriptionResult(text: finalText, isFinal: true)
-                        }
-
-                        recognizer.reset() // Start new utterance
-                        lastPartialResult = ""
-                    } else {
-                        // Emit partial results if changed
-                        let result = recognizer.getResult()
-                        let partial = result.text.trimmingCharacters(in: .whitespacesAndNewlines)
-
-                        if partial != lastPartialResult, !partial.isEmpty {
-                            handleTranscriptionResult(text: partial, isFinal: false)
-                            lastPartialResult = partial
-                        }
-                    }
-                } catch {
-                    Bridge.log("❌ Error processing audio: \(error.localizedDescription)")
-                }
-            } else {
-                // Sleep briefly to avoid tight CPU loop if no audio is available
-                Thread.sleep(forTimeInterval: 0.01)
+        guard worker.isCurrent(generation) else { return }
+        let text = recognizer.getResult().text.trimmingCharacters(in: .whitespacesAndNewlines)
+        if recognizer.isEndpoint() {
+            if !text.isEmpty {
+                handleTranscriptionResult(text: text, isFinal: true, generation: generation)
             }
+            recognizer.reset()
+            session.lastPartialResult = ""
+        } else if !text.isEmpty, text != session.lastPartialResult {
+            handleTranscriptionResult(text: text, isFinal: false, generation: generation)
+            session.lastPartialResult = text
         }
-
-        Bridge.log("ASR processing thread stopped")
     }
 
     /**
@@ -377,45 +271,11 @@ class SherpaOnnxTranscriber {
         return samples
     }
 
-    /**
-     * Stop transcription processing.
-     * This shuts down the processing thread and releases Sherpa-ONNX resources.
-     */
     func shutdown() {
-        Bridge.log("🛑 Shutting down SherpaOnnxTranscriber...")
-
-        isRunning = false
-        processingTask?.cancel()
-
-        // Synchronize access to recognizer during shutdown
-        objc_sync_enter(self)
-        defer { objc_sync_exit(self) }
-
-        // The recognizer will be automatically cleaned up by ARC when set to nil
-        if recognizer != nil {
-            Bridge.log("🧹 Cleaning up Sherpa-ONNX recognizer")
-            recognizer = nil
-        }
-
-        // Clear any remaining audio buffers
-        pcmQueue.sync {
-            let remainingBuffers = self.pcmBuffers.count
-            if remainingBuffers > 0 {
-                Bridge.log("🗑️ Clearing \(remainingBuffers) remaining audio buffers")
-            }
-            self.pcmBuffers.removeAll()
-        }
-
-        Bridge.log("✅ SherpaOnnxTranscriber shutdown complete")
+        worker.setActive(false)
     }
 
-    /**
-     * Restarts the transcriber after a model change.
-     * Shuts down existing resources, clears buffers, and reinitializes the recognizer.
-     */
     func restart() {
-        Bridge.log("♻️ Restarting SherpaOnnxTranscriber...")
-        shutdown()
-        initialize()
+        worker.restart()
     }
 }