@mentra/bluetooth-sdk 0.1.8 → 0.1.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (147) hide show
  1. package/README.md +116 -18
  2. package/android/build.gradle +289 -16
  3. package/android/gradle.properties.example +9 -0
  4. package/android/lc3Lib/build.gradle +95 -6
  5. package/android/settings.gradle +3 -0
  6. package/android/silero/LICENSE.md +17 -0
  7. package/android/silero/build.gradle +42 -0
  8. package/android/silero/proguard-rules.pro +24 -0
  9. package/android/silero/src/androidTest/assets/hello.wav +0 -0
  10. package/android/silero/src/androidTest/java/com/konovalov/vad/silero/VadSileroTest.kt +79 -0
  11. package/android/silero/src/main/assets/LICENSE +21 -0
  12. package/android/silero/src/main/assets/silero_vad.onnx +0 -0
  13. package/android/silero/src/main/java/com/konovalov/vad/silero/Vad.kt +160 -0
  14. package/android/silero/src/main/java/com/konovalov/vad/silero/VadSilero.kt +449 -0
  15. package/android/silero/src/main/java/com/konovalov/vad/silero/config/FrameSize.kt +15 -0
  16. package/android/silero/src/main/java/com/konovalov/vad/silero/config/Mode.kt +14 -0
  17. package/android/silero/src/main/java/com/konovalov/vad/silero/config/SampleRate.kt +12 -0
  18. package/android/silero/src/main/java/com/konovalov/vad/silero/utils/AudioUtils.kt +47 -0
  19. package/android/silero/src/main/java/com/konovalov/vad/silero/utils/TensorMap.kt +41 -0
  20. package/android/silero/src/test/java/com/konovalov/vad/silero/utils/AudioUtilsTest.kt +44 -0
  21. package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkDefaults.kt +6 -0
  22. package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkModule.kt +165 -27
  23. package/android/src/main/java/com/mentra/bluetoothsdk/Bridge.kt +89 -21
  24. package/android/src/main/java/com/mentra/bluetoothsdk/DeviceManager.kt +189 -16
  25. package/android/src/main/java/com/mentra/bluetoothsdk/DeviceStore.kt +22 -2
  26. package/android/src/main/java/com/mentra/bluetoothsdk/MentraBluetoothSdk.kt +787 -85
  27. package/android/src/main/java/com/mentra/bluetoothsdk/audio/AudioModels.kt +6 -2
  28. package/android/src/main/java/com/mentra/bluetoothsdk/camera/CameraModels.kt +123 -10
  29. package/android/src/main/java/com/mentra/bluetoothsdk/controllers/ControllerManager.kt +12 -2
  30. package/android/src/main/java/com/mentra/bluetoothsdk/controllers/R1.kt +8 -2
  31. package/android/src/main/java/com/mentra/bluetoothsdk/debug/BleTraceLogger.kt +33 -17
  32. package/android/src/main/java/com/mentra/bluetoothsdk/events/BluetoothEvents.kt +101 -0
  33. package/android/src/main/java/com/mentra/bluetoothsdk/internal/MapParsing.kt +8 -0
  34. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoReceiverRegistry.java +108 -0
  35. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoUploadServer.kt +399 -74
  36. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/MentraPhotoReceiverModule.kt +100 -17
  37. package/android/src/main/java/com/mentra/bluetoothsdk/requests/DisplayRequests.kt +17 -0
  38. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G1.java +2 -2
  39. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G2.kt +569 -220
  40. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Mach1.java +2 -2
  41. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraLive.java +271 -66
  42. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraNex.kt +176 -104
  43. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraosBle.java +12753 -27036
  44. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/SGCManager.kt +54 -2
  45. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Simulated.kt +16 -4
  46. package/android/src/main/java/com/mentra/bluetoothsdk/status/DeviceStatus.kt +60 -3
  47. package/android/src/main/java/com/mentra/bluetoothsdk/streaming/StreamModels.kt +12 -3
  48. package/android/src/main/java/com/mentra/bluetoothsdk/stt/STTTools.kt +87 -36
  49. package/android/src/main/java/com/mentra/bluetoothsdk/stt/SherpaOnnxTranscriber.kt +13 -4
  50. package/android/src/main/java/com/mentra/bluetoothsdk/stt/VadGateSpeechPolicy.kt +229 -0
  51. package/android/src/main/java/com/mentra/bluetoothsdk/tts/TTSTools.kt +180 -0
  52. package/android/src/main/java/com/mentra/bluetoothsdk/utils/AvifExifStripper.java +573 -0
  53. package/android/src/main/java/com/mentra/bluetoothsdk/utils/BlePhotoUploadService.java +488 -50
  54. package/android/src/main/java/com/mentra/bluetoothsdk/utils/HeifExifTagReader.java +199 -0
  55. package/android/src/main/java/com/mentra/bluetoothsdk/utils/MessageChunkReassembler.java +18 -1
  56. package/android/src/main/java/com/mentra/bluetoothsdk/utils/MessageChunker.java +64 -18
  57. package/android/src/main/java/com/mentra/bluetoothsdk/utils/NexSGCUtils.kt +48 -39
  58. package/android/src/test/java/com/mentra/bluetoothsdk/camera/PhotoRequestTest.kt +23 -0
  59. package/android/src/test/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoReceiverRegistryTest.java +75 -0
  60. package/android/src/test/java/com/mentra/bluetoothsdk/utils/AvifExifStripperTest.java +30 -0
  61. package/android/src/test/java/com/mentra/bluetoothsdk/utils/BlePhotoUploadServiceTest.java +98 -0
  62. package/android/src/test/resources/avif_with_exif.avif +0 -0
  63. package/build/BluetoothSdk.types.d.ts +241 -43
  64. package/build/BluetoothSdk.types.d.ts.map +1 -1
  65. package/build/BluetoothSdk.types.js +11 -10
  66. package/build/BluetoothSdk.types.js.map +1 -1
  67. package/build/_private/BluetoothSdkModule.d.ts +36 -23
  68. package/build/_private/BluetoothSdkModule.d.ts.map +1 -1
  69. package/build/_private/BluetoothSdkModule.js +30 -52
  70. package/build/_private/BluetoothSdkModule.js.map +1 -1
  71. package/build/_private/photoRequestPayload.d.ts +4 -0
  72. package/build/_private/photoRequestPayload.d.ts.map +1 -0
  73. package/build/_private/photoRequestPayload.js +25 -0
  74. package/build/_private/photoRequestPayload.js.map +1 -0
  75. package/build/index.d.ts +1 -1
  76. package/build/index.d.ts.map +1 -1
  77. package/build/index.js +23 -0
  78. package/build/index.js.map +1 -1
  79. package/build/react/useMentraBluetooth.d.ts +2 -2
  80. package/build/react/useMentraBluetooth.d.ts.map +1 -1
  81. package/build/react/useMentraBluetooth.js +3 -3
  82. package/build/react/useMentraBluetooth.js.map +1 -1
  83. package/ios/BluetoothSdkModule.swift +174 -70
  84. package/ios/LocalPhotoUploadServer.swift +370 -105
  85. package/ios/MentraBluetoothSDK.podspec +3 -1
  86. package/ios/MentraPhotoReceiverModule.swift +62 -10
  87. package/ios/Packages/CoreObjC/include/PcmConverter.h +22 -0
  88. package/ios/Packages/SherpaOnnx/SherpaOnnx.swift +668 -26
  89. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Info.plist +13 -9
  90. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/Headers/sherpa-onnx/c-api/c-api.h +4551 -0
  91. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/Headers/sherpa-onnx/c-api/cxx-api.h +1745 -0
  92. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/libsherpa-onnx.a +0 -0
  93. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/Headers/sherpa-onnx/c-api/c-api.h +4551 -0
  94. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/Headers/sherpa-onnx/c-api/cxx-api.h +1745 -0
  95. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/libsherpa-onnx.a +0 -0
  96. package/ios/Source/BluetoothSdkDefaults.swift +6 -0
  97. package/ios/Source/Bridge.swift +58 -19
  98. package/ios/Source/DeviceManager.swift +149 -17
  99. package/ios/Source/DeviceStore.swift +16 -6
  100. package/ios/Source/MentraBluetoothSDK.swift +803 -47
  101. package/ios/Source/ObservableStore.swift +8 -2
  102. package/ios/Source/{Audio → audio}/AudioModels.swift +9 -7
  103. package/ios/Source/{Camera → camera}/CameraModels.swift +225 -16
  104. package/ios/Source/controllers/ControllerManager.swift +4 -3
  105. package/ios/Source/controllers/R1.swift +3 -3
  106. package/ios/Source/{Events → events}/BluetoothEvents.swift +183 -1
  107. package/ios/Source/{Internal → internal}/ValueParsing.swift +10 -1
  108. package/ios/Source/{Requests → requests}/DisplayRequests.swift +28 -4
  109. package/ios/Source/services/PhoneMic.swift +1 -0
  110. package/ios/Source/sgcs/G1.swift +2 -2
  111. package/ios/Source/sgcs/G2.swift +645 -403
  112. package/ios/Source/sgcs/Mach1.swift +2 -2
  113. package/ios/Source/sgcs/MentraLive.swift +812 -319
  114. package/ios/Source/sgcs/MentraNex.swift +412 -182
  115. package/ios/Source/sgcs/SGCManager.swift +57 -3
  116. package/ios/Source/sgcs/Simulated.swift +7 -3
  117. package/ios/Source/sgcs/mentraos_ble.pb.swift +3073 -3134
  118. package/ios/Source/status/DeviceStatus.swift +974 -0
  119. package/ios/Source/{Status → status}/WifiHotspotStatus.swift +4 -4
  120. package/ios/Source/{Streaming → streaming}/StreamModels.swift +24 -9
  121. package/ios/Source/stt/STTTools.swift +27 -12
  122. package/ios/Source/stt/SherpaOnnxTranscriber.swift +31 -8
  123. package/ios/Source/tts/TTSTools.swift +164 -0
  124. package/ios/Source/utils/JSCExperiment.swift +7 -7
  125. package/ios/Source/utils/MessageChunkReassembler.swift +20 -1
  126. package/ios/Source/utils/MessageChunker.swift +78 -21
  127. package/ios/Source/utils/TarBz2Extractor.swift +53 -2
  128. package/package.json +1 -1
  129. package/plugin/build/withAndroid.js +68 -4
  130. package/src/BluetoothSdk.types.ts +331 -59
  131. package/src/_private/BluetoothSdkModule.ts +113 -102
  132. package/src/_private/photoRequestPayload.ts +28 -0
  133. package/src/index.ts +58 -15
  134. package/src/react/useMentraBluetooth.ts +6 -4
  135. package/android/.project +0 -28
  136. package/android/lc3Lib/.project +0 -28
  137. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/cargs.h +0 -162
  138. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/sherpa-onnx/c-api/c-api.h +0 -1852
  139. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/sherpa-onnx/c-api/cxx-api.h +0 -674
  140. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/sherpa-onnx.a +0 -0
  141. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/sherpa-onnx.a +0 -0
  142. package/ios/Source/Status/DeviceStatus.swift +0 -471
  143. /package/ios/Source/{Connection → connection}/ScanSession.swift +0 -0
  144. /package/ios/Source/{Errors → errors}/BluetoothError.swift +0 -0
  145. /package/ios/Source/{Internal → internal}/BluetoothAvailability.swift +0 -0
  146. /package/ios/Source/{Status → status}/RuntimeState.swift +0 -0
  147. /package/ios/Source/{Types → types}/DeviceModels.swift +0 -0
@@ -138,11 +138,11 @@ public struct WifiStatusEvent: CustomStringConvertible {
138
138
  }
139
139
 
140
140
  init(connected: Bool, ssid: String?, localIp: String?) {
141
- self.status = WifiStatus.fromStoreFields(connected: connected, ssid: ssid, localIp: localIp) ?? .disconnected
141
+ status = WifiStatus.fromStoreFields(connected: connected, ssid: ssid, localIp: localIp) ?? .disconnected
142
142
  }
143
143
 
144
144
  init(values: [String: Any]) {
145
- self.status = WifiStatus(values: values) ?? .disconnected
145
+ status = WifiStatus(values: values) ?? .disconnected
146
146
  }
147
147
 
148
148
  public var values: [String: Any] {
@@ -289,11 +289,11 @@ public struct HotspotStatusEvent: CustomStringConvertible {
289
289
  }
290
290
 
291
291
  init(enabled: Bool, ssid: String?, password: String?, localIp: String?) {
292
- self.status = HotspotStatus.fromStoreFields(enabled: enabled, ssid: ssid, password: password, localIp: localIp) ?? .disabled
292
+ status = HotspotStatus.fromStoreFields(enabled: enabled, ssid: ssid, password: password, localIp: localIp) ?? .disabled
293
293
  }
294
294
 
295
295
  init(values: [String: Any]) {
296
- self.status = HotspotStatus(values: values) ?? .disabled
296
+ status = HotspotStatus(values: values) ?? .disabled
297
297
  }
298
298
 
299
299
  public var values: [String: Any] {
@@ -76,13 +76,20 @@ public struct StreamAudioConfig {
76
76
  }
77
77
  }
78
78
 
79
+ /// Effective video settings reported by the glasses after defaults and clamps.
79
80
  public struct StreamResolvedVideoConfig: Equatable {
81
+ /// Encoded output width sent to the stream endpoint.
80
82
  public let width: Int
83
+ /// Encoded output height sent to the stream endpoint.
81
84
  public let height: Int
85
+ /// Native camera buffer width selected before crop/downscale.
82
86
  public let captureWidth: Int?
87
+ /// Native camera buffer height selected before crop/downscale.
83
88
  public let captureHeight: Int?
89
+ /// Encoded video bitrate in bits per second.
84
90
  public let bitrate: Int
85
- public let fps: Int
91
+ /// Resolved capture/encode frame rate.
92
+ public let fps: Double
86
93
 
87
94
  public init(
88
95
  width: Int,
@@ -90,7 +97,7 @@ public struct StreamResolvedVideoConfig: Equatable {
90
97
  captureWidth: Int? = nil,
91
98
  captureHeight: Int? = nil,
92
99
  bitrate: Int,
93
- fps: Int
100
+ fps: Double
94
101
  ) {
95
102
  self.width = width
96
103
  self.height = height
@@ -105,7 +112,7 @@ public struct StreamResolvedVideoConfig: Equatable {
105
112
  let width = intValue(values["width"]),
106
113
  let height = intValue(values["height"]),
107
114
  let bitrate = intValue(values["bitrate"]),
108
- let fps = intValue(values["fps"])
115
+ let fps = doubleValue(values["fps"])
109
116
  else {
110
117
  return nil
111
118
  }
@@ -382,11 +389,19 @@ public enum StreamStatus: CustomStringConvertible, Equatable {
382
389
  let resolvedConfig = StreamResolvedConfig(values: values["resolvedConfig"] as? [String: Any])
383
390
  let attempt = optionalIntValue(values, "attempt")
384
391
  let maxAttempts = optionalIntValue(values, "maxAttempts") ?? 0
392
+ let parsedState = StreamState.from(rawState)
385
393
 
386
394
  if hasAnyKey(values, "streaming") || hasAnyKey(values, "reconnecting") {
387
395
  let streaming = boolValue(values, "streaming") == true
388
396
  let reconnecting = boolValue(values, "reconnecting") == true
389
- let snapshotState: StreamState = reconnecting ? .reconnecting : (streaming ? .streaming : .stopped)
397
+ let snapshotState: StreamState
398
+ if reconnecting {
399
+ snapshotState = .reconnecting
400
+ } else if streaming {
401
+ snapshotState = .streaming
402
+ } else {
403
+ snapshotState = parsedState ?? .stopped
404
+ }
390
405
  self = .snapshot(
391
406
  state: snapshotState,
392
407
  streaming: streaming,
@@ -399,7 +414,7 @@ public enum StreamStatus: CustomStringConvertible, Equatable {
399
414
  return
400
415
  }
401
416
 
402
- guard let state = StreamState.from(rawState) else {
417
+ guard let state = parsedState else {
403
418
  self = .error(
404
419
  streamId: streamId,
405
420
  errorDetails: rawState.map { "Unknown stream status: \($0)" } ?? "Missing stream status",
@@ -569,7 +584,7 @@ public struct StreamStatusEvent: CustomStringConvertible {
569
584
  }
570
585
 
571
586
  public init(values: [String: Any]) {
572
- self.status = StreamStatus(values: values)
587
+ status = StreamStatus(values: values)
573
588
  }
574
589
 
575
590
  public var state: StreamState {
@@ -607,9 +622,9 @@ public struct KeepAliveAckEvent: CustomStringConvertible, Equatable {
607
622
  }
608
623
 
609
624
  public init(values: [String: Any]) {
610
- self.streamId = stringValue(values, "streamId") ?? ""
611
- self.ackId = stringValue(values, "ackId") ?? ""
612
- self.timestamp = intValue(values["timestamp"])
625
+ streamId = stringValue(values, "streamId") ?? ""
626
+ ackId = stringValue(values, "ackId") ?? ""
627
+ timestamp = intValue(values["timestamp"])
613
628
  }
614
629
 
615
630
  public var values: [String: Any] {
@@ -67,16 +67,18 @@ class STTTools {
67
67
  }
68
68
 
69
69
  // Check for CTC model
70
- let ctcModelPath = (modelPath as NSString).appendingPathComponent("model.int8.onnx")
71
- if fileManager.fileExists(atPath: ctcModelPath) {
70
+ if firstExistingFile(in: modelPath, candidates: ["model.int8.onnx", "model.onnx"]) != nil {
72
71
  return true
73
72
  }
74
73
 
75
74
  // Check for transducer model
76
- let transducerFiles = ["encoder.onnx", "decoder.onnx", "joiner.onnx"]
77
- for file in transducerFiles {
78
- let filePath = (modelPath as NSString).appendingPathComponent(file)
79
- if !fileManager.fileExists(atPath: filePath) {
75
+ let transducerFiles = [
76
+ ["encoder.onnx", "encoder.int8.onnx"],
77
+ ["decoder.onnx", "decoder.int8.onnx"],
78
+ ["joiner.onnx", "joiner.int8.onnx"],
79
+ ]
80
+ for candidates in transducerFiles {
81
+ if firstExistingFile(in: modelPath, candidates: candidates) == nil {
80
82
  return false
81
83
  }
82
84
  }
@@ -95,18 +97,20 @@ class STTTools {
95
97
  }
96
98
 
97
99
  // Check for CTC model
98
- let ctcModelPath = (path as NSString).appendingPathComponent("model.int8.onnx")
99
- if fileManager.fileExists(atPath: ctcModelPath) {
100
+ if firstExistingFile(in: path, candidates: ["model.int8.onnx", "model.onnx"]) != nil {
100
101
  return true
101
102
  }
102
103
 
103
104
  // Check for transducer model
104
- let transducerFiles = ["encoder.onnx", "decoder.onnx", "joiner.onnx"]
105
+ let transducerFiles = [
106
+ ["encoder.onnx", "encoder.int8.onnx"],
107
+ ["decoder.onnx", "decoder.int8.onnx"],
108
+ ["joiner.onnx", "joiner.int8.onnx"],
109
+ ]
105
110
  var allTransducerFilesPresent = true
106
111
 
107
- for file in transducerFiles {
108
- let filePath = (path as NSString).appendingPathComponent(file)
109
- if !fileManager.fileExists(atPath: filePath) {
112
+ for candidates in transducerFiles {
113
+ if firstExistingFile(in: path, candidates: candidates) == nil {
110
114
  allTransducerFilesPresent = false
111
115
  break
112
116
  }
@@ -151,4 +155,15 @@ class STTTools {
151
155
  }
152
156
  return true
153
157
  }
158
+
159
+ private static func firstExistingFile(in directory: String, candidates: [String]) -> String? {
160
+ let fileManager = FileManager.default
161
+ for candidate in candidates {
162
+ let path = (directory as NSString).appendingPathComponent(candidate)
163
+ if fileManager.fileExists(atPath: path) {
164
+ return path
165
+ }
166
+ }
167
+ return nil
168
+ }
154
169
  }
@@ -54,6 +54,17 @@ class SherpaOnnxTranscriber {
54
54
  return storedPath
55
55
  }
56
56
 
57
+ private static func firstExistingFile(in directory: String, candidates: [String]) -> String? {
58
+ let fileManager = FileManager.default
59
+ for candidate in candidates {
60
+ let path = (directory as NSString).appendingPathComponent(candidate)
61
+ if fileManager.fileExists(atPath: path) {
62
+ return path
63
+ }
64
+ }
65
+ return nil
66
+ }
67
+
57
68
  /**
58
69
  * Constructor that accepts a UIViewController to load model assets.
59
70
  */
@@ -79,8 +90,14 @@ class SherpaOnnxTranscriber {
79
90
  // Check if we have a custom model path set
80
91
  if let customPath = SherpaOnnxTranscriber.customModelPath {
81
92
  // Detect model type based on available files
82
- let ctcModelPath = (customPath as NSString).appendingPathComponent("model.int8.onnx")
83
- let transducerEncoderPath = (customPath as NSString).appendingPathComponent("encoder.onnx")
93
+ let ctcModelPath = Self.firstExistingFile(
94
+ in: customPath,
95
+ candidates: ["model.int8.onnx", "model.onnx"]
96
+ )
97
+ let transducerEncoderPath = Self.firstExistingFile(
98
+ in: customPath,
99
+ candidates: ["encoder.int8.onnx", "encoder.onnx"]
100
+ )
84
101
 
85
102
  tokensPath = (customPath as NSString).appendingPathComponent("tokens.txt")
86
103
 
@@ -91,7 +108,7 @@ class SherpaOnnxTranscriber {
91
108
  ])
92
109
  }
93
110
 
94
- if fileManager.fileExists(atPath: ctcModelPath) {
111
+ if let ctcModelPath {
95
112
  // CTC model detected
96
113
  modelType = "ctc"
97
114
  Bridge.log("Detected CTC model at \(customPath)")
@@ -123,17 +140,23 @@ class SherpaOnnxTranscriber {
123
140
  // Create recognizer with the wrapper
124
141
  recognizer = SherpaOnnxRecognizer(config: &config)
125
142
 
126
- } else if fileManager.fileExists(atPath: transducerEncoderPath) {
143
+ } else if let transducerEncoderPath {
127
144
  // Transducer model detected
128
145
  modelType = "transducer"
129
146
  Bridge.log("Detected transducer model at \(customPath)")
130
147
 
131
- let decoderPath = (customPath as NSString).appendingPathComponent("decoder.onnx")
132
- let joinerPath = (customPath as NSString).appendingPathComponent("joiner.onnx")
148
+ let decoderPath = Self.firstExistingFile(
149
+ in: customPath,
150
+ candidates: ["decoder.int8.onnx", "decoder.onnx"]
151
+ )
152
+ let joinerPath = Self.firstExistingFile(
153
+ in: customPath,
154
+ candidates: ["joiner.int8.onnx", "joiner.onnx"]
155
+ )
133
156
 
134
157
  // Verify all transducer files exist
135
- guard fileManager.fileExists(atPath: decoderPath),
136
- fileManager.fileExists(atPath: joinerPath)
158
+ guard let decoderPath,
159
+ let joinerPath
137
160
  else {
138
161
  throw NSError(domain: "SherpaOnnxTranscriber", code: 1, userInfo: [
139
162
  NSLocalizedDescriptionKey: "Transducer model files incomplete at path: \(customPath)",
@@ -0,0 +1,164 @@
1
+ import Foundation
2
+
3
+ /// Utilities for offline Sherpa-ONNX Supertonic 3 TTS.
4
+ ///
5
+ /// Supertonic 3 is a single multilingual model bundle (31 languages) with
6
+ /// 10 preset voice styles packed into voice.bin. Speaker IDs are indexed
7
+ /// alphabetically by JSON filename, so the order is F1, F2, F3, F4, F5,
8
+ /// M1, M2, M3, M4, M5 (sids 0..9). We default to F1 (sid 0) for a female
9
+ /// voice.
10
+ class TTSTools {
11
+ private static let kFileDurationPredictor = "duration_predictor.int8.onnx"
12
+ private static let kFileTextEncoder = "text_encoder.int8.onnx"
13
+ private static let kFileVectorEstimator = "vector_estimator.int8.onnx"
14
+ private static let kFileVocoder = "vocoder.int8.onnx"
15
+ private static let kFileTtsJson = "tts.json"
16
+ private static let kFileUnicodeIndexer = "unicode_indexer.bin"
17
+ private static let kFileVoiceStyle = "voice.bin"
18
+
19
+ private static let kRequiredFiles: [String] = [
20
+ kFileDurationPredictor,
21
+ kFileTextEncoder,
22
+ kFileVectorEstimator,
23
+ kFileVocoder,
24
+ kFileTtsJson,
25
+ kFileUnicodeIndexer,
26
+ kFileVoiceStyle,
27
+ ]
28
+
29
+ private static let kSupportedLangs: Set<String> = [
30
+ "en", "ko", "ja", "ar", "bg", "cs", "da", "de", "el", "es", "et",
31
+ "fi", "fr", "hi", "hr", "hu", "id", "it", "lt", "lv", "nl", "pl",
32
+ "pt", "ro", "ru", "sk", "sl", "sv", "tr", "uk", "vi",
33
+ ]
34
+
35
+ static func setTtsModelDetails(_ path: String, _ languageCode: String) {
36
+ UserDefaults.standard.set(path, forKey: "TTSModelPath")
37
+ UserDefaults.standard.set(languageCode, forKey: "TTSModelLanguageCode")
38
+ UserDefaults.standard.synchronize()
39
+ }
40
+
41
+ static func getTtsModelPath() -> String {
42
+ return UserDefaults.standard.string(forKey: "TTSModelPath") ?? ""
43
+ }
44
+
45
+ static func getTtsModelLanguage() -> String {
46
+ return UserDefaults.standard.string(forKey: "TTSModelLanguageCode") ?? "en-US"
47
+ }
48
+
49
+ static func checkTTSModelAvailable() -> Bool {
50
+ guard let modelPath = UserDefaults.standard.string(forKey: "TTSModelPath") else {
51
+ return false
52
+ }
53
+ return validateTTSModel(modelPath)
54
+ }
55
+
56
+ static func validateTTSModel(_ path: String) -> Bool {
57
+ let fileManager = FileManager.default
58
+ var isDirectory: ObjCBool = false
59
+ if !fileManager.fileExists(atPath: path, isDirectory: &isDirectory) || !isDirectory.boolValue {
60
+ Bridge.log("TTS model path does not exist or is not a directory: \(path)")
61
+ return false
62
+ }
63
+
64
+ for name in kRequiredFiles {
65
+ let filePath = (path as NSString).appendingPathComponent(name)
66
+ if !fileManager.fileExists(atPath: filePath) {
67
+ Bridge.log("TTS model missing required file '\(name)' at: \(path)")
68
+ return false
69
+ }
70
+ let size = (try? fileManager.attributesOfItem(atPath: filePath)[.size] as? UInt64) ?? 0
71
+ if size == 0 {
72
+ Bridge.log("TTS model file '\(name)' is empty at: \(path)")
73
+ return false
74
+ }
75
+ }
76
+
77
+ return true
78
+ }
79
+
80
+ static func generateTtsAudio(
81
+ text: String,
82
+ modelPath: String,
83
+ outputPath: String,
84
+ speakerId: Int,
85
+ speed: Double
86
+ ) -> Bool {
87
+ guard !text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
88
+ Bridge.log("TTS_ERROR: text is empty")
89
+ return false
90
+ }
91
+ guard validateTTSModel(modelPath) else {
92
+ Bridge.log("TTS_ERROR: model is invalid: \(modelPath)")
93
+ return false
94
+ }
95
+
96
+ do {
97
+ let outputURL = URL(fileURLWithPath: outputPath)
98
+ try FileManager.default.createDirectory(
99
+ at: outputURL.deletingLastPathComponent(),
100
+ withIntermediateDirectories: true,
101
+ attributes: nil
102
+ )
103
+
104
+ let supertonic = sherpaOnnxOfflineTtsSupertonicModelConfig(
105
+ durationPredictor: (modelPath as NSString).appendingPathComponent(kFileDurationPredictor),
106
+ textEncoder: (modelPath as NSString).appendingPathComponent(kFileTextEncoder),
107
+ vectorEstimator: (modelPath as NSString).appendingPathComponent(kFileVectorEstimator),
108
+ vocoder: (modelPath as NSString).appendingPathComponent(kFileVocoder),
109
+ ttsJson: (modelPath as NSString).appendingPathComponent(kFileTtsJson),
110
+ unicodeIndexer: (modelPath as NSString).appendingPathComponent(kFileUnicodeIndexer),
111
+ voiceStyle: (modelPath as NSString).appendingPathComponent(kFileVoiceStyle)
112
+ )
113
+ var modelConfig = sherpaOnnxOfflineTtsModelConfig(
114
+ numThreads: 2,
115
+ supertonic: supertonic
116
+ )
117
+ var config = sherpaOnnxOfflineTtsConfig(model: modelConfig, maxNumSentences: 1)
118
+ let tts = SherpaOnnxOfflineTtsWrapper(config: &config)
119
+
120
+ // The new wrapper dropped numSpeakers() in v1.13.2; call the C API
121
+ // directly. The C signature accepts the underlying OpaquePointer.
122
+ let numSpeakers = max(1, Int(SherpaOnnxOfflineTtsNumSpeakers(tts.tts)))
123
+ let sid = max(0, min(speakerId, numSpeakers - 1))
124
+ let clampedSpeed = Float(min(max(speed, 0.5), 2.0))
125
+ let lang = languageTagToSupertonicLang(getTtsModelLanguage())
126
+
127
+ // generate(...) no longer accepts `extra`; that field moved onto
128
+ // SherpaOnnxGenerationConfigSwift and is delivered via
129
+ // generateWithConfig(...).
130
+ //
131
+ // numSteps controls the Supertonic flow-matching vocoder's
132
+ // iteration count. The Swift wrapper defaults to 1, which produces
133
+ // unintelligible robot audio — sherpa-onnx's actual C-side default
134
+ // is 5. Pin it here so we don't inherit the wrapper's bug.
135
+ let genConfig = SherpaOnnxGenerationConfigSwift(
136
+ speed: clampedSpeed,
137
+ sid: sid,
138
+ numSteps: 5,
139
+ extra: ["lang": lang]
140
+ )
141
+ let audio = tts.generateWithConfig(
142
+ text: text,
143
+ config: genConfig,
144
+ callback: nil,
145
+ arg: nil
146
+ )
147
+ let saved = audio.save(filename: outputPath) == 1
148
+ Bridge.log(
149
+ "TTS generated \(outputPath): saved=\(saved) sid=\(sid) lang=\(lang) " +
150
+ "sr=\(audio.sampleRate) samples=\(audio.n) speed=\(clampedSpeed)"
151
+ )
152
+ return saved
153
+ } catch {
154
+ Bridge.log("TTS_ERROR: \(error.localizedDescription)")
155
+ return false
156
+ }
157
+ }
158
+
159
+ /// Convert a BCP-47 tag (e.g. "en-US", "fr-FR") to a Supertonic 2-letter code.
160
+ private static func languageTagToSupertonicLang(_ tag: String) -> String {
161
+ let primary = tag.split(separator: "-").first.map(String.init)?.lowercased() ?? "en"
162
+ return kSupportedLangs.contains(primary) ? primary : "en"
163
+ }
164
+ }
@@ -79,7 +79,7 @@ private func jlog(_ message: String) {
79
79
  let beforeMB = MemoryMonitor.currentMemoryMB()
80
80
  jlog("🧪 JSCExperiment.spawn(\(count)) starting; baseline \(String(format: "%.1f", beforeMB)) MB")
81
81
  var spawned = 0
82
- for i in 0..<count {
82
+ for i in 0 ..< count {
83
83
  let id = "spike-\(UUID().uuidString.prefix(8))-\(i)"
84
84
  if spawnOne(id: id) {
85
85
  spawned += 1
@@ -102,19 +102,19 @@ private func jlog(_ message: String) {
102
102
  ctx.name = "MentraJS: \(id)"
103
103
  // Inspectable in dev builds only. iOS 16.4+ guarded.
104
104
  #if DEBUG
105
- if #available(iOS 16.4, *) {
106
- ctx.isInspectable = true
107
- }
105
+ if #available(iOS 16.4, *) {
106
+ ctx.isInspectable = true
107
+ }
108
108
  #endif
109
109
 
110
110
  // Single-dispatcher bridge. Per Pebble's CrashReproducer doc, never
111
111
  // bind individual native callbacks as JSValue properties — JSC's GC
112
112
  // races with ARC and crashes. One C-callable function only.
113
- let dispatch: @convention(block) (String, String, [Any]?) -> Any? = { iface, method, args in
113
+ let dispatch: @convention(block) (String, String, [Any]?) -> Any? = { _, _, _ in
114
114
  // Stub: real impl routes to native services.
115
115
  // For the spike we just need __dispatch to exist so the
116
116
  // workload can call it without throwing.
117
- return NSNull()
117
+ NSNull()
118
118
  }
119
119
  ctx.setObject(dispatch, forKeyedSubscript: "__dispatch" as NSString)
120
120
 
@@ -147,7 +147,7 @@ private func jlog(_ message: String) {
147
147
  // setInterval that fires the JS callback. The JS code already
148
148
  // expects setInterval to exist as a global.
149
149
  let setInterval: @convention(block) (JSValue, Double) -> Int = { fn, ms in
150
- let id = Int.random(in: 1...Int.max)
150
+ let id = Int.random(in: 1 ... Int.max)
151
151
  // Stub: we won't actually fire in the spike, but the JS
152
152
  // callback existing is what we want for memory measurement.
153
153
  _ = fn // hold a ref to keep the JS callback alive
@@ -9,22 +9,41 @@ final class MessageChunkReassembler {
9
9
  func addChunk(_ info: MessageChunker.ChunkInfo) -> String? {
10
10
  cleanupTimedOutSessions()
11
11
 
12
+ guard !info.chunkId.isEmpty,
13
+ info.totalChunks > 0,
14
+ info.chunkIndex >= 0,
15
+ info.chunkIndex < info.totalChunks
16
+ else {
17
+ print("MessageChunkReassembler: Dropping invalid chunk metadata for \(info.chunkId)")
18
+ return nil
19
+ }
20
+
21
+ if let existing = activeSessions[info.chunkId], existing.totalChunks != info.totalChunks {
22
+ print(
23
+ "MessageChunkReassembler: totalChunks mismatch for \(info.chunkId) (expected \(existing.totalChunks), got \(info.totalChunks)); resetting session"
24
+ )
25
+ activeSessions.removeValue(forKey: info.chunkId)
26
+ }
27
+
12
28
  if activeSessions.count >= Self.maxConcurrentSessions,
13
29
  activeSessions[info.chunkId] == nil
14
30
  {
15
31
  removeOldestSession()
16
32
  }
17
33
 
34
+ let isNewSession = activeSessions[info.chunkId] == nil
18
35
  let session = activeSessions[info.chunkId] ?? ChunkSession(
19
36
  chunkId: info.chunkId,
20
37
  totalChunks: info.totalChunks
21
38
  )
22
- activeSessions[info.chunkId] = session
23
39
 
24
40
  guard session.addChunk(index: info.chunkIndex, data: info.data) else {
25
41
  print("MessageChunkReassembler: Failed to add chunk \(info.chunkIndex) for \(info.chunkId)")
26
42
  return nil
27
43
  }
44
+ if isNewSession {
45
+ activeSessions[info.chunkId] = session
46
+ }
28
47
 
29
48
  guard session.isComplete else {
30
49
  return nil
@@ -12,17 +12,18 @@ import Foundation
12
12
  * d = chunk data payload
13
13
  *
14
14
  * Each chunk after C-wrapping + K900 framing must fit within the BES2700's
15
- * 253-byte BLE write limit. With compact keys, 80 bytes of raw data produces
16
- * a final packed size of ~245 bytes worst-case (with heavy JSON escaping).
15
+ * 253-byte BLE write limit. Payload size is selected by measuring the final
16
+ * packed chunk so JSON escaping cannot push a chunk over the BLE limit.
17
17
  */
18
18
  class MessageChunker {
19
19
  // Threshold: if C-wrapped message exceeds this, chunking is triggered.
20
20
  // BES2700 limit is 253 bytes; anything over ~200 bytes packed needs chunking.
21
21
  private static let MESSAGE_SIZE_THRESHOLD = 200
22
22
 
23
- /// Maximum raw bytes per chunk. After double JSON escaping + compact envelope
24
- /// + C-wrapper + K900 framing, 80 bytes stays under the 253-byte BLE limit.
25
- private static let CHUNK_DATA_SIZE = 80
23
+ private static let INITIAL_CHUNK_DATA_SIZE = 80
24
+ private static let MIN_CHUNK_DATA_SIZE = 4
25
+ private static let MAX_PACKED_CHUNK_SIZE = 253
26
+ private static let K900_FRAME_OVERHEAD = 7
26
27
 
27
28
  /**
28
29
  * Check if a message needs to be chunked
@@ -51,34 +52,36 @@ class MessageChunker {
51
52
  * @param messageId The message ID for ACK tracking (if applicable)
52
53
  * @return Array of chunk dictionaries ready to be C-wrapped and sent
53
54
  */
54
- static func createChunks(originalJson: String, messageId: Int64 = -1) -> [[String: Any]] {
55
+ static func createChunks(originalJson: String, messageId: Int64 = -1, wakeUp: Bool = false) -> [[String: Any]] {
55
56
  guard let messageData = originalJson.data(using: .utf8) else {
56
57
  print("MessageChunker: Failed to convert message to data")
57
58
  return []
58
59
  }
59
60
 
60
- var chunks: [[String: Any]] = []
61
61
  let totalBytes = messageData.count
62
62
 
63
63
  // Compact chunk session ID: messageId_timestamp (no "chunk_" prefix)
64
64
  let chunkId = "\(messageId)_\(Int(Date().timeIntervalSince1970 * 1000))"
65
65
 
66
- // Calculate total chunks needed
67
- let totalChunks = Int(ceil(Double(totalBytes) / Double(CHUNK_DATA_SIZE)))
66
+ for chunkSize in stride(from: INITIAL_CHUNK_DATA_SIZE, through: MIN_CHUNK_DATA_SIZE, by: -1) {
67
+ let chunks = buildChunks(messageData, chunkId: chunkId, messageId: messageId, chunkSize: chunkSize)
68
+ if allChunksFit(chunks, wakeUp: wakeUp) {
69
+ print("MessageChunker: Creating \(chunks.count) chunks for message of size \(totalBytes) bytes using \(chunkSize)-byte UTF-8 slices")
70
+ return chunks
71
+ }
72
+ }
73
+
74
+ print("MessageChunker: Unable to create K900 chunks within \(MAX_PACKED_CHUNK_SIZE) bytes")
75
+ return []
76
+ }
68
77
 
69
- print("MessageChunker: Creating \(totalChunks) chunks for message of size \(totalBytes) bytes")
78
+ private static func buildChunks(_ messageData: Data, chunkId: String, messageId: Int64, chunkSize: Int) -> [[String: Any]] {
79
+ var chunks: [[String: Any]] = []
80
+ let chunkStrings = splitUtf8(messageData, chunkSize: chunkSize)
81
+ let totalChunks = chunkStrings.count
70
82
 
71
83
  for i in 0 ..< totalChunks {
72
- let startIndex = i * CHUNK_DATA_SIZE
73
- let endIndex = min(startIndex + CHUNK_DATA_SIZE, totalBytes)
74
- let chunkRange = startIndex ..< endIndex
75
-
76
- // Extract chunk data as string
77
- let chunkData = messageData.subdata(in: chunkRange)
78
- guard let chunkString = String(data: chunkData, encoding: .utf8) else {
79
- print("MessageChunker: Failed to convert chunk \(i) to string")
80
- continue
81
- }
84
+ let chunkString = chunkStrings[i]
82
85
 
83
86
  // Create chunk dictionary with compact keys
84
87
  var chunk: [String: Any] = [
@@ -96,12 +99,66 @@ class MessageChunker {
96
99
 
97
100
  chunks.append(chunk)
98
101
 
99
- print("MessageChunker: Created chunk \(i)/\(totalChunks - 1) with \(chunkData.count) bytes")
102
+ print("MessageChunker: Created chunk \(i)/\(totalChunks - 1) with \(chunkString.data(using: .utf8)?.count ?? 0) bytes")
100
103
  }
101
104
 
102
105
  return chunks
103
106
  }
104
107
 
108
+ private static func allChunksFit(_ chunks: [[String: Any]], wakeUp: Bool) -> Bool {
109
+ for (index, chunk) in chunks.enumerated() {
110
+ let packedLength = packedK900Length(chunk, wakeUp: wakeUp && index == 0)
111
+ if packedLength == nil || packedLength! > MAX_PACKED_CHUNK_SIZE {
112
+ print("MessageChunker: Chunk \(index) packed to \(packedLength ?? 0) bytes, exceeding \(MAX_PACKED_CHUNK_SIZE)")
113
+ return false
114
+ }
115
+ }
116
+ return true
117
+ }
118
+
119
+ private static func packedK900Length(_ chunk: [String: Any], wakeUp: Bool) -> Int? {
120
+ guard let chunkData = try? JSONSerialization.data(withJSONObject: chunk),
121
+ let chunkString = String(data: chunkData, encoding: .utf8)
122
+ else {
123
+ return nil
124
+ }
125
+
126
+ var wrapper: [String: Any] = ["C": chunkString]
127
+ if wakeUp {
128
+ wrapper["W"] = 1
129
+ }
130
+
131
+ guard let wrappedData = try? JSONSerialization.data(withJSONObject: wrapper) else {
132
+ return nil
133
+ }
134
+
135
+ return wrappedData.count + K900_FRAME_OVERHEAD
136
+ }
137
+
138
+ private static func splitUtf8(_ messageData: Data, chunkSize: Int) -> [String] {
139
+ var chunkStrings: [String] = []
140
+ var offset = 0
141
+ while offset < messageData.count {
142
+ let endIndex = findUtf8ChunkEnd(messageData, startIndex: offset, chunkSize: chunkSize)
143
+ let chunkData = messageData.subdata(in: offset ..< endIndex)
144
+ chunkStrings.append(String(data: chunkData, encoding: .utf8)!)
145
+ offset = endIndex
146
+ }
147
+ return chunkStrings
148
+ }
149
+
150
+ private static func findUtf8ChunkEnd(_ messageData: Data, startIndex: Int, chunkSize: Int) -> Int {
151
+ var endIndex = min(startIndex + chunkSize, messageData.count)
152
+ while endIndex > startIndex, endIndex < messageData.count, isUtf8ContinuationByte(messageData[endIndex]) {
153
+ endIndex -= 1
154
+ }
155
+ return endIndex > startIndex ? endIndex : min(startIndex + chunkSize, messageData.count)
156
+ }
157
+
158
+ private static func isUtf8ContinuationByte(_ value: UInt8) -> Bool {
159
+ (value & 0xC0) == 0x80
160
+ }
161
+
105
162
  /**
106
163
  * Check if a received message is a chunked message.
107
164
  * Supports both verbose ("type":"chunked_msg") and compact ("t":"ck") formats.