@mentra/bluetooth-sdk 0.1.9 → 0.1.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (147) hide show
  1. package/README.md +116 -18
  2. package/android/build.gradle +289 -16
  3. package/android/gradle.properties.example +9 -0
  4. package/android/lc3Lib/build.gradle +95 -6
  5. package/android/settings.gradle +3 -0
  6. package/android/silero/LICENSE.md +17 -0
  7. package/android/silero/build.gradle +42 -0
  8. package/android/silero/proguard-rules.pro +24 -0
  9. package/android/silero/src/androidTest/assets/hello.wav +0 -0
  10. package/android/silero/src/androidTest/java/com/konovalov/vad/silero/VadSileroTest.kt +79 -0
  11. package/android/silero/src/main/assets/LICENSE +21 -0
  12. package/android/silero/src/main/assets/silero_vad.onnx +0 -0
  13. package/android/silero/src/main/java/com/konovalov/vad/silero/Vad.kt +160 -0
  14. package/android/silero/src/main/java/com/konovalov/vad/silero/VadSilero.kt +449 -0
  15. package/android/silero/src/main/java/com/konovalov/vad/silero/config/FrameSize.kt +15 -0
  16. package/android/silero/src/main/java/com/konovalov/vad/silero/config/Mode.kt +14 -0
  17. package/android/silero/src/main/java/com/konovalov/vad/silero/config/SampleRate.kt +12 -0
  18. package/android/silero/src/main/java/com/konovalov/vad/silero/utils/AudioUtils.kt +47 -0
  19. package/android/silero/src/main/java/com/konovalov/vad/silero/utils/TensorMap.kt +41 -0
  20. package/android/silero/src/test/java/com/konovalov/vad/silero/utils/AudioUtilsTest.kt +44 -0
  21. package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkDefaults.kt +6 -0
  22. package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkModule.kt +195 -34
  23. package/android/src/main/java/com/mentra/bluetoothsdk/Bridge.kt +103 -21
  24. package/android/src/main/java/com/mentra/bluetoothsdk/DeviceManager.kt +210 -18
  25. package/android/src/main/java/com/mentra/bluetoothsdk/DeviceStore.kt +30 -2
  26. package/android/src/main/java/com/mentra/bluetoothsdk/MentraBluetoothSdk.kt +787 -85
  27. package/android/src/main/java/com/mentra/bluetoothsdk/audio/AudioModels.kt +6 -2
  28. package/android/src/main/java/com/mentra/bluetoothsdk/camera/CameraModels.kt +125 -11
  29. package/android/src/main/java/com/mentra/bluetoothsdk/controllers/ControllerManager.kt +12 -2
  30. package/android/src/main/java/com/mentra/bluetoothsdk/controllers/R1.kt +8 -2
  31. package/android/src/main/java/com/mentra/bluetoothsdk/debug/BleTraceLogger.kt +33 -17
  32. package/android/src/main/java/com/mentra/bluetoothsdk/events/BluetoothEvents.kt +101 -0
  33. package/android/src/main/java/com/mentra/bluetoothsdk/internal/MapParsing.kt +8 -0
  34. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoReceiverRegistry.java +108 -0
  35. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoUploadServer.kt +399 -74
  36. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/MentraPhotoReceiverModule.kt +100 -17
  37. package/android/src/main/java/com/mentra/bluetoothsdk/requests/DisplayRequests.kt +17 -0
  38. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G1.java +2 -2
  39. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G2.kt +1293 -758
  40. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Mach1.java +2 -2
  41. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraLive.java +271 -66
  42. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraNex.kt +197 -149
  43. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraosBle.java +12753 -27036
  44. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/SGCManager.kt +64 -2
  45. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Simulated.kt +16 -4
  46. package/android/src/main/java/com/mentra/bluetoothsdk/status/DeviceStatus.kt +60 -3
  47. package/android/src/main/java/com/mentra/bluetoothsdk/streaming/StreamModels.kt +12 -3
  48. package/android/src/main/java/com/mentra/bluetoothsdk/stt/STTTools.kt +87 -36
  49. package/android/src/main/java/com/mentra/bluetoothsdk/stt/SherpaOnnxTranscriber.kt +13 -4
  50. package/android/src/main/java/com/mentra/bluetoothsdk/stt/VadGateSpeechPolicy.kt +229 -0
  51. package/android/src/main/java/com/mentra/bluetoothsdk/tts/TTSTools.kt +180 -0
  52. package/android/src/main/java/com/mentra/bluetoothsdk/utils/AvifExifStripper.java +573 -0
  53. package/android/src/main/java/com/mentra/bluetoothsdk/utils/BlePhotoUploadService.java +488 -50
  54. package/android/src/main/java/com/mentra/bluetoothsdk/utils/HeifExifTagReader.java +199 -0
  55. package/android/src/main/java/com/mentra/bluetoothsdk/utils/MessageChunkReassembler.java +18 -1
  56. package/android/src/main/java/com/mentra/bluetoothsdk/utils/MessageChunker.java +64 -18
  57. package/android/src/main/java/com/mentra/bluetoothsdk/utils/NexSGCUtils.kt +48 -39
  58. package/android/src/test/java/com/mentra/bluetoothsdk/camera/PhotoRequestTest.kt +23 -0
  59. package/android/src/test/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoReceiverRegistryTest.java +75 -0
  60. package/android/src/test/java/com/mentra/bluetoothsdk/utils/AvifExifStripperTest.java +30 -0
  61. package/android/src/test/java/com/mentra/bluetoothsdk/utils/BlePhotoUploadServiceTest.java +98 -0
  62. package/android/src/test/resources/avif_with_exif.avif +0 -0
  63. package/build/BluetoothSdk.types.d.ts +253 -44
  64. package/build/BluetoothSdk.types.d.ts.map +1 -1
  65. package/build/BluetoothSdk.types.js +11 -10
  66. package/build/BluetoothSdk.types.js.map +1 -1
  67. package/build/_private/BluetoothSdkModule.d.ts +37 -23
  68. package/build/_private/BluetoothSdkModule.d.ts.map +1 -1
  69. package/build/_private/BluetoothSdkModule.js +43 -52
  70. package/build/_private/BluetoothSdkModule.js.map +1 -1
  71. package/build/_private/photoRequestPayload.d.ts +4 -0
  72. package/build/_private/photoRequestPayload.d.ts.map +1 -0
  73. package/build/_private/photoRequestPayload.js +25 -0
  74. package/build/_private/photoRequestPayload.js.map +1 -0
  75. package/build/index.d.ts +1 -1
  76. package/build/index.d.ts.map +1 -1
  77. package/build/index.js +25 -0
  78. package/build/index.js.map +1 -1
  79. package/build/react/useMentraBluetooth.d.ts +2 -2
  80. package/build/react/useMentraBluetooth.d.ts.map +1 -1
  81. package/build/react/useMentraBluetooth.js +3 -3
  82. package/build/react/useMentraBluetooth.js.map +1 -1
  83. package/ios/BluetoothSdkModule.swift +177 -70
  84. package/ios/LocalPhotoUploadServer.swift +370 -105
  85. package/ios/MentraBluetoothSDK.podspec +3 -1
  86. package/ios/MentraPhotoReceiverModule.swift +62 -10
  87. package/ios/Packages/CoreObjC/include/PcmConverter.h +22 -0
  88. package/ios/Packages/SherpaOnnx/SherpaOnnx.swift +668 -26
  89. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Info.plist +13 -9
  90. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/Headers/sherpa-onnx/c-api/c-api.h +4551 -0
  91. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/Headers/sherpa-onnx/c-api/cxx-api.h +1745 -0
  92. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/libsherpa-onnx.a +0 -0
  93. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/Headers/sherpa-onnx/c-api/c-api.h +4551 -0
  94. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/Headers/sherpa-onnx/c-api/cxx-api.h +1745 -0
  95. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/libsherpa-onnx.a +0 -0
  96. package/ios/Source/BluetoothSdkDefaults.swift +6 -0
  97. package/ios/Source/Bridge.swift +68 -19
  98. package/ios/Source/DeviceManager.swift +167 -32
  99. package/ios/Source/DeviceStore.swift +24 -8
  100. package/ios/Source/MentraBluetoothSDK.swift +803 -47
  101. package/ios/Source/ObservableStore.swift +6 -0
  102. package/ios/Source/{Audio → audio}/AudioModels.swift +9 -7
  103. package/ios/Source/{Camera → camera}/CameraModels.swift +226 -16
  104. package/ios/Source/controllers/ControllerManager.swift +4 -3
  105. package/ios/Source/controllers/R1.swift +3 -3
  106. package/ios/Source/{Events → events}/BluetoothEvents.swift +183 -1
  107. package/ios/Source/{Internal → internal}/ValueParsing.swift +10 -1
  108. package/ios/Source/{Requests → requests}/DisplayRequests.swift +28 -4
  109. package/ios/Source/services/PhoneMic.swift +1 -0
  110. package/ios/Source/sgcs/G1.swift +5 -5
  111. package/ios/Source/sgcs/G2.swift +1494 -870
  112. package/ios/Source/sgcs/Mach1.swift +4 -4
  113. package/ios/Source/sgcs/MentraLive.swift +814 -321
  114. package/ios/Source/sgcs/MentraNex.swift +604 -244
  115. package/ios/Source/sgcs/SGCManager.swift +68 -5
  116. package/ios/Source/sgcs/Simulated.swift +9 -5
  117. package/ios/Source/sgcs/mentraos_ble.pb.swift +3073 -3134
  118. package/ios/Source/status/DeviceStatus.swift +974 -0
  119. package/ios/Source/{Status → status}/WifiHotspotStatus.swift +4 -4
  120. package/ios/Source/{Streaming → streaming}/StreamModels.swift +24 -9
  121. package/ios/Source/stt/STTTools.swift +27 -12
  122. package/ios/Source/stt/SherpaOnnxTranscriber.swift +31 -8
  123. package/ios/Source/tts/TTSTools.swift +164 -0
  124. package/ios/Source/utils/JSCExperiment.swift +7 -7
  125. package/ios/Source/utils/MessageChunkReassembler.swift +20 -1
  126. package/ios/Source/utils/MessageChunker.swift +78 -21
  127. package/ios/Source/utils/TarBz2Extractor.swift +53 -2
  128. package/package.json +1 -1
  129. package/plugin/build/withAndroid.js +68 -4
  130. package/src/BluetoothSdk.types.ts +344 -60
  131. package/src/_private/BluetoothSdkModule.ts +132 -102
  132. package/src/_private/photoRequestPayload.ts +28 -0
  133. package/src/index.ts +61 -15
  134. package/src/react/useMentraBluetooth.ts +6 -4
  135. package/android/.project +0 -28
  136. package/android/lc3Lib/.project +0 -28
  137. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/cargs.h +0 -162
  138. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/sherpa-onnx/c-api/c-api.h +0 -1852
  139. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/sherpa-onnx/c-api/cxx-api.h +0 -674
  140. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/sherpa-onnx.a +0 -0
  141. package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/sherpa-onnx.a +0 -0
  142. package/ios/Source/Status/DeviceStatus.swift +0 -471
  143. /package/ios/Source/{Connection → connection}/ScanSession.swift +0 -0
  144. /package/ios/Source/{Errors → errors}/BluetoothError.swift +0 -0
  145. /package/ios/Source/{Internal → internal}/BluetoothAvailability.swift +0 -0
  146. /package/ios/Source/{Status → status}/RuntimeState.swift +0 -0
  147. /package/ios/Source/{Types → types}/DeviceModels.swift +0 -0
@@ -3,6 +3,7 @@ import ExpoModulesCore
3
3
  import Foundation
4
4
 
5
5
  public class MentraPhotoReceiverModule: Module {
6
+ private let receiverLock = NSLock()
6
7
  private var photoUploadServer: LocalPhotoUploadServer?
7
8
 
8
9
  public func definition() -> ModuleDefinition {
@@ -28,6 +29,9 @@ public class MentraPhotoReceiverModule: Module {
28
29
  }
29
30
 
30
31
  private func startPhotoReceiver() throws -> [String: Any] {
32
+ receiverLock.lock()
33
+ defer { receiverLock.unlock() }
34
+
31
35
  guard let host = bestLocalIPv4Address() else {
32
36
  throw PhotoReceiverError("No Wi-Fi/LAN IPv4 address found for this phone.")
33
37
  }
@@ -42,17 +46,19 @@ public class MentraPhotoReceiverModule: Module {
42
46
  )
43
47
  photoUploadServer = server
44
48
 
49
+ if let activePort = server.activePort {
50
+ let uploadUrl = "http://\(host):\(activePort)/upload"
51
+ emitStatus(message: "Photo receiver ready at \(uploadUrl)")
52
+ return receiverResult(uploadUrl: uploadUrl, host: host, port: activePort)
53
+ }
54
+
45
55
  var lastError: Error?
46
56
  for port in photoPorts {
47
57
  do {
48
58
  let actualPort = try server.start(port: UInt16(port))
49
59
  let uploadUrl = "http://\(host):\(actualPort)/upload"
50
60
  emitStatus(message: "Photo receiver ready at \(uploadUrl)")
51
- return [
52
- "uploadUrl": uploadUrl,
53
- "host": host,
54
- "port": actualPort,
55
- ]
61
+ return receiverResult(uploadUrl: uploadUrl, host: host, port: actualPort)
56
62
  } catch {
57
63
  lastError = error
58
64
  emitStatus(message: "Port \(port) unavailable: \(error.localizedDescription)")
@@ -65,6 +71,9 @@ public class MentraPhotoReceiverModule: Module {
65
71
  }
66
72
 
67
73
  private func stopPhotoReceiverInternal() {
74
+ receiverLock.lock()
75
+ defer { receiverLock.unlock() }
76
+
68
77
  photoUploadServer?.stop()
69
78
  emitStatus(message: "Photo receiver stopped")
70
79
  }
@@ -79,6 +88,7 @@ public class MentraPhotoReceiverModule: Module {
79
88
  }
80
89
 
81
90
  private func emitStatus(message: String) {
91
+ NSLog("[MentraPhotoReceiver] %@", message)
82
92
  sendEvent("receiverStatus", [
83
93
  "message": message,
84
94
  ])
@@ -91,6 +101,9 @@ public class MentraPhotoReceiverModule: Module {
91
101
  }
92
102
  defer { freeifaddrs(interfaces) }
93
103
 
104
+ var preferredPrivate: String?
105
+ var preferred: String?
106
+ var privateFallback: String?
94
107
  var fallback: String?
95
108
  var cursor: UnsafeMutablePointer<ifaddrs>? = first
96
109
  while let current = cursor {
@@ -102,6 +115,12 @@ public class MentraPhotoReceiverModule: Module {
102
115
  }
103
116
 
104
117
  let name = String(cString: interface.ifa_name)
118
+ let flags = interface.ifa_flags
119
+ guard (flags & UInt32(IFF_UP)) != 0,
120
+ (flags & UInt32(IFF_LOOPBACK)) == 0 else {
121
+ continue
122
+ }
123
+
105
124
  var address = addressPointer.pointee
106
125
  var hostname = [CChar](repeating: 0, count: Int(NI_MAXHOST))
107
126
  let result = getnameinfo(
@@ -118,16 +137,49 @@ public class MentraPhotoReceiverModule: Module {
118
137
  }
119
138
 
120
139
  let ip = String(cString: hostname)
121
- guard ip != "127.0.0.1" else {
140
+ guard ip != "127.0.0.1", !isLinkLocalIPv4(ip) else {
122
141
  continue
123
142
  }
124
- if name == "en0" {
125
- return ip
143
+ if isPreferredLocalInterface(name), isPrivateIPv4(ip) {
144
+ preferredPrivate = preferredPrivate ?? ip
145
+ } else if isPreferredLocalInterface(name) {
146
+ preferred = preferred ?? ip
147
+ } else if isPrivateIPv4(ip) {
148
+ privateFallback = privateFallback ?? ip
149
+ } else {
150
+ fallback = fallback ?? ip
126
151
  }
127
- fallback = fallback ?? ip
128
152
  }
129
153
 
130
- return fallback
154
+ return preferredPrivate ?? preferred ?? privateFallback ?? fallback
155
+ }
156
+
157
+ private func receiverResult(uploadUrl: String, host: String, port: UInt16) -> [String: Any] {
158
+ [
159
+ "uploadUrl": uploadUrl,
160
+ "host": host,
161
+ "port": port,
162
+ ]
163
+ }
164
+
165
+ private func isPreferredLocalInterface(_ name: String) -> Bool {
166
+ name == "en0" ||
167
+ name.hasPrefix("bridge") ||
168
+ name.hasPrefix("ap") ||
169
+ name.hasPrefix("awdl") ||
170
+ name.hasPrefix("llw")
171
+ }
172
+
173
+ private func isPrivateIPv4(_ ip: String) -> Bool {
174
+ if ip.hasPrefix("192.168.") || ip.hasPrefix("10.") {
175
+ return true
176
+ }
177
+ let parts = ip.split(separator: ".").compactMap { Int($0) }
178
+ return parts.count == 4 && parts[0] == 172 && (16...31).contains(parts[1])
179
+ }
180
+
181
+ private func isLinkLocalIPv4(_ ip: String) -> Bool {
182
+ ip.hasPrefix("169.254.")
131
183
  }
132
184
 
133
185
  private let photoPorts = [8787, 8788, 8789, 8790]
@@ -0,0 +1,22 @@
1
+ //
2
+ // PcmConverter.h
3
+ // Runner
4
+ //
5
+ // Created by Hawk on 2024/3/14.
6
+ //
7
+
8
+ #import <Foundation/Foundation.h>
9
+
10
+ NS_ASSUME_NONNULL_BEGIN
11
+
12
+ @interface PcmConverter : NSObject
13
+ + (void)setupStaticEncoderAndDecoder;
14
+ - (NSMutableData *)decode:(NSData *)lc3data frameSize:(NSInteger)frameSize;
15
+ - (NSMutableData *)encode:(NSData *)pcmdata frameSize:(NSInteger)frameSize;
16
+ - (void)setOutputFrameSize:(NSInteger)frameSize;
17
+ - (NSInteger)getOutputFrameSize;
18
+ - (void)resetEncoder;
19
+ - (void)resetDecoder;
20
+ @end
21
+
22
+ NS_ASSUME_NONNULL_END