@mentra/bluetooth-sdk 0.1.9 → 0.1.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +116 -18
- package/android/build.gradle +289 -16
- package/android/gradle.properties.example +9 -0
- package/android/lc3Lib/build.gradle +95 -6
- package/android/settings.gradle +3 -0
- package/android/silero/LICENSE.md +17 -0
- package/android/silero/build.gradle +42 -0
- package/android/silero/proguard-rules.pro +24 -0
- package/android/silero/src/androidTest/assets/hello.wav +0 -0
- package/android/silero/src/androidTest/java/com/konovalov/vad/silero/VadSileroTest.kt +79 -0
- package/android/silero/src/main/assets/LICENSE +21 -0
- package/android/silero/src/main/assets/silero_vad.onnx +0 -0
- package/android/silero/src/main/java/com/konovalov/vad/silero/Vad.kt +160 -0
- package/android/silero/src/main/java/com/konovalov/vad/silero/VadSilero.kt +449 -0
- package/android/silero/src/main/java/com/konovalov/vad/silero/config/FrameSize.kt +15 -0
- package/android/silero/src/main/java/com/konovalov/vad/silero/config/Mode.kt +14 -0
- package/android/silero/src/main/java/com/konovalov/vad/silero/config/SampleRate.kt +12 -0
- package/android/silero/src/main/java/com/konovalov/vad/silero/utils/AudioUtils.kt +47 -0
- package/android/silero/src/main/java/com/konovalov/vad/silero/utils/TensorMap.kt +41 -0
- package/android/silero/src/test/java/com/konovalov/vad/silero/utils/AudioUtilsTest.kt +44 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkDefaults.kt +6 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkModule.kt +195 -34
- package/android/src/main/java/com/mentra/bluetoothsdk/Bridge.kt +103 -21
- package/android/src/main/java/com/mentra/bluetoothsdk/DeviceManager.kt +210 -18
- package/android/src/main/java/com/mentra/bluetoothsdk/DeviceStore.kt +30 -2
- package/android/src/main/java/com/mentra/bluetoothsdk/MentraBluetoothSdk.kt +787 -85
- package/android/src/main/java/com/mentra/bluetoothsdk/audio/AudioModels.kt +6 -2
- package/android/src/main/java/com/mentra/bluetoothsdk/camera/CameraModels.kt +125 -11
- package/android/src/main/java/com/mentra/bluetoothsdk/controllers/ControllerManager.kt +12 -2
- package/android/src/main/java/com/mentra/bluetoothsdk/controllers/R1.kt +8 -2
- package/android/src/main/java/com/mentra/bluetoothsdk/debug/BleTraceLogger.kt +33 -17
- package/android/src/main/java/com/mentra/bluetoothsdk/events/BluetoothEvents.kt +101 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/internal/MapParsing.kt +8 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoReceiverRegistry.java +108 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoUploadServer.kt +399 -74
- package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/MentraPhotoReceiverModule.kt +100 -17
- package/android/src/main/java/com/mentra/bluetoothsdk/requests/DisplayRequests.kt +17 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G1.java +2 -2
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G2.kt +1293 -758
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Mach1.java +2 -2
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraLive.java +271 -66
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraNex.kt +197 -149
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraosBle.java +12753 -27036
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/SGCManager.kt +64 -2
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Simulated.kt +16 -4
- package/android/src/main/java/com/mentra/bluetoothsdk/status/DeviceStatus.kt +60 -3
- package/android/src/main/java/com/mentra/bluetoothsdk/streaming/StreamModels.kt +12 -3
- package/android/src/main/java/com/mentra/bluetoothsdk/stt/STTTools.kt +87 -36
- package/android/src/main/java/com/mentra/bluetoothsdk/stt/SherpaOnnxTranscriber.kt +13 -4
- package/android/src/main/java/com/mentra/bluetoothsdk/stt/VadGateSpeechPolicy.kt +229 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/tts/TTSTools.kt +180 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/utils/AvifExifStripper.java +573 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/utils/BlePhotoUploadService.java +488 -50
- package/android/src/main/java/com/mentra/bluetoothsdk/utils/HeifExifTagReader.java +199 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/utils/MessageChunkReassembler.java +18 -1
- package/android/src/main/java/com/mentra/bluetoothsdk/utils/MessageChunker.java +64 -18
- package/android/src/main/java/com/mentra/bluetoothsdk/utils/NexSGCUtils.kt +48 -39
- package/android/src/test/java/com/mentra/bluetoothsdk/camera/PhotoRequestTest.kt +23 -0
- package/android/src/test/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoReceiverRegistryTest.java +75 -0
- package/android/src/test/java/com/mentra/bluetoothsdk/utils/AvifExifStripperTest.java +30 -0
- package/android/src/test/java/com/mentra/bluetoothsdk/utils/BlePhotoUploadServiceTest.java +98 -0
- package/android/src/test/resources/avif_with_exif.avif +0 -0
- package/build/BluetoothSdk.types.d.ts +253 -44
- package/build/BluetoothSdk.types.d.ts.map +1 -1
- package/build/BluetoothSdk.types.js +11 -10
- package/build/BluetoothSdk.types.js.map +1 -1
- package/build/_private/BluetoothSdkModule.d.ts +37 -23
- package/build/_private/BluetoothSdkModule.d.ts.map +1 -1
- package/build/_private/BluetoothSdkModule.js +43 -52
- package/build/_private/BluetoothSdkModule.js.map +1 -1
- package/build/_private/photoRequestPayload.d.ts +4 -0
- package/build/_private/photoRequestPayload.d.ts.map +1 -0
- package/build/_private/photoRequestPayload.js +25 -0
- package/build/_private/photoRequestPayload.js.map +1 -0
- package/build/index.d.ts +1 -1
- package/build/index.d.ts.map +1 -1
- package/build/index.js +25 -0
- package/build/index.js.map +1 -1
- package/build/react/useMentraBluetooth.d.ts +2 -2
- package/build/react/useMentraBluetooth.d.ts.map +1 -1
- package/build/react/useMentraBluetooth.js +3 -3
- package/build/react/useMentraBluetooth.js.map +1 -1
- package/ios/BluetoothSdkModule.swift +177 -70
- package/ios/LocalPhotoUploadServer.swift +370 -105
- package/ios/MentraBluetoothSDK.podspec +3 -1
- package/ios/MentraPhotoReceiverModule.swift +62 -10
- package/ios/Packages/CoreObjC/include/PcmConverter.h +22 -0
- package/ios/Packages/SherpaOnnx/SherpaOnnx.swift +668 -26
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Info.plist +13 -9
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/Headers/sherpa-onnx/c-api/c-api.h +4551 -0
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/Headers/sherpa-onnx/c-api/cxx-api.h +1745 -0
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/libsherpa-onnx.a +0 -0
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/Headers/sherpa-onnx/c-api/c-api.h +4551 -0
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/Headers/sherpa-onnx/c-api/cxx-api.h +1745 -0
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/libsherpa-onnx.a +0 -0
- package/ios/Source/BluetoothSdkDefaults.swift +6 -0
- package/ios/Source/Bridge.swift +68 -19
- package/ios/Source/DeviceManager.swift +167 -32
- package/ios/Source/DeviceStore.swift +24 -8
- package/ios/Source/MentraBluetoothSDK.swift +803 -47
- package/ios/Source/ObservableStore.swift +6 -0
- package/ios/Source/{Audio → audio}/AudioModels.swift +9 -7
- package/ios/Source/{Camera → camera}/CameraModels.swift +226 -16
- package/ios/Source/controllers/ControllerManager.swift +4 -3
- package/ios/Source/controllers/R1.swift +3 -3
- package/ios/Source/{Events → events}/BluetoothEvents.swift +183 -1
- package/ios/Source/{Internal → internal}/ValueParsing.swift +10 -1
- package/ios/Source/{Requests → requests}/DisplayRequests.swift +28 -4
- package/ios/Source/services/PhoneMic.swift +1 -0
- package/ios/Source/sgcs/G1.swift +5 -5
- package/ios/Source/sgcs/G2.swift +1494 -870
- package/ios/Source/sgcs/Mach1.swift +4 -4
- package/ios/Source/sgcs/MentraLive.swift +814 -321
- package/ios/Source/sgcs/MentraNex.swift +604 -244
- package/ios/Source/sgcs/SGCManager.swift +68 -5
- package/ios/Source/sgcs/Simulated.swift +9 -5
- package/ios/Source/sgcs/mentraos_ble.pb.swift +3073 -3134
- package/ios/Source/status/DeviceStatus.swift +974 -0
- package/ios/Source/{Status → status}/WifiHotspotStatus.swift +4 -4
- package/ios/Source/{Streaming → streaming}/StreamModels.swift +24 -9
- package/ios/Source/stt/STTTools.swift +27 -12
- package/ios/Source/stt/SherpaOnnxTranscriber.swift +31 -8
- package/ios/Source/tts/TTSTools.swift +164 -0
- package/ios/Source/utils/JSCExperiment.swift +7 -7
- package/ios/Source/utils/MessageChunkReassembler.swift +20 -1
- package/ios/Source/utils/MessageChunker.swift +78 -21
- package/ios/Source/utils/TarBz2Extractor.swift +53 -2
- package/package.json +1 -1
- package/plugin/build/withAndroid.js +68 -4
- package/src/BluetoothSdk.types.ts +344 -60
- package/src/_private/BluetoothSdkModule.ts +132 -102
- package/src/_private/photoRequestPayload.ts +28 -0
- package/src/index.ts +61 -15
- package/src/react/useMentraBluetooth.ts +6 -4
- package/android/.project +0 -28
- package/android/lc3Lib/.project +0 -28
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/cargs.h +0 -162
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/sherpa-onnx/c-api/c-api.h +0 -1852
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/sherpa-onnx/c-api/cxx-api.h +0 -674
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64/sherpa-onnx.a +0 -0
- package/ios/Packages/SherpaOnnx/sherpa-onnx.xcframework/ios-arm64_x86_64-simulator/sherpa-onnx.a +0 -0
- package/ios/Source/Status/DeviceStatus.swift +0 -471
- /package/ios/Source/{Connection → connection}/ScanSession.swift +0 -0
- /package/ios/Source/{Errors → errors}/BluetoothError.swift +0 -0
- /package/ios/Source/{Internal → internal}/BluetoothAvailability.swift +0 -0
- /package/ios/Source/{Status → status}/RuntimeState.swift +0 -0
- /package/ios/Source/{Types → types}/DeviceModels.swift +0 -0
|
@@ -3,6 +3,7 @@ import ExpoModulesCore
|
|
|
3
3
|
import Foundation
|
|
4
4
|
|
|
5
5
|
public class MentraPhotoReceiverModule: Module {
|
|
6
|
+
private let receiverLock = NSLock()
|
|
6
7
|
private var photoUploadServer: LocalPhotoUploadServer?
|
|
7
8
|
|
|
8
9
|
public func definition() -> ModuleDefinition {
|
|
@@ -28,6 +29,9 @@ public class MentraPhotoReceiverModule: Module {
|
|
|
28
29
|
}
|
|
29
30
|
|
|
30
31
|
private func startPhotoReceiver() throws -> [String: Any] {
|
|
32
|
+
receiverLock.lock()
|
|
33
|
+
defer { receiverLock.unlock() }
|
|
34
|
+
|
|
31
35
|
guard let host = bestLocalIPv4Address() else {
|
|
32
36
|
throw PhotoReceiverError("No Wi-Fi/LAN IPv4 address found for this phone.")
|
|
33
37
|
}
|
|
@@ -42,17 +46,19 @@ public class MentraPhotoReceiverModule: Module {
|
|
|
42
46
|
)
|
|
43
47
|
photoUploadServer = server
|
|
44
48
|
|
|
49
|
+
if let activePort = server.activePort {
|
|
50
|
+
let uploadUrl = "http://\(host):\(activePort)/upload"
|
|
51
|
+
emitStatus(message: "Photo receiver ready at \(uploadUrl)")
|
|
52
|
+
return receiverResult(uploadUrl: uploadUrl, host: host, port: activePort)
|
|
53
|
+
}
|
|
54
|
+
|
|
45
55
|
var lastError: Error?
|
|
46
56
|
for port in photoPorts {
|
|
47
57
|
do {
|
|
48
58
|
let actualPort = try server.start(port: UInt16(port))
|
|
49
59
|
let uploadUrl = "http://\(host):\(actualPort)/upload"
|
|
50
60
|
emitStatus(message: "Photo receiver ready at \(uploadUrl)")
|
|
51
|
-
return
|
|
52
|
-
"uploadUrl": uploadUrl,
|
|
53
|
-
"host": host,
|
|
54
|
-
"port": actualPort,
|
|
55
|
-
]
|
|
61
|
+
return receiverResult(uploadUrl: uploadUrl, host: host, port: actualPort)
|
|
56
62
|
} catch {
|
|
57
63
|
lastError = error
|
|
58
64
|
emitStatus(message: "Port \(port) unavailable: \(error.localizedDescription)")
|
|
@@ -65,6 +71,9 @@ public class MentraPhotoReceiverModule: Module {
|
|
|
65
71
|
}
|
|
66
72
|
|
|
67
73
|
private func stopPhotoReceiverInternal() {
|
|
74
|
+
receiverLock.lock()
|
|
75
|
+
defer { receiverLock.unlock() }
|
|
76
|
+
|
|
68
77
|
photoUploadServer?.stop()
|
|
69
78
|
emitStatus(message: "Photo receiver stopped")
|
|
70
79
|
}
|
|
@@ -79,6 +88,7 @@ public class MentraPhotoReceiverModule: Module {
|
|
|
79
88
|
}
|
|
80
89
|
|
|
81
90
|
private func emitStatus(message: String) {
|
|
91
|
+
NSLog("[MentraPhotoReceiver] %@", message)
|
|
82
92
|
sendEvent("receiverStatus", [
|
|
83
93
|
"message": message,
|
|
84
94
|
])
|
|
@@ -91,6 +101,9 @@ public class MentraPhotoReceiverModule: Module {
|
|
|
91
101
|
}
|
|
92
102
|
defer { freeifaddrs(interfaces) }
|
|
93
103
|
|
|
104
|
+
var preferredPrivate: String?
|
|
105
|
+
var preferred: String?
|
|
106
|
+
var privateFallback: String?
|
|
94
107
|
var fallback: String?
|
|
95
108
|
var cursor: UnsafeMutablePointer<ifaddrs>? = first
|
|
96
109
|
while let current = cursor {
|
|
@@ -102,6 +115,12 @@ public class MentraPhotoReceiverModule: Module {
|
|
|
102
115
|
}
|
|
103
116
|
|
|
104
117
|
let name = String(cString: interface.ifa_name)
|
|
118
|
+
let flags = interface.ifa_flags
|
|
119
|
+
guard (flags & UInt32(IFF_UP)) != 0,
|
|
120
|
+
(flags & UInt32(IFF_LOOPBACK)) == 0 else {
|
|
121
|
+
continue
|
|
122
|
+
}
|
|
123
|
+
|
|
105
124
|
var address = addressPointer.pointee
|
|
106
125
|
var hostname = [CChar](repeating: 0, count: Int(NI_MAXHOST))
|
|
107
126
|
let result = getnameinfo(
|
|
@@ -118,16 +137,49 @@ public class MentraPhotoReceiverModule: Module {
|
|
|
118
137
|
}
|
|
119
138
|
|
|
120
139
|
let ip = String(cString: hostname)
|
|
121
|
-
guard ip != "127.0.0.1" else {
|
|
140
|
+
guard ip != "127.0.0.1", !isLinkLocalIPv4(ip) else {
|
|
122
141
|
continue
|
|
123
142
|
}
|
|
124
|
-
if name
|
|
125
|
-
|
|
143
|
+
if isPreferredLocalInterface(name), isPrivateIPv4(ip) {
|
|
144
|
+
preferredPrivate = preferredPrivate ?? ip
|
|
145
|
+
} else if isPreferredLocalInterface(name) {
|
|
146
|
+
preferred = preferred ?? ip
|
|
147
|
+
} else if isPrivateIPv4(ip) {
|
|
148
|
+
privateFallback = privateFallback ?? ip
|
|
149
|
+
} else {
|
|
150
|
+
fallback = fallback ?? ip
|
|
126
151
|
}
|
|
127
|
-
fallback = fallback ?? ip
|
|
128
152
|
}
|
|
129
153
|
|
|
130
|
-
return fallback
|
|
154
|
+
return preferredPrivate ?? preferred ?? privateFallback ?? fallback
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
private func receiverResult(uploadUrl: String, host: String, port: UInt16) -> [String: Any] {
|
|
158
|
+
[
|
|
159
|
+
"uploadUrl": uploadUrl,
|
|
160
|
+
"host": host,
|
|
161
|
+
"port": port,
|
|
162
|
+
]
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
private func isPreferredLocalInterface(_ name: String) -> Bool {
|
|
166
|
+
name == "en0" ||
|
|
167
|
+
name.hasPrefix("bridge") ||
|
|
168
|
+
name.hasPrefix("ap") ||
|
|
169
|
+
name.hasPrefix("awdl") ||
|
|
170
|
+
name.hasPrefix("llw")
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
private func isPrivateIPv4(_ ip: String) -> Bool {
|
|
174
|
+
if ip.hasPrefix("192.168.") || ip.hasPrefix("10.") {
|
|
175
|
+
return true
|
|
176
|
+
}
|
|
177
|
+
let parts = ip.split(separator: ".").compactMap { Int($0) }
|
|
178
|
+
return parts.count == 4 && parts[0] == 172 && (16...31).contains(parts[1])
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
private func isLinkLocalIPv4(_ ip: String) -> Bool {
|
|
182
|
+
ip.hasPrefix("169.254.")
|
|
131
183
|
}
|
|
132
184
|
|
|
133
185
|
private let photoPorts = [8787, 8788, 8789, 8790]
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
//
|
|
2
|
+
// PcmConverter.h
|
|
3
|
+
// Runner
|
|
4
|
+
//
|
|
5
|
+
// Created by Hawk on 2024/3/14.
|
|
6
|
+
//
|
|
7
|
+
|
|
8
|
+
#import <Foundation/Foundation.h>
|
|
9
|
+
|
|
10
|
+
NS_ASSUME_NONNULL_BEGIN
|
|
11
|
+
|
|
12
|
+
@interface PcmConverter : NSObject
|
|
13
|
+
+ (void)setupStaticEncoderAndDecoder;
|
|
14
|
+
- (NSMutableData *)decode:(NSData *)lc3data frameSize:(NSInteger)frameSize;
|
|
15
|
+
- (NSMutableData *)encode:(NSData *)pcmdata frameSize:(NSInteger)frameSize;
|
|
16
|
+
- (void)setOutputFrameSize:(NSInteger)frameSize;
|
|
17
|
+
- (NSInteger)getOutputFrameSize;
|
|
18
|
+
- (void)resetEncoder;
|
|
19
|
+
- (void)resetDecoder;
|
|
20
|
+
@end
|
|
21
|
+
|
|
22
|
+
NS_ASSUME_NONNULL_END
|