@mentra/bluetooth-sdk 0.1.19 → 0.1.21-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -2
- package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkModule.kt +239 -52
- package/android/src/main/java/com/mentra/bluetoothsdk/Bridge.kt +65 -14
- package/android/src/main/java/com/mentra/bluetoothsdk/DeviceManager.kt +47 -4
- package/android/src/main/java/com/mentra/bluetoothsdk/DeviceStore.kt +6 -1
- package/android/src/main/java/com/mentra/bluetoothsdk/MentraBluetoothSdk.kt +381 -97
- package/android/src/main/java/com/mentra/bluetoothsdk/MentraLocalNetworkModule.kt +333 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/camera/CameraModels.kt +16 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G1.kt +2 -2
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G1TextSanitizer.kt +38 -0
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G2.kt +16 -9
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Mach1.kt +1 -1
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraLive.kt +293 -174
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraNex.kt +1 -1
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Nimo.kt +1 -1
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/SGCManager.kt +4 -2
- package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Simulated.kt +1 -1
- package/android/src/main/java/com/mentra/bluetoothsdk/status/WifiHotspotStatus.kt +19 -3
- package/android/src/main/java/com/mentra/bluetoothsdk/streaming/StreamModels.kt +15 -2
- package/android/src/main/java/com/mentra/bluetoothsdk/tts/TTSTools.kt +115 -47
- package/android/src/main/java/com/mentra/bluetoothsdk/types/DeviceModels.kt +2 -3
- package/android/src/main/java/com/mentra/bluetoothsdk/utils/BlePhotoUploadService.java +65 -8
- package/android/src/main/java/com/mentra/bluetoothsdk/utils/K900ProtocolUtils.java +6 -5
- package/android/src/main/java/com/mentra/bluetoothsdk/utils/audio/PcmStreamPlayer.kt +328 -0
- package/android/src/test/java/com/mentra/bluetoothsdk/BluetoothSdkExceptionTest.kt +18 -0
- package/android/src/test/java/com/mentra/bluetoothsdk/camera/PhotoRequestTest.kt +31 -0
- package/android/src/test/java/com/mentra/bluetoothsdk/sgcs/G1TextSanitizerTest.kt +31 -0
- package/android/src/test/java/com/mentra/bluetoothsdk/utils/BlePhotoUploadServiceTest.java +9 -0
- package/build/BluetoothSdk.types.d.ts +71 -1
- package/build/BluetoothSdk.types.d.ts.map +1 -1
- package/build/BluetoothSdk.types.js.map +1 -1
- package/build/_internal.d.ts +2 -0
- package/build/_internal.d.ts.map +1 -1
- package/build/_internal.js +2 -0
- package/build/_internal.js.map +1 -1
- package/build/_private/BluetoothSdkModule.d.ts +27 -2
- package/build/_private/BluetoothSdkModule.d.ts.map +1 -1
- package/build/_private/BluetoothSdkModule.js +20 -0
- package/build/_private/BluetoothSdkModule.js.map +1 -1
- package/build/_private/MentraLocalNetworkModule.d.ts +37 -0
- package/build/_private/MentraLocalNetworkModule.d.ts.map +1 -0
- package/build/_private/MentraLocalNetworkModule.js +3 -0
- package/build/_private/MentraLocalNetworkModule.js.map +1 -0
- package/build/_private/cameraRequestPayload.d.ts.map +1 -1
- package/build/_private/cameraRequestPayload.js +1 -0
- package/build/_private/cameraRequestPayload.js.map +1 -1
- package/build/_private/photoRequestPayload.d.ts.map +1 -1
- package/build/_private/photoRequestPayload.js +2 -0
- package/build/_private/photoRequestPayload.js.map +1 -1
- package/build/index.d.ts +1 -1
- package/build/index.d.ts.map +1 -1
- package/build/index.js +9 -0
- package/build/index.js.map +1 -1
- package/expo-module.config.json +1 -0
- package/ios/BluetoothSdkModule.swift +72 -0
- package/ios/Source/BluetoothSdkDefaults.swift +1 -1
- package/ios/Source/Bridge.swift +72 -18
- package/ios/Source/DeviceManager.swift +52 -3
- package/ios/Source/DeviceStore.swift +6 -1
- package/ios/Source/MentraBluetoothSDK.swift +298 -21
- package/ios/Source/audio/PcmStreamPlayer.swift +410 -0
- package/ios/Source/camera/CameraModels.swift +22 -3
- package/ios/Source/sgcs/G1.swift +2 -2
- package/ios/Source/sgcs/G2.swift +19 -7
- package/ios/Source/sgcs/Mach1.swift +1 -1
- package/ios/Source/sgcs/MentraLive.swift +456 -52
- package/ios/Source/sgcs/MentraLiveL2capChannel.swift +183 -0
- package/ios/Source/sgcs/MentraNex.swift +1 -1
- package/ios/Source/sgcs/Nimo.swift +1 -1
- package/ios/Source/sgcs/SGCManager.swift +5 -2
- package/ios/Source/sgcs/Simulated.swift +1 -1
- package/ios/Source/status/WifiHotspotStatus.swift +21 -2
- package/ios/Source/streaming/StreamModels.swift +9 -0
- package/ios/Source/tts/TTSTools.swift +1 -1
- package/ios/Source/utils/G1Text.swift +30 -1
- package/package.json +1 -2
- package/src/BluetoothSdk.types.ts +75 -0
- package/src/_internal.ts +2 -0
- package/src/_private/BluetoothSdkModule.ts +68 -13
- package/src/_private/MentraLocalNetworkModule.ts +50 -0
- package/src/_private/cameraRequestPayload.ts +1 -0
- package/src/_private/photoRequestPayload.ts +3 -3
- package/src/index.ts +10 -0
|
@@ -0,0 +1,328 @@
|
|
|
1
|
+
package com.mentra.bluetoothsdk.utils.audio
|
|
2
|
+
|
|
3
|
+
import android.media.AudioAttributes
|
|
4
|
+
import android.media.AudioFormat
|
|
5
|
+
import android.media.AudioTrack
|
|
6
|
+
import android.util.Base64
|
|
7
|
+
import android.util.Log
|
|
8
|
+
import java.util.ArrayDeque
|
|
9
|
+
import java.util.concurrent.locks.ReentrantLock
|
|
10
|
+
import kotlin.concurrent.withLock
|
|
11
|
+
import kotlin.math.max
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Live PCM chunk player for miniapp `speaker.createStream()`.
|
|
15
|
+
*
|
|
16
|
+
* One instance = one stream session. 16-bit LE PCM only. Unlike the SCO-tuned
|
|
17
|
+
* [PCMAudioPlayer], this uses USAGE_MEDIA so the OS routes it like any media
|
|
18
|
+
* playback (A2DP to connected glasses, else phone speaker).
|
|
19
|
+
*
|
|
20
|
+
* Threading: a dedicated feeder thread drains an in-memory chunk queue into a
|
|
21
|
+
* MODE_STREAM AudioTrack (AudioTrack.write blocks there, not on callers).
|
|
22
|
+
* `write` blocks the CALLING thread only while the backlog exceeds
|
|
23
|
+
* [BACKPRESSURE_CEILING_MS] — call it from a background dispatcher. When the
|
|
24
|
+
* track underruns (queue empty), AudioTrack plays silence and playback resumes
|
|
25
|
+
* seamlessly on the next chunk; no stop/start pops.
|
|
26
|
+
*/
|
|
27
|
+
class PcmStreamPlayer(
|
|
28
|
+
private val streamId: String,
|
|
29
|
+
private val sampleRate: Int,
|
|
30
|
+
channels: Int,
|
|
31
|
+
volume: Float,
|
|
32
|
+
) {
|
|
33
|
+
companion object {
|
|
34
|
+
private const val TAG = "PcmStreamPlayer"
|
|
35
|
+
|
|
36
|
+
/** write() blocks while more than this is queued but unplayed. */
|
|
37
|
+
private const val BACKPRESSURE_CEILING_MS = 2_000L
|
|
38
|
+
|
|
39
|
+
/** Hard cap — writes that would exceed this are rejected outright. */
|
|
40
|
+
private const val MAX_BACKLOG_MS = 10_000L
|
|
41
|
+
|
|
42
|
+
/** AudioTrack buffer: max(4x minimum, 500ms) of jitter headroom. */
|
|
43
|
+
private const val TRACK_BUFFER_MS = 500
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
private val bytesPerFrame = 2 * channels
|
|
47
|
+
private val channelMask =
|
|
48
|
+
if (channels >= 2) AudioFormat.CHANNEL_OUT_STEREO else AudioFormat.CHANNEL_OUT_MONO
|
|
49
|
+
|
|
50
|
+
private val lock = ReentrantLock()
|
|
51
|
+
private val queueChanged = lock.newCondition()
|
|
52
|
+
|
|
53
|
+
/** Pending chunks not yet handed to AudioTrack. Guarded by [lock]. */
|
|
54
|
+
private val queue = ArrayDeque<ByteArray>()
|
|
55
|
+
private var queuedBytes = 0L
|
|
56
|
+
|
|
57
|
+
/** Total frames handed to AudioTrack.write. Guarded by [lock]. */
|
|
58
|
+
private var framesWritten = 0L
|
|
59
|
+
|
|
60
|
+
private var closing = false
|
|
61
|
+
private var aborted = false
|
|
62
|
+
private var writeInProgress = false
|
|
63
|
+
private var released = false
|
|
64
|
+
private var finalPlayedFrames: Long? = null
|
|
65
|
+
private var endedError: String? = null
|
|
66
|
+
|
|
67
|
+
private val audioTrack: AudioTrack
|
|
68
|
+
private val feeder: Thread
|
|
69
|
+
|
|
70
|
+
init {
|
|
71
|
+
val minBuf =
|
|
72
|
+
AudioTrack.getMinBufferSize(sampleRate, channelMask, AudioFormat.ENCODING_PCM_16BIT)
|
|
73
|
+
val jitterBuf = sampleRate * bytesPerFrame * TRACK_BUFFER_MS / 1000
|
|
74
|
+
audioTrack =
|
|
75
|
+
AudioTrack.Builder()
|
|
76
|
+
.setAudioAttributes(
|
|
77
|
+
AudioAttributes.Builder()
|
|
78
|
+
.setUsage(AudioAttributes.USAGE_MEDIA)
|
|
79
|
+
.setContentType(AudioAttributes.CONTENT_TYPE_SPEECH)
|
|
80
|
+
.build()
|
|
81
|
+
)
|
|
82
|
+
.setAudioFormat(
|
|
83
|
+
AudioFormat.Builder()
|
|
84
|
+
.setEncoding(AudioFormat.ENCODING_PCM_16BIT)
|
|
85
|
+
.setSampleRate(sampleRate)
|
|
86
|
+
.setChannelMask(channelMask)
|
|
87
|
+
.build()
|
|
88
|
+
)
|
|
89
|
+
.setBufferSizeInBytes(max(minBuf * 4, jitterBuf))
|
|
90
|
+
.setTransferMode(AudioTrack.MODE_STREAM)
|
|
91
|
+
.build()
|
|
92
|
+
if (audioTrack.state != AudioTrack.STATE_INITIALIZED) {
|
|
93
|
+
audioTrack.release()
|
|
94
|
+
throw IllegalStateException("AudioTrack failed to initialize (rate=$sampleRate)")
|
|
95
|
+
}
|
|
96
|
+
audioTrack.setVolume(volume.coerceIn(0f, 1f))
|
|
97
|
+
audioTrack.play()
|
|
98
|
+
|
|
99
|
+
feeder =
|
|
100
|
+
Thread({ feedLoop() }, "PcmStreamPlayer-$streamId").apply {
|
|
101
|
+
isDaemon = true
|
|
102
|
+
start()
|
|
103
|
+
}
|
|
104
|
+
Log.d(TAG, "[$streamId] opened: rate=$sampleRate ch=$channels vol=$volume")
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** Milliseconds of audio accepted but not yet played out. */
|
|
108
|
+
private fun backlogMsLocked(): Long {
|
|
109
|
+
val playedFrames = audioTrack.playbackHeadPosition.toLong() and 0xFFFFFFFFL
|
|
110
|
+
val queuedFrames = queuedBytes / bytesPerFrame
|
|
111
|
+
return max(0L, (framesWritten - playedFrames + queuedFrames)) * 1000L / sampleRate
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* Append base64-encoded 16-bit LE PCM. Blocks while the backlog is above
|
|
116
|
+
* the backpressure ceiling. Returns the backlog in ms after this chunk.
|
|
117
|
+
*/
|
|
118
|
+
fun write(base64: String): Long {
|
|
119
|
+
val bytes = Base64.decode(base64, Base64.DEFAULT)
|
|
120
|
+
require(bytes.isNotEmpty()) { "PCM chunk is empty" }
|
|
121
|
+
require(bytes.size % bytesPerFrame == 0) {
|
|
122
|
+
"PCM chunk length ${bytes.size} is not aligned to a $bytesPerFrame-byte frame"
|
|
123
|
+
}
|
|
124
|
+
lock.withLock {
|
|
125
|
+
check(!closing && !aborted) { "stream is closed" }
|
|
126
|
+
endedError?.let { throw IllegalStateException(it) }
|
|
127
|
+
check(backlogMsLocked() + bytes.size / bytesPerFrame * 1000L / sampleRate <= MAX_BACKLOG_MS) {
|
|
128
|
+
"backlog exceeds ${MAX_BACKLOG_MS}ms — producer is not throttling"
|
|
129
|
+
}
|
|
130
|
+
queue.add(bytes)
|
|
131
|
+
queuedBytes += bytes.size
|
|
132
|
+
queueChanged.signalAll()
|
|
133
|
+
// Backpressure: hold the caller until the feeder drains below the
|
|
134
|
+
// ceiling. Caller is a background dispatcher, never the UI thread.
|
|
135
|
+
while (backlogMsLocked() > BACKPRESSURE_CEILING_MS && !closing && !aborted && endedError == null) {
|
|
136
|
+
queueChanged.await()
|
|
137
|
+
}
|
|
138
|
+
endedError?.let { throw IllegalStateException(it) }
|
|
139
|
+
return backlogMsLocked()
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/**
|
|
144
|
+
* Drain everything queued, then stop. Blocks until played out. Returns the
|
|
145
|
+
* total played duration in ms.
|
|
146
|
+
*/
|
|
147
|
+
fun close(): Long {
|
|
148
|
+
lock.withLock {
|
|
149
|
+
if (aborted) return playedMsLocked()
|
|
150
|
+
closing = true
|
|
151
|
+
queueChanged.signalAll()
|
|
152
|
+
// Wait for the feeder to hand everything to the track...
|
|
153
|
+
while ((queue.isNotEmpty() || writeInProgress) && endedError == null && !aborted) {
|
|
154
|
+
queueChanged.await()
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
// ...then for the track to play it out while AudioTrack is still in
|
|
158
|
+
// PLAYSTATE_PLAYING. Calling stop() first flushes pending MODE_STREAM
|
|
159
|
+
// audio on several Android builds and truncates the final chunk.
|
|
160
|
+
try {
|
|
161
|
+
val deadline = System.currentTimeMillis() + MAX_BACKLOG_MS
|
|
162
|
+
while (System.currentTimeMillis() < deadline) {
|
|
163
|
+
val drained =
|
|
164
|
+
lock.withLock {
|
|
165
|
+
val played = audioTrack.playbackHeadPosition.toLong() and 0xFFFFFFFFL
|
|
166
|
+
played >= framesWritten || endedError != null
|
|
167
|
+
}
|
|
168
|
+
if (drained) break
|
|
169
|
+
Thread.sleep(20)
|
|
170
|
+
}
|
|
171
|
+
audioTrack.stop()
|
|
172
|
+
} catch (e: Exception) {
|
|
173
|
+
Log.w(TAG, "[$streamId] drain-on-close failed", e)
|
|
174
|
+
} finally {
|
|
175
|
+
releaseInternals()
|
|
176
|
+
}
|
|
177
|
+
return playedMsLocked()
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
/** Stop now, dropping the backlog. Idempotent. */
|
|
181
|
+
fun abort() {
|
|
182
|
+
lock.withLock {
|
|
183
|
+
if (aborted) return
|
|
184
|
+
aborted = true
|
|
185
|
+
queue.clear()
|
|
186
|
+
queuedBytes = 0
|
|
187
|
+
queueChanged.signalAll()
|
|
188
|
+
}
|
|
189
|
+
try {
|
|
190
|
+
audioTrack.pause()
|
|
191
|
+
audioTrack.flush()
|
|
192
|
+
} catch (e: Exception) {
|
|
193
|
+
Log.w(TAG, "[$streamId] abort flush failed", e)
|
|
194
|
+
}
|
|
195
|
+
releaseInternals()
|
|
196
|
+
Log.d(TAG, "[$streamId] aborted")
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
/** Played duration so far, in ms. */
|
|
200
|
+
fun playedMs(): Long = lock.withLock { playedMsLocked() }
|
|
201
|
+
|
|
202
|
+
private fun playedMsLocked(): Long {
|
|
203
|
+
val played =
|
|
204
|
+
finalPlayedFrames
|
|
205
|
+
?: try {
|
|
206
|
+
audioTrack.playbackHeadPosition.toLong() and 0xFFFFFFFFL
|
|
207
|
+
} catch (e: Exception) {
|
|
208
|
+
framesWritten
|
|
209
|
+
}
|
|
210
|
+
return played * 1000L / sampleRate
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
private fun feedLoop() {
|
|
214
|
+
try {
|
|
215
|
+
while (true) {
|
|
216
|
+
val chunk: ByteArray =
|
|
217
|
+
lock.withLock {
|
|
218
|
+
while (queue.isEmpty() && !closing && !aborted) {
|
|
219
|
+
queueChanged.await()
|
|
220
|
+
}
|
|
221
|
+
if (aborted) return
|
|
222
|
+
if (queue.isEmpty() && closing) return
|
|
223
|
+
queue.poll()!!.also { writeInProgress = true }
|
|
224
|
+
}
|
|
225
|
+
// Blocking write OUTSIDE the lock so write()/abort() stay responsive.
|
|
226
|
+
var off = 0
|
|
227
|
+
while (off < chunk.size) {
|
|
228
|
+
val n = audioTrack.write(chunk, off, chunk.size - off)
|
|
229
|
+
if (n < 0) throw IllegalStateException("AudioTrack.write error $n")
|
|
230
|
+
off += n
|
|
231
|
+
}
|
|
232
|
+
lock.withLock {
|
|
233
|
+
queuedBytes -= chunk.size
|
|
234
|
+
framesWritten += chunk.size / bytesPerFrame
|
|
235
|
+
writeInProgress = false
|
|
236
|
+
queueChanged.signalAll()
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
} catch (e: Exception) {
|
|
240
|
+
if (!aborted) {
|
|
241
|
+
Log.e(TAG, "[$streamId] feeder failed", e)
|
|
242
|
+
lock.withLock {
|
|
243
|
+
endedError = e.message ?: "feeder failed"
|
|
244
|
+
queue.clear()
|
|
245
|
+
queuedBytes = 0
|
|
246
|
+
writeInProgress = false
|
|
247
|
+
queueChanged.signalAll()
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
} finally {
|
|
251
|
+
lock.withLock { queueChanged.signalAll() }
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
private fun releaseInternals() {
|
|
256
|
+
val shouldRelease =
|
|
257
|
+
lock.withLock {
|
|
258
|
+
if (released) {
|
|
259
|
+
false
|
|
260
|
+
} else {
|
|
261
|
+
finalPlayedFrames =
|
|
262
|
+
try {
|
|
263
|
+
audioTrack.playbackHeadPosition.toLong() and 0xFFFFFFFFL
|
|
264
|
+
} catch (_: Exception) {
|
|
265
|
+
framesWritten
|
|
266
|
+
}
|
|
267
|
+
released = true
|
|
268
|
+
true
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
if (!shouldRelease) return
|
|
272
|
+
try {
|
|
273
|
+
audioTrack.release()
|
|
274
|
+
} catch (e: Exception) {
|
|
275
|
+
Log.w(TAG, "[$streamId] release failed", e)
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
/**
|
|
281
|
+
* Registry of active [PcmStreamPlayer] sessions, keyed by streamId. All
|
|
282
|
+
* methods are safe to call with unknown ids (write throws, abort no-ops) so
|
|
283
|
+
* races with teardown resolve cleanly.
|
|
284
|
+
*/
|
|
285
|
+
object PcmStreamManager {
|
|
286
|
+
private val players = java.util.concurrent.ConcurrentHashMap<String, PcmStreamPlayer>()
|
|
287
|
+
|
|
288
|
+
fun open(streamId: String, sampleRate: Int, channels: Int, volume: Float) {
|
|
289
|
+
require(streamId.isNotBlank()) { "streamId is required" }
|
|
290
|
+
require(sampleRate == 16000 || sampleRate == 24000 || sampleRate == 48000) {
|
|
291
|
+
"unsupported PCM sample rate $sampleRate"
|
|
292
|
+
}
|
|
293
|
+
require(channels == 1) { "only mono PCM is supported" }
|
|
294
|
+
// Replacing an id is a caller bug, but never leak the old track.
|
|
295
|
+
players.remove(streamId)?.abort()
|
|
296
|
+
players[streamId] = PcmStreamPlayer(streamId, sampleRate, channels, volume)
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
fun write(streamId: String, base64: String): Long {
|
|
300
|
+
val player =
|
|
301
|
+
players[streamId] ?: throw IllegalStateException("pcm stream $streamId is not open")
|
|
302
|
+
return player.write(base64)
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
fun close(streamId: String): Long {
|
|
306
|
+
val player =
|
|
307
|
+
players[streamId]
|
|
308
|
+
?: throw IllegalStateException("pcm stream $streamId is not open")
|
|
309
|
+
return try {
|
|
310
|
+
player.close()
|
|
311
|
+
} finally {
|
|
312
|
+
// Leave a draining player addressable so teardown can still abort
|
|
313
|
+
// it. The conditional remove preserves a replacement with the
|
|
314
|
+
// same id if one was opened while this close was completing.
|
|
315
|
+
players.remove(streamId, player)
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
fun abort(streamId: String) {
|
|
320
|
+
players.remove(streamId)?.abort()
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
fun abortAll() {
|
|
324
|
+
val active = players.values.toList()
|
|
325
|
+
players.clear()
|
|
326
|
+
active.forEach { it.abort() }
|
|
327
|
+
}
|
|
328
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
package com.mentra.bluetoothsdk
|
|
2
|
+
|
|
3
|
+
import org.assertj.core.api.Assertions.assertThat
|
|
4
|
+
import org.junit.Test
|
|
5
|
+
|
|
6
|
+
class BluetoothSdkExceptionTest {
|
|
7
|
+
@Test
|
|
8
|
+
fun `exception is usable without the Expo runtime`() {
|
|
9
|
+
val cause = IllegalArgumentException("cause")
|
|
10
|
+
val error = BluetoothSdkException("capture_failed", "Capture failed", cause)
|
|
11
|
+
|
|
12
|
+
assertThat(error.code).isEqualTo("capture_failed")
|
|
13
|
+
assertThat(error.message).isEqualTo("Capture failed")
|
|
14
|
+
assertThat(error.cause).isSameAs(cause)
|
|
15
|
+
assertThat(error).isInstanceOf(IllegalStateException::class.java)
|
|
16
|
+
assertThat(error.javaClass.superclass.name).doesNotStartWith("expo.modules")
|
|
17
|
+
}
|
|
18
|
+
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
package com.mentra.bluetoothsdk.camera
|
|
2
2
|
|
|
3
3
|
import com.mentra.bluetoothsdk.PhotoCompression
|
|
4
|
+
import com.mentra.bluetoothsdk.PhotoMode
|
|
4
5
|
import com.mentra.bluetoothsdk.PhotoRequest
|
|
5
6
|
import com.mentra.bluetoothsdk.PhotoSize
|
|
6
7
|
import org.assertj.core.api.Assertions.assertThat
|
|
@@ -34,6 +35,8 @@ class PhotoRequestTest {
|
|
|
34
35
|
)
|
|
35
36
|
|
|
36
37
|
assertThat(request.exposureTimeNs).isNull()
|
|
38
|
+
assertThat(request.mode).isEqualTo(PhotoMode.PHOTO)
|
|
39
|
+
assertThat(request.transferMethod).isEqualTo("auto")
|
|
37
40
|
}
|
|
38
41
|
|
|
39
42
|
@Test
|
|
@@ -77,4 +80,32 @@ class PhotoRequestTest {
|
|
|
77
80
|
|
|
78
81
|
assertThat(request.requestId).isEqualTo("photo-1")
|
|
79
82
|
}
|
|
83
|
+
|
|
84
|
+
@Test
|
|
85
|
+
fun `fromMap preserves text mode`() {
|
|
86
|
+
val request =
|
|
87
|
+
PhotoRequest.fromMap(
|
|
88
|
+
mapOf(
|
|
89
|
+
"size" to "medium",
|
|
90
|
+
"mode" to "text",
|
|
91
|
+
"webhookUrl" to "https://example.com/upload",
|
|
92
|
+
)
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
assertThat(request.mode).isEqualTo(PhotoMode.TEXT)
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
@Test
|
|
99
|
+
fun `fromMap preserves forced BLE transfer`() {
|
|
100
|
+
val request =
|
|
101
|
+
PhotoRequest.fromMap(
|
|
102
|
+
mapOf(
|
|
103
|
+
"size" to "medium",
|
|
104
|
+
"transferMethod" to "ble",
|
|
105
|
+
"webhookUrl" to "https://example.com/upload",
|
|
106
|
+
)
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
assertThat(request.transferMethod).isEqualTo("ble")
|
|
110
|
+
}
|
|
80
111
|
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
package com.mentra.bluetoothsdk.sgcs
|
|
2
|
+
|
|
3
|
+
import org.assertj.core.api.Assertions.assertThat
|
|
4
|
+
import org.junit.Test
|
|
5
|
+
|
|
6
|
+
class G1TextSanitizerTest {
|
|
7
|
+
|
|
8
|
+
@Test
|
|
9
|
+
fun `strips Vietnamese diacritics unsupported by G1`() {
|
|
10
|
+
assertThat(sanitizeG1DisplayText("Tiếng Việt: Đặng, Nguyễn, phở bò"))
|
|
11
|
+
.isEqualTo("Tieng Viet: Dang, Nguyen, pho bo")
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
@Test
|
|
15
|
+
fun `handles decomposed marks and preserves layout`() {
|
|
16
|
+
assertThat(sanitizeG1DisplayText("Cafe\u0301\nĐa Nang"))
|
|
17
|
+
.isEqualTo("Cafe\nDa Nang")
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
@Test
|
|
21
|
+
fun `expands Latin letters that do not decompose into combining marks`() {
|
|
22
|
+
assertThat(sanitizeG1DisplayText("Øresund Łódź Æsir Œuvre Straße Ðingvellir Þingvellir"))
|
|
23
|
+
.isEqualTo("Oresund Lodz AEsir OEuvre Strasse Dingvellir THingvellir")
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
@Test
|
|
27
|
+
fun `leaves text without diacritics unchanged`() {
|
|
28
|
+
assertThat(sanitizeG1DisplayText("Hello, world! 123"))
|
|
29
|
+
.isEqualTo("Hello, world! 123")
|
|
30
|
+
}
|
|
31
|
+
}
|
|
@@ -57,6 +57,15 @@ public class BlePhotoUploadServiceTest {
|
|
|
57
57
|
assertThat(new JSONObject(json).getInt("sampleCount")).isEqualTo(5);
|
|
58
58
|
}
|
|
59
59
|
|
|
60
|
+
@Test
|
|
61
|
+
public void convertToJpeg_passesThroughExistingJpegWithoutReencode() throws Exception {
|
|
62
|
+
File input = tempFolder.newFile("input.jpg");
|
|
63
|
+
writeMinimalJpeg(input);
|
|
64
|
+
byte[] inputBytes = java.nio.file.Files.readAllBytes(input.toPath());
|
|
65
|
+
byte[] outputBytes = BlePhotoUploadService.convertToJpegPreservingExif(inputBytes);
|
|
66
|
+
assertThat(outputBytes).isEqualTo(inputBytes);
|
|
67
|
+
}
|
|
68
|
+
|
|
60
69
|
@Test
|
|
61
70
|
public void convertToJpegPreservesUserComment() throws Exception {
|
|
62
71
|
File input = tempFolder.newFile("input.jpg");
|
|
@@ -88,6 +88,16 @@ export type ConnectedWifiStatus = Extract<WifiStatus, {
|
|
|
88
88
|
export declare function isConnectedWifiStatus(status: WifiStatus): status is ConnectedWifiStatus;
|
|
89
89
|
export type WifiStatusChangeEvent = WifiStatus & {
|
|
90
90
|
type: "wifi_status_change";
|
|
91
|
+
/**
|
|
92
|
+
* Glasses-reported provisioning failure reason when THIS event is the verdict of a
|
|
93
|
+
* failed connect attempt; absent on routine link-state updates. An attempt property,
|
|
94
|
+
* not a link property — which is why it lives on the event, not on WifiStatus:
|
|
95
|
+
* "connect_timeout" arrives on a disconnected status (never associated), while
|
|
96
|
+
* "connected_to_other_network" arrives on a *connected* status (the attempt failed
|
|
97
|
+
* and the glasses ended up on / fell back to a different SSID than requested).
|
|
98
|
+
* Requires ASG client v40+ — older glasses never send it.
|
|
99
|
+
*/
|
|
100
|
+
error?: string;
|
|
91
101
|
};
|
|
92
102
|
export type HotspotStatus = {
|
|
93
103
|
state: "disabled";
|
|
@@ -317,7 +327,7 @@ export type RgbLedControlSuccessResponseEvent = Extract<RgbLedControlResponseEve
|
|
|
317
327
|
state: "success";
|
|
318
328
|
}>;
|
|
319
329
|
export type SettingsAckStatus = "applied" | "ready" | "error" | "failed" | "failure" | "rejected";
|
|
320
|
-
export type SettingsAckSetting = "gallery_mode" | "button_photo" | "button_video_recording" | "button_max_recording_time" | "camera_fov" | "camera_tuning";
|
|
330
|
+
export type SettingsAckSetting = "gallery_mode" | "button_photo" | "button_video_recording" | "button_max_recording_time" | "camera_fov" | "camera_fov_override" | "camera_tuning";
|
|
321
331
|
export type SettingsAckEvent = {
|
|
322
332
|
type: "settings_ack";
|
|
323
333
|
requestId: string;
|
|
@@ -327,6 +337,7 @@ export type SettingsAckEvent = {
|
|
|
327
337
|
fov?: number;
|
|
328
338
|
roiPosition?: CameraRoiPositionValue;
|
|
329
339
|
hardwareApplied?: boolean;
|
|
340
|
+
leaseId?: string;
|
|
330
341
|
active?: boolean;
|
|
331
342
|
size?: ButtonPhotoSize | string;
|
|
332
343
|
width?: number;
|
|
@@ -348,7 +359,13 @@ export type SettingsAckSuccessEvent = Omit<SettingsAckEvent, "status"> & {
|
|
|
348
359
|
export type RgbLedAction = "on" | "off";
|
|
349
360
|
export type RgbLedColor = "red" | "green" | "blue" | "orange" | "white";
|
|
350
361
|
export type PhotoSize = "low" | "medium" | "high" | "max";
|
|
362
|
+
export type PhotoMode = "photo" | "text";
|
|
351
363
|
export type ButtonPhotoSize = "low" | "medium" | "high" | "max";
|
|
364
|
+
/**
|
|
365
|
+
* @deprecated Sticky action-button photo presets via {@link BluetoothSdkPublicModule.setPhotoCaptureDefaults}
|
|
366
|
+
* are deprecated. Prefer per-request {@link BluetoothSdkPublicModule.requestPhoto} options
|
|
367
|
+
* (e.g. `mode: "text"` for AE ÷3) instead of persisting button-photo tuning on the glasses.
|
|
368
|
+
*/
|
|
352
369
|
export type PhotoCaptureDefaults = {
|
|
353
370
|
size?: PhotoSize;
|
|
354
371
|
mfnr?: boolean;
|
|
@@ -423,6 +440,12 @@ export type CameraFovResult = {
|
|
|
423
440
|
roiPosition: CameraRoiPosition;
|
|
424
441
|
timestamp: number;
|
|
425
442
|
};
|
|
443
|
+
export type CameraFovOverrideRequest = CameraFovRequest & {
|
|
444
|
+
/** Phone-owned lease used to make delayed releases safe. */
|
|
445
|
+
leaseId: string;
|
|
446
|
+
/** Safety TTL; refresh the same lease/configuration to extend without a HAL restart. */
|
|
447
|
+
ttlMs?: number;
|
|
448
|
+
};
|
|
426
449
|
export type CameraFovSetting = {
|
|
427
450
|
fov: number;
|
|
428
451
|
roiPosition: CameraRoiPositionValue;
|
|
@@ -437,6 +460,9 @@ export type PhotoRequestParams = {
|
|
|
437
460
|
requestId?: string;
|
|
438
461
|
appId?: string;
|
|
439
462
|
size: PhotoSize;
|
|
463
|
+
mode?: PhotoMode;
|
|
464
|
+
/** `ble` skips direct upload and forces phone-relayed BLE image transfer. */
|
|
465
|
+
transferMethod?: "auto" | "ble";
|
|
440
466
|
webhookUrl: string | null;
|
|
441
467
|
authToken: string | null;
|
|
442
468
|
compress: PhotoCompression;
|
|
@@ -458,9 +484,12 @@ export type PhotoRequestParams = {
|
|
|
458
484
|
ispAnalogGain?: string;
|
|
459
485
|
};
|
|
460
486
|
export type WarmUpCameraParams = {
|
|
487
|
+
/** Supply this when the owner needs to call stopCameraWarmUp during teardown. */
|
|
461
488
|
requestId?: string;
|
|
462
489
|
size: PhotoSize;
|
|
490
|
+
mode?: PhotoMode;
|
|
463
491
|
exposureTimeNs?: number | null;
|
|
492
|
+
/** Ready-state hold; defaults to 15 seconds and is capped at 60 seconds by ASG. */
|
|
464
493
|
durationMs?: number;
|
|
465
494
|
};
|
|
466
495
|
export type StreamVideoConfig = {
|
|
@@ -566,11 +595,24 @@ export type StreamResolvedConfig = {
|
|
|
566
595
|
noiseSuppression?: boolean;
|
|
567
596
|
};
|
|
568
597
|
};
|
|
598
|
+
/** Live encoder telemetry, when the glasses emit it (no firmware does yet). */
|
|
599
|
+
export type StreamLiveStats = {
|
|
600
|
+
/** Current encoded video bitrate in bits per second. */
|
|
601
|
+
bitrate?: number;
|
|
602
|
+
/** Current encode frame rate. */
|
|
603
|
+
fps?: number;
|
|
604
|
+
droppedFrames?: number;
|
|
605
|
+
/** Seconds since the stream started. */
|
|
606
|
+
duration?: number;
|
|
607
|
+
/** Device temperature in °C, if the hardware reports it. */
|
|
608
|
+
temperatureC?: number;
|
|
609
|
+
};
|
|
569
610
|
type StreamStatusCommon = {
|
|
570
611
|
type: "stream_status";
|
|
571
612
|
streamId?: string;
|
|
572
613
|
timestamp?: number;
|
|
573
614
|
resolvedConfig?: StreamResolvedConfig;
|
|
615
|
+
stats?: StreamLiveStats;
|
|
574
616
|
};
|
|
575
617
|
export type StreamStatusEvent = (StreamStatusCommon & {
|
|
576
618
|
kind: "lifecycle";
|
|
@@ -675,6 +717,8 @@ export type BluetoothSdkModuleEvents = {
|
|
|
675
717
|
speaking_status: (event: SpeakingStatusEvent) => void;
|
|
676
718
|
battery_status: (event: BatteryStatusEvent) => void;
|
|
677
719
|
local_transcription: (event: LocalTranscriptionEvent) => void;
|
|
720
|
+
phone_notification: (event: PhoneNotificationEvent) => void;
|
|
721
|
+
phone_notification_dismissed: (event: PhoneNotificationDismissedEvent) => void;
|
|
678
722
|
wifi_status_change: (event: WifiStatusChangeEvent) => void;
|
|
679
723
|
wifi_scan_result: (event: WifiScanResultEvent) => void;
|
|
680
724
|
hotspot_status_change: (event: HotspotStatusChangeEvent) => void;
|
|
@@ -719,6 +763,21 @@ export interface ExtractionProgressEvent {
|
|
|
719
763
|
bytesRead: number;
|
|
720
764
|
totalBytes: number;
|
|
721
765
|
}
|
|
766
|
+
export interface PhoneNotificationEvent {
|
|
767
|
+
notificationId: string;
|
|
768
|
+
app: string;
|
|
769
|
+
title: string;
|
|
770
|
+
content: string;
|
|
771
|
+
priority: string;
|
|
772
|
+
timestamp: number;
|
|
773
|
+
packageName: string;
|
|
774
|
+
}
|
|
775
|
+
export interface PhoneNotificationDismissedEvent {
|
|
776
|
+
notificationId: string;
|
|
777
|
+
notificationKey: string;
|
|
778
|
+
packageName: string;
|
|
779
|
+
timestamp: number;
|
|
780
|
+
}
|
|
722
781
|
export type PublicGlassesStatus = Omit<GlassesStatus, "otaUpdateAvailable" | "otaProgress" | "otaInProgress" | "otaVersionUrl">;
|
|
723
782
|
export type PublicBluetoothStatus = Pick<BluetoothStatus, "searching" | "searchingController" | "systemMicUnavailable" | "micRanking" | "currentMic" | "searchResults" | "wifiScanResults" | "lastLog" | "otherBtConnected" | "galleryModeEnabled">;
|
|
724
783
|
export type BluetoothSdkEventMap = {
|
|
@@ -798,10 +857,19 @@ export interface BluetoothSdkPublicModule {
|
|
|
798
857
|
setHotspotState(enabled: boolean): Promise<HotspotStatusChangeEvent>;
|
|
799
858
|
setGalleryModeEnabled(enabled: boolean): Promise<SettingsAckSuccessEvent>;
|
|
800
859
|
setVoiceActivityDetectionEnabled(enabled: boolean): Promise<void>;
|
|
860
|
+
/**
|
|
861
|
+
* @deprecated Sticky action-button photo presets are deprecated. Prefer per-request
|
|
862
|
+
* `requestPhoto(...)` options (e.g. `mode: "text"` for AE ÷3, or explicit per-shot
|
|
863
|
+
* fields). Still functional until removed in a future release.
|
|
864
|
+
*/
|
|
801
865
|
setPhotoCaptureDefaults(settings: PhotoCaptureDefaults): Promise<SettingsAckSuccessEvent>;
|
|
802
866
|
setVideoRecordingDefaults(settings: VideoRecordingDefaults): Promise<SettingsAckSuccessEvent>;
|
|
803
867
|
setMaxVideoRecordingDuration(minutes: number): Promise<SettingsAckSuccessEvent>;
|
|
804
868
|
setCameraFov(request: CameraFovRequest): Promise<CameraFovResult>;
|
|
869
|
+
/** One-way FOV command for legacy ASG clients that do not send settings acknowledgements. */
|
|
870
|
+
setLegacyCameraFov(request: CameraFovRequest): Promise<CameraFovResult>;
|
|
871
|
+
setCameraFovOverride(request: CameraFovOverrideRequest): Promise<CameraFovResult>;
|
|
872
|
+
releaseCameraFovOverride(leaseId: string): Promise<SettingsAckSuccessEvent>;
|
|
805
873
|
/**
|
|
806
874
|
* Configure camera HAL tuning (ANR / gain) on Mentra Live glasses.
|
|
807
875
|
*
|
|
@@ -819,6 +887,8 @@ export interface BluetoothSdkPublicModule {
|
|
|
819
887
|
queryGalleryStatus(): Promise<GalleryStatusEvent>;
|
|
820
888
|
requestPhoto(params: PhotoRequestParams): Promise<PhotoSuccessResponseEvent>;
|
|
821
889
|
warmUpCamera(params: WarmUpCameraParams): Promise<CameraStatusEvent>;
|
|
890
|
+
/** Release one request-owned warm-up. Opening requests reject with camera_warm_up_cancelled. */
|
|
891
|
+
stopCameraWarmUp(requestId: string): Promise<void>;
|
|
822
892
|
startVideoRecording(requestId: string, save: boolean, sound: boolean, settings?: VideoRecordingSettings): Promise<VideoRecordingStartedStatusEvent>;
|
|
823
893
|
/**
|
|
824
894
|
* Stop the active recording. When {@link webhookUrl} is provided, the glasses
|