@mentra/bluetooth-sdk 0.1.6 → 0.1.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/README.md +26 -22
  2. package/android/.project +28 -0
  3. package/android/build.gradle +5 -9
  4. package/android/lc3Lib/.project +28 -0
  5. package/android/src/main/java/com/mentra/bluetoothsdk/BluetoothSdkModule.kt +54 -33
  6. package/android/src/main/java/com/mentra/bluetoothsdk/Bridge.kt +48 -10
  7. package/android/src/main/java/com/mentra/bluetoothsdk/DeviceManager.kt +27 -36
  8. package/android/src/main/java/com/mentra/bluetoothsdk/DeviceStore.kt +8 -4
  9. package/android/src/main/java/com/mentra/bluetoothsdk/MentraBluetoothSdk.kt +170 -14
  10. package/android/src/main/java/com/mentra/bluetoothsdk/audio/AudioModels.kt +6 -6
  11. package/android/src/main/java/com/mentra/bluetoothsdk/camera/CameraModels.kt +69 -10
  12. package/android/src/main/java/com/mentra/bluetoothsdk/controllers/ControllerManager.kt +5 -3
  13. package/android/src/main/java/com/mentra/bluetoothsdk/controllers/R1.kt +2 -1
  14. package/android/src/main/java/com/mentra/bluetoothsdk/debug/BleTraceLogger.kt +149 -0
  15. package/android/src/main/java/com/mentra/bluetoothsdk/events/BluetoothEvents.kt +13 -0
  16. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/LocalPhotoUploadServer.kt +297 -0
  17. package/android/src/main/java/com/mentra/bluetoothsdk/photoreceiver/MentraPhotoReceiverModule.kt +120 -0
  18. package/android/src/main/java/com/mentra/bluetoothsdk/services/Foreground.kt +13 -0
  19. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G1.java +1 -1
  20. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/G2.kt +3 -1
  21. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Mach1.java +1 -1
  22. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraLive.java +269 -28
  23. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/MentraNex.kt +1 -1
  24. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/SGCManager.kt +8 -3
  25. package/android/src/main/java/com/mentra/bluetoothsdk/sgcs/Simulated.kt +4 -1
  26. package/android/src/main/java/com/mentra/bluetoothsdk/status/DeviceStatus.kt +14 -12
  27. package/android/src/main/java/com/mentra/bluetoothsdk/status/RuntimeState.kt +4 -2
  28. package/android/src/main/java/com/mentra/bluetoothsdk/streaming/StreamModels.kt +131 -6
  29. package/android/src/main/java/com/mentra/bluetoothsdk/stt/SherpaOnnxTranscriber.kt +1 -3
  30. package/android/src/main/java/com/mentra/bluetoothsdk/utils/MessageChunkReassembler.java +105 -0
  31. package/build/BluetoothSdk.types.d.ts +107 -48
  32. package/build/BluetoothSdk.types.d.ts.map +1 -1
  33. package/build/BluetoothSdk.types.js +4 -0
  34. package/build/BluetoothSdk.types.js.map +1 -1
  35. package/build/_private/BluetoothSdkModule.d.ts +8 -6
  36. package/build/_private/BluetoothSdkModule.d.ts.map +1 -1
  37. package/build/_private/BluetoothSdkModule.js +58 -9
  38. package/build/_private/BluetoothSdkModule.js.map +1 -1
  39. package/build/index.d.ts +2 -2
  40. package/build/index.d.ts.map +1 -1
  41. package/build/index.js +6 -5
  42. package/build/index.js.map +1 -1
  43. package/build/photo-receiver/MentraPhotoReceiver.types.d.ts +18 -0
  44. package/build/photo-receiver/MentraPhotoReceiver.types.d.ts.map +1 -0
  45. package/build/photo-receiver/MentraPhotoReceiver.types.js +2 -0
  46. package/build/photo-receiver/MentraPhotoReceiver.types.js.map +1 -0
  47. package/build/photo-receiver/MentraPhotoReceiverModule.d.ts +10 -0
  48. package/build/photo-receiver/MentraPhotoReceiverModule.d.ts.map +1 -0
  49. package/build/photo-receiver/MentraPhotoReceiverModule.js +3 -0
  50. package/build/photo-receiver/MentraPhotoReceiverModule.js.map +1 -0
  51. package/build/photo-receiver/MentraPhotoReceiverModule.web.d.ts +10 -0
  52. package/build/photo-receiver/MentraPhotoReceiverModule.web.d.ts.map +1 -0
  53. package/build/photo-receiver/MentraPhotoReceiverModule.web.js +12 -0
  54. package/build/photo-receiver/MentraPhotoReceiverModule.web.js.map +1 -0
  55. package/build/photo-receiver/index.d.ts +3 -0
  56. package/build/photo-receiver/index.d.ts.map +1 -0
  57. package/build/photo-receiver/index.js +3 -0
  58. package/build/photo-receiver/index.js.map +1 -0
  59. package/build/react/useMentraBluetooth.d.ts +5 -3
  60. package/build/react/useMentraBluetooth.d.ts.map +1 -1
  61. package/build/react/useMentraBluetooth.js +15 -4
  62. package/build/react/useMentraBluetooth.js.map +1 -1
  63. package/expo-module.config.json +5 -2
  64. package/ios/BluetoothSdkModule.swift +59 -26
  65. package/ios/LocalPhotoUploadServer.swift +411 -0
  66. package/ios/MentraBluetoothSDK.podspec +8 -5
  67. package/ios/MentraPhotoReceiverModule.swift +147 -0
  68. package/ios/Source/Audio/AudioModels.swift +10 -10
  69. package/ios/Source/Bridge.swift +42 -12
  70. package/ios/Source/Camera/CameraModels.swift +74 -19
  71. package/ios/Source/DeviceManager.swift +23 -98
  72. package/ios/Source/DeviceStore.swift +12 -4
  73. package/ios/Source/Events/BluetoothEvents.swift +37 -0
  74. package/ios/Source/MentraBluetoothSDK.swift +135 -14
  75. package/ios/Source/Status/DeviceStatus.swift +12 -4
  76. package/ios/Source/Status/RuntimeState.swift +22 -13
  77. package/ios/Source/Streaming/StreamModels.swift +239 -45
  78. package/ios/Source/controllers/ControllerManager.swift +3 -3
  79. package/ios/Source/controllers/R1.swift +9 -3
  80. package/ios/Source/sgcs/G1.swift +5 -4
  81. package/ios/Source/sgcs/G2.swift +5 -6
  82. package/ios/Source/sgcs/Mach1.swift +1 -1
  83. package/ios/Source/sgcs/MentraLive.swift +183 -22
  84. package/ios/Source/sgcs/MentraNex.swift +1 -1
  85. package/ios/Source/sgcs/SGCManager.swift +11 -3
  86. package/ios/Source/sgcs/Simulated.swift +2 -2
  87. package/ios/Source/stt/SherpaOnnxTranscriber.swift +1 -4
  88. package/ios/Source/utils/MessageChunkReassembler.swift +89 -0
  89. package/package.json +6 -1
  90. package/src/BluetoothSdk.types.ts +133 -62
  91. package/src/_private/BluetoothSdkModule.ts +79 -26
  92. package/src/index.ts +15 -8
  93. package/src/photo-receiver/MentraPhotoReceiver.types.ts +20 -0
  94. package/src/photo-receiver/MentraPhotoReceiverModule.ts +14 -0
  95. package/src/photo-receiver/MentraPhotoReceiverModule.web.ts +20 -0
  96. package/src/photo-receiver/index.ts +2 -0
  97. package/src/react/useMentraBluetooth.ts +19 -7
  98. package/android/src/main/java/com/mentra/bluetoothsdk/stt/VadGateSpeechPolicy.kt +0 -221
  99. package/ios/Packages/VAD/Common/VAD/VADQuality.swift +0 -41
  100. package/ios/Packages/VAD/Common/VAD/VADState.swift +0 -26
  101. package/ios/Packages/VAD/Common/VAD/VADStrategy.swift +0 -29
  102. package/ios/Packages/VAD/Common/VAD/VADType.swift +0 -14
  103. package/ios/Packages/VAD/Data/Configuration.swift +0 -68
  104. package/ios/Packages/VAD/Data/FrameSize.swift +0 -39
  105. package/ios/Packages/VAD/Data/Record.swift +0 -13
  106. package/ios/Packages/VAD/Data/Result.swift +0 -22
  107. package/ios/Packages/VAD/Data/SampleRate.swift +0 -48
  108. package/ios/Packages/VAD/Silero/Model/silero_vad.onnx +0 -0
  109. package/ios/Packages/VAD/Silero/SileroVAD.swift +0 -284
  110. package/ios/Packages/VAD/Silero/SileroVADStrategy.swift +0 -64
@@ -0,0 +1,411 @@
1
+ import Foundation
2
+ import Network
3
+
4
+ struct PhotoUpload {
5
+ let requestId: String?
6
+ let photoFile: URL
7
+ let byteCount: Int
8
+ let fields: [String: String]
9
+ }
10
+
11
+ final class LocalPhotoUploadServer {
12
+ private let queue = DispatchQueue(label: "com.mentra.examples.photo-upload")
13
+ private let uploadDirectory: URL
14
+ private let onLog: (String) -> Void
15
+ private let onUpload: (PhotoUpload) -> Void
16
+ private var listener: NWListener?
17
+
18
+ init(onLog: @escaping (String) -> Void, onUpload: @escaping (PhotoUpload) -> Void) {
19
+ self.onLog = onLog
20
+ self.onUpload = onUpload
21
+ uploadDirectory = FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask)[0]
22
+ .appendingPathComponent("mentra-photo-uploads", isDirectory: true)
23
+ }
24
+
25
+ func start(port: UInt16) throws -> UInt16 {
26
+ stop()
27
+ try FileManager.default.createDirectory(
28
+ at: uploadDirectory,
29
+ withIntermediateDirectories: true
30
+ )
31
+
32
+ let parameters = NWParameters.tcp
33
+ parameters.allowLocalEndpointReuse = true
34
+ guard let endpointPort = NWEndpoint.Port(rawValue: port) else {
35
+ throw PhotoUploadServerError("Invalid port \(port)")
36
+ }
37
+
38
+ let listener = try NWListener(using: parameters, on: endpointPort)
39
+ let started = DispatchSemaphore(value: 0)
40
+ var startError: Error?
41
+ var isReady = false
42
+
43
+ listener.stateUpdateHandler = { [weak self] state in
44
+ switch state {
45
+ case .ready:
46
+ isReady = true
47
+ self?.onLog("Listening on 0.0.0.0:\(port)")
48
+ started.signal()
49
+ case .failed(let error):
50
+ startError = error
51
+ self?.onLog("Photo receiver failed on \(port): \(error)")
52
+ started.signal()
53
+ case .cancelled:
54
+ if !isReady {
55
+ startError = PhotoUploadServerError("Listener cancelled")
56
+ started.signal()
57
+ }
58
+ default:
59
+ break
60
+ }
61
+ }
62
+
63
+ listener.newConnectionHandler = { [weak self] connection in
64
+ self?.handle(connection)
65
+ }
66
+
67
+ self.listener = listener
68
+ listener.start(queue: queue)
69
+
70
+ if started.wait(timeout: .now() + 2) == .timedOut {
71
+ throw PhotoUploadServerError("Timed out starting listener")
72
+ }
73
+ if let startError {
74
+ self.listener = nil
75
+ listener.cancel()
76
+ throw startError
77
+ }
78
+
79
+ return port
80
+ }
81
+
82
+ func stop() {
83
+ listener?.cancel()
84
+ listener = nil
85
+ }
86
+
87
+ private func handle(_ connection: NWConnection) {
88
+ connection.start(queue: queue)
89
+ receive(connection, state: RequestReadState())
90
+ }
91
+
92
+ private func receive(_ connection: NWConnection, state: RequestReadState) {
93
+ connection.receive(minimumIncompleteLength: 1, maximumLength: 64 * 1024) {
94
+ [weak self] data, _, isComplete, error in
95
+ guard let self else {
96
+ connection.cancel()
97
+ return
98
+ }
99
+ if let error {
100
+ self.onLog("request failed: \(error.localizedDescription)")
101
+ connection.cancel()
102
+ return
103
+ }
104
+ if let data, !data.isEmpty {
105
+ state.buffer.append(data)
106
+ }
107
+
108
+ do {
109
+ if try self.handleIfComplete(connection, state: state) {
110
+ return
111
+ }
112
+ } catch {
113
+ self.onLog("request failed: \(error.localizedDescription)")
114
+ self.writeJson(connection, status: 500, body: #"{"ok":false,"error":"server_error"}"#)
115
+ return
116
+ }
117
+
118
+ if isComplete {
119
+ self.writeJson(connection, status: 400, body: #"{"ok":false,"error":"incomplete_request"}"#)
120
+ return
121
+ }
122
+ self.receive(connection, state: state)
123
+ }
124
+ }
125
+
126
+ private func handleIfComplete(_ connection: NWConnection, state: RequestReadState) throws -> Bool {
127
+ if state.request == nil {
128
+ guard let headerRange = state.buffer.range(of: Data([13, 10, 13, 10])) else {
129
+ if state.buffer.count > Self.maxHeaderBytes {
130
+ writeJson(connection, status: 413, body: #"{"ok":false,"error":"headers_too_large"}"#)
131
+ return true
132
+ }
133
+ return false
134
+ }
135
+
136
+ let headerData = state.buffer[state.buffer.startIndex..<headerRange.lowerBound]
137
+ guard let headerText = String(data: headerData, encoding: .isoLatin1) else {
138
+ writeJson(connection, status: 400, body: #"{"ok":false,"error":"invalid_headers"}"#)
139
+ return true
140
+ }
141
+ let request = HttpRequest.parse(headerText)
142
+ state.request = request
143
+ state.bodyStartIndex = headerRange.upperBound
144
+ onLog("\(request.method) \(request.path)")
145
+
146
+ if request.method == "GET", request.path == "/" || request.path == "/health" {
147
+ writeJson(connection, status: 200, body: #"{"ok":true,"service":"mentra-photo-upload-receiver"}"#)
148
+ return true
149
+ }
150
+
151
+ guard request.method == "POST", request.path == "/upload" else {
152
+ writeJson(connection, status: 404, body: #"{"ok":false,"error":"not_found"}"#)
153
+ return true
154
+ }
155
+
156
+ guard let contentLength = request.headers["content-length"].flatMap(Int.init), contentLength > 0 else {
157
+ writeJson(connection, status: 411, body: #"{"ok":false,"error":"content_length_required"}"#)
158
+ return true
159
+ }
160
+ guard contentLength <= Self.maxUploadBytes else {
161
+ writeJson(connection, status: 413, body: #"{"ok":false,"error":"upload_too_large"}"#)
162
+ return true
163
+ }
164
+
165
+ state.contentLength = contentLength
166
+ if request.headers["expect"]?.localizedCaseInsensitiveContains("100-continue") == true {
167
+ write(connection, text: "HTTP/1.1 100 Continue\r\n\r\n", closeAfterSend: false)
168
+ }
169
+ }
170
+
171
+ guard let request = state.request, let bodyStartIndex = state.bodyStartIndex else {
172
+ return false
173
+ }
174
+ guard state.buffer.count >= bodyStartIndex + state.contentLength else {
175
+ return false
176
+ }
177
+
178
+ let bodyEndIndex = bodyStartIndex + state.contentLength
179
+ let body = Data(state.buffer[bodyStartIndex..<bodyEndIndex])
180
+ let contentType = request.headers["content-type"] ?? ""
181
+ onLog("headers contentType=\(contentType) contentLength=\(state.contentLength)")
182
+
183
+ guard let boundary = multipartBoundary(contentType) else {
184
+ writeJson(connection, status: 400, body: #"{"ok":false,"error":"multipart_boundary_required"}"#)
185
+ return true
186
+ }
187
+
188
+ let parsed = parseMultipart(body, boundary: boundary)
189
+ guard let photoBytes = parsed.files["photo"] ?? parsed.files.values.first else {
190
+ writeJson(connection, status: 400, body: #"{"ok":false,"error":"photo_field_required"}"#)
191
+ return true
192
+ }
193
+
194
+ let requestId = parsed.fields["requestId"] ?? parsed.fields["request_id"]
195
+ let filename = safeFilePart(requestId ?? "photo-\(Int(Date().timeIntervalSince1970 * 1000))")
196
+ let photoFile = uploadDirectory.appendingPathComponent("\(filename).jpg")
197
+ try photoBytes.write(to: photoFile, options: .atomic)
198
+
199
+ onLog("upload fields=\(parsed.fields.keys.joined(separator: ",")) requestId=\(requestId ?? "") bytes=\(photoBytes.count) saved=\(photoFile.path)")
200
+ onUpload(PhotoUpload(
201
+ requestId: requestId,
202
+ photoFile: photoFile,
203
+ byteCount: photoBytes.count,
204
+ fields: parsed.fields
205
+ ))
206
+ writeJson(
207
+ connection,
208
+ status: 200,
209
+ body: #"{"ok":true,"requestId":"\#(jsonEscape(requestId ?? ""))","bytes":\#(photoBytes.count)}"#
210
+ )
211
+ return true
212
+ }
213
+
214
+ private func writeJson(_ connection: NWConnection, status: Int, body: String) {
215
+ let reason: String
216
+ switch status {
217
+ case 200:
218
+ reason = "OK"
219
+ case 400:
220
+ reason = "Bad Request"
221
+ case 404:
222
+ reason = "Not Found"
223
+ case 411:
224
+ reason = "Length Required"
225
+ case 413:
226
+ reason = "Payload Too Large"
227
+ default:
228
+ reason = "Internal Server Error"
229
+ }
230
+
231
+ let bodyData = Data(body.utf8)
232
+ let header = """
233
+ HTTP/1.1 \(status) \(reason)\r
234
+ Content-Type: application/json\r
235
+ Content-Length: \(bodyData.count)\r
236
+ Connection: close\r
237
+ \r
238
+ """
239
+ var response = Data(header.utf8)
240
+ response.append(bodyData)
241
+ connection.send(content: response, completion: .contentProcessed { _ in
242
+ connection.cancel()
243
+ })
244
+ }
245
+
246
+ private func write(
247
+ _ connection: NWConnection,
248
+ text: String,
249
+ closeAfterSend: Bool
250
+ ) {
251
+ connection.send(content: Data(text.utf8), completion: .contentProcessed { _ in
252
+ if closeAfterSend {
253
+ connection.cancel()
254
+ }
255
+ })
256
+ }
257
+
258
+ private func parseMultipart(_ body: Data, boundary: String) -> ParsedMultipart {
259
+ let marker = Data("--\(boundary)".utf8)
260
+ let nextMarkerPrefix = Data("\r\n--\(boundary)".utf8)
261
+ let headerDelimiter = Data([13, 10, 13, 10])
262
+ let crlf = Data([13, 10])
263
+ let closing = Data([45, 45])
264
+ var fields: [String: String] = [:]
265
+ var files: [String: Data] = [:]
266
+ var cursor = body.startIndex
267
+
268
+ while let markerRange = body.range(of: marker, options: [], in: cursor..<body.endIndex) {
269
+ var partStart = markerRange.upperBound
270
+ if body.starts(with: closing, at: partStart) {
271
+ break
272
+ }
273
+ if body.starts(with: crlf, at: partStart) {
274
+ partStart += crlf.count
275
+ }
276
+ guard
277
+ let headerRange = body.range(of: headerDelimiter, options: [], in: partStart..<body.endIndex)
278
+ else {
279
+ break
280
+ }
281
+
282
+ let dataStart = headerRange.upperBound
283
+ guard
284
+ let nextMarkerRange = body.range(
285
+ of: nextMarkerPrefix,
286
+ options: [],
287
+ in: dataStart..<body.endIndex
288
+ )
289
+ else {
290
+ break
291
+ }
292
+
293
+ let headerData = body[partStart..<headerRange.lowerBound]
294
+ let headerBlock = String(data: headerData, encoding: .isoLatin1) ?? ""
295
+ let disposition = headerBlock
296
+ .components(separatedBy: "\r\n")
297
+ .first { $0.caseInsensitiveHasPrefix("Content-Disposition:") } ?? ""
298
+ if let name = firstCapture(#"name="([^"]+)""#, in: disposition) {
299
+ let bytes = Data(body[dataStart..<nextMarkerRange.lowerBound])
300
+ if firstCapture(#"filename="([^"]*)""#, in: disposition) != nil {
301
+ files[name] = bytes
302
+ } else {
303
+ fields[name] = String(data: bytes, encoding: .utf8)?
304
+ .trimmingCharacters(in: .whitespacesAndNewlines)
305
+ }
306
+ }
307
+ cursor = nextMarkerRange.lowerBound + crlf.count
308
+ }
309
+
310
+ return ParsedMultipart(fields: fields, files: files)
311
+ }
312
+
313
+ private func multipartBoundary(_ contentType: String) -> String? {
314
+ contentType
315
+ .split(separator: ";")
316
+ .map { $0.trimmingCharacters(in: .whitespacesAndNewlines) }
317
+ .first { $0.caseInsensitiveHasPrefix("boundary=") }?
318
+ .split(separator: "=", maxSplits: 1)
319
+ .last
320
+ .map(String.init)?
321
+ .trimmingCharacters(in: CharacterSet(charactersIn: "\""))
322
+ }
323
+
324
+ private func firstCapture(_ pattern: String, in text: String) -> String? {
325
+ guard
326
+ let regex = try? NSRegularExpression(pattern: pattern),
327
+ let match = regex.firstMatch(in: text, range: NSRange(text.startIndex..., in: text)),
328
+ match.numberOfRanges > 1,
329
+ let range = Range(match.range(at: 1), in: text)
330
+ else {
331
+ return nil
332
+ }
333
+ return String(text[range])
334
+ }
335
+
336
+ private func safeFilePart(_ value: String) -> String {
337
+ let allowed = CharacterSet(charactersIn: "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._-")
338
+ let sanitized = String(value.unicodeScalars.map { allowed.contains($0) ? Character($0) : "_" })
339
+ return sanitized.isEmpty ? "photo" : sanitized
340
+ }
341
+
342
+ private func jsonEscape(_ value: String) -> String {
343
+ value
344
+ .replacingOccurrences(of: "\\", with: "\\\\")
345
+ .replacingOccurrences(of: "\"", with: "\\\"")
346
+ }
347
+
348
+ private struct HttpRequest {
349
+ let method: String
350
+ let path: String
351
+ let headers: [String: String]
352
+
353
+ static func parse(_ headerText: String) -> HttpRequest {
354
+ let lines = headerText.components(separatedBy: "\r\n").filter { !$0.isEmpty }
355
+ let requestParts = lines.first?.split(separator: " ").map(String.init) ?? []
356
+ let method = requestParts.first?.uppercased() ?? ""
357
+ let path = requestParts.dropFirst().first?.split(separator: "?").first.map(String.init) ?? ""
358
+ let headers: [String: String] = Dictionary(uniqueKeysWithValues: lines.dropFirst().compactMap { line in
359
+ guard let separator = line.firstIndex(of: ":") else {
360
+ return nil
361
+ }
362
+ let key = line[..<separator].lowercased()
363
+ let value = line[line.index(after: separator)...].trimmingCharacters(in: .whitespaces)
364
+ return (String(key), value)
365
+ })
366
+ return HttpRequest(method: method, path: path, headers: headers)
367
+ }
368
+ }
369
+
370
+ private final class RequestReadState {
371
+ var buffer = Data()
372
+ var request: HttpRequest?
373
+ var bodyStartIndex: Data.Index?
374
+ var contentLength = 0
375
+ }
376
+
377
+ private struct ParsedMultipart {
378
+ let fields: [String: String]
379
+ let files: [String: Data]
380
+ }
381
+
382
+ private struct PhotoUploadServerError: LocalizedError {
383
+ let message: String
384
+
385
+ init(_ message: String) {
386
+ self.message = message
387
+ }
388
+
389
+ var errorDescription: String? {
390
+ message
391
+ }
392
+ }
393
+
394
+ private static let maxHeaderBytes = 64 * 1024
395
+ private static let maxUploadBytes = 25 * 1024 * 1024
396
+ }
397
+
398
+ private extension Data {
399
+ func starts(with prefix: Data, at index: Data.Index) -> Bool {
400
+ guard index >= startIndex, index + prefix.count <= endIndex else {
401
+ return false
402
+ }
403
+ return self[index..<index + prefix.count].elementsEqual(prefix)
404
+ }
405
+ }
406
+
407
+ private extension String {
408
+ func caseInsensitiveHasPrefix(_ prefix: String) -> Bool {
409
+ range(of: prefix, options: [.anchored, .caseInsensitive]) != nil
410
+ }
411
+ }
@@ -37,7 +37,9 @@ Pod::Spec.new do |s|
37
37
  }
38
38
 
39
39
  # iOS frameworks required by Bluetooth SDK
40
- s.frameworks = 'AVFoundation', 'CoreBluetooth', 'UIKit', 'CoreGraphics'
40
+ ios_frameworks = ['AVFoundation', 'CoreBluetooth', 'UIKit', 'CoreGraphics']
41
+ ios_frameworks << 'Network' if include_expo_adapter
42
+ s.frameworks = ios_frameworks
41
43
 
42
44
  # System libraries required by MentraOS
43
45
  s.library = 'bz2'
@@ -45,8 +47,6 @@ Pod::Spec.new do |s|
45
47
  # Vendored frameworks
46
48
  s.vendored_frameworks = 'Packages/SherpaOnnx/sherpa-onnx.xcframework'
47
49
 
48
- # Resources (model files)
49
- s.resources = 'Packages/VAD/Silero/Model/*.onnx'
50
50
  s.resource_bundles = {
51
51
  'BluetoothSDKPrivacy' => ['Source/PrivacyInfo.xcprivacy']
52
52
  }
@@ -56,10 +56,13 @@ Pod::Spec.new do |s|
56
56
  "Packages/CoreObjC/**/*.{h,m,mm,hpp,cpp,c}",
57
57
  "Packages/SherpaOnnx/SherpaOnnx.swift",
58
58
  "Packages/SherpaOnnx/sherpa-onnx.xcframework/Headers/**/*.{h,hpp}",
59
- "Packages/VAD/**/*.swift",
60
59
  "Packages/libbz2/shim.h"
61
60
  ]
62
- native_source_files << "BluetoothSdkModule.swift" if include_expo_adapter
61
+ native_source_files.concat([
62
+ "BluetoothSdkModule.swift",
63
+ "LocalPhotoUploadServer.swift",
64
+ "MentraPhotoReceiverModule.swift"
65
+ ]) if include_expo_adapter
63
66
  s.source_files = native_source_files
64
67
 
65
68
  # Explicitly mark C++ headers and internal headers as private to prevent exposure in public interface
@@ -0,0 +1,147 @@
1
+ import Darwin
2
+ import ExpoModulesCore
3
+ import Foundation
4
+
5
+ public class MentraPhotoReceiverModule: Module {
6
+ private var photoUploadServer: LocalPhotoUploadServer?
7
+
8
+ public func definition() -> ModuleDefinition {
9
+ Name("MentraPhotoReceiver")
10
+
11
+ Events("photoUpload", "receiverStatus")
12
+
13
+ AsyncFunction("isSupported") {
14
+ true
15
+ }
16
+
17
+ AsyncFunction("startPhotoReceiver") { () -> [String: Any] in
18
+ try startPhotoReceiver()
19
+ }
20
+
21
+ AsyncFunction("stopPhotoReceiver") {
22
+ stopPhotoReceiverInternal()
23
+ }
24
+
25
+ OnDestroy {
26
+ stopPhotoReceiverInternal()
27
+ }
28
+ }
29
+
30
+ private func startPhotoReceiver() throws -> [String: Any] {
31
+ guard let host = bestLocalIPv4Address() else {
32
+ throw PhotoReceiverError("No Wi-Fi/LAN IPv4 address found for this phone.")
33
+ }
34
+
35
+ let server = photoUploadServer ?? LocalPhotoUploadServer(
36
+ onLog: { [weak self] message in
37
+ self?.emitStatus(message: message)
38
+ },
39
+ onUpload: { [weak self] upload in
40
+ self?.handlePhotoUpload(upload)
41
+ }
42
+ )
43
+ photoUploadServer = server
44
+
45
+ var lastError: Error?
46
+ for port in photoPorts {
47
+ do {
48
+ let actualPort = try server.start(port: UInt16(port))
49
+ let uploadUrl = "http://\(host):\(actualPort)/upload"
50
+ emitStatus(message: "Photo receiver ready at \(uploadUrl)")
51
+ return [
52
+ "uploadUrl": uploadUrl,
53
+ "host": host,
54
+ "port": actualPort,
55
+ ]
56
+ } catch {
57
+ lastError = error
58
+ emitStatus(message: "Port \(port) unavailable: \(error.localizedDescription)")
59
+ }
60
+ }
61
+
62
+ throw PhotoReceiverError(
63
+ "Could not start phone photo receiver: \(lastError?.localizedDescription ?? "all ports unavailable")"
64
+ )
65
+ }
66
+
67
+ private func stopPhotoReceiverInternal() {
68
+ photoUploadServer?.stop()
69
+ emitStatus(message: "Photo receiver stopped")
70
+ }
71
+
72
+ private func handlePhotoUpload(_ upload: PhotoUpload) {
73
+ sendEvent("photoUpload", [
74
+ "requestId": upload.requestId as Any,
75
+ "fileUri": upload.photoFile.absoluteString,
76
+ "byteCount": upload.byteCount,
77
+ ])
78
+ emitStatus(message: "Photo uploaded (\(upload.byteCount) bytes)")
79
+ }
80
+
81
+ private func emitStatus(message: String) {
82
+ sendEvent("receiverStatus", [
83
+ "message": message,
84
+ ])
85
+ }
86
+
87
+ private func bestLocalIPv4Address() -> String? {
88
+ var interfaces: UnsafeMutablePointer<ifaddrs>?
89
+ guard getifaddrs(&interfaces) == 0, let first = interfaces else {
90
+ return nil
91
+ }
92
+ defer { freeifaddrs(interfaces) }
93
+
94
+ var fallback: String?
95
+ var cursor: UnsafeMutablePointer<ifaddrs>? = first
96
+ while let current = cursor {
97
+ defer { cursor = current.pointee.ifa_next }
98
+ let interface = current.pointee
99
+ guard let addressPointer = interface.ifa_addr,
100
+ addressPointer.pointee.sa_family == UInt8(AF_INET) else {
101
+ continue
102
+ }
103
+
104
+ let name = String(cString: interface.ifa_name)
105
+ var address = addressPointer.pointee
106
+ var hostname = [CChar](repeating: 0, count: Int(NI_MAXHOST))
107
+ let result = getnameinfo(
108
+ &address,
109
+ socklen_t(address.sa_len),
110
+ &hostname,
111
+ socklen_t(hostname.count),
112
+ nil,
113
+ 0,
114
+ NI_NUMERICHOST
115
+ )
116
+ guard result == 0 else {
117
+ continue
118
+ }
119
+
120
+ let ip = String(cString: hostname)
121
+ guard ip != "127.0.0.1" else {
122
+ continue
123
+ }
124
+ if name == "en0" {
125
+ return ip
126
+ }
127
+ fallback = fallback ?? ip
128
+ }
129
+
130
+ return fallback
131
+ }
132
+
133
+ private let photoPorts = [8787, 8788, 8789, 8790]
134
+ }
135
+
136
+ final class PhotoReceiverError: Exception, @unchecked Sendable {
137
+ private let message: String
138
+
139
+ init(_ message: String) {
140
+ self.message = message
141
+ super.init()
142
+ }
143
+
144
+ override var reason: String {
145
+ message
146
+ }
147
+ }
@@ -34,7 +34,7 @@ public struct MicPcmEvent: CustomStringConvertible {
34
34
  public let bitsPerSample: Int
35
35
  public let channels: Int
36
36
  public let encoding: String
37
- public let vadGated: Bool
37
+ public let voiceActivityDetectionEnabled: Bool
38
38
  public let values: [String: Any]
39
39
 
40
40
  public init(values: [String: Any]) {
@@ -43,7 +43,7 @@ public struct MicPcmEvent: CustomStringConvertible {
43
43
  let bitsPerSample = intValue(values["bitsPerSample"]) ?? Self.bitsPerSample
44
44
  let channels = intValue(values["channels"]) ?? Self.channels
45
45
  let encoding = values["encoding"] as? String ?? Self.encoding
46
- let vadGated = boolValue(values, "vadGated") ?? false
46
+ let voiceActivityDetectionEnabled = boolValue(values, "voiceActivityDetectionEnabled") ?? true
47
47
 
48
48
  var normalized = values
49
49
  normalized["type"] = "mic_pcm"
@@ -52,19 +52,19 @@ public struct MicPcmEvent: CustomStringConvertible {
52
52
  normalized["bitsPerSample"] = bitsPerSample
53
53
  normalized["channels"] = channels
54
54
  normalized["encoding"] = encoding
55
- normalized["vadGated"] = vadGated
55
+ normalized["voiceActivityDetectionEnabled"] = voiceActivityDetectionEnabled
56
56
 
57
57
  self.pcm = pcm
58
58
  self.sampleRate = sampleRate
59
59
  self.bitsPerSample = bitsPerSample
60
60
  self.channels = channels
61
61
  self.encoding = encoding
62
- self.vadGated = vadGated
62
+ self.voiceActivityDetectionEnabled = voiceActivityDetectionEnabled
63
63
  self.values = normalized
64
64
  }
65
65
 
66
66
  public var description: String {
67
- "MicPcmEvent(bytes: \(pcm.count), sampleRate: \(sampleRate), bitsPerSample: \(bitsPerSample), channels: \(channels), encoding: \(encoding), vadGated: \(vadGated))"
67
+ "MicPcmEvent(bytes: \(pcm.count), sampleRate: \(sampleRate), bitsPerSample: \(bitsPerSample), channels: \(channels), encoding: \(encoding), voiceActivityDetectionEnabled: \(voiceActivityDetectionEnabled))"
68
68
  }
69
69
  }
70
70
 
@@ -83,7 +83,7 @@ public struct MicLc3Event: CustomStringConvertible {
83
83
  public let frameSizeBytes: Int
84
84
  public let bitrate: Int
85
85
  public let packetizedFromGlasses: Bool
86
- public let vadGated: Bool
86
+ public let voiceActivityDetectionEnabled: Bool
87
87
  public let values: [String: Any]
88
88
 
89
89
  public init(values: [String: Any]) {
@@ -95,7 +95,7 @@ public struct MicLc3Event: CustomStringConvertible {
95
95
  let frameSizeBytes = intValue(values["frameSizeBytes"]) ?? Self.defaultFrameSizeBytes
96
96
  let bitrate = intValue(values["bitrate"]) ?? frameSizeBytes * 8 * (1000 / frameDurationMs)
97
97
  let packetizedFromGlasses = boolValue(values, "packetizedFromGlasses") ?? false
98
- let vadGated = boolValue(values, "vadGated") ?? false
98
+ let voiceActivityDetectionEnabled = boolValue(values, "voiceActivityDetectionEnabled") ?? true
99
99
 
100
100
  var normalized = values
101
101
  normalized["type"] = "mic_lc3"
@@ -107,7 +107,7 @@ public struct MicLc3Event: CustomStringConvertible {
107
107
  normalized["frameSizeBytes"] = frameSizeBytes
108
108
  normalized["bitrate"] = bitrate
109
109
  normalized["packetizedFromGlasses"] = packetizedFromGlasses
110
- normalized["vadGated"] = vadGated
110
+ normalized["voiceActivityDetectionEnabled"] = voiceActivityDetectionEnabled
111
111
 
112
112
  self.lc3 = lc3
113
113
  self.sampleRate = sampleRate
@@ -117,12 +117,12 @@ public struct MicLc3Event: CustomStringConvertible {
117
117
  self.frameSizeBytes = frameSizeBytes
118
118
  self.bitrate = bitrate
119
119
  self.packetizedFromGlasses = packetizedFromGlasses
120
- self.vadGated = vadGated
120
+ self.voiceActivityDetectionEnabled = voiceActivityDetectionEnabled
121
121
  self.values = normalized
122
122
  }
123
123
 
124
124
  public var description: String {
125
- "MicLc3Event(bytes: \(lc3.count), sampleRate: \(sampleRate), channels: \(channels), frameDurationMs: \(frameDurationMs), frameSizeBytes: \(frameSizeBytes), bitrate: \(bitrate), packetizedFromGlasses: \(packetizedFromGlasses), vadGated: \(vadGated))"
125
+ "MicLc3Event(bytes: \(lc3.count), sampleRate: \(sampleRate), channels: \(channels), frameDurationMs: \(frameDurationMs), frameSizeBytes: \(frameSizeBytes), bitrate: \(bitrate), packetizedFromGlasses: \(packetizedFromGlasses), voiceActivityDetectionEnabled: \(voiceActivityDetectionEnabled))"
126
126
  }
127
127
  }
128
128