@glassly/bluetooth-sdk 0.1.0-dev.26

package/android/src/main/java/com/glassly/bluetoothsdk/sgcs/ar99/ota/OtaProtocol.java

@@ -0,0 +1,92 @@
+package com.glassly.bluetoothsdk.sgcs.ar99.ota;
+
+import java.io.ByteArrayOutputStream;
+
+public final class OtaProtocol {
+  private OtaProtocol() {}
+
+  public static byte[] buildFrame(byte cmd, byte paramType, byte[] payload) {
+    ByteArrayOutputStream out = new ByteArrayOutputStream();
+    out.write(OtaCommandConstants.OTA.SVC_ID & 0xFF);
+    out.write(cmd & 0xFF);
+    out.write(paramType & 0xFF);
+    int len = payload != null ? payload.length : 0;
+    out.write(len & 0xFF);
+    out.write((len >> 8) & 0xFF);
+    if (payload != null && payload.length > 0) {
+      try {
+        out.write(payload);
+      } catch (Exception ignored) {
+      }
+    }
+    return out.toByteArray();
+  }
+
+  public static byte[] buildRequestUpgrade(int packageSize, int blockSize, long crc32) {
+    // Wire layout matches the vendor bring-up frame: type marker, then TLV
+    // 0x0A/len4 = package size (LE). CRC is validated per-block in SEND_IMAGE_DATA;
+    // blockSize is negotiated separately in CONNECT_NEGOTIATION.
+    ByteArrayOutputStream payload = new ByteArrayOutputStream();
+    payload.write(0x09);
+    payload.write(0x01);
+    payload.write(0x00);
+    payload.write(0x01);
+    payload.write(0x0A);
+    payload.write(0x04);
+    payload.write(packageSize & 0xFF);
+    payload.write((packageSize >> 8) & 0xFF);
+    payload.write((packageSize >> 16) & 0xFF);
+    payload.write((packageSize >> 24) & 0xFF);
+    payload.write(0x00);
+    return buildFrame(OtaCommandConstants.OTA.REQUEST_UPGRADE, (byte) 0x80, payload.toByteArray());
+  }
+
+  public static byte[] buildConnectNegotiation(int blockSize) {
+    return buildFrame(OtaCommandConstants.OTA.CONNECT_NEGOTIATION, (byte) 0x80, null);
+  }
+
+  public static byte[] buildNegotiationResult() {
+    ByteArrayOutputStream payload = new ByteArrayOutputStream();
+    payload.write(0x01);
+    payload.write(0x01);
+    payload.write(0x00);
+    payload.write(0x01);
+    return buildFrame(OtaCommandConstants.OTA.NEGOTIATION_RESULT, (byte) 0x80, payload.toByteArray());
+  }
+
+  public static byte[] buildSendImageData(int psn, byte[] data) {
+    ByteArrayOutputStream payload = new ByteArrayOutputStream();
+    payload.write(psn & 0xFF);
+    long crc32 = OtaCrc32Util.calculate(data);
+    payload.write((byte) (crc32 & 0xFF));
+    payload.write((byte) ((crc32 >> 8) & 0xFF));
+    payload.write((byte) ((crc32 >> 16) & 0xFF));
+    payload.write((byte) ((crc32 >> 24) & 0xFF));
+    if (data != null && data.length > 0) {
+      try {
+        payload.write(data);
+      } catch (Exception ignored) {
+      }
+    }
+    return buildFrame(OtaCommandConstants.OTA.SEND_IMAGE_DATA, (byte) 0x80, payload.toByteArray());
+  }
+
+  public static FrameHeader parseFrameHeader(byte[] frame) {
+    if (frame == null || frame.length < 5) return null;
+    FrameHeader header = new FrameHeader();
+    header.svcId = frame[0];
+    header.cmd = frame[1];
+    header.paramType = frame[2];
+    header.paramLen = OtaByteUtils.readShortLE(frame, 3) & 0xFFFF;
+    header.payloadOffset = 5;
+    return header;
+  }
+
+  public static class FrameHeader {
+    public byte svcId;
+    public byte cmd;
+    public byte paramType;
+    public int paramLen;
+    public int payloadOffset;
+  }
+}

package/android/src/main/java/com/glassly/bluetoothsdk/speakerid/SpeakerIdEngine.kt

@@ -0,0 +1,525 @@
+package com.glassly.bluetoothsdk.speakerid
+
+import android.util.Base64
+import com.glassly.bluetoothsdk.Bridge
+import com.k2fsa.sherpa.onnx.SpeakerEmbeddingExtractor
+import com.k2fsa.sherpa.onnx.SpeakerEmbeddingExtractorConfig
+import java.io.File
+import java.nio.ByteBuffer
+import java.nio.ByteOrder
+import java.util.concurrent.Executors
+import java.util.concurrent.atomic.AtomicBoolean
+import kotlin.math.sqrt
+
+/**
+ * On-device speaker identification over the canonical 16 kHz mono PCM stream.
+ *
+ * Taps DeviceManager.handlePcm (the single audio choke point), windows voiced
+ * audio with a local energy gate, computes a speaker embedding per
+ * utterance-ish window (sherpa-onnx CAM++ model), matches against enrolled
+ * profile centroids by cosine similarity, and emits `speaker_detected` bridge
+ * events. Raw audio never leaves the phone for speaker ID; enrollment audio is
+ * discarded as soon as its embeddings are computed.
+ *
+ * Profiles are pushed in from JS (`setSpeakerProfiles`) — MMKV in the engine
+ * layer is the source of truth; native holds them in memory only.
+ *
+ * Ported 1:1 from iOS SpeakerIdEngine.swift.
+ */
+object SpeakerIdEngine {
+    private const val TAG = "SpeakerIdEngine"
+
+    // Benchmarked cross-speaker ceiling for this model is ~0.46 (the MAX over
+    // clean-audio pairs; typical impostors score far lower), while on-device
+    // same-speaker scores run below the 0.67+ clean bench. Owner-only voice
+    // recognition favors recall: a borderline accept just labels a line "You",
+    // and the JS segment resolver's overlap voting + conflict damping still
+    // filter isolated stray windows — so sit at the edge of the impostor range
+    // rather than above it. (With one enrolled profile the margin rule is
+    // vacuous.)
+    private const val MATCH_THRESHOLD = 0.45f
+    private const val MATCH_MARGIN = 0.08f
+    private const val WINDOW_VOICED_TARGET_MS = 3000.0
+    private const val END_OF_SPEECH_SILENCE_MS = 500.0
+    private const val MIN_VOICED_MS = 1000.0
+    private const val MAX_WINDOW_SAMPLES = 16000 * 10
+    private const val VOICED_RMS_THRESHOLD = 0.01f
+    private const val ENROLL_CONSISTENCY_THRESHOLD = 0.6f
+    private const val ENROLL_CHUNK_COUNT = 3
+
+    private const val SAMPLE_RATE = 16000
+    private const val PREFS_NAME = "GlasslyPrefs"
+    private const val KEY_MODEL_PATH = "SpeakerIdModelPath"
+
+    data class Profile(val id: String, val name: String, val embedding: FloatArray)
+
+    data class Match(
+        val profileId: String,
+        val name: String,
+        val similarity: Float,
+        /** Epoch ms when the window producing this match ended. */
+        val endMs: Long,
+    )
+
+    private val executor = Executors.newSingleThreadExecutor { runnable ->
+        Thread(runnable, "SpeakerIdProcessor").apply { isDaemon = true }
+    }
+
+    private val identifyEnabled = AtomicBoolean(false)
+    private val enrolling = AtomicBoolean(false)
+
+    // Everything below is touched on the executor thread only.
+    private var extractor: SpeakerEmbeddingExtractor? = null
+    private var extractorLoadAttempted = false
+    @Volatile private var profiles: List<Profile> = emptyList()
+
+    private var windowSamples = ArrayList<Float>()
+    private var windowStartMs = 0L
+    private var windowVoicedMs = 0.0
+    private var trailingSilenceMs = 0.0
+
+    private class EnrollmentSession(
+        val targetVoicedMs: Double,
+        val source: String,
+    ) {
+        val samples = ArrayList<Float>()
+        var voicedMs = 0.0
+        var lastProgressAt = 0L
+    }
+
+    private var enrollment: EnrollmentSession? = null
+    private var enrollmentEnded: (() -> Unit)? = null
+
+    @Volatile private var lastMatch: Match? = null
+
+    // MARK: - Public API (any thread)
+
+    fun setIdentifyEnabled(enabled: Boolean) {
+        identifyEnabled.set(enabled)
+        Bridge.log("SpeakerID: identify ${if (enabled) "enabled" else "disabled"}")
+        executor.execute {
+            if (enabled) ensureExtractor() else resetWindow()
+        }
+    }
+
+    fun setProfiles(next: List<Profile>) {
+        profiles = next
+        Bridge.log(
+            "SpeakerID: profiles set — ${next.size}: " +
+                next.joinToString(", ") { "${it.name}(${it.embedding.size}d)" }
+        )
+    }
+
+    fun setModelPath(path: String) {
+        try {
+            val prefs = Bridge.getContext().getSharedPreferences(PREFS_NAME, android.content.Context.MODE_PRIVATE)
+            prefs.edit().putString(KEY_MODEL_PATH, path).apply()
+            Bridge.log("SpeakerID: model path set → $path")
+        } catch (e: Exception) {
+            Bridge.log("SpeakerID: failed to persist model path: ${e.message}")
+        }
+        executor.execute {
+            // Force a reload on next use — the model may have just arrived.
+            extractor?.release()
+            extractor = null
+            extractorLoadAttempted = false
+        }
+    }
+
+    fun isModelAvailable(): Boolean {
+        val path = resolvedModelPath() ?: return false
+        return File(path).let { it.exists() && it.length() > 0 }
+    }
+
+    /**
+     * [onEnded] fires (on the engine thread) whenever the enrollment session
+     * finishes or fails, so the caller can release the mic requirement.
+     */
+    fun startEnrollment(targetSpeechSeconds: Double, source: String, onEnded: () -> Unit) {
+        enrolling.set(true)
+        executor.execute {
+            ensureExtractor()
+            if (extractor == null) {
+                Bridge.log("SpeakerID: enrollment aborted — model unavailable")
+                finishEnrollment(ok = false, reason = "model_unavailable", embeddings = emptyList(), source = source)
+                onEnded()
+                return@execute
+            }
+            Bridge.log("SpeakerID: enrollment started — target ${targetSpeechSeconds}s voiced, mic=$source")
+            enrollmentEnded = onEnded
+            enrollment = EnrollmentSession(
+                targetVoicedMs = maxOf(5.0, targetSpeechSeconds) * 1000.0,
+                source = source,
+            )
+        }
+    }
+
+    fun cancelEnrollment() {
+        enrolling.set(false)
+        executor.execute {
+            enrollment = null
+            val onEnded = enrollmentEnded
+            enrollmentEnded = null
+            onEnded?.invoke()
+        }
+    }
+
+    /**
+     * Latest accepted identification, if it ended within [withinMs] of now.
+     * Used by the local STT path to stamp transcripts (native-side join).
+     */
+    fun recentMatch(withinMs: Long): Match? {
+        val match = lastMatch ?: return null
+        return if (System.currentTimeMillis() - match.endMs <= withinMs) match else null
+    }
+
+    /** Called from DeviceManager.handlePcm with 16 kHz mono s16le PCM. */
+    fun acceptAudio(pcm16le: ByteArray) {
+        if (!identifyEnabled.get() && !enrolling.get()) return
+        if (pcm16le.size < 2) return
+        val copy = pcm16le.copyOf()
+        executor.execute { process(copy) }
+    }
+
+    // MARK: - Processing (executor thread only)
+
+    private fun process(pcmData: ByteArray) {
+        val samples = toFloatArray(pcmData)
+        if (samples.isEmpty()) return
+        val chunkMs = samples.size.toDouble() / SAMPLE_RATE * 1000.0
+        val rms = rms(samples)
+        val voiced = rms > VOICED_RMS_THRESHOLD
+
+        if (enrollment != null) {
+            processEnrollment(samples, chunkMs, rms, voiced)
+            return
+        }
+        if (!identifyEnabled.get()) return
+        processIdentification(samples, chunkMs, voiced)
+    }
+
+    private fun processIdentification(samples: FloatArray, chunkMs: Double, voiced: Boolean) {
+        val nowMs = System.currentTimeMillis()
+
+        if (windowSamples.isEmpty()) {
+            if (!voiced) return
+            windowStartMs = nowMs - chunkMs.toLong()
+            windowVoicedMs = 0.0
+            trailingSilenceMs = 0.0
+        }
+
+        // Accumulate VOICED samples only — enrollment computes centroids from
+        // voiced-only audio, so identification must embed like-for-like.
+        // Windows padded with inter-phrase silence score far below the same
+        // speaker's centroid (observed 0.1–0.48 vs the 0.67+ bench).
+        if (voiced) {
+            if (windowSamples.size < MAX_WINDOW_SAMPLES) {
+                windowSamples.ensureCapacity(windowSamples.size + samples.size)
+                for (s in samples) windowSamples.add(s)
+            }
+            windowVoicedMs += chunkMs
+            trailingSilenceMs = 0.0
+        } else {
+            trailingSilenceMs += chunkMs
+        }
+
+        val endOfSpeech = trailingSilenceMs >= END_OF_SPEECH_SILENCE_MS
+        val windowFull = windowVoicedMs >= WINDOW_VOICED_TARGET_MS
+        if (!endOfSpeech && !windowFull) return
+
+        val startMs = windowStartMs
+        val voicedMs = windowVoicedMs
+        val window = windowSamples.toFloatArray()
+        resetWindow()
+
+        if (voicedMs < MIN_VOICED_MS) {
+            Bridge.log("SpeakerID: window skipped — only ${voicedMs.toInt()}ms voiced (<${MIN_VOICED_MS.toInt()}ms)")
+            return
+        }
+        identify(window, startMs, nowMs, voicedMs)
+    }
+
+    private fun identify(window: FloatArray, startMs: Long, endMs: Long, voicedMs: Double) {
+        ensureExtractor()
+        val extractor = extractor
+        if (extractor == null) {
+            Bridge.log("SpeakerID: window dropped — extractor not loaded (model missing?)")
+            return
+        }
+        val currentProfiles = profiles
+        if (currentProfiles.isEmpty()) {
+            Bridge.log("SpeakerID: window dropped — no enrolled profiles pushed")
+            return
+        }
+
+        val stream = extractor.createStream()
+        val embedding: FloatArray
+        try {
+            stream.acceptWaveform(window, SAMPLE_RATE)
+            stream.inputFinished()
+            if (!extractor.isReady(stream)) {
+                Bridge.log("SpeakerID: window dropped — extractor not ready (${window.size} samples)")
+                return
+            }
+            embedding = l2Normalize(extractor.compute(stream))
+        } finally {
+            stream.release()
+        }
+        if (embedding.isEmpty()) {
+            Bridge.log("SpeakerID: window dropped — empty embedding")
+            return
+        }
+
+        var best: Profile? = null
+        var bestCos = -1f
+        var second = -1f
+        for (profile in currentProfiles) {
+            if (profile.embedding.size != embedding.size) continue
+            val cos = dot(profile.embedding, embedding)
+            if (best == null || cos > bestCos) {
+                second = bestCos
+                best = profile
+                bestCos = cos
+            } else if (cos > second) {
+                second = cos
+            }
+        }
+
+        var matched: Profile? = null
+        if (best != null && bestCos >= MATCH_THRESHOLD) {
+            // Margin rule: reject ambiguous windows where the runner-up is
+            // close (overlapping talkers produce garbage embeddings).
+            if (second < 0 || bestCos - second >= MATCH_MARGIN) {
+                matched = best
+            }
+        }
+
+        if (matched != null) {
+            lastMatch = Match(matched.id, matched.name, bestCos, endMs)
+        }
+
+        val durationMs = endMs - startMs
+        val speech = "speech=${voicedMs.toInt()}ms window=${durationMs}ms"
+        if (matched != null) {
+            Bridge.log(
+                "SpeakerID: matched ${matched.name} cos=${"%.3f".format(bestCos)} " +
+                    "second=${"%.3f".format(second)} $speech"
+            )
+        } else if (best != null) {
+            val reason =
+                if (bestCos < MATCH_THRESHOLD) "below threshold $MATCH_THRESHOLD"
+                else "margin to runner-up < $MATCH_MARGIN"
+            Bridge.log(
+                "SpeakerID: no match — best ${best.name} cos=${"%.3f".format(bestCos)} " +
+                    "second=${"%.3f".format(second)} ($reason) $speech"
+            )
+        }
+
+        Bridge.sendSpeakerDetected(
+            speakerId = matched?.id,
+            name = matched?.name,
+            similarity = (if (matched != null) bestCos else maxOf(bestCos, 0f)).toDouble(),
+            startMs = startMs,
+            endMs = endMs,
+        )
+    }
+
+    // MARK: - Enrollment (executor thread only)
+
+    private fun processEnrollment(samples: FloatArray, chunkMs: Double, rms: Float, voiced: Boolean) {
+        val session = enrollment ?: return
+        if (voiced) {
+            session.samples.ensureCapacity(session.samples.size + samples.size)
+            for (s in samples) session.samples.add(s)
+            session.voicedMs += chunkMs
+        }
+
+        val now = System.currentTimeMillis()
+        if (now - session.lastProgressAt >= 250) {
+            session.lastProgressAt = now
+            Bridge.sendSpeakerEnrollProgress(
+                voicedSeconds = session.voicedMs / 1000.0,
+                targetSeconds = session.targetVoicedMs / 1000.0,
+                level = rms.toDouble(),
+            )
+        }
+
+        if (session.voicedMs >= session.targetVoicedMs) {
+            enrollment = null
+            enrolling.set(false)
+            completeEnrollment(session)
+        }
+    }
+
+    private fun completeEnrollment(session: EnrollmentSession) {
+        val extractor = extractor
+        if (extractor == null) {
+            finishEnrollment(ok = false, reason = "model_unavailable", embeddings = emptyList(), source = session.source)
+            return
+        }
+
+        // One embedding per prompt-sized chunk; internal consistency check
+        // catches wind/noise/multiple talkers.
+        val chunkSize = session.samples.size / ENROLL_CHUNK_COUNT
+        if (chunkSize < SAMPLE_RATE) {
+            finishEnrollment(ok = false, reason = "too_short", embeddings = emptyList(), source = session.source)
+            return
+        }
+
+        val embeddings = ArrayList<FloatArray>()
+        for (i in 0 until ENROLL_CHUNK_COUNT) {
+            val start = i * chunkSize
+            val end = if (i == ENROLL_CHUNK_COUNT - 1) session.samples.size else start + chunkSize
+            val chunk = FloatArray(end - start) { session.samples[start + it] }
+            val stream = extractor.createStream()
+            try {
+                stream.acceptWaveform(chunk, SAMPLE_RATE)
+                stream.inputFinished()
+                if (!extractor.isReady(stream)) continue
+                val embedding = l2Normalize(extractor.compute(stream))
+                if (embedding.isNotEmpty()) embeddings.add(embedding)
+            } finally {
+                stream.release()
+            }
+        }
+
+        if (embeddings.size < 2) {
+            finishEnrollment(ok = false, reason = "too_short", embeddings = emptyList(), source = session.source)
+            return
+        }
+
+        for (i in embeddings.indices) {
+            for (j in i + 1 until embeddings.size) {
+                val cos = dot(embeddings[i], embeddings[j])
+                if (cos < ENROLL_CONSISTENCY_THRESHOLD) {
+                    Bridge.log(
+                        "SpeakerID: enrollment chunks $i/$j inconsistent — cos=${"%.3f".format(cos)} " +
+                            "< $ENROLL_CONSISTENCY_THRESHOLD (noise / multiple talkers?)"
+                    )
+                    finishEnrollment(ok = false, reason = "inconsistent", embeddings = emptyList(), source = session.source)
+                    return
+                }
+            }
+        }
+
+        finishEnrollment(
+            ok = true,
+            reason = null,
+            embeddings = embeddings.map { encodeBase64(it) },
+            source = session.source,
+            dim = embeddings[0].size,
+        )
+    }
+
+    private fun finishEnrollment(ok: Boolean, reason: String?, embeddings: List<String>, source: String, dim: Int = 0) {
+        enrolling.set(false)
+        Bridge.log(
+            if (ok) "SpeakerID: enrollment ok — ${embeddings.size} embeddings (dim=$dim, mic=$source)"
+            else "SpeakerID: enrollment failed — ${reason ?: "unknown"} (mic=$source)"
+        )
+        Bridge.sendSpeakerEnrollResult(ok = ok, reason = reason, embeddings = embeddings, dim = dim, source = source)
+        val onEnded = enrollmentEnded
+        enrollmentEnded = null
+        onEnded?.invoke()
+    }
+
+    // MARK: - Model loading (executor thread only)
+
+    private fun ensureExtractor() {
+        if (extractor != null || extractorLoadAttempted) return
+        extractorLoadAttempted = true
+        val modelPath = resolvedModelPath()
+        if (modelPath == null || !File(modelPath).exists()) {
+            Bridge.log("SpeakerID: model not downloaded yet — engine idle")
+            return
+        }
+        try {
+            val config = SpeakerEmbeddingExtractorConfig(modelPath, 2, false, "cpu")
+            extractor = SpeakerEmbeddingExtractor(null, config)
+            Bridge.log("SpeakerID: embedding extractor initialized (dim=${extractor?.dim()})")
+        } catch (e: Exception) {
+            Bridge.log("SpeakerID: failed to initialize extractor: ${e.message}")
+            extractor = null
+        } catch (e: LinkageError) {
+            Bridge.log("SpeakerID: sherpa-onnx runtime unavailable: ${e.message}")
+            extractor = null
+        }
+    }
+
+    /**
+     * Same Documents-relative reconstruction as the STT model path (the app
+     * id changes between dev builds; iOS parity).
+     */
+    private fun resolvedModelPath(): String? {
+        return try {
+            val context = Bridge.getContext()
+            val prefs = context.getSharedPreferences(PREFS_NAME, android.content.Context.MODE_PRIVATE)
+            val storedPath = prefs.getString(KEY_MODEL_PATH, null) ?: return null
+            if (storedPath.contains("/Documents/")) {
+                File(context.filesDir, storedPath.substringAfter("/Documents/")).absolutePath
+            } else {
+                storedPath
+            }
+        } catch (e: Exception) {
+            null
+        }
+    }
+
+    // MARK: - Helpers
+
+    private fun resetWindow() {
+        windowSamples = ArrayList()
+        windowStartMs = 0
+        windowVoicedMs = 0.0
+        trailingSilenceMs = 0.0
+    }
+
+    private fun toFloatArray(pcmData: ByteArray): FloatArray {
+        val count = pcmData.size / 2
+        val samples = FloatArray(count)
+        val buffer = ByteBuffer.wrap(pcmData).order(ByteOrder.LITTLE_ENDIAN)
+        for (i in 0 until count) {
+            samples[i] = buffer.short / 32768.0f
+        }
+        return samples
+    }
+
+    private fun rms(samples: FloatArray): Float {
+        if (samples.isEmpty()) return 0f
+        var sum = 0f
+        for (s in samples) sum += s * s
+        return sqrt(sum / samples.size)
+    }
+
+    private fun dot(a: FloatArray, b: FloatArray): Float {
+        if (a.size != b.size) return 0f
+        var sum = 0f
+        for (i in a.indices) sum += a[i] * b[i]
+        return sum
+    }
+
+    private fun l2Normalize(v: FloatArray): FloatArray {
+        var sum = 0f
+        for (x in v) sum += x * x
+        val norm = sqrt(sum)
+        if (norm <= 0f) return v
+        return FloatArray(v.size) { v[it] / norm }
+    }
+
+    private fun encodeBase64(embedding: FloatArray): String {
+        val bytes = ByteBuffer.allocate(embedding.size * 4).order(ByteOrder.LITTLE_ENDIAN)
+        for (f in embedding) bytes.putFloat(f)
+        return Base64.encodeToString(bytes.array(), Base64.NO_WRAP)
+    }
+
+    fun decodeEmbedding(base64: String): FloatArray? {
+        return try {
+            val bytes = Base64.decode(base64, Base64.DEFAULT)
+            if (bytes.isEmpty() || bytes.size % 4 != 0) return null
+            val buffer = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN)
+            FloatArray(bytes.size / 4) { buffer.getFloat(it * 4) }
+        } catch (e: IllegalArgumentException) {
+            null
+        }
+    }
+}