@glassly/engine 0.1.0-dev.0

package/src/services/SpeakerLabelResolver.ts

@@ -0,0 +1,140 @@
+/**
+ * SpeakerLabelResolver — engine wiring around SpeakerSegmentResolver.
+ *
+ * Subscribes to native `speaker_detected` bridge events (on-device speaker
+ * identification windows) and annotates cloud transcription events with a
+ * user/other/unknown determination — WITHOUT mutating the raw provider
+ * `speakerId` label. The two label spaces stay distinguishable; resolution is
+ * additive.
+ *
+ * Voice recognition is owner-only (the store holds the user's own "Your
+ * voice" profile), so the determination is framed as `isUser`:
+ *   - matched the enrolled voice            → isUser: true
+ *   - confidently a different voice         → isUser: false
+ *   - ambiguous / no evidence               → no annotation (honest unknown)
+ *
+ * "Confidently different" uses a second, lower similarity gate: a no-match
+ * window only counts as OTHER evidence when its cosine similarity is clearly
+ * below the enrolled centroid (OTHER_MAX_SIMILARITY), not merely below the
+ * native match threshold (0.45). Windows in the ambiguous band in between —
+ * the user's own degraded audio (wind, short utterances, overlap) lands there
+ * — assert nothing.
+ *
+ * Cloud segment times (`startMs`/`endMs`) are stream-relative, not epoch, so
+ * segments are anchored on the JS receive time: the segment is assumed to
+ * have just ended at receipt, spanning [receivedAt - duration, receivedAt].
+ * Transport latency makes this approximate; the resolver's overlap voting
+ * absorbs it (accepted v1 caveat — no sample-accurate alignment).
+ */
+
+import BluetoothSdk from "@glassly/bluetooth-sdk/internal"
+
+import {SpeakerSegmentResolver} from "./SpeakerSegmentResolver"
+
+/** Clamp for implausible provider durations (protects the epoch estimate). */
+const MAX_SEGMENT_DURATION_MS = 30_000
+
+/**
+ * No-match windows at or below this cosine similarity are affirmative OTHER
+ * evidence. The model's benchmarked cross-speaker range tops out near 0.46
+ * and same-speaker scores sit well above it; 0.35 leaves an ambiguous band
+ * (0.35–0.45, up to the native match threshold) where degraded own-voice
+ * windows fail to match without being mislabeled as someone else.
+ */
+export const OTHER_MAX_SIMILARITY = 0.35
+
+export interface CloudSegmentLike {
+  isFinal: boolean
+  speakerId?: string
+  startMs?: number
+  endMs?: number
+  durationMs?: number
+}
+
+export interface SpeakerDetermination {
+  /**
+   * true = the enrolled voice (the user themself); false = confidently a
+   * different voice. Unknown segments never produce a determination.
+   */
+  isUser: boolean
+  /** Enrolled profile id when isUser (stable opaque id for downstream). */
+  resolvedSpeakerId?: string
+  /** Enrolled display name when isUser — on-device PII, stripped before miniapp fan-out. */
+  resolvedSpeakerName?: string
+}
+
+class SpeakerLabelResolver {
+  private readonly resolver = new SpeakerSegmentResolver()
+  private sub: {remove: () => void} | null = null
+
+  start(): void {
+    if (this.sub) return
+    this.sub = BluetoothSdk.addListener("speaker_detected", (event) => {
+      const profileId = typeof event.speakerId === "string" && event.speakerId.length > 0 ? event.speakerId : null
+      const name = typeof event.name === "string" && event.name.length > 0 ? event.name : null
+      const similarity = typeof event.similarity === "number" ? event.similarity : null
+      const confidentOther = profileId === null && similarity !== null && similarity <= OTHER_MAX_SIMILARITY
+      console.log(
+        `SpeakerID: window ${profileId ? (name ?? "user") : confidentOther ? "other" : "ambiguous"} ` +
+          `sim=${similarity?.toFixed?.(3) ?? similarity} ` +
+          `span=${event.endMs - event.startMs}ms endAge=${Date.now() - event.endMs}ms`,
+      )
+      this.resolver.addWindow({profileId, name, confidentOther, startMs: event.startMs, endMs: event.endMs})
+    })
+  }
+
+  stop(): void {
+    this.sub?.remove()
+    this.sub = null
+    this.resolver.reset()
+  }
+
+  /** Per audio session: drop identification evidence and carry-forward state. */
+  reset(): void {
+    this.resolver.reset()
+  }
+
+  /**
+   * Resolve a FINAL cloud segment received just now. Returns null when there
+   * is no confident determination for it.
+   */
+  resolveCloudSegment(d: CloudSegmentLike, receivedAtMs: number = Date.now()): SpeakerDetermination | null {
+    if (!d.isFinal) return null
+    const rawDuration = d.durationMs ?? (d.endMs != null && d.startMs != null ? d.endMs - d.startMs : 0)
+    const duration = Math.min(Math.max(rawDuration, 0), MAX_SEGMENT_DURATION_MS)
+    const resolution = this.resolver.resolve({
+      startMs: receivedAtMs - duration,
+      endMs: receivedAtMs,
+      label: d.speakerId,
+    })
+    const determination: SpeakerDetermination | null = resolution
+      ? resolution.resolvedSpeakerId != null
+        ? // Enrollment is owner-only (speakers.enrollSelf), so any enrolled
+          // match is the user — no owner-id lookup needed.
+          {
+            isUser: true,
+            resolvedSpeakerId: resolution.resolvedSpeakerId,
+            resolvedSpeakerName: resolution.resolvedSpeakerName ?? undefined,
+          }
+        : {isUser: false}
+      : null
+    console.log(
+      `SpeakerID: cloud final label=${d.speakerId ?? "-"} dur=${duration}ms → ` +
+        (determination ? (determination.isUser ? "user" : "other") : "unknown"),
+    )
+    return determination
+  }
+
+  /**
+   * Local-STT transcripts are stamped natively (SpeakerIdEngine.recentMatch →
+   * `resolvedSpeakerId`); that join only produces positive matches, so this
+   * adds `isUser: true` when the native stamp is present and nothing else.
+   */
+  annotateLocalTranscript<T extends {resolvedSpeakerId?: unknown}>(event: T): T & {isUser?: boolean} {
+    const matched = typeof event.resolvedSpeakerId === "string" && event.resolvedSpeakerId.length > 0
+    return matched ? {...event, isUser: true} : event
+  }
+}
+
+const speakerLabelResolver = new SpeakerLabelResolver()
+export default speakerLabelResolver

package/src/services/SpeakerProfileStore.ts

@@ -0,0 +1,265 @@
+/**
+ * SpeakerProfileStore — device-local voice profiles for on-device speaker
+ * identification (Settings > Voice Recognition). Since v2 the product surface
+ * is owner-only ("Your voice"), enrolled through `speakers.enrollSelf`; the
+ * store keeps its generic multi-profile shape underneath.
+ *
+ * Voice embeddings are biometric data: profiles are NEVER synced (no
+ * saveOnServer anywhere near this store), mirroring the repo's device-local
+ * conventions for memory and API keys. Raw enrollment audio is discarded by
+ * the native layer; only embeddings and their mean centroid persist here.
+ *
+ * Pure class + injected backend (same pattern as MemoryStore) so the store
+ * stays dependency-free for tests; speakerProfileStoreInstance.ts binds MMKV
+ * and the user-id resolver.
+ */
+
+import {embeddingFromBase64, embeddingToBase64, meanEmbedding} from "../utils/speakerEmbedding"
+
+const KEY_ROOT = "speaker_profiles_"
+const MAX_PROFILES = 50
+
+/** The model whose embedding space current profiles live in. Embeddings from
+ *  different models are incompatible — a model swap must trigger re-enrollment
+ *  (detectable via each profile's `modelId`). */
+export const SPEAKER_ID_MODEL_ID = "3dspeaker_speech_campplus_sv_zh_en_16k-common_advanced"
+export const SPEAKER_ID_EMBEDDING_DIM = 192
+
+export type SpeakerSampleSource = "glasses" | "phone" | "unknown"
+
+export interface SpeakerSample {
+  /** base64 Float32Array (little-endian), length = dim. */
+  embedding: string
+  source: SpeakerSampleSource
+  createdAt: number
+}
+
+export interface SpeakerProfile {
+  /** "spk_"-prefixed id — this is what flows downstream as a speaker id. */
+  id: string
+  /** Display name, user-entered. On-device PII — never exposed to miniapps. */
+  name: string
+  modelId: string
+  dim: number
+  /** base64 Float32Array — the matching vector (mean of samples, normalized). */
+  centroid: string
+  samples: SpeakerSample[]
+  createdAt: number
+  updatedAt: number
+}
+
+export interface SpeakerProfileStoreState {
+  /**
+   * v2 = owner-only voice recognition (2026-08-10): the store holds the
+   * device owner's "Your voice" profile, enrolled via `speakers.enrollSelf`.
+   * v1 multi-speaker rosters are cleared on load rather than migrated — the
+   * roster UI is gone and a v1 profile can't be assumed to be the owner.
+   */
+  version: 2
+  profiles: SpeakerProfile[]
+}
+
+export interface SpeakerProfileStoreBackend {
+  get(key: string): unknown | null
+  set(key: string, value: unknown): void
+  remove(key: string): void
+}
+
+export interface SpeakerProfileStoreHooks {
+  getUserId: () => Promise<string>
+  backend: SpeakerProfileStoreBackend
+  now?: () => number
+}
+
+function makeId(now: number): string {
+  return `spk_${now.toString(36)}${Math.random().toString(36).slice(2, 10)}`
+}
+
+export class SpeakerProfileStore {
+  private profiles: SpeakerProfile[] = []
+  private loadedUserId: string | null = null
+  private loadPromise: Promise<void> | null = null
+  private readonly changeListeners = new Set<() => void>()
+
+  constructor(private readonly hooks: SpeakerProfileStoreHooks) {}
+
+  onChanged(listener: () => void): () => void {
+    this.changeListeners.add(listener)
+    return () => this.changeListeners.delete(listener)
+  }
+
+  async list(): Promise<SpeakerProfile[]> {
+    await this.ensureLoaded()
+    return [...this.profiles]
+  }
+
+  async get(id: string): Promise<SpeakerProfile | null> {
+    await this.ensureLoaded()
+    return this.profiles.find((p) => p.id === id) ?? null
+  }
+
+  /**
+   * Create a profile from enrollment chunk embeddings (base64 Float32Arrays).
+   * The centroid is their normalized mean. Throws when the cap is reached or
+   * no valid embeddings were provided.
+   */
+  async add(name: string, embeddings: string[], source: SpeakerSampleSource): Promise<SpeakerProfile> {
+    const displayName = name.trim()
+    if (!displayName) throw new Error("Speaker name is required")
+    await this.ensureLoaded()
+    if (this.profiles.length >= MAX_PROFILES) {
+      throw new Error(`Speaker profile limit reached (${MAX_PROFILES})`)
+    }
+    const vectors = decodeAll(embeddings)
+    if (vectors.length === 0) throw new Error("No valid embeddings provided")
+
+    const now = this.now()
+    const profile: SpeakerProfile = {
+      id: makeId(now),
+      name: displayName,
+      modelId: SPEAKER_ID_MODEL_ID,
+      dim: vectors[0].length,
+      centroid: embeddingToBase64(meanEmbedding(vectors)),
+      samples: embeddings.map((embedding) => ({embedding, source, createdAt: now})),
+      createdAt: now,
+      updatedAt: now,
+    }
+    this.profiles.push(profile)
+    this.persist()
+    this.emitChanged()
+    return profile
+  }
+
+  /** Append "Improve recognition" samples and recompute the centroid. */
+  async addSamples(id: string, embeddings: string[], source: SpeakerSampleSource): Promise<SpeakerProfile | null> {
+    await this.ensureLoaded()
+    const profile = this.profiles.find((p) => p.id === id)
+    if (!profile) return null
+    const now = this.now()
+    profile.samples.push(...embeddings.map((embedding) => ({embedding, source, createdAt: now})))
+    const vectors = decodeAll(profile.samples.map((s) => s.embedding))
+    if (vectors.length > 0) {
+      profile.centroid = embeddingToBase64(meanEmbedding(vectors))
+      profile.dim = vectors[0].length
+    }
+    profile.updatedAt = now
+    this.persist()
+    this.emitChanged()
+    return profile
+  }
+
+  async rename(id: string, name: string): Promise<boolean> {
+    const displayName = name.trim()
+    if (!displayName) return false
+    await this.ensureLoaded()
+    const profile = this.profiles.find((p) => p.id === id)
+    if (!profile) return false
+    profile.name = displayName
+    profile.updatedAt = this.now()
+    this.persist()
+    this.emitChanged()
+    return true
+  }
+
+  async remove(id: string): Promise<boolean> {
+    await this.ensureLoaded()
+    const before = this.profiles.length
+    this.profiles = this.profiles.filter((p) => p.id !== id)
+    if (this.profiles.length === before) return false
+    this.persist()
+    this.emitChanged()
+    return true
+  }
+
+  /** Ids of profiles enrolled under a different (incompatible) model. */
+  async staleProfileIds(currentModelId: string = SPEAKER_ID_MODEL_ID): Promise<string[]> {
+    await this.ensureLoaded()
+    return this.profiles.filter((p) => p.modelId !== currentModelId).map((p) => p.id)
+  }
+
+  // === Load / persist ===
+
+  private now(): number {
+    return this.hooks.now?.() ?? Date.now()
+  }
+
+  private async ensureLoaded(): Promise<void> {
+    const userId = (await this.hooks.getUserId()).trim()
+    if (!userId) throw new Error("Glassly user identity is unavailable")
+    if (this.loadedUserId === userId) return
+    if (!this.loadPromise) {
+      this.loadPromise = this.loadFor(userId).finally(() => {
+        this.loadPromise = null
+      })
+    }
+    await this.loadPromise
+  }
+
+  private async loadFor(userId: string): Promise<void> {
+    this.profiles = []
+    const raw = this.hooks.backend.get(this.key(userId))
+    const state = parseState(raw)
+    if (state) this.profiles = state.profiles
+    this.loadedUserId = userId
+  }
+
+  private key(userId: string): string {
+    return `${KEY_ROOT}${userId}`
+  }
+
+  private persist(): void {
+    if (this.loadedUserId == null) return
+    const state: SpeakerProfileStoreState = {version: 2, profiles: this.profiles}
+    this.hooks.backend.set(this.key(this.loadedUserId), JSON.stringify(state))
+  }
+
+  private emitChanged(): void {
+    for (const listener of this.changeListeners) {
+      try {
+        listener()
+      } catch (err) {
+        console.warn("SpeakerProfileStore: onChanged listener threw:", err instanceof Error ? err.message : err)
+      }
+    }
+  }
+}
+
+function decodeAll(embeddings: string[]): Float32Array[] {
+  const out: Float32Array[] = []
+  for (const b64 of embeddings) {
+    try {
+      const v = embeddingFromBase64(b64)
+      if (v.length > 0) out.push(v)
+    } catch {
+      // skip malformed sample
+    }
+  }
+  return out
+}
+
+/** Stored value may be a JSON string or a raw object (backend-dependent). */
+function parseState(raw: unknown): SpeakerProfileStoreState | null {
+  try {
+    const value = typeof raw === "string" ? (JSON.parse(raw) as unknown) : raw
+    if (!value || typeof value !== "object") return null
+    const state = value as Record<string, unknown>
+    // version 1 (multi-speaker roster) intentionally falls through to null —
+    // see SpeakerProfileStoreState: old rosters are cleared, not migrated.
+    if (state.version !== 2 || !Array.isArray(state.profiles)) return null
+    return {version: 2, profiles: (state.profiles as unknown[]).filter(isProfile)}
+  } catch {
+    return null
+  }
+}
+
+function isProfile(v: unknown): v is SpeakerProfile {
+  if (!v || typeof v !== "object") return false
+  const p = v as Record<string, unknown>
+  return (
+    typeof p.id === "string" &&
+    typeof p.name === "string" &&
+    typeof p.modelId === "string" &&
+    typeof p.centroid === "string" &&
+    Array.isArray(p.samples)
+  )
+}

package/src/services/SpeakerSegmentResolver.ts

@@ -0,0 +1,194 @@
+/**
+ * SpeakerSegmentResolver — pure per-segment speaker resolution logic.
+ *
+ * Joins on-device speaker-identification windows (native SpeakerIdEngine
+ * `speaker_detected` events) against transcription segments by time overlap.
+ *
+ * Design constraint (see notes/speaker-id-plan.md §3.3): provider diarization
+ * labels (Soniox "Speaker N") are NOT reliable — the same person can flicker
+ * across labels and one label can absorb two people. The label is never an
+ * identity and no label→profile map is kept; it is used only as a hint that
+ * the speaker MAY have changed:
+ *   1. Carry-forward: a segment with no overlapping ID window may inherit the
+ *      previous segment's resolution only if its label is unchanged and the
+ *      gap is small. A label change breaks carry-forward.
+ *   2. Conflict damping: if the resolved identity changes while the label did
+ *      NOT change, a stronger vote is required before switching — this absorbs
+ *      phone/cloud clock skew at turn boundaries without ever letting the
+ *      label veto strong embedding evidence.
+ * Segments without sufficient ID evidence carry no resolution — unknown is
+ * the honest default, never "whatever this label resolved to last time".
+ *
+ * Resolution is tri-state: an enrolled identity, a confident OTHER
+ * (`resolvedSpeakerId: null` — a voice that is affirmatively not enrolled;
+ * windows in the ambiguous similarity band never produce it), or null.
+ *
+ * Clock caveat accepted for v1: phone/cloud alignment is approximate
+ * (transport latency); overlap voting over ≥2 s windows absorbs it. Do not
+ * attempt sample-accurate alignment.
+ */
+
+export interface SpeakerIdWindow {
+  /** Matched profile id, or null when the voice matched no enrolled profile. */
+  profileId: string | null
+  name: string | null
+  /**
+   * For null-profile windows only: true when the similarity was clearly BELOW
+   * the enrolled voice (not merely below the match threshold) — affirmative
+   * evidence of a different person. Null-profile windows without this flag are
+   * ambiguous: they block carry-forward when dominant but assert nothing.
+   */
+  confidentOther?: boolean
+  /** Epoch ms (native clock — same epoch as JS Date.now()). */
+  startMs: number
+  endMs: number
+}
+
+export interface SegmentInput {
+  /** Estimated epoch ms span of the spoken segment. */
+  startMs: number
+  endMs: number
+  /** Raw provider diarization label — hint only, never an identity. */
+  label?: string
+}
+
+export interface SegmentResolution {
+  /**
+   * Matched enrolled profile id — or null for a confident OTHER: the segment
+   * was dominated by a voice that is affirmatively not any enrolled profile.
+   */
+  resolvedSpeakerId: string | null
+  resolvedSpeakerName: string | null
+}
+
+/** Windows older than this (vs. the resolver's newest timestamp) are dropped. */
+export const WINDOW_RETENTION_MS = 60_000
+/** Carry-forward dies when the gap since the previous segment exceeds this. */
+export const CARRY_FORWARD_MAX_GAP_MS = 5_000
+/** A winning vote needs at least this much absolute overlap. */
+export const MIN_WINNING_OVERLAP_MS = 250
+
+interface Candidate {
+  /** "profile" = enrolled match; "other" = confident non-enrolled voice; "unknown" = ambiguous no-match. */
+  kind: "profile" | "other" | "unknown"
+  profileId: string | null
+  name: string | null
+  overlapMs: number
+  windowCount: number
+}
+
+/** Damping-comparison key: OTHER counts as its own identity, distinct from every profile. */
+function identityKey(res: SegmentResolution): string {
+  return res.resolvedSpeakerId ?? "\0other"
+}
+
+interface LastSegmentState {
+  endMs: number
+  label: string | undefined
+  resolution: SegmentResolution | null
+}
+
+export class SpeakerSegmentResolver {
+  private windows: SpeakerIdWindow[] = []
+  private last: LastSegmentState | null = null
+
+  addWindow(window: SpeakerIdWindow): void {
+    if (!(window.endMs > window.startMs)) return
+    this.windows.push(window)
+    const cutoff = window.endMs - WINDOW_RETENTION_MS
+    this.windows = this.windows.filter((w) => w.endMs >= cutoff)
+  }
+
+  /** Per audio session: drop all identification evidence and segment state. */
+  reset(): void {
+    this.windows = []
+    this.last = null
+  }
+
+  /**
+   * Resolve one FINAL segment. Also advances the carry-forward state, so call
+   * exactly once per final segment, in delivery order.
+   */
+  resolve(segment: SegmentInput): SegmentResolution | null {
+    const resolution = this.resolveInner(segment)
+    this.last = {endMs: segment.endMs, label: segment.label, resolution}
+    return resolution
+  }
+
+  private resolveInner(segment: SegmentInput): SegmentResolution | null {
+    const candidates = this.voteCandidates(segment)
+
+    if (candidates.length === 0) {
+      return this.carryForward(segment)
+    }
+
+    candidates.sort((a, b) => b.overlapMs - a.overlapMs)
+    const winner = candidates[0]
+    const runnerUp = candidates[1]
+
+    // Too little overlap to mean anything (clock-skew sliver) — treat as no
+    // evidence rather than a confident unknown.
+    const minOverlap = Math.min(MIN_WINNING_OVERLAP_MS, Math.max(1, (segment.endMs - segment.startMs) * 0.3))
+    if (winner.overlapMs < minOverlap) {
+      return this.carryForward(segment)
+    }
+
+    // Ambiguous-voice-dominant segment: honest null (no carry-forward — a
+    // voice failed to match but wasn't clearly different either).
+    if (winner.kind === "unknown") return null
+
+    // Confident-other-dominant → an OTHER resolution (null id); a profile
+    // winner resolves to that identity. Both pass through conflict damping.
+    const proposed: SegmentResolution =
+      winner.kind === "other"
+        ? {resolvedSpeakerId: null, resolvedSpeakerName: null}
+        : {resolvedSpeakerId: winner.profileId, resolvedSpeakerName: winner.name}
+
+    // Conflict damping: identity flip while the provider label did NOT change
+    // needs a stronger vote (≥2 agreeing windows, or a clear overlap majority)
+    // — otherwise keep the previous binding for this segment. A label change
+    // is the provider hinting the speaker may have changed, so the flip
+    // stands on normal evidence.
+    const prev = this.last?.resolution ?? null
+    const labelUnchanged = this.last != null && this.last.label === segment.label
+    const identityFlipped = prev != null && identityKey(prev) !== identityKey(proposed)
+    if (identityFlipped && labelUnchanged) {
+      const strongEnough =
+        winner.windowCount >= 2 || (runnerUp === undefined ? true : winner.overlapMs >= runnerUp.overlapMs * 2)
+      if (!strongEnough) return prev
+    }
+
+    return proposed
+  }
+
+  private voteCandidates(segment: SegmentInput): Candidate[] {
+    const byKey = new Map<string, Candidate>()
+    for (const w of this.windows) {
+      const overlap = Math.min(segment.endMs, w.endMs) - Math.max(segment.startMs, w.startMs)
+      if (overlap <= 0) continue
+      const kind: Candidate["kind"] = w.profileId != null ? "profile" : w.confidentOther ? "other" : "unknown"
+      const key = w.profileId ?? `\0${kind}`
+      const existing = byKey.get(key)
+      if (existing) {
+        existing.overlapMs += overlap
+        existing.windowCount += 1
+      } else {
+        byKey.set(key, {kind, profileId: w.profileId, name: w.name, overlapMs: overlap, windowCount: 1})
+      }
+    }
+    return [...byKey.values()]
+  }
+
+  /**
+   * No overlapping ID window (short interjection, window not flushed yet):
+   * inherit the previous segment's resolution only while the label is
+   * unchanged and the gap is small. A label change is the "speaker may have
+   * changed" hint — the segment stays unresolved.
+   */
+  private carryForward(segment: SegmentInput): SegmentResolution | null {
+    if (!this.last?.resolution) return null
+    if (this.last.label !== segment.label) return null
+    if (segment.startMs - this.last.endMs > CARRY_FORWARD_MAX_GAP_MS) return null
+    return this.last.resolution
+  }
+}