@glassly/engine 0.1.0-dev.0

package/src/services/soniox/SonioxDirectService.ts

@@ -0,0 +1,517 @@
+/**
+ * SonioxDirectService
+ *
+ * Phone → Soniox direct transcription with a user-supplied API key
+ * (Settings > Keys). Enabled iff the `soniox_api_key` setting is non-empty.
+ *
+ * While enabled, LocalMiniappRuntime routes transcription/translation
+ * subscriptions HERE instead of to the cloud (and pushes an empty cloud
+ * subscription set, which also silences the LC3 audio uplink) — audio for
+ * transcription never touches the Glassly cloud. This service opens one
+ * Soniox realtime WebSocket session per subscription, feeds it decoded PCM
+ * from the glasses mic (`mic_pcm`), assembles tokens into utterances, and
+ * emits the same TranscriptionData / TranslationData shapes the cloud client
+ * emits so the miniapp fan-out is provider-agnostic.
+ *
+ * Failure policy (per product decision): a failing key degrades to on-device
+ * sherpa STT via LocalSttFallbackCoordinator — never back to the cloud.
+ * Reconnect semantics mirror the cloud provider: transient errors back off
+ * 500ms → 8s for 10 attempts, billing/auth (non-transient) errors retry on a
+ * slow 60s cadence forever; after transient attempts are exhausted the slow
+ * cadence takes over too (the phone has no worker-recreate path of last
+ * resort, so it must self-recover).
+ */
+
+import BluetoothSdk from "@glassly/bluetooth-sdk/internal"
+import type {AudioSubscription, TranscriptionData, TranslationData} from "@glassly/cloud-protocol"
+
+import {SETTINGS, useSettingsStore} from "../../stores/settings"
+import {BgTimer} from "../../utils/timers"
+import localSttFallbackCoordinator from "../LocalSttFallbackCoordinator"
+import {
+  SonioxClientError,
+  SonioxRealtimeClient,
+  validateSonioxApiKey,
+  type SonioxSessionOptions,
+} from "./SonioxRealtimeClient"
+import {SonioxTokenAssembler} from "./SonioxTokenAssembler"
+
+const LOG_TAG = "SONIOX_DIRECT"
+
+const RECONNECT_BASE_MS = 500
+const RECONNECT_MAX_MS = 8_000
+const RECONNECT_MAX_ATTEMPTS = 10
+const NON_TRANSIENT_RETRY_MS = 60_000
+
+export type SonioxDirectState = "inactive" | "connecting" | "active" | "error"
+export type SonioxDirectErrorKind = "invalid_key" | "quota" | "unreachable"
+
+export interface SonioxDirectStatus {
+  enabled: boolean
+  state: SonioxDirectState
+  errorKind?: SonioxDirectErrorKind
+  lastErrorAt?: number
+}
+
+interface SessionSlot {
+  key: string
+  sub: AudioSubscription
+  client: SonioxRealtimeClient | null
+  assembler: SonioxTokenAssembler
+  open: boolean
+  /** ≥1 failed connect/drop since the last successful open. */
+  failed: boolean
+  attempts: number
+  nonTransient: boolean
+  errorKind?: SonioxDirectErrorKind
+  retryTimer: number | null
+}
+
+function subLanguage(sub: AudioSubscription): {language: string; hints?: string[]} {
+  const source = sub.kind === "transcription" ? sub.language : sub.source
+  if (source.mode === "specific") return {language: source.code}
+  return {language: "auto", hints: source.hints}
+}
+
+function classifyError(err: SonioxClientError): SonioxDirectErrorKind {
+  if (err.code === 401 || err.code === 403) return "invalid_key"
+  if (err.code === 402 || err.code === 429 || err.code === "quota_exceeded") return "quota"
+  return "unreachable"
+}
+
+class SonioxDirectService {
+  private static instance: SonioxDirectService | null = null
+
+  private started = false
+  private apiKey = ""
+  private desiredSubs: AudioSubscription[] = []
+  private transcriptionHints: string[] = []
+  private slots = new Map<string, SessionSlot>()
+
+  private micSub: {remove: () => void} | null = null
+  private unsubscribeSettings: (() => void) | null = null
+
+  private transcriptListeners = new Set<(d: TranscriptionData) => void>()
+  private translationListeners = new Set<(d: TranslationData) => void>()
+  private statusListeners = new Set<(s: SonioxDirectStatus) => void>()
+  private lastStatus: SonioxDirectStatus = {enabled: false, state: "inactive"}
+  private lastErrorAt: number | undefined
+
+  private constructor() {}
+
+  /** Test seam: swap the realtime-client constructor (bun module mocks leak
+   *  across test files, so injection beats mock.module here). */
+  private clientFactory: (opts: SonioxSessionOptions) => SonioxRealtimeClient = (opts) => new SonioxRealtimeClient(opts)
+  setClientFactoryForTests(factory: ((opts: SonioxSessionOptions) => SonioxRealtimeClient) | null): void {
+    this.clientFactory = factory ?? ((opts) => new SonioxRealtimeClient(opts))
+  }
+
+  static getInstance(): SonioxDirectService {
+    if (!SonioxDirectService.instance) {
+      SonioxDirectService.instance = new SonioxDirectService()
+    }
+    return SonioxDirectService.instance
+  }
+
+  // ===========================================================================
+  // Lifecycle
+  // ===========================================================================
+
+  start(): void {
+    if (this.started) return
+    this.started = true
+    this.apiKey = this.readApiKey()
+    this.unsubscribeSettings = useSettingsStore.subscribe(
+      (s) => s.settings[SETTINGS.soniox_api_key.key] as string | undefined,
+      () => {
+        const next = this.readApiKey()
+        if (next === this.apiKey) return
+        const enabledChanged = !!next !== !!this.apiKey
+        this.apiKey = next
+        console.log(`${LOG_TAG}: api key ${next ? "set" : "cleared"}`)
+        // Existing sessions carry the old key in their config; rebuild.
+        this.teardownAllSlots()
+        this.reconcile()
+        if (enabledChanged) {
+          // Subscriptions must move between cloud and direct; the runtime
+          // re-partitions and calls setDesiredSubscriptions back into us.
+          this.routingChangedListener?.()
+        }
+      },
+    )
+    this.reconcile()
+  }
+
+  stop(): void {
+    if (!this.started) return
+    this.started = false
+    this.unsubscribeSettings?.()
+    this.unsubscribeSettings = null
+    this.teardownAllSlots()
+    this.reconcile()
+  }
+
+  private readApiKey(): string {
+    const raw = useSettingsStore.getState().getSetting(SETTINGS.soniox_api_key.key)
+    return typeof raw === "string" ? raw.trim() : ""
+  }
+
+  // ===========================================================================
+  // Public surface
+  // ===========================================================================
+
+  isEnabled(): boolean {
+    return this.started && this.apiKey.length > 0
+  }
+
+  /**
+   * Whether the primary (Soniox-direct) transcription path is believed usable.
+   * False once any desired session has failed and none are open — that is the
+   * signal for the sherpa fallback. While disabled this is vacuously true
+   * (callers should gate on isEnabled()).
+   */
+  isHealthy(): boolean {
+    if (!this.isEnabled()) return true
+    if (this.slots.size === 0) return true
+    let anyOpen = false
+    let anyFailed = false
+    for (const slot of this.slots.values()) {
+      if (slot.open) anyOpen = true
+      if (slot.failed) anyFailed = true
+    }
+    return anyOpen || !anyFailed
+  }
+
+  getStatus(): SonioxDirectStatus {
+    return this.lastStatus
+  }
+
+  onTranscript(cb: (d: TranscriptionData) => void): () => void {
+    this.transcriptListeners.add(cb)
+    return () => this.transcriptListeners.delete(cb)
+  }
+
+  onTranslation(cb: (d: TranslationData) => void): () => void {
+    this.translationListeners.add(cb)
+    return () => this.translationListeners.delete(cb)
+  }
+
+  onStatusChanged(cb: (s: SonioxDirectStatus) => void): () => void {
+    this.statusListeners.add(cb)
+    return () => this.statusListeners.delete(cb)
+  }
+
+  /** Registered by LocalMiniappRuntime; invoked when enablement flips so the
+   *  runtime re-partitions subscriptions between cloud and direct. A callback
+   *  (not an import) to avoid a service import cycle. */
+  private routingChangedListener: (() => void) | null = null
+  setRoutingChangedListener(cb: (() => void) | null): void {
+    this.routingChangedListener = cb
+  }
+
+  /**
+   * The full transcription/translation subscription set this service should
+   * serve (same AudioSubscription[] the cloud would otherwise get), plus the
+   * union of auto-mode detection hints. Called by LocalMiniappRuntime on every
+   * subscription recompute; an empty array tears everything down.
+   */
+  setDesiredSubscriptions(subs: AudioSubscription[], transcriptionHints: string[] = []): void {
+    this.desiredSubs = subs
+    this.transcriptionHints = transcriptionHints
+    this.reconcile()
+  }
+
+  validateApiKey(key: string): Promise<{ok: true} | {ok: false; reason: "invalid_key" | "network"}> {
+    return validateSonioxApiKey(key.trim())
+  }
+
+  // ===========================================================================
+  // Session reconciliation
+  // ===========================================================================
+
+  private subKey(sub: AudioSubscription): string {
+    return sub.kind === "transcription"
+      ? `tx:${JSON.stringify(sub.language)}`
+      : `tr:${JSON.stringify(sub.source)}:${sub.target}`
+  }
+
+  private reconcile(): void {
+    const desired = new Map<string, AudioSubscription>()
+    if (this.isEnabled()) {
+      for (const sub of this.desiredSubs) desired.set(this.subKey(sub), sub)
+    }
+
+    for (const [key, slot] of this.slots) {
+      if (!desired.has(key)) {
+        this.teardownSlot(slot)
+        this.slots.delete(key)
+      }
+    }
+    for (const [key, sub] of desired) {
+      if (!this.slots.has(key)) {
+        this.slots.set(key, this.createSlot(key, sub))
+      }
+    }
+
+    this.syncMicListener()
+    this.publishStatus()
+  }
+
+  private createSlot(key: string, sub: AudioSubscription): SessionSlot {
+    const {language} = subLanguage(sub)
+    const isTranslation = sub.kind === "translation"
+    const slot: SessionSlot = {
+      key,
+      sub,
+      client: null,
+      assembler: null as unknown as SonioxTokenAssembler,
+      open: false,
+      failed: false,
+      attempts: 0,
+      nonTransient: false,
+      retryTimer: null,
+    }
+    slot.assembler = new SonioxTokenAssembler({
+      scope: key,
+      isTranslation,
+      fallbackLanguage: language === "auto" ? undefined : language,
+      onTranscript: (t) => this.deliver(slot, t),
+    })
+    this.connectSlot(slot)
+    return slot
+  }
+
+  private connectSlot(slot: SessionSlot): void {
+    const {language, hints} = subLanguage(slot.sub)
+    const client = this.clientFactory({
+      apiKey: this.apiKey,
+      language,
+      languageHints: hints ?? (language === "auto" ? this.transcriptionHints : undefined),
+      targetLanguage: slot.sub.kind === "translation" ? slot.sub.target : undefined,
+      onOpen: () => {
+        slot.open = true
+        slot.failed = false
+        slot.attempts = 0
+        slot.nonTransient = false
+        slot.errorKind = undefined
+        console.log(`${LOG_TAG}: session open ${slot.key}`)
+        this.publishStatus()
+      },
+      onResult: (result) => slot.assembler.handleResult(result),
+      onEndpoint: () => slot.assembler.handleEndpoint(),
+      onFinalized: () => slot.assembler.handleFinalized(),
+      onError: (err) => this.handleSlotError(slot, err),
+      onClose: () => this.handleSlotClosed(slot),
+    })
+    slot.client = client
+    client.connect()
+  }
+
+  private handleSlotError(slot: SessionSlot, err: SonioxClientError): void {
+    if (!this.slots.has(slot.key)) return
+    slot.errorKind = classifyError(err)
+    slot.nonTransient = err.nonTransient
+    this.lastErrorAt = Date.now()
+    console.warn(`${LOG_TAG}: session error ${slot.key} code=${err.code ?? "?"} ${err.message}`)
+    // A server-side error message usually precedes the socket dying; close
+    // proactively (close() suppresses the client's own onClose callback) and
+    // drive the retry from here.
+    slot.client?.close()
+    this.scheduleReconnect(slot)
+  }
+
+  private handleSlotClosed(slot: SessionSlot): void {
+    if (!this.slots.has(slot.key)) return
+    this.scheduleReconnect(slot)
+  }
+
+  private scheduleReconnect(slot: SessionSlot): void {
+    if (slot.retryTimer !== null) return
+    slot.open = false
+    slot.failed = true
+    slot.client = null
+
+    let delay: number
+    if (slot.nonTransient) {
+      slot.attempts = 0
+      delay = NON_TRANSIENT_RETRY_MS
+    } else {
+      slot.attempts += 1
+      if (slot.attempts > RECONNECT_MAX_ATTEMPTS) {
+        // Unlike the cloud worker (whose detach/attach recreates providers),
+        // nothing else will ever rebuild these sessions — fall back to the
+        // slow cadence instead of giving up outright.
+        delay = NON_TRANSIENT_RETRY_MS
+      } else {
+        delay = Math.min(RECONNECT_MAX_MS, RECONNECT_BASE_MS * 2 ** (slot.attempts - 1))
+      }
+    }
+    console.log(`${LOG_TAG}: reconnect ${slot.key} attempt=${slot.attempts} delayMs=${delay}`)
+    this.publishStatus()
+
+    slot.retryTimer = BgTimer.setTimeout(() => {
+      slot.retryTimer = null
+      if (!this.slots.has(slot.key) || !this.isEnabled()) return
+      this.connectSlot(slot)
+    }, delay)
+  }
+
+  private teardownSlot(slot: SessionSlot): void {
+    if (slot.retryTimer !== null) {
+      BgTimer.clearTimeout(slot.retryTimer)
+      slot.retryTimer = null
+    }
+    // Flush the in-flight utterance so a mid-update caption commits rather
+    // than dangling (mirrors the cloud provider's close()).
+    try {
+      slot.assembler.flush()
+    } catch {
+      /* best-effort */
+    }
+    slot.client?.close()
+    slot.client = null
+    slot.open = false
+  }
+
+  private teardownAllSlots(): void {
+    for (const slot of this.slots.values()) this.teardownSlot(slot)
+    this.slots.clear()
+  }
+
+  // ===========================================================================
+  // Audio feed
+  // ===========================================================================
+
+  private syncMicListener(): void {
+    const wantMic = this.slots.size > 0
+    if (wantMic && !this.micSub) {
+      this.micSub = BluetoothSdk.addListener("mic_pcm", (event: {pcm: ArrayBuffer}) => {
+        if (!event?.pcm) return
+        for (const slot of this.slots.values()) {
+          slot.client?.sendAudio(event.pcm)
+        }
+      })
+    } else if (!wantMic && this.micSub) {
+      this.micSub.remove()
+      this.micSub = null
+    }
+  }
+
+  // ===========================================================================
+  // Transcript delivery
+  // ===========================================================================
+
+  /**
+   * Map an assembled utterance to the cloud wire shapes. Language resolution
+   * mirrors cloud result.ts: a specific-language subscription's code WINS over
+   * the provider-detected code (miniapp handlers key on the exact stream, and
+   * Soniox reports bare ISO codes while subscriptions carry BCP-47 tags).
+   */
+  private deliver(
+    slot: SessionSlot,
+    t: {
+      text: string
+      isFinal: boolean
+      utteranceId?: string
+      speakerId?: string
+      language?: string
+      originalText?: string
+      sourceLanguage?: string
+      startMs?: number
+      endMs?: number
+    },
+  ): void {
+    const timestamp = Date.now()
+    const startMs = t.startMs ?? 0
+    const endMs = t.endMs ?? 0
+    const durationMs = t.startMs != null && t.endMs != null ? t.endMs - t.startMs : undefined
+
+    if (slot.sub.kind === "translation") {
+      const source = slot.sub.source
+      const resolved =
+        source.mode === "specific"
+          ? {language: source.code, detected: false}
+          : {language: t.sourceLanguage ?? "auto", detected: true}
+      const data: TranslationData = {
+        userId: "local",
+        subscription: slot.sub,
+        text: t.text,
+        originalText: t.originalText,
+        isFinal: t.isFinal,
+        utteranceId: t.utteranceId,
+        speakerId: t.speakerId,
+        startMs,
+        endMs,
+        durationMs,
+        source: {language: resolved.language, detected: resolved.detected},
+        target: {language: slot.sub.target},
+        provider: "soniox-direct",
+        timestamp,
+      }
+      for (const cb of this.translationListeners) cb(data)
+      return
+    }
+
+    const source = slot.sub.language
+    const resolved =
+      source.mode === "specific"
+        ? {language: source.code, detected: false}
+        : {language: t.language ?? "auto", detected: true}
+    const data: TranscriptionData = {
+      userId: "local",
+      subscription: slot.sub,
+      text: t.text,
+      isFinal: t.isFinal,
+      utteranceId: t.utteranceId,
+      speakerId: t.speakerId,
+      startMs,
+      endMs,
+      durationMs,
+      resolvedLanguage: resolved.language,
+      languageDetected: resolved.detected,
+      tokens: [],
+      provider: "soniox-direct",
+      timestamp,
+    }
+    for (const cb of this.transcriptListeners) cb(data)
+  }
+
+  // ===========================================================================
+  // Status
+  // ===========================================================================
+
+  private computeStatus(): SonioxDirectStatus {
+    const enabled = this.isEnabled()
+    if (!enabled) return {enabled: false, state: "inactive"}
+    if (this.slots.size === 0) return {enabled: true, state: "inactive"}
+
+    let anyOpen = false
+    let errorKind: SonioxDirectErrorKind | undefined
+    let anyFailed = false
+    for (const slot of this.slots.values()) {
+      if (slot.open) anyOpen = true
+      if (slot.failed) anyFailed = true
+      // invalid_key > quota > unreachable, so the UI surfaces the actionable one.
+      if (slot.errorKind === "invalid_key" || (slot.errorKind && errorKind !== "invalid_key")) {
+        errorKind = errorKind === "quota" && slot.errorKind === "unreachable" ? errorKind : slot.errorKind
+      }
+    }
+    if (anyOpen) return {enabled: true, state: "active"}
+    if (anyFailed) return {enabled: true, state: "error", errorKind, lastErrorAt: this.lastErrorAt}
+    return {enabled: true, state: "connecting"}
+  }
+
+  private publishStatus(): void {
+    const next = this.computeStatus()
+    const prev = this.lastStatus
+    const changed = next.enabled !== prev.enabled || next.state !== prev.state || next.errorKind !== prev.errorKind
+    this.lastStatus = next
+    if (!changed) return
+    console.log(`${LOG_TAG}: status -> ${next.state}${next.errorKind ? ` (${next.errorKind})` : ""}`)
+    localSttFallbackCoordinator.onSonioxDirectStateChange(this.isEnabled(), this.isHealthy())
+    for (const cb of this.statusListeners) cb(next)
+  }
+}
+
+const sonioxDirectService = SonioxDirectService.getInstance()
+export default sonioxDirectService