@glassly/engine 0.1.0-dev.0

package/src/services/soniox/SonioxRealtimeClient.ts

@@ -0,0 +1,349 @@
+/**
+ * SonioxRealtimeClient
+ *
+ * Minimal React Native WebSocket client for Soniox realtime STT
+ * (wss://stt-rt.soniox.com/transcribe-websocket). Used by SonioxDirectService
+ * when the user supplies their own Soniox API key: audio streams from the
+ * phone straight to Soniox, bypassing the Glassly cloud.
+ *
+ * Deliberately hand-rolled rather than depending on `@soniox/node` (node HTTP
+ * machinery) or `@soniox/web` (MediaStream capture): the wire protocol is one
+ * JSON config message, then binary PCM frames up / JSON token results down.
+ * Protocol shape mirrors the vendored SDK's buildConfigMessage /
+ * parseResultMessage (cloud-v2 runtime's provider is the reference
+ * implementation: cloud-v2/packages/runtime/src/services/audio/providers/soniox.ts).
+ */
+
+import {BgTimer} from "../../utils/timers"
+
+export const SONIOX_STT_WS_URL = "wss://stt-rt.soniox.com/transcribe-websocket"
+export const SONIOX_MODEL = "stt-rt-v4"
+/** REST endpoint used only for cheap key validation (Save & Test). */
+export const SONIOX_MODELS_URL = "https://api.soniox.com/v1/models"
+
+const CONNECT_TIMEOUT_MS = 20_000
+const KEEPALIVE_INTERVAL_MS = 5_000
+
+export interface SonioxToken {
+  text: string
+  start_ms?: number
+  end_ms?: number
+  confidence?: number
+  is_final: boolean
+  speaker?: string
+  language?: string
+  translation_status?: "none" | "original" | "translation"
+  source_language?: string
+}
+
+export interface SonioxResult {
+  tokens: SonioxToken[]
+  finished?: boolean
+}
+
+export class SonioxClientError extends Error {
+  readonly code: number | string | undefined
+  /**
+   * Billing/auth failures (bad key, quota exhausted) are not healed by
+   * reconnecting — Soniox accepts the socket then rejects the session.
+   * Mirrors the cloud provider's isNonTransientProviderError: 401/402/403
+   * plus the quota codes (the raw wire uses 429 for quota).
+   */
+  readonly nonTransient: boolean
+
+  constructor(message: string, code?: number | string) {
+    super(message)
+    this.name = "SonioxClientError"
+    this.code = code
+    this.nonTransient = code === 401 || code === 402 || code === 403 || code === 429 || code === "quota_exceeded"
+  }
+}
+
+export interface SonioxSessionOptions {
+  apiKey: string
+  /** Subscription language ("auto", "en-US", ...). Hints are derived from it. */
+  language?: string
+  /** Detection hints for auto-mode subscriptions (already bare or BCP-47; normalized here). */
+  languageHints?: string[]
+  /** When set, configure Soniox one-way translation to this target (BCP-47 ok; normalized). */
+  targetLanguage?: string
+  onOpen?: () => void
+  onResult: (result: SonioxResult) => void
+  onEndpoint: () => void
+  onFinalized: () => void
+  onError: (err: SonioxClientError) => void
+  onClose?: () => void
+  /** Test seam: WebSocket constructor override. */
+  webSocketFactory?: (url: string) => WebSocket
+}
+
+/**
+ * Build Soniox `language_hints`. A specific language is its own hint (region
+ * stripped: "en-US" -> "en"). For "auto" the miniapp's detection hints are
+ * reduced to bare codes and deduped. Passing a BCP-47 tag or unknown code
+ * makes Soniox reject the session ("Invalid language hint"), so everything is
+ * normalized here. Port of the cloud provider's sonioxLanguageHints.
+ */
+export function sonioxLanguageHints(language: string | undefined, hints: string[] | undefined): string[] | undefined {
+  const toBareCode = (code: string) => code.split("-")[0].toLowerCase()
+  if (language && language !== "auto") {
+    return [toBareCode(language)]
+  }
+  if (!hints || hints.length === 0) return undefined
+  const bare = [...new Set(hints.map(toBareCode).filter(Boolean))]
+  return bare.length > 0 ? bare : undefined
+}
+
+/**
+ * Soniox one-way translation target takes a bare ISO 639-1 code — a BCP-47
+ * tag ("es-ES") kills the session. Subscription keys keep the full tag; only
+ * the wire config is reduced. Port of the cloud provider's sonioxTranslationTarget.
+ */
+export function sonioxTranslationTarget(target: string): string {
+  return target.split("-")[0].toLowerCase()
+}
+
+/** The first (and only) config message sent after the socket opens. */
+export function buildSonioxConfigMessage(opts: {
+  apiKey: string
+  language?: string
+  languageHints?: string[]
+  targetLanguage?: string
+}): Record<string, unknown> {
+  return {
+    api_key: opts.apiKey,
+    model: SONIOX_MODEL,
+    audio_format: "pcm_s16le",
+    sample_rate: 16_000,
+    num_channels: 1,
+    enable_language_identification: true,
+    enable_endpoint_detection: true,
+    enable_speaker_diarization: true,
+    max_endpoint_delay_ms: 2_000,
+    language_hints: sonioxLanguageHints(opts.language, opts.languageHints),
+    context: {
+      terms: ["Glassly", "Hey Glassly"],
+      text: "Glassly, Hey Glassly (an AI assistant)",
+    },
+    translation: opts.targetLanguage
+      ? {type: "one_way", target_language: sonioxTranslationTarget(opts.targetLanguage)}
+      : undefined,
+  }
+}
+
+/**
+ * One realtime session: connect, stream audio, receive tokens. A dead session
+ * is not reusable — SonioxDirectService replaces it on reconnect.
+ */
+export class SonioxRealtimeClient {
+  private readonly opts: SonioxSessionOptions
+  private ws: WebSocket | null = null
+  private opened = false
+  private closed = false
+  private connectTimer: number | null = null
+  private keepaliveTimer: number | null = null
+  private lastAudioSentAt = 0
+
+  constructor(opts: SonioxSessionOptions) {
+    this.opts = opts
+  }
+
+  isOpen(): boolean {
+    return this.opened && !this.closed
+  }
+
+  connect(): void {
+    if (this.ws) return
+    const factory = this.opts.webSocketFactory ?? ((url: string) => new WebSocket(url))
+    let ws: WebSocket
+    try {
+      ws = factory(SONIOX_STT_WS_URL)
+    } catch (err) {
+      this.opts.onError(new SonioxClientError(`websocket construction failed: ${(err as Error)?.message ?? err}`))
+      return
+    }
+    this.ws = ws
+    ws.binaryType = "arraybuffer"
+
+    this.connectTimer = BgTimer.setTimeout(() => {
+      this.connectTimer = null
+      if (!this.opened && !this.closed) {
+        this.opts.onError(new SonioxClientError("connect timeout", 503))
+        this.close()
+      }
+    }, CONNECT_TIMEOUT_MS)
+
+    ws.onopen = () => {
+      if (this.closed) return
+      this.clearConnectTimer()
+      this.opened = true
+      const config = buildSonioxConfigMessage({
+        apiKey: this.opts.apiKey,
+        language: this.opts.language,
+        languageHints: this.opts.languageHints,
+        targetLanguage: this.opts.targetLanguage,
+      })
+      try {
+        ws.send(JSON.stringify(config))
+      } catch (err) {
+        this.opts.onError(new SonioxClientError(`config send failed: ${(err as Error)?.message ?? err}`))
+        this.close()
+        return
+      }
+      this.startKeepalive()
+      this.opts.onOpen?.()
+    }
+
+    ws.onmessage = (event) => {
+      if (this.closed || typeof event.data !== "string") return
+      this.handleMessage(event.data)
+    }
+
+    ws.onerror = () => {
+      if (this.closed) return
+      // RN's error event carries no useful detail; the close event follows.
+      this.opts.onError(new SonioxClientError("websocket error", 503))
+    }
+
+    ws.onclose = () => {
+      const wasClosed = this.closed
+      this.teardown()
+      if (!wasClosed) this.opts.onClose?.()
+    }
+  }
+
+  /** Binary PCM (s16le 16kHz mono) frame. Dropped when the socket isn't open. */
+  sendAudio(pcm: ArrayBuffer): void {
+    if (!this.isOpen() || !this.ws) return
+    this.lastAudioSentAt = Date.now()
+    try {
+      this.ws.send(pcm)
+    } catch (err) {
+      this.opts.onError(new SonioxClientError(`audio send failed: ${(err as Error)?.message ?? err}`))
+    }
+  }
+
+  finalize(): void {
+    this.sendControl({type: "finalize"})
+  }
+
+  close(): void {
+    if (this.closed) {
+      return
+    }
+    this.closed = true
+    this.clearConnectTimer()
+    this.stopKeepalive()
+    const ws = this.ws
+    this.ws = null
+    this.opened = false
+    if (ws) {
+      try {
+        ws.close()
+      } catch {
+        /* best-effort */
+      }
+    }
+  }
+
+  private handleMessage(data: string): void {
+    let raw: Record<string, unknown>
+    try {
+      raw = JSON.parse(data) as Record<string, unknown>
+    } catch {
+      return
+    }
+    if ("error_code" in raw || "error_message" in raw) {
+      const code = raw.error_code as number | string | undefined
+      const message = typeof raw.error_message === "string" ? raw.error_message : "soniox error"
+      this.opts.onError(new SonioxClientError(message, code))
+      return
+    }
+
+    const allTokens = Array.isArray(raw.tokens) ? (raw.tokens as SonioxToken[]) : []
+    let sawEndpoint = false
+    let sawFinalized = false
+    const tokens: SonioxToken[] = []
+    for (const t of allTokens) {
+      if (typeof t?.text !== "string") continue
+      if (t.text === "<end>") {
+        sawEndpoint = true
+      } else if (t.text === "<fin>") {
+        sawFinalized = true
+      } else {
+        tokens.push(t)
+      }
+    }
+
+    if (tokens.length > 0) {
+      this.opts.onResult({tokens, finished: raw.finished === true})
+    }
+    // Order matters: tokens in the same message belong to the utterance the
+    // endpoint closes, so deliver them before the boundary signal.
+    if (sawEndpoint) this.opts.onEndpoint()
+    if (sawFinalized) this.opts.onFinalized()
+  }
+
+  private sendControl(msg: {type: string}): void {
+    if (!this.isOpen() || !this.ws) return
+    try {
+      this.ws.send(JSON.stringify(msg))
+    } catch {
+      /* best-effort */
+    }
+  }
+
+  private startKeepalive(): void {
+    this.stopKeepalive()
+    this.keepaliveTimer = BgTimer.setInterval(() => {
+      // Only needed when no audio has been sent recently — Soniox times out
+      // idle sessions. Continuous audio makes this a safety net.
+      if (Date.now() - this.lastAudioSentAt >= KEEPALIVE_INTERVAL_MS) {
+        this.sendControl({type: "keepalive"})
+      }
+    }, KEEPALIVE_INTERVAL_MS)
+  }
+
+  private stopKeepalive(): void {
+    if (this.keepaliveTimer !== null) {
+      BgTimer.clearInterval(this.keepaliveTimer)
+      this.keepaliveTimer = null
+    }
+  }
+
+  private clearConnectTimer(): void {
+    if (this.connectTimer !== null) {
+      BgTimer.clearTimeout(this.connectTimer)
+      this.connectTimer = null
+    }
+  }
+
+  private teardown(): void {
+    this.closed = true
+    this.opened = false
+    this.clearConnectTimer()
+    this.stopKeepalive()
+    this.ws = null
+  }
+}
+
+/**
+ * Cheap key validation for the Keys screen: GET /v1/models with the key as a
+ * Bearer token. 401/403 → invalid key; network failure → indeterminate (the
+ * caller may offer "save anyway").
+ */
+export async function validateSonioxApiKey(
+  apiKey: string,
+): Promise<{ok: true} | {ok: false; reason: "invalid_key" | "network"}> {
+  try {
+    const res = await fetch(SONIOX_MODELS_URL, {
+      method: "GET",
+      headers: {Authorization: `Bearer ${apiKey}`},
+    })
+    if (res.ok) return {ok: true}
+    if (res.status === 401 || res.status === 403) return {ok: false, reason: "invalid_key"}
+    return {ok: false, reason: "network"}
+  } catch {
+    return {ok: false, reason: "network"}
+  }
+}