@huynle/hub 0.0.0-stage → 0.6.0

package/src/central/speech/models.ts

@@ -0,0 +1,89 @@
+// Speech model lists of OpenAI-compatible providers. Their GET /models usually lists every model of the
+// gateway (chat, image, embedding, music…) without a speech filter, so the models are classified here: speech-
+// to-text by name ("transcribe", "whisper", "stt", "asr") or as audio-in/text-out audio models; text-to-speech by
+// name ("tts", "text-to-speech", "speech", "voice") or by speech/audio output, never music or audio chat models.
+// The configured model is always offered first (gateways may serve models they do not list), and when nothing is
+// labeled for a kind the full list is shown with a note.
+
+export type SpeechModelKind = "stt" | "tts"
+
+export interface SpeechModelInfo {
+  readonly id: string
+  readonly name: string
+  /** The model configured for this kind right now. */
+  readonly current?: true
+}
+
+export interface SpeechModelList {
+  readonly models: SpeechModelInfo[]
+  /** Set when the list could not be narrowed to speech models. */
+  readonly note?: string
+}
+
+export const SPEECH_UNLABELED_NOTE = "This provider doesn't label speech models; showing all"
+
+const STT_NAME = /transcri|whisper|(^|[^a-z])stt|speech-to-text|(^|[^a-z])asr/i
+const TTS_NAME = /(^|[^a-z])tts|text-to-speech|speech(?!-to-text)|voice/i
+const EXPLICIT_TTS = /(^|[^a-z])tts|text-to-speech/i
+const MUSIC = /lyria|music|gpt-audio\b/i
+const NOT_SPEECH_TYPES = new Set(["chat", "embedding", "image", "rerank"])
+
+interface RawModel {
+  readonly id: string
+  readonly name: string
+  readonly type?: string
+  readonly output: readonly string[] | undefined
+}
+
+const strings = (value: unknown): string[] | undefined => (Array.isArray(value) ? value.filter((x): x is string => typeof x === "string").map((x) => x.toLowerCase()) : undefined)
+
+/** `{data: [{id, name?, type?, output_modalities?}]}` entries with an id. */
+function rawModels(body: unknown): RawModel[] {
+  const data = (body as { data?: unknown } | null | undefined)?.data
+  if (!Array.isArray(data)) return []
+  const out: RawModel[] = []
+  const seen = new Set<string>()
+  for (const entry of data) {
+    const e = entry as Record<string, unknown> | null
+    const id = typeof e?.id === "string" ? e.id : ""
+    if (!id || seen.has(id)) continue
+    seen.add(id)
+    const name = typeof e!.name === "string" && e!.name ? e!.name : id
+    out.push({ id, name, ...(typeof e!.type === "string" ? { type: e!.type.toLowerCase() } : {}), output: strings(e!.output_modalities) })
+  }
+  return out
+}
+
+function isStt(model: RawModel): boolean {
+  if (model.type && NOT_SPEECH_TYPES.has(model.type)) return false
+  const label = `${model.id} ${model.name}`
+  if (STT_NAME.test(label) || model.output?.includes("transcription")) return true
+  // Unlabeled audio models that only answer with text (nova, parakeet, chirp…).
+  return model.type === "audio" && !TTS_NAME.test(label) && !MUSIC.test(label) && (!model.output || model.output.every((x) => x === "text"))
+}
+
+function isTts(model: RawModel): boolean {
+  if (model.type && NOT_SPEECH_TYPES.has(model.type)) return false
+  const label = `${model.id} ${model.name}`
+  if (STT_NAME.test(label)) return false
+  if (MUSIC.test(label) && !EXPLICIT_TTS.test(label)) return false
+  if (TTS_NAME.test(label) || model.output?.includes("speech")) return true
+  return model.type === "audio" && !!model.output?.includes("audio")
+}
+
+/** The models of `kind` in a provider's /models answer, with the configured model `current` first. */
+export function speechModelList(kind: SpeechModelKind, body: unknown, current: string): SpeechModelList {
+  const all = rawModels(body)
+  const matching = all.filter(kind === "stt" ? isStt : isTts)
+  const unlabeled = all.length > 0 && matching.length === 0
+  return withCurrent((unlabeled ? all : matching).map(({ id, name }) => ({ id, name })), current, unlabeled ? SPEECH_UNLABELED_NOTE : undefined)
+}
+
+/** Puts the configured model first, marked current (added when the list does not have it). */
+export function withCurrent(models: readonly SpeechModelInfo[], current: string, note?: string): SpeechModelList {
+  const id = current.trim()
+  const rest = models.filter((m) => m.id !== id).map(({ id, name }) => ({ id, name }))
+  const found = models.find((m) => m.id === id)
+  const list = id ? [{ id, name: found?.name ?? id, current: true as const }, ...rest] : rest
+  return { models: list, ...(note ? { note } : {}) }
+}

package/src/central/speech/openai.ts

@@ -0,0 +1,66 @@
+// OpenAI-compatible speech endpoints (OpenAI, Groq, local whisper/kokoro servers, ...):
+// STT as multipart POST {baseURL}/audio/transcriptions (file, model, language?), TTS as
+// POST {baseURL}/audio/speech {model, input, voice, response_format: "mp3", speed?}.
+import { audioMime, SpeechError, upstreamDetail, withSpeedRetry, type SpeechProvider } from "./provider"
+
+export interface OpenAICompatibleOptions {
+  /** Base URL including the version path, e.g. https://api.openai.com/v1 (no trailing slash). */
+  readonly baseURL: string
+  /** Optional: local servers often need no key. */
+  readonly apiKey: () => Promise<string | undefined>
+  readonly sttModel: string
+  readonly ttsModel: string
+  readonly voice: string
+  readonly language?: string
+  readonly fetch?: typeof globalThis.fetch
+}
+
+export class OpenAICompatible implements SpeechProvider {
+  private readonly fetch: typeof globalThis.fetch
+
+  constructor(private readonly options: OpenAICompatibleOptions) {
+    this.fetch = options.fetch ?? globalThis.fetch
+  }
+
+  async transcribe(audio: Uint8Array, format: string, signal?: AbortSignal): Promise<string> {
+    const form = new FormData()
+    form.append("file", new File([new Uint8Array(audio)], `audio.${format}`, { type: audioMime(format) }))
+    form.append("model", this.options.sttModel)
+    if (this.options.language) form.append("language", this.options.language)
+    const response = await this.fetch(`${this.options.baseURL}/audio/transcriptions`, {
+      method: "POST",
+      headers: await this.headers(),
+      body: form,
+      signal,
+    })
+    if (!response.ok) throw await failure("transcription", response)
+    const body = (await response.json()) as { text?: unknown }
+    return String(body.text ?? "").trim()
+  }
+
+  async speak(text: string, speed?: number, signal?: AbortSignal): Promise<Uint8Array<ArrayBuffer>> {
+    const { ttsModel: model, voice } = this.options
+    const headers = { ...(await this.headers()), "content-type": "application/json" }
+    const response = await withSpeedRetry(
+      (s) =>
+        this.fetch(`${this.options.baseURL}/audio/speech`, {
+          method: "POST",
+          headers,
+          body: JSON.stringify({ model, input: text, voice, response_format: "mp3", ...(s !== undefined ? { speed: s } : {}) }),
+          signal,
+        }),
+      speed,
+    )
+    if (!response.ok) throw await failure("speech", response)
+    return new Uint8Array(await response.arrayBuffer())
+  }
+
+  private async headers(): Promise<Record<string, string>> {
+    const key = await this.options.apiKey()
+    return key ? { authorization: `Bearer ${key}` } : {}
+  }
+}
+
+async function failure(kind: string, response: Response): Promise<SpeechError> {
+  return new SpeechError(`Speech provider ${kind} failed (${response.status}): ${await upstreamDetail(response)}`, response.status)
+}

package/src/central/speech/openrouter.ts

@@ -0,0 +1,71 @@
+// OpenRouter speech-to-text (/audio/transcriptions, JSON with base64 input_audio) and text-to-speech
+// (/audio/speech).
+import { SpeechError, upstreamDetail, withSpeedRetry, type SpeechProvider } from "./provider"
+
+export const OPENROUTER_BASE_URL = "https://openrouter.ai/api/v1"
+
+export interface OpenRouterOptions {
+  readonly apiKey: () => Promise<string | undefined>
+  readonly sttModel: string
+  readonly ttsModel: string
+  readonly voice: string
+  readonly language?: string
+  readonly baseURL?: string
+  readonly fetch?: typeof globalThis.fetch
+}
+
+export class OpenRouterError extends SpeechError {
+  constructor(message: string, status?: number) {
+    super(message, status)
+    this.name = "OpenRouterError"
+  }
+}
+
+export class OpenRouter implements SpeechProvider {
+  private readonly baseURL: string
+  private readonly fetch: typeof globalThis.fetch
+
+  constructor(private readonly options: OpenRouterOptions) {
+    this.baseURL = options.baseURL ?? OPENROUTER_BASE_URL
+    this.fetch = options.fetch ?? globalThis.fetch
+  }
+
+  async transcribe(audio: Uint8Array, format: string, signal?: AbortSignal): Promise<string> {
+    const { sttModel: model, language } = this.options
+    const input_audio = { data: Buffer.from(audio).toString("base64"), format }
+    const response = await this.post("/audio/transcriptions", { model, input_audio, ...(language ? { language } : {}) }, signal)
+    if (!response.ok) throw await failure("transcription", response)
+    const body = (await response.json()) as { text?: unknown }
+    return String(body.text ?? "").trim()
+  }
+
+  async speak(text: string, speed?: number, signal?: AbortSignal): Promise<Uint8Array<ArrayBuffer>> {
+    const { ttsModel: model, voice } = this.options
+    const response = await withSpeedRetry(
+      (s) =>
+        this.post("/audio/speech", { model, input: text, voice, response_format: "mp3", ...(s !== undefined ? { speed: s } : {}) }, signal),
+      speed,
+    )
+    if (!response.ok) throw await failure("speech", response)
+    return new Uint8Array(await response.arrayBuffer())
+  }
+
+  private async post(path: string, body: unknown, signal?: AbortSignal): Promise<Response> {
+    const key = await this.options.apiKey()
+    if (!key) {
+      throw new OpenRouterError(
+        "No OpenRouter API key: set one in the speech settings, set OPENROUTER_API_KEY, or connect OpenRouter in OpenCode",
+      )
+    }
+    return this.fetch(`${this.baseURL}${path}`, {
+      method: "POST",
+      headers: { authorization: `Bearer ${key}`, "content-type": "application/json" },
+      body: JSON.stringify(body),
+      signal,
+    })
+  }
+}
+
+async function failure(kind: string, response: Response): Promise<OpenRouterError> {
+  return new OpenRouterError(`OpenRouter ${kind} failed (${response.status}): ${await upstreamDetail(response)}`, response.status)
+}

package/src/central/speech/provider.ts

@@ -0,0 +1,74 @@
+// Speech provider interface shared by OpenRouter and OpenAI-compatible endpoints, the typed upstream
+// error (mapped to 502 by the hub), audio format detection, and the speed-retry rule.
+
+/** Speech-to-text and text-to-speech against one configured provider. */
+export interface SpeechProvider {
+  transcribe(audio: Uint8Array, format: string, signal?: AbortSignal): Promise<string>
+  /** MP3 audio. `speed` is optional; providers that reject it get one retry without it. */
+  speak(text: string, speed?: number, signal?: AbortSignal): Promise<Uint8Array<ArrayBuffer>>
+}
+
+/** The speech provider refused or failed a request (or is not configured). Never carries the API key. */
+export class SpeechError extends Error {
+  constructor(
+    message: string,
+    readonly status?: number,
+  ) {
+    super(message)
+    this.name = "SpeechError"
+  }
+}
+
+const FORMATS: Record<string, string> = {
+  "audio/webm": "webm",
+  "audio/mp4": "m4a",
+  "audio/x-m4a": "m4a",
+  "audio/m4a": "m4a",
+  "audio/aac": "aac",
+  "audio/ogg": "ogg",
+  "audio/wav": "wav",
+  "audio/x-wav": "wav",
+  "audio/wave": "wav",
+  "audio/mpeg": "mp3",
+  "audio/mp3": "mp3",
+  "audio/flac": "flac",
+}
+
+const MIME: Record<string, string> = {
+  webm: "audio/webm",
+  m4a: "audio/mp4",
+  aac: "audio/aac",
+  ogg: "audio/ogg",
+  wav: "audio/wav",
+  mp3: "audio/mpeg",
+  flac: "audio/flac",
+}
+
+/** Maps a browser recording's Content-Type to an audio format name (webm, m4a, wav, ...). */
+export function audioFormat(contentType: string | null | undefined): string | undefined {
+  const mime = contentType?.split(";")[0]?.trim().toLowerCase()
+  return mime ? FORMATS[mime] : undefined
+}
+
+/** Content type for an audio format name. */
+export const audioMime = (format: string): string => MIME[format] ?? "application/octet-stream"
+
+/** Upstream error message: the JSON `error.message` when present, else the body, capped. */
+export async function upstreamDetail(response: Response): Promise<string> {
+  const raw = await response.text().catch(() => "")
+  try {
+    const parsed = JSON.parse(raw) as { error?: { message?: unknown } | string; message?: unknown }
+    if (typeof parsed.error === "string") return parsed.error.slice(0, 300)
+    if (parsed.error?.message) return String(parsed.error.message).slice(0, 300)
+    if (typeof parsed.message === "string") return parsed.message.slice(0, 300)
+  } catch {}
+  return raw.slice(0, 300)
+}
+
+/** Sends with `speed`; when that is refused with 400, sends once more without it. */
+export async function withSpeedRetry(send: (speed?: number) => Promise<Response>, speed?: number): Promise<Response> {
+  const response = await send(speed)
+  if (response.status !== 400 || speed === undefined) return response
+  await response.body?.cancel().catch(() => {})
+  return send(undefined)
+}

package/src/central/speech/service.ts

@@ -0,0 +1,168 @@
+// Speech for the hub: builds the configured provider for each request from the stored settings (so
+// changes apply at once), proxies model lists for the settings page, and caches them for 10 minutes.
+import { OpenAICompatible } from "./openai"
+import { OpenRouter } from "./openrouter"
+import { speechModelList, withCurrent, type SpeechModelInfo, type SpeechModelList } from "./models"
+import { SpeechError, upstreamDetail, type SpeechProvider } from "./provider"
+import {
+  lowerLayers,
+  resolveStt,
+  resolveTts,
+  speechView,
+  updateSpeech,
+  userSpeechView,
+  type Resolved,
+  type SpeechDefaults,
+  type SpeechView,
+  type StoredSpeech,
+  type UserSpeechView,
+} from "./settings"
+
+export const MODELS_CACHE_MS = 10 * 60_000
+const MODELS_TIMEOUT_MS = 10_000
+
+export interface SpeechSettingsStore {
+  /** The server default. */
+  speechSettings(): StoredSpeech
+  setSpeechSettings(settings: StoredSpeech): void
+  /** A user's own settings (multi-user hubs). */
+  userSpeech?(userID: string): StoredSpeech
+  setUserSpeech?(userID: string, settings: StoredSpeech): void
+}
+
+/** Speech for one scope: a user (their settings over the default) or, without a user, the server default. */
+export interface ScopedSpeech {
+  transcribe(audio: Uint8Array, format: string): Promise<string>
+  speak(text: string): Promise<Uint8Array<ArrayBuffer>>
+}
+
+export interface SpeechServiceOptions {
+  readonly store: SpeechSettingsStore
+  readonly defaults: SpeechDefaults
+  readonly fetch?: typeof globalThis.fetch
+  readonly now?: () => number
+}
+
+export type SpeechKind = "stt" | "tts"
+
+export type ModelInfo = SpeechModelInfo
+export type ModelList = SpeechModelList
+
+export class SpeechService {
+  private readonly fetch: typeof globalThis.fetch
+  private readonly now: () => number
+  private readonly cache = new Map<string, { at: number; body: unknown }>()
+
+  constructor(private readonly options: SpeechServiceOptions) {
+    this.fetch = options.fetch ?? globalThis.fetch
+    this.now = options.now ?? Date.now
+  }
+
+  /** The server default's view, or a user's view (own settings over the default). */
+  view(): SpeechView
+  view(userID: string | undefined): SpeechView | UserSpeechView
+  view(userID?: string): SpeechView | UserSpeechView {
+    const stored = this.options.store.speechSettings()
+    if (userID === undefined) return speechView(stored, this.options.defaults)
+    return userSpeechView(stored, this.options.defaults, this.userSettings(userID) ?? {})
+  }
+
+  /** Partial update of the server default, or of a user's own settings; throws ValidationError without saving anything. */
+  update(input: unknown, userID?: string): SpeechView | UserSpeechView {
+    const { store, defaults } = this.options
+    if (userID === undefined) store.setSpeechSettings(updateSpeech(store.speechSettings(), input, defaults.file))
+    else {
+      if (!store.setUserSpeech) throw new Error("per-user speech settings are not supported by this store")
+      // Validated against (and keys dropped relative to) the layers below: server default and hub.json.
+      store.setUserSpeech(userID, updateSpeech(this.userSettings(userID) ?? {}, input, lowerLayers(store.speechSettings(), defaults)))
+    }
+    this.cache.clear()
+    return this.view(userID)
+  }
+
+  /** Speech with a user's effective settings (undefined: the server default). */
+  forUser(userID: string | undefined): ScopedSpeech {
+    return {
+      transcribe: (audio, format) => this.transcribe(audio, format, undefined, userID),
+      speak: (text) => this.speak(text, undefined, userID),
+    }
+  }
+
+  transcribe(audio: Uint8Array, format: string, signal?: AbortSignal, userID?: string): Promise<string> {
+    return this.provider("stt", userID).transcribe(audio, format, signal)
+  }
+
+  speak(text: string, signal?: AbortSignal, userID?: string): Promise<Uint8Array<ArrayBuffer>> {
+    const tts = resolveTts(this.options.store.speechSettings(), this.options.defaults, this.userSettings(userID))
+    return this.provider("tts", userID).speak(text, tts.speed, signal)
+  }
+
+  /**
+   * Models offered by the configured provider for `kind` (a user's effective provider when given), the
+   * configured model first and marked `current`. OpenRouter filters by output modality; other providers list
+   * everything, so their models are classified here (the full list with a `note` when nothing is labeled).
+   */
+  async models(kind: SpeechKind, userID?: string): Promise<ModelList> {
+    const settings = this.options.store.speechSettings()
+    const user = this.userSettings(userID)
+    const resolved: Resolved = kind === "stt" ? resolveStt(settings, this.options.defaults, user) : resolveTts(settings, this.options.defaults, user)
+    const openrouter = resolved.provider === "openrouter"
+    const url = openrouter
+      ? `${resolved.baseURL}/models?output_modalities=${kind === "stt" ? "transcription" : "speech"}`
+      : `${resolved.baseURL}/models`
+    // Per scope: a list fetched with one user's key is never served to another.
+    const cacheKey = `${userID ?? ""} ${url}`
+    const cached = this.cache.get(cacheKey)
+    let body: unknown
+    if (cached && this.now() - cached.at < MODELS_CACHE_MS) body = cached.body
+    else {
+      try {
+        // OpenRouter's list is public: no key is sent. Other providers usually need theirs.
+        const headers: Record<string, string> = !openrouter && resolved.apiKey ? { authorization: `Bearer ${resolved.apiKey}` } : {}
+        const response = await this.fetch(url, { headers, signal: AbortSignal.timeout(MODELS_TIMEOUT_MS) })
+        if (!response.ok) throw new SpeechError(`Model list failed (${response.status}): ${await upstreamDetail(response)}`, response.status)
+        body = await response.json()
+      } catch (cause) {
+        if (!openrouter) return withCurrent([], resolved.model)
+        throw cause instanceof SpeechError ? cause : new SpeechError(`Model list failed: ${cause instanceof Error ? cause.message : cause}`)
+      }
+      this.cache.set(cacheKey, { at: this.now(), body })
+    }
+    return openrouter ? withCurrent(parseModels(body), resolved.model) : speechModelList(kind, body, resolved.model)
+  }
+
+  private userSettings(userID: string | undefined): StoredSpeech | undefined {
+    return userID === undefined ? undefined : (this.options.store.userSpeech?.(userID) ?? {})
+  }
+
+  private provider(kind: SpeechKind, userID?: string): SpeechProvider {
+    const settings = this.options.store.speechSettings()
+    const user = this.userSettings(userID)
+    const stt = resolveStt(settings, this.options.defaults, user)
+    const tts = resolveTts(settings, this.options.defaults, user)
+    const active = kind === "stt" ? stt : tts
+    const options = {
+      apiKey: async () => active.apiKey,
+      sttModel: stt.model,
+      ttsModel: tts.model,
+      voice: tts.voice,
+      ...(stt.language ? { language: stt.language } : {}),
+      baseURL: active.baseURL,
+      fetch: this.fetch,
+    }
+    return active.provider === "openrouter" ? new OpenRouter(options) : new OpenAICompatible(options)
+  }
+}
+
+function parseModels(body: unknown): ModelInfo[] {
+  const data = (body as { data?: unknown } | null)?.data
+  if (!Array.isArray(data)) return []
+  const models: ModelInfo[] = []
+  for (const entry of data) {
+    const id = (entry as { id?: unknown } | null)?.id
+    if (typeof id !== "string" || !id) continue
+    const name = (entry as { name?: unknown }).name
+    models.push({ id, name: typeof name === "string" && name ? name : id })
+  }
+  return models
+}