iobroker.assistant 1.2.0 → 1.3.0

package/build/lib/voice/openrouter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"openrouter.js","sourceRoot":"","sources":["../../../src/lib/voice/openrouter.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAuDA,gEA8BC;AASD,8CAOC;AASD,0CAiBC;AA/HD;;;;;;;;;GASG;AACH,iDAAwC;AACxC,gCAA8D;AAC9D,+BAAiD;AAGjD,kHAAkH;AACrG,QAAA,4BAA4B,GAAG,+BAA+B,CAAC;AAC5E;;;GAGG;AACU,QAAA,4BAA4B,GAAG,8BAA8B,CAAC;AAE3E,sFAAsF;AACtF,MAAM,QAAQ,GAAG,KAAK,CAAC;AAEvB,iFAAiF;AACjF,MAAM,QAAQ,GAAG,CAAC,UAAU,CAAC,CAAC;AAE9B,6CAA6C;AAC7C,MAAM,iBAAiB,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAazC,SAAS,MAAM,CAAC,MAAc;IAC1B,OAAO,IAAI,gBAAM,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,sBAAgB,CAAC,UAAU,CAAC,OAAO,EAAE,cAAc,EAAE,wBAAkB,EAAE,CAAC,CAAC;AACpH,CAAC;AAED,MAAM,UAAU,GAAG,IAAI,GAAG,EAAoE,CAAC;AAE/F;;;;;GAKG;AACH,SAAgB,0BAA0B,CACtC,IAAgC,EAChC,YAA0B,KAAK;IAE/B,MAAM,MAAM,GAAG,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACpC,IAAI,MAAM,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,EAAE,GAAG,iBAAiB,EAAE,CAAC;QACvD,OAAO,MAAM,CAAC,MAAM,CAAC;IACzB,CAAC;IACD,MAAM,MAAM,GAAG,CAAC,KAAK,IAAI,EAAE;QACvB,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,GAAG,sBAAgB,CAAC,UAAU,CAAC,OAAO,6BAA6B,IAAI,EAAE,EAAE;YACnG,OAAO,EAAE,wBAAkB;YAC3B,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC;SACrC,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACV,MAAM,IAAI,KAAK,CAAC,+BAA+B,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;QACjE,CAAC;QACD,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAA8E,CAAC;QAC7G,OAAO,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;aACnB,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC,EAAE,KAAK,QAAQ,IAAI,CAAC,CAAC,EAAE,CAAC;aAC7C,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACP,EAAE,EAAE,CAAC,CAAC,EAAY;YAClB,IAAI,EAAE,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC,CAAC,EAAa;YAC5D,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,gBAAgB,CAAC;gBACrC,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC;gBACtE,CAAC,CAAC,EAAE;SACX,CAAC,CAAC,CAAC;IACZ,CAAC,CAAC,EAAE,CAAC;IACL,UAAU,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;IACjD,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IAC5C,OAAO,MAAM,CAAC;AAClB,CAAC;AAED;;;;;;GAMG;AACH,SAAgB,iBAAiB,CAAC,MAAgB,EAAE,QAAgB;IAChE,MAAM,IAAI,GAAG,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;IAC1D,IAAI,CAAC,IAAI,EAAE,CAAC;QACR,OAAO,MAAM,CAAC;IAClB,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,IAAI,IAAI,iBAAiB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IACnF,OAAO,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC;AAC/C,CAAC;AAED;;;;;;GAMG;AACH,SAAgB,eAAe,CAAC,KAAa,EAAE,WAAmB;IAC9D,IAAI,KAAK,CAAC,MAAM,GAAG,EAAE,IAAI,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,KAAK,MAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC,KAAK,MAAM,EAAE,CAAC;QAC/G,MAAM,UAAU,GAAG,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;QAC1C,iEAAiE;QACjE,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,OAAO,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;YAChC,MAAM,EAAE,GAAG,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC;YACxD,MAAM,IAAI,GAAG,KAAK,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAC5C,IAAI,EAAE,KAAK,MAAM,EAAE,CAAC;gBAChB,OAAO,EAAE,GAAG,EAAE,KAAK,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC;YACtG,CAAC;YACD,MAAM,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC;QACpC,CAAC;QACD,OAAO,EAAE,GAAG,EAAE,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC;IACnD,CAAC;IACD,MAAM,IAAI,GAAG,yCAAyC,CAAC,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC;IAC/E,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,UAAU,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;AACzE,CAAC;AAED,0EAA0E;AAC1E,MAAa,aAAa;IACL,MAAM,CAAS;IACf,KAAK,CAAS;IAE/B,YAAY,MAAc,EAAE,KAAK,GAAG,EAAE;QAClC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;QAC7B,IAAI,CAAC,KAAK,GAAG,KAAK,IAAI,oCAA4B,CAAC;IACvD,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,GAAW,EAAE,UAAkB,EAAE,IAAY;QAC1D,MAAM,IAAI,GAAG,MAAM,IAAA,eAAM,EAAC,IAAA,cAAQ,EAAC,GAAG,EAAE,UAAU,CAAC,EAAE,WAAW,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;QACzF,mEAAmE;QACnE,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,MAAM,CAAC;YACtD,IAAI;YACJ,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,QAAQ,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,SAAS;SACpD,CAAC,CAAC;QACH,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IACnC,CAAC;CACJ;AAnBD,sCAmBC;AAED,qCAAqC;AACrC,MAAa,aAAa;IACL,MAAM,CAAS;IACf,KAAK,CAAS;IACd,KAAK,CAAS;IACd,MAAM,CAAgB;IACtB,SAAS,CAAgB;IAE1C;;;;;;OAMG;IACH,YAAY,MAAc,EAAE,KAAK,GAAG,EAAE,EAAE,KAAK,GAAG,EAAE,EAAE,MAAqB,EAAE,SAAwB;QAC/F,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;QAC7B,IAAI,CAAC,KAAK,GAAG,KAAK,IAAI,oCAA4B,CAAC;QACnD,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;IAC/B,CAAC;IAED,0GAA0G;IAClG,KAAK,CAAC,QAAQ,CAAC,IAAY;QAC/B,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACb,OAAO,IAAI,CAAC,KAAK,CAAC;QACtB,CAAC;QACD,IAAI,CAAC;YACD,MAAM,MAAM,GAAG,MAAM,0BAA0B,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;YAC1E,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC,KAAK,CAAC,EAAE,MAAM,IAAI,EAAE,CAAC;YACnE,OAAO,iBAAiB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACpD,CAAC;QAAC,MAAM,CAAC;YACL,OAAO,EAAE,CAAC;QACd,CAAC;IACL,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,IAAY,EAAE,IAAY;QACvC,MAAM,GAAG,GAAG,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC;QACnE,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,CAAC,KAAK,uCAAuC,CAAC,CAAC;QAC1E,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACxC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;YAC9C,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,KAAK,EAAE,IAAI;YACX,mGAAmG;YACnG,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3B,eAAe,EAAE,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK;SACJ,CAAC,CAAC;QACtC,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC;QACnD,IAAI,GAAG,EAAE,CAAC;YACN,OAAO,IAAI,CAAC,MAAO,CAAC,KAAK,CAAC,CAAC;QAC/B,CAAC;QACD,OAAO,eAAe,CAAC,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC;IACzE,CAAC;CACJ;AAvDD,sCAuDC","sourcesContent":["/**\n * Speech over OpenRouter: one key for the speech-to-text and text-to-speech models of many vendors (OpenAI,\n * ElevenLabs, Google, Deepgram, Mistral …) behind the OpenAI audio API at https://openrouter.ai/api/v1.\n * Model ids carry the vendor (`openai/whisper-large-v3-turbo`), and the voices depend on the model — the public\n * model list names them (`supported_voices`), so the settings offer exactly the voices of the chosen model.\n *\n * Differences to OpenAI itself: transcription ignores `prompt` (no vocabulary hints), and speech comes back as\n * `pcm` (most models, the default) or `mp3` (MiniMax only). The PCM rate is not part of the API — OpenRouter\n * documents 24 kHz, the rate of OpenAI, Gemini and Azure — so a rate in the content type wins over that default.\n */\nimport OpenAI, { toFile } from 'openai';\nimport { OPENROUTER_HEADERS, PROVIDER_PRESETS } from '../llm';\nimport { pcmToWav, type SttEngine } from './stt';\nimport type { TtsEngine, TtsResult } from './tts';\n\n/** Speech-to-text model when none is configured: Whisper large v3 turbo, multilingual and the cheapest Whisper */\nexport const DEFAULT_OPENROUTER_STT_MODEL = 'openai/whisper-large-v3-turbo';\n/**\n * Text-to-speech model when none is configured: multilingual, PCM (no decoder needed) and fast — in a test about\n * 0.8 s for a sentence where Gemini TTS took 2.5 s, which a spoken answer waits for.\n */\nexport const DEFAULT_OPENROUTER_TTS_MODEL = 'elevenlabs/eleven-flash-v2.5';\n\n/** Rate of the PCM OpenRouter returns when the content type does not say otherwise */\nconst PCM_RATE = 24000;\n\n/** Vendors whose speech models only return mp3 (they answer `pcm` with a 400) */\nconst MP3_ONLY = ['minimax/'];\n\n/** How long the public model list is kept */\nconst MODEL_LIST_TTL_MS = 60 * 60 * 1000;\n\n/** Decode compressed audio (mp3) to mono 16-bit PCM — ffmpeg in the adapter */\nexport type AudioDecoder = (audio: Buffer) => Promise<TtsResult>;\n\n/** A speech model of OpenRouter */\nexport interface OpenRouterSpeechModel {\n    id: string;\n    name: string;\n    /** Voices of a text-to-speech model, empty when the model names none */\n    voices: string[];\n}\n\nfunction client(apiKey: string): OpenAI {\n    return new OpenAI({ apiKey, baseURL: PROVIDER_PRESETS.openrouter.baseUrl, defaultHeaders: OPENROUTER_HEADERS });\n}\n\nconst modelLists = new Map<string, { at: number; models: Promise<OpenRouterSpeechModel[]> }>();\n\n/**\n * The speech models of OpenRouter. The list is public (no key), cached for an hour; a failed request is not kept.\n *\n * @param kind `speech` = text-to-speech, `transcription` = speech-to-text\n * @param fetchImpl fetch (tests)\n */\nexport function listOpenRouterSpeechModels(\n    kind: 'speech' | 'transcription',\n    fetchImpl: typeof fetch = fetch,\n): Promise<OpenRouterSpeechModel[]> {\n    const cached = modelLists.get(kind);\n    if (cached && Date.now() - cached.at < MODEL_LIST_TTL_MS) {\n        return cached.models;\n    }\n    const models = (async () => {\n        const res = await fetchImpl(`${PROVIDER_PRESETS.openrouter.baseUrl}/models?output_modalities=${kind}`, {\n            headers: OPENROUTER_HEADERS,\n            signal: AbortSignal.timeout(15000),\n        });\n        if (!res.ok) {\n            throw new Error(`OpenRouter model list: HTTP ${res.status}`);\n        }\n        const body = (await res.json()) as { data?: { id?: unknown; name?: unknown; supported_voices?: unknown }[] };\n        return (body.data || [])\n            .filter(m => typeof m.id === 'string' && m.id)\n            .map(m => ({\n                id: m.id as string,\n                name: typeof m.name === 'string' ? m.name : (m.id as string),\n                voices: Array.isArray(m.supported_voices)\n                    ? m.supported_voices.filter((v): v is string => typeof v === 'string')\n                    : [],\n            }));\n    })();\n    modelLists.set(kind, { at: Date.now(), models });\n    models.catch(() => modelLists.delete(kind));\n    return models;\n}\n\n/**\n * The voices of a text-to-speech model. Voices named after a locale (`de-DE-Klaus:MAI-Voice-2`) are narrowed to\n * the language, if that leaves any; other names (`Kore`, `george`) say nothing about the language.\n *\n * @param voices the voices of the model\n * @param language ISO-639-1 language ('' = all)\n */\nexport function voicesForLanguage(voices: string[], language: string): string[] {\n    const lang = (language || '').split('-')[0].toLowerCase();\n    if (!lang) {\n        return voices;\n    }\n    const matching = voices.filter(v => new RegExp(`^${lang}[-_][A-Z]{2}\\\\b`).test(v));\n    return matching.length ? matching : voices;\n}\n\n/**\n * Raw PCM out of a speech response: WAV is unwrapped, the rate comes from the content type (`audio/pcm;rate=…`)\n * or is the documented 24 kHz.\n *\n * @param audio the response body\n * @param contentType the content type of the response\n */\nexport function pcmFromResponse(audio: Buffer, contentType: string): TtsResult {\n    if (audio.length > 44 && audio.toString('latin1', 0, 4) === 'RIFF' && audio.toString('latin1', 8, 12) === 'WAVE') {\n        const sampleRate = audio.readUInt32LE(24);\n        // the data chunk is not always right after the 16-byte fmt chunk\n        let offset = 12;\n        while (offset + 8 <= audio.length) {\n            const id = audio.toString('latin1', offset, offset + 4);\n            const size = audio.readUInt32LE(offset + 4);\n            if (id === 'data') {\n                return { pcm: audio.subarray(offset + 8, Math.min(audio.length, offset + 8 + size)), sampleRate };\n            }\n            offset += 8 + size + (size % 2);\n        }\n        return { pcm: audio.subarray(44), sampleRate };\n    }\n    const rate = /(?:^|[;\\s])(?:rate|sample_?rate)=(\\d+)/i.exec(contentType || '');\n    return { pcm: audio, sampleRate: rate ? Number(rate[1]) : PCM_RATE };\n}\n\n/** Speech-to-text over OpenRouter (OpenAI-compatible multipart upload) */\nexport class OpenRouterStt implements SttEngine {\n    private readonly client: OpenAI;\n    private readonly model: string;\n\n    constructor(apiKey: string, model = '') {\n        this.client = client(apiKey);\n        this.model = model || DEFAULT_OPENROUTER_STT_MODEL;\n    }\n\n    async transcribe(pcm: Buffer, sampleRate: number, lang: string): Promise<string> {\n        const file = await toFile(pcmToWav(pcm, sampleRate), 'audio.wav', { type: 'audio/wav' });\n        // no vocabulary hints: OpenRouter accepts `prompt`, but ignores it\n        const res = await this.client.audio.transcriptions.create({\n            file,\n            model: this.model,\n            language: (lang || '').split('-')[0] || undefined,\n        });\n        return (res.text || '').trim();\n    }\n}\n\n/** Text-to-speech over OpenRouter */\nexport class OpenRouterTts implements TtsEngine {\n    private readonly client: OpenAI;\n    private readonly model: string;\n    private readonly voice: string;\n    private readonly decode?: AudioDecoder;\n    private readonly fetchImpl?: typeof fetch;\n\n    /**\n     * @param apiKey OpenRouter key\n     * @param model model id ('' = {@link DEFAULT_OPENROUTER_TTS_MODEL})\n     * @param voice voice of the model ('' = the first voice of the model for the language)\n     * @param decode decoder for the models that only return mp3\n     * @param fetchImpl fetch for the model list (tests)\n     */\n    constructor(apiKey: string, model = '', voice = '', decode?: AudioDecoder, fetchImpl?: typeof fetch) {\n        this.client = client(apiKey);\n        this.model = model || DEFAULT_OPENROUTER_TTS_MODEL;\n        this.voice = voice;\n        this.decode = decode;\n        this.fetchImpl = fetchImpl;\n    }\n\n    /** The configured voice, or the first one the model offers for the language — most models need a voice */\n    private async voiceFor(lang: string): Promise<string> {\n        if (this.voice) {\n            return this.voice;\n        }\n        try {\n            const models = await listOpenRouterSpeechModels('speech', this.fetchImpl);\n            const voices = models.find(m => m.id === this.model)?.voices || [];\n            return voicesForLanguage(voices, lang)[0] || '';\n        } catch {\n            return '';\n        }\n    }\n\n    async synthesize(text: string, lang: string): Promise<TtsResult> {\n        const mp3 = MP3_ONLY.some(prefix => this.model.startsWith(prefix));\n        if (mp3 && !this.decode) {\n            throw new Error(`${this.model} only returns mp3, which needs ffmpeg`);\n        }\n        const voice = await this.voiceFor(lang);\n        const res = await this.client.audio.speech.create({\n            model: this.model,\n            input: text,\n            // `voice` is required by the OpenAI types; an empty one is left out (a model with a default voice)\n            ...(voice ? { voice } : {}),\n            response_format: mp3 ? 'mp3' : 'pcm',\n        } as OpenAI.Audio.SpeechCreateParams);\n        const audio = Buffer.from(await res.arrayBuffer());\n        if (mp3) {\n            return this.decode!(audio);\n        }\n        return pcmFromResponse(audio, res.headers.get('content-type') || '');\n    }\n}\n"]}