@getpaseo/server 0.1.2

package/dist/server/server/speech/providers/local/sherpa/sherpa-tts.d.ts

@@ -0,0 +1,21 @@
+import type pino from "pino";
+import type { SpeechStreamResult, TextToSpeechProvider } from "../../../speech-provider.js";
+export type SherpaTtsPreset = "kokoro-en-v0_19" | "kitten-nano-en-v0_1-fp16";
+export type SherpaTtsConfig = {
+    preset: SherpaTtsPreset;
+    modelDir: string;
+    speakerId?: number;
+    speed?: number;
+    lengthScale?: number;
+    numThreads?: number;
+};
+export declare class SherpaOnnxTTS implements TextToSpeechProvider {
+    private readonly tts;
+    private readonly speakerId;
+    private readonly speed;
+    private readonly logger;
+    constructor(config: SherpaTtsConfig, logger: pino.Logger);
+    synthesizeSpeech(text: string): Promise<SpeechStreamResult>;
+    free(): void;
+}
+//# sourceMappingURL=sherpa-tts.d.ts.map

package/dist/server/server/speech/providers/local/sherpa/sherpa-tts.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sherpa-tts.d.ts","sourceRoot":"","sources":["../../../../../../../src/server/speech/providers/local/sherpa/sherpa-tts.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAI7B,OAAO,KAAK,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AAI5F,MAAM,MAAM,eAAe,GAAG,iBAAiB,GAAG,0BAA0B,CAAC;AAE7E,MAAM,MAAM,eAAe,GAAG;IAC5B,MAAM,EAAE,eAAe,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,CAAC;AAQF,qBAAa,aAAc,YAAW,oBAAoB;IACxD,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAM;IAC1B,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAc;gBAEzB,MAAM,EAAE,eAAe,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM;IAwDlD,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAkCjE,IAAI,IAAI,IAAI;CAOb"}

package/dist/server/server/speech/providers/local/sherpa/sherpa-tts.js

@@ -0,0 +1,95 @@
+import { Readable } from "node:stream";
+import { existsSync } from "node:fs";
+import { chunkBuffer, float32ToPcm16le } from "../../../audio.js";
+import { loadSherpaOnnxNode } from "./sherpa-onnx-node-loader.js";
+function assertFileExists(filePath, label) {
+    if (!existsSync(filePath)) {
+        throw new Error(`Missing ${label}: ${filePath}`);
+    }
+}
+export class SherpaOnnxTTS {
+    constructor(config, logger) {
+        if (config.preset !== "kokoro-en-v0_19" && config.preset !== "kitten-nano-en-v0_1-fp16") {
+            throw new Error(`Unsupported Sherpa TTS preset: ${config.preset}`);
+        }
+        this.logger = logger.child({ module: "speech", provider: "local", component: "tts" });
+        this.speakerId = config.speakerId ?? 0;
+        this.speed = config.speed ?? 1.0;
+        const sherpa = loadSherpaOnnxNode();
+        if (typeof sherpa.OfflineTts !== "function") {
+            throw new Error("sherpa-onnx-node OfflineTts is unavailable");
+        }
+        const modelFile = config.preset === "kokoro-en-v0_19" ? "model.onnx" : "model.fp16.onnx";
+        const modelPath = `${config.modelDir}/${modelFile}`;
+        const voicesPath = `${config.modelDir}/voices.bin`;
+        const tokensPath = `${config.modelDir}/tokens.txt`;
+        const dataDir = `${config.modelDir}/espeak-ng-data`;
+        assertFileExists(modelPath, "TTS model");
+        assertFileExists(voicesPath, "TTS voices");
+        assertFileExists(tokensPath, "TTS tokens");
+        assertFileExists(dataDir, "TTS espeak-ng dataDir");
+        const modelConfig = config.preset === "kokoro-en-v0_19"
+            ? {
+                kokoro: {
+                    model: modelPath,
+                    voices: voicesPath,
+                    tokens: tokensPath,
+                    dataDir,
+                    lengthScale: config.lengthScale ?? 1.0,
+                },
+            }
+            : {
+                kitten: {
+                    model: modelPath,
+                    voices: voicesPath,
+                    tokens: tokensPath,
+                    dataDir,
+                    lengthScale: config.lengthScale ?? 1.0,
+                },
+            };
+        const offlineTtsConfig = {
+            model: modelConfig,
+            numThreads: config.numThreads ?? 2,
+            provider: "cpu",
+            maxNumSentences: 1,
+        };
+        this.tts = new sherpa.OfflineTts(offlineTtsConfig);
+        this.logger.info({ preset: config.preset, modelDir: config.modelDir }, "Sherpa offline TTS initialized");
+    }
+    async synthesizeSpeech(text) {
+        const trimmed = text.trim();
+        if (!trimmed) {
+            throw new Error("Cannot synthesize empty text");
+        }
+        const audio = this.tts.generate({ text: trimmed, sid: this.speakerId, speed: this.speed });
+        const samples = audio && audio.samples instanceof Float32Array
+            ? audio.samples
+            : audio && Array.isArray(audio.samples)
+                ? Float32Array.from(audio.samples)
+                : null;
+        const sampleRate = audio && typeof audio.sampleRate === "number" && Number.isFinite(audio.sampleRate) && audio.sampleRate > 0
+            ? audio.sampleRate
+            : typeof this.tts.sampleRate === "number"
+                ? this.tts.sampleRate
+                : 24000;
+        if (!samples) {
+            throw new Error("Unexpected sherpa TTS output: missing Float32 samples");
+        }
+        const pcm16 = float32ToPcm16le(samples);
+        const chunkBytes = Math.max(2, Math.round(sampleRate * 0.05) * 2); // ~50ms
+        const chunks = chunkBuffer(pcm16, chunkBytes);
+        return {
+            stream: Readable.from(chunks),
+            format: `pcm;rate=${sampleRate}`,
+        };
+    }
+    free() {
+        try {
+            this.tts?.free?.();
+        }
+        catch {
+            // ignore
+        }
+    }
+}
+//# sourceMappingURL=sherpa-tts.js.map

package/dist/server/server/speech/providers/local/sherpa/sherpa-tts.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sherpa-tts.js","sourceRoot":"","sources":["../../../../../../../src/server/speech/providers/local/sherpa/sherpa-tts.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAGrC,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAClE,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AAalE,SAAS,gBAAgB,CAAC,QAAgB,EAAE,KAAa;IACvD,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,WAAW,KAAK,KAAK,QAAQ,EAAE,CAAC,CAAC;IACnD,CAAC;AACH,CAAC;AAED,MAAM,OAAO,aAAa;IAMxB,YAAY,MAAuB,EAAE,MAAmB;QACtD,IAAI,MAAM,CAAC,MAAM,KAAK,iBAAiB,IAAI,MAAM,CAAC,MAAM,KAAK,0BAA0B,EAAE,CAAC;YACxF,MAAM,IAAI,KAAK,CAAC,kCAAkC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QACrE,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;QACtF,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,CAAC,CAAC;QACvC,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,GAAG,CAAC;QAEjC,MAAM,MAAM,GAAG,kBAAkB,EAAE,CAAC;QACpC,IAAI,OAAO,MAAM,CAAC,UAAU,KAAK,UAAU,EAAE,CAAC;YAC5C,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;QAChE,CAAC;QAED,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,KAAK,iBAAiB,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,iBAAiB,CAAC;QACzF,MAAM,SAAS,GAAG,GAAG,MAAM,CAAC,QAAQ,IAAI,SAAS,EAAE,CAAC;QACpD,MAAM,UAAU,GAAG,GAAG,MAAM,CAAC,QAAQ,aAAa,CAAC;QACnD,MAAM,UAAU,GAAG,GAAG,MAAM,CAAC,QAAQ,aAAa,CAAC;QACnD,MAAM,OAAO,GAAG,GAAG,MAAM,CAAC,QAAQ,iBAAiB,CAAC;QAEpD,gBAAgB,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;QACzC,gBAAgB,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;QAC3C,gBAAgB,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;QAC3C,gBAAgB,CAAC,OAAO,EAAE,uBAAuB,CAAC,CAAC;QAEnD,MAAM,WAAW,GACf,MAAM,CAAC,MAAM,KAAK,iBAAiB;YACjC,CAAC,CAAC;gBACE,MAAM,EAAE;oBACN,KAAK,EAAE,SAAS;oBAChB,MAAM,EAAE,UAAU;oBAClB,MAAM,EAAE,UAAU;oBAClB,OAAO;oBACP,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,GAAG;iBACvC;aACF;YACH,CAAC,CAAC;gBACE,MAAM,EAAE;oBACN,KAAK,EAAE,SAAS;oBAChB,MAAM,EAAE,UAAU;oBAClB,MAAM,EAAE,UAAU;oBAClB,OAAO;oBACP,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,GAAG;iBACvC;aACF,CAAC;QAER,MAAM,gBAAgB,GAAG;YACvB,KAAK,EAAE,WAAW;YAClB,UAAU,EAAE,MAAM,CAAC,UAAU,IAAI,CAAC;YAClC,QAAQ,EAAE,KAAK;YACf,eAAe,EAAE,CAAC;SACnB,CAAC;QAEF,IAAI,CAAC,GAAG,GAAG,IAAI,MAAM,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC;QACnD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,EAAE,gCAAgC,CAAC,CAAC;IAC3G,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,IAAY;QACjC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAC5B,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;QAClD,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,SAAS,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;QAC3F,MAAM,OAAO,GACX,KAAK,IAAI,KAAK,CAAC,OAAO,YAAY,YAAY;YAC5C,CAAC,CAAC,KAAK,CAAC,OAAO;YACf,CAAC,CAAC,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC;gBACrC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,OAAmB,CAAC;gBAC9C,CAAC,CAAC,IAAI,CAAC;QACb,MAAM,UAAU,GACd,KAAK,IAAI,OAAO,KAAK,CAAC,UAAU,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,UAAU,GAAG,CAAC;YACxG,CAAC,CAAC,KAAK,CAAC,UAAU;YAClB,CAAC,CAAC,OAAO,IAAI,CAAC,GAAG,CAAC,UAAU,KAAK,QAAQ;gBACvC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU;gBACrB,CAAC,CAAC,KAAK,CAAC;QAEd,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAC;QAC3E,CAAC;QAED,MAAM,KAAK,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;QACxC,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ;QAC3E,MAAM,MAAM,GAAG,WAAW,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QAE9C,OAAO;YACL,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC;YAC7B,MAAM,EAAE,YAAY,UAAU,EAAE;SACjC,CAAC;IACJ,CAAC;IAED,IAAI;QACF,IAAI,CAAC;YACH,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC;QACrB,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;IACH,CAAC;CACF"}

package/dist/server/server/speech/providers/openai/config.d.ts

@@ -0,0 +1,22 @@
+import type { PersistedConfig } from "../../../persisted-config.js";
+import type { RequestedSpeechProviders } from "../../speech-types.js";
+import type { STTConfig } from "./stt.js";
+import type { TTSConfig } from "./tts.js";
+export declare const DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL = "gpt-4o-transcribe";
+export declare const DEFAULT_OPENAI_TTS_MODEL = "tts-1";
+export type OpenAiSpeechProviderConfig = {
+    apiKey?: string;
+    stt?: Partial<STTConfig> & {
+        apiKey?: string;
+    };
+    tts?: Partial<TTSConfig> & {
+        apiKey?: string;
+    };
+    realtimeTranscriptionModel?: string;
+};
+export declare function resolveOpenAiSpeechConfig(params: {
+    env: NodeJS.ProcessEnv;
+    persisted: PersistedConfig;
+    providers: RequestedSpeechProviders;
+}): OpenAiSpeechProviderConfig | undefined;
+//# sourceMappingURL=config.d.ts.map

package/dist/server/server/speech/providers/openai/config.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../../../../../src/server/speech/providers/openai/config.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,uBAAuB,CAAC;AACtE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAC1C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAE1C,eAAO,MAAM,2CAA2C,sBAAsB,CAAC;AAC/E,eAAO,MAAM,wBAAwB,UAAU,CAAC;AAEhD,MAAM,MAAM,0BAA0B,GAAG;IACvC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,GAAG,CAAC,EAAE,OAAO,CAAC,SAAS,CAAC,GAAG;QAAE,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC/C,GAAG,CAAC,EAAE,OAAO,CAAC,SAAS,CAAC,GAAG;QAAE,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC/C,0BAA0B,CAAC,EAAE,MAAM,CAAC;CACrC,CAAC;AAiDF,wBAAgB,yBAAyB,CAAC,MAAM,EAAE;IAChD,GAAG,EAAE,MAAM,CAAC,UAAU,CAAC;IACvB,SAAS,EAAE,eAAe,CAAC;IAC3B,SAAS,EAAE,wBAAwB,CAAC;CACrC,GAAG,0BAA0B,GAAG,SAAS,CAyDzC"}

package/dist/server/server/speech/providers/openai/config.js

@@ -0,0 +1,94 @@
+import { z } from "zod";
+export const DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL = "gpt-4o-transcribe";
+export const DEFAULT_OPENAI_TTS_MODEL = "tts-1";
+const OpenAiTtsVoiceSchema = z.enum([
+    "alloy",
+    "echo",
+    "fable",
+    "onyx",
+    "nova",
+    "shimmer",
+]);
+const OpenAiTtsModelSchema = z.enum(["tts-1", "tts-1-hd"]);
+const NumberLikeSchema = z.union([
+    z.number(),
+    z.string().trim().min(1),
+]);
+const OptionalFiniteNumberSchema = NumberLikeSchema
+    .pipe(z.coerce.number().finite())
+    .optional();
+const OptionalTrimmedStringSchema = z
+    .string()
+    .trim()
+    .optional()
+    .transform((value) => (value && value.length > 0 ? value : undefined));
+const OpenAiSpeechResolutionSchema = z.object({
+    apiKey: OptionalTrimmedStringSchema,
+    sttConfidenceThreshold: OptionalFiniteNumberSchema,
+    sttModel: OptionalTrimmedStringSchema,
+    ttsVoice: z
+        .string()
+        .trim()
+        .toLowerCase()
+        .pipe(OpenAiTtsVoiceSchema)
+        .default("alloy"),
+    ttsModel: z
+        .string()
+        .trim()
+        .toLowerCase()
+        .pipe(OpenAiTtsModelSchema)
+        .default(DEFAULT_OPENAI_TTS_MODEL),
+    realtimeTranscriptionModel: OptionalTrimmedStringSchema.default(DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL),
+});
+export function resolveOpenAiSpeechConfig(params) {
+    const parsed = OpenAiSpeechResolutionSchema.parse({
+        apiKey: params.env.OPENAI_API_KEY ?? params.persisted.providers?.openai?.apiKey,
+        sttConfidenceThreshold: params.env.STT_CONFIDENCE_THRESHOLD ??
+            params.persisted.features?.dictation?.stt?.confidenceThreshold,
+        sttModel: params.env.STT_MODEL ??
+            (params.providers.voiceStt.provider === "openai"
+                ? params.persisted.features?.voiceMode?.stt?.model
+                : undefined) ??
+            (params.providers.dictationStt.provider === "openai"
+                ? params.persisted.features?.dictation?.stt?.model
+                : undefined),
+        ttsVoice: params.env.TTS_VOICE ??
+            (params.providers.voiceTts.provider === "openai"
+                ? params.persisted.features?.voiceMode?.tts?.voice
+                : undefined) ??
+            "alloy",
+        ttsModel: params.env.TTS_MODEL ??
+            (params.providers.voiceTts.provider === "openai"
+                ? params.persisted.features?.voiceMode?.tts?.model
+                : undefined) ??
+            DEFAULT_OPENAI_TTS_MODEL,
+        realtimeTranscriptionModel: params.env.OPENAI_REALTIME_TRANSCRIPTION_MODEL ??
+            (params.providers.dictationStt.provider === "openai"
+                ? params.persisted.features?.dictation?.stt?.model
+                : undefined) ??
+            DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL,
+    });
+    if (!parsed.apiKey) {
+        return undefined;
+    }
+    return {
+        apiKey: parsed.apiKey,
+        stt: {
+            apiKey: parsed.apiKey,
+            ...(parsed.sttConfidenceThreshold !== undefined
+                ? { confidenceThreshold: parsed.sttConfidenceThreshold }
+                : {}),
+            ...(parsed.sttModel
+                ? { model: parsed.sttModel }
+                : {}),
+        },
+        tts: {
+            apiKey: parsed.apiKey,
+            voice: parsed.ttsVoice,
+            model: parsed.ttsModel,
+            responseFormat: "pcm",
+        },
+        realtimeTranscriptionModel: parsed.realtimeTranscriptionModel,
+    };
+}
+//# sourceMappingURL=config.js.map

package/dist/server/server/speech/providers/openai/config.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.js","sourceRoot":"","sources":["../../../../../../src/server/speech/providers/openai/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAOxB,MAAM,CAAC,MAAM,2CAA2C,GAAG,mBAAmB,CAAC;AAC/E,MAAM,CAAC,MAAM,wBAAwB,GAAG,OAAO,CAAC;AAShD,MAAM,oBAAoB,GAAG,CAAC,CAAC,IAAI,CAAC;IAClC,OAAO;IACP,MAAM;IACN,OAAO;IACP,MAAM;IACN,MAAM;IACN,SAAS;CACV,CAAC,CAAC;AAEH,MAAM,oBAAoB,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC;AAE3D,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC;IAC/B,CAAC,CAAC,MAAM,EAAE;IACV,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;CACzB,CAAC,CAAC;AAEH,MAAM,0BAA0B,GAAG,gBAAgB;KAChD,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC;KAChC,QAAQ,EAAE,CAAC;AAEd,MAAM,2BAA2B,GAAG,CAAC;KAClC,MAAM,EAAE;KACR,IAAI,EAAE;KACN,QAAQ,EAAE;KACV,SAAS,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;AAEzE,MAAM,4BAA4B,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,MAAM,EAAE,2BAA2B;IACnC,sBAAsB,EAAE,0BAA0B;IAClD,QAAQ,EAAE,2BAA2B;IACrC,QAAQ,EAAE,CAAC;SACR,MAAM,EAAE;SACR,IAAI,EAAE;SACN,WAAW,EAAE;SACb,IAAI,CAAC,oBAAoB,CAAC;SAC1B,OAAO,CAAC,OAAO,CAAC;IACnB,QAAQ,EAAE,CAAC;SACR,MAAM,EAAE;SACR,IAAI,EAAE;SACN,WAAW,EAAE;SACb,IAAI,CAAC,oBAAoB,CAAC;SAC1B,OAAO,CAAC,wBAAwB,CAAC;IACpC,0BAA0B,EAAE,2BAA2B,CAAC,OAAO,CAC7D,2CAA2C,CAC5C;CACF,CAAC,CAAC;AAEH,MAAM,UAAU,yBAAyB,CAAC,MAIzC;IACC,MAAM,MAAM,GAAG,4BAA4B,CAAC,KAAK,CAAC;QAChD,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,cAAc,IAAI,MAAM,CAAC,SAAS,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM;QAC/E,sBAAsB,EACpB,MAAM,CAAC,GAAG,CAAC,wBAAwB;YACnC,MAAM,CAAC,SAAS,CAAC,QAAQ,EAAE,SAAS,EAAE,GAAG,EAAE,mBAAmB;QAChE,QAAQ,EACN,MAAM,CAAC,GAAG,CAAC,SAAS;YACpB,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,QAAQ,KAAK,QAAQ;gBAC9C,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,EAAE,SAAS,EAAE,GAAG,EAAE,KAAK;gBAClD,CAAC,CAAC,SAAS,CAAC;YACd,CAAC,MAAM,CAAC,SAAS,CAAC,YAAY,CAAC,QAAQ,KAAK,QAAQ;gBAClD,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,EAAE,SAAS,EAAE,GAAG,EAAE,KAAK;gBAClD,CAAC,CAAC,SAAS,CAAC;QAChB,QAAQ,EACN,MAAM,CAAC,GAAG,CAAC,SAAS;YACpB,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,QAAQ,KAAK,QAAQ;gBAC9C,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,EAAE,SAAS,EAAE,GAAG,EAAE,KAAK;gBAClD,CAAC,CAAC,SAAS,CAAC;YACd,OAAO;QACT,QAAQ,EACN,MAAM,CAAC,GAAG,CAAC,SAAS;YACpB,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,QAAQ,KAAK,QAAQ;gBAC9C,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,EAAE,SAAS,EAAE,GAAG,EAAE,KAAK;gBAClD,CAAC,CAAC,SAAS,CAAC;YACd,wBAAwB;QAC1B,0BAA0B,EACxB,MAAM,CAAC,GAAG,CAAC,mCAAmC;YAC9C,CAAC,MAAM,CAAC,SAAS,CAAC,YAAY,CAAC,QAAQ,KAAK,QAAQ;gBAClD,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,EAAE,SAAS,EAAE,GAAG,EAAE,KAAK;gBAClD,CAAC,CAAC,SAAS,CAAC;YACd,2CAA2C;KAC9C,CAAC,CAAC;IAEH,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QACnB,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,OAAO;QACL,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,GAAG,EAAE;YACH,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,GAAG,CAAC,MAAM,CAAC,sBAAsB,KAAK,SAAS;gBAC7C,CAAC,CAAC,EAAE,mBAAmB,EAAE,MAAM,CAAC,sBAAsB,EAAE;gBACxD,CAAC,CAAC,EAAE,CAAC;YACP,GAAG,CAAC,MAAM,CAAC,QAAQ;gBACjB,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,QAAQ,EAAE;gBAC5B,CAAC,CAAC,EAAE,CAAC;SACR;QACD,GAAG,EAAE;YACH,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,KAAK,EAAE,MAAM,CAAC,QAAQ;YACtB,KAAK,EAAE,MAAM,CAAC,QAAQ;YACtB,cAAc,EAAE,KAAK;SACtB;QACD,0BAA0B,EAAE,MAAM,CAAC,0BAA0B;KAC9D,CAAC;AACJ,CAAC"}

package/dist/server/server/speech/providers/openai/realtime-transcription-session.d.ts

@@ -0,0 +1,42 @@
+import type pino from "pino";
+import { EventEmitter } from "node:events";
+import type { StreamingTranscriptionSession } from "../../speech-provider.js";
+type OpenAITurnDetection = null | {
+    type: "server_vad";
+    create_response?: boolean;
+    threshold?: number;
+    prefix_padding_ms?: number;
+    silence_duration_ms?: number;
+} | {
+    type: "semantic_vad";
+    create_response?: boolean;
+    eagerness?: "low" | "medium" | "high";
+};
+export declare class OpenAIRealtimeTranscriptionSession extends EventEmitter implements StreamingTranscriptionSession {
+    readonly requiredSampleRate = 24000;
+    private readonly apiKey;
+    private readonly logger;
+    private readonly transcriptionModel;
+    private readonly language?;
+    private readonly prompt?;
+    private readonly turnDetection;
+    private ws;
+    private ready;
+    private closing;
+    private partialByItemId;
+    constructor(params: {
+        apiKey: string;
+        logger: pino.Logger;
+        transcriptionModel: string;
+        language?: string;
+        prompt?: string;
+        turnDetection?: OpenAITurnDetection;
+    });
+    connect(): Promise<void>;
+    appendPcm16(pcm16le: Buffer): void;
+    commit(): void;
+    clear(): void;
+    close(): void;
+}
+export {};
+//# sourceMappingURL=realtime-transcription-session.d.ts.map

package/dist/server/server/speech/providers/openai/realtime-transcription-session.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"realtime-transcription-session.d.ts","sourceRoot":"","sources":["../../../../../../src/server/speech/providers/openai/realtime-transcription-session.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAE7B,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,KAAK,EAAE,6BAA6B,EAAE,MAAM,0BAA0B,CAAC;AAE9E,KAAK,mBAAmB,GACpB,IAAI,GACJ;IACE,IAAI,EAAE,YAAY,CAAC;IACnB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC9B,GACD;IACE,IAAI,EAAE,cAAc,CAAC;IACrB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,SAAS,CAAC,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;CACvC,CAAC;AA6CN,qBAAa,kCACX,SAAQ,YACR,YAAW,6BAA6B;IAExC,SAAgB,kBAAkB,SAAS;IAC3C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAc;IACrC,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAS;IAC5C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAsB;IAEpD,OAAO,CAAC,EAAE,CAA0B;IACpC,OAAO,CAAC,KAAK,CAA8B;IAC3C,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,eAAe,CAA6B;gBAExC,MAAM,EAAE;QAClB,MAAM,EAAE,MAAM,CAAC;QACf,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC;QACpB,kBAAkB,EAAE,MAAM,CAAC;QAC3B,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,aAAa,CAAC,EAAE,mBAAmB,CAAC;KACrC;IAUY,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAgI9B,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI;IASlC,MAAM,IAAI,IAAI;IAQd,KAAK,IAAI,IAAI;IAQb,KAAK,IAAI,IAAI;CAWrB"}

package/dist/server/server/speech/providers/openai/realtime-transcription-session.js

@@ -0,0 +1,165 @@
+import WebSocket from "ws";
+import { EventEmitter } from "node:events";
+export class OpenAIRealtimeTranscriptionSession extends EventEmitter {
+    constructor(params) {
+        super();
+        this.requiredSampleRate = 24000;
+        this.ws = null;
+        this.ready = null;
+        this.closing = false;
+        this.partialByItemId = new Map();
+        this.apiKey = params.apiKey;
+        this.logger = params.logger.child({ provider: "openai", component: "realtime-transcription" });
+        this.transcriptionModel = params.transcriptionModel;
+        this.language = params.language;
+        this.prompt = params.prompt;
+        this.turnDetection = params.turnDetection ?? null;
+    }
+    async connect() {
+        if (this.ready) {
+            return this.ready;
+        }
+        this.closing = false;
+        this.ready = new Promise((resolve, reject) => {
+            const url = "wss://api.openai.com/v1/realtime?intent=transcription";
+            const ws = new WebSocket(url, {
+                headers: {
+                    Authorization: `Bearer ${this.apiKey}`,
+                },
+            });
+            this.ws = ws;
+            let resolved = false;
+            const fail = (error) => {
+                if (resolved) {
+                    this.emit("error", error);
+                    return;
+                }
+                resolved = true;
+                reject(error);
+            };
+            ws.on("open", () => {
+                this.logger.debug("OpenAI realtime transcription websocket connected");
+                const update = {
+                    type: "session.update",
+                    session: {
+                        type: "transcription",
+                        audio: {
+                            input: {
+                                format: { type: "audio/pcm", rate: 24000 },
+                                transcription: {
+                                    model: this.transcriptionModel,
+                                    ...(this.language ? { language: this.language } : {}),
+                                    ...(this.prompt ? { prompt: this.prompt } : {}),
+                                },
+                                turn_detection: this.turnDetection,
+                            },
+                        },
+                    },
+                };
+                ws.send(JSON.stringify(update));
+            });
+            ws.on("message", (data) => {
+                const text = typeof data === "string" ? data : data.toString("utf-8");
+                let parsed;
+                try {
+                    parsed = JSON.parse(text);
+                }
+                catch {
+                    return;
+                }
+                const event = parsed;
+                if (event.type === "session.created" ||
+                    event.type === "session.updated") {
+                    if (!resolved) {
+                        resolved = true;
+                        resolve();
+                    }
+                    return;
+                }
+                if (event.type === "input_audio_buffer.committed") {
+                    this.emit("committed", {
+                        segmentId: event.item_id,
+                        previousSegmentId: event.previous_item_id,
+                    });
+                    return;
+                }
+                if (event.type === "input_audio_buffer.speech_started") {
+                    this.emit("speech_started");
+                    return;
+                }
+                if (event.type === "input_audio_buffer.speech_stopped") {
+                    this.emit("speech_stopped");
+                    return;
+                }
+                if (event.type === "conversation.item.input_audio_transcription.delta") {
+                    const replaceDelta = this.transcriptionModel === "whisper-1";
+                    const prev = this.partialByItemId.get(event.item_id) ?? "";
+                    const next = replaceDelta ? event.delta : prev + event.delta;
+                    this.partialByItemId.set(event.item_id, next);
+                    this.emit("transcript", { segmentId: event.item_id, transcript: next, isFinal: false });
+                    return;
+                }
+                if (event.type === "conversation.item.input_audio_transcription.completed") {
+                    this.partialByItemId.set(event.item_id, event.transcript);
+                    this.emit("transcript", { segmentId: event.item_id, transcript: event.transcript, isFinal: true });
+                    return;
+                }
+                if (event.type === "error") {
+                    const message = event.error?.message ?? "OpenAI realtime error";
+                    fail(new Error(message));
+                }
+            });
+            ws.on("error", (err) => {
+                fail(err instanceof Error ? err : new Error(String(err)));
+            });
+            ws.on("close", () => {
+                this.logger.debug("OpenAI realtime websocket closed");
+                if (this.closing) {
+                    return;
+                }
+                if (!resolved) {
+                    fail(new Error("OpenAI realtime websocket closed before ready"));
+                    return;
+                }
+                fail(new Error("OpenAI realtime websocket closed"));
+            });
+        });
+        return this.ready;
+    }
+    appendPcm16(pcm16le) {
+        if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
+            throw new Error("OpenAI realtime websocket not connected");
+        }
+        const base64Audio = pcm16le.toString("base64");
+        const event = { type: "input_audio_buffer.append", audio: base64Audio };
+        this.ws.send(JSON.stringify(event));
+    }
+    commit() {
+        if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
+            throw new Error("OpenAI realtime websocket not connected");
+        }
+        const event = { type: "input_audio_buffer.commit" };
+        this.ws.send(JSON.stringify(event));
+    }
+    clear() {
+        if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
+            return;
+        }
+        const event = { type: "input_audio_buffer.clear" };
+        this.ws.send(JSON.stringify(event));
+    }
+    close() {
+        try {
+            this.closing = true;
+            this.ws?.close();
+        }
+        catch {
+            // no-op
+        }
+        finally {
+            this.ws = null;
+            this.ready = null;
+        }
+    }
+}
+//# sourceMappingURL=realtime-transcription-session.js.map

package/dist/server/server/speech/providers/openai/realtime-transcription-session.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"realtime-transcription-session.js","sourceRoot":"","sources":["../../../../../../src/server/speech/providers/openai/realtime-transcription-session.ts"],"names":[],"mappings":"AACA,OAAO,SAAS,MAAM,IAAI,CAAC;AAC3B,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AA6D3C,MAAM,OAAO,kCACX,SAAQ,YAAY;IAgBpB,YAAY,MAOX;QACC,KAAK,EAAE,CAAC;QArBM,uBAAkB,GAAG,KAAK,CAAC;QAQnC,OAAE,GAAqB,IAAI,CAAC;QAC5B,UAAK,GAAyB,IAAI,CAAC;QACnC,YAAO,GAAG,KAAK,CAAC;QAChB,oBAAe,GAAG,IAAI,GAAG,EAAkB,CAAC;QAWlD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QAC5B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,wBAAwB,EAAE,CAAC,CAAC;QAC/F,IAAI,CAAC,kBAAkB,GAAG,MAAM,CAAC,kBAAkB,CAAC;QACpD,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;QAChC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QAC5B,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,IAAI,IAAI,CAAC;IACpD,CAAC;IAEM,KAAK,CAAC,OAAO;QAClB,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,OAAO,IAAI,CAAC,KAAK,CAAC;QACpB,CAAC;QAED,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;QACrB,IAAI,CAAC,KAAK,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACjD,MAAM,GAAG,GAAG,uDAAuD,CAAC;YACpE,MAAM,EAAE,GAAG,IAAI,SAAS,CAAC,GAAG,EAAE;gBAC5B,OAAO,EAAE;oBACP,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;iBACvC;aACF,CAAC,CAAC;YACH,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;YAEb,IAAI,QAAQ,GAAG,KAAK,CAAC;YAErB,MAAM,IAAI,GAAG,CAAC,KAAY,EAAE,EAAE;gBAC5B,IAAI,QAAQ,EAAE,CAAC;oBACb,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;oBAC1B,OAAO;gBACT,CAAC;gBACD,QAAQ,GAAG,IAAI,CAAC;gBAChB,MAAM,CAAC,KAAK,CAAC,CAAC;YAChB,CAAC,CAAC;YAEF,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE;gBACjB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,mDAAmD,CAAC,CAAC;gBACvE,MAAM,MAAM,GAAsB;oBAChC,IAAI,EAAE,gBAAgB;oBACtB,OAAO,EAAE;wBACP,IAAI,EAAE,eAAe;wBACrB,KAAK,EAAE;4BACL,KAAK,EAAE;gCACL,MAAM,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,KAAK,EAAE;gCAC1C,aAAa,EAAE;oCACb,KAAK,EAAE,IAAI,CAAC,kBAAkB;oCAC9B,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oCACrD,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iCAChD;gCACD,cAAc,EAAE,IAAI,CAAC,aAAa;6BACnC;yBACF;qBACF;iBACF,CAAC;gBACF,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;YAClC,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,IAAI,EAAE,EAAE;gBACxB,MAAM,IAAI,GAAG,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;gBACtE,IAAI,MAAe,CAAC;gBACpB,IAAI,CAAC;oBACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAC5B,CAAC;gBAAC,MAAM,CAAC;oBACP,OAAO;gBACT,CAAC;gBAED,MAAM,KAAK,GAAG,MAA2B,CAAC;gBAC1C,IACE,KAAK,CAAC,IAAI,KAAK,iBAAiB;oBAChC,KAAK,CAAC,IAAI,KAAK,iBAAiB,EAChC,CAAC;oBACD,IAAI,CAAC,QAAQ,EAAE,CAAC;wBACd,QAAQ,GAAG,IAAI,CAAC;wBAChB,OAAO,EAAE,CAAC;oBACZ,CAAC;oBACD,OAAO;gBACT,CAAC;gBAED,IAAI,KAAK,CAAC,IAAI,KAAK,8BAA8B,EAAE,CAAC;oBAClD,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE;wBACrB,SAAS,EAAE,KAAK,CAAC,OAAO;wBACxB,iBAAiB,EAAE,KAAK,CAAC,gBAAgB;qBAC1C,CAAC,CAAC;oBACH,OAAO;gBACT,CAAC;gBAED,IAAI,KAAK,CAAC,IAAI,KAAK,mCAAmC,EAAE,CAAC;oBACvD,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;oBAC5B,OAAO;gBACT,CAAC;gBAED,IAAI,KAAK,CAAC,IAAI,KAAK,mCAAmC,EAAE,CAAC;oBACvD,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;oBAC5B,OAAO;gBACT,CAAC;gBAED,IAAI,KAAK,CAAC,IAAI,KAAK,mDAAmD,EAAE,CAAC;oBACvE,MAAM,YAAY,GAAG,IAAI,CAAC,kBAAkB,KAAK,WAAW,CAAC;oBAC7D,MAAM,IAAI,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;oBAC3D,MAAM,IAAI,GAAG,YAAY,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC;oBAC7D,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;oBAC9C,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,SAAS,EAAE,KAAK,CAAC,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;oBACxF,OAAO;gBACT,CAAC;gBAED,IAAI,KAAK,CAAC,IAAI,KAAK,uDAAuD,EAAE,CAAC;oBAC3E,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC;oBAC1D,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,SAAS,EAAE,KAAK,CAAC,OAAO,EAAE,UAAU,EAAE,KAAK,CAAC,UAAU,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;oBACnG,OAAO;gBACT,CAAC;gBAED,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;oBAC3B,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,EAAE,OAAO,IAAI,uBAAuB,CAAC;oBAChE,IAAI,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;gBAC3B,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;gBACrB,IAAI,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAC5D,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;gBAClB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC;gBACtD,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;oBACjB,OAAO;gBACT,CAAC;gBACD,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACd,IAAI,CAAC,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC,CAAC;oBACjE,OAAO;gBACT,CAAC;gBACD,IAAI,CAAC,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC,CAAC;YACtD,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAEM,WAAW,CAAC,OAAe;QAChC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,IAAI,CAAC,EAAE,CAAC,UAAU,KAAK,SAAS,CAAC,IAAI,EAAE,CAAC;YACtD,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;QAC7D,CAAC;QACD,MAAM,WAAW,GAAG,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAC/C,MAAM,KAAK,GAAsB,EAAE,IAAI,EAAE,2BAA2B,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC;QAC3F,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;IACtC,CAAC;IAEM,MAAM;QACX,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,IAAI,CAAC,EAAE,CAAC,UAAU,KAAK,SAAS,CAAC,IAAI,EAAE,CAAC;YACtD,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;QAC7D,CAAC;QACD,MAAM,KAAK,GAAsB,EAAE,IAAI,EAAE,2BAA2B,EAAE,CAAC;QACvE,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;IACtC,CAAC;IAEM,KAAK;QACV,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,IAAI,CAAC,EAAE,CAAC,UAAU,KAAK,SAAS,CAAC,IAAI,EAAE,CAAC;YACtD,OAAO;QACT,CAAC;QACD,MAAM,KAAK,GAAsB,EAAE,IAAI,EAAE,0BAA0B,EAAE,CAAC;QACtE,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;IACtC,CAAC;IAEM,KAAK;QACV,IAAI,CAAC;YACH,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;YACpB,IAAI,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC;QACnB,CAAC;QAAC,MAAM,CAAC;YACP,QAAQ;QACV,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC;YACf,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QACpB,CAAC;IACH,CAAC;CACF"}

package/dist/server/server/speech/providers/openai/runtime.d.ts

@@ -0,0 +1,27 @@
+import type { Logger } from "pino";
+import type { SpeechToTextProvider, TextToSpeechProvider } from "../../speech-provider.js";
+import type { RequestedSpeechProviders } from "../../speech-types.js";
+import { type OpenAiSpeechProviderConfig } from "./config.js";
+export type OpenAiSpeechAvailability = {
+    stt: boolean;
+    tts: boolean;
+    dictationStt: boolean;
+};
+export type SpeechServices = {
+    sttService: SpeechToTextProvider | null;
+    ttsService: TextToSpeechProvider | null;
+    dictationSttService: SpeechToTextProvider | null;
+};
+export declare function getOpenAiSpeechAvailability(openaiConfig: OpenAiSpeechProviderConfig | undefined): OpenAiSpeechAvailability;
+export declare function validateOpenAiCredentialRequirements(params: {
+    providers: RequestedSpeechProviders;
+    openaiConfig: OpenAiSpeechProviderConfig | undefined;
+    logger: Logger;
+}): void;
+export declare function initializeOpenAiSpeechServices(params: {
+    providers: RequestedSpeechProviders;
+    openaiConfig: OpenAiSpeechProviderConfig | undefined;
+    existing: SpeechServices;
+    logger: Logger;
+}): SpeechServices;
+//# sourceMappingURL=runtime.d.ts.map

package/dist/server/server/speech/providers/openai/runtime.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runtime.d.ts","sourceRoot":"","sources":["../../../../../../src/server/speech/providers/openai/runtime.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AAEnC,OAAO,KAAK,EAAE,oBAAoB,EAAE,oBAAoB,EAAE,MAAM,0BAA0B,CAAC;AAC3F,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,uBAAuB,CAAC;AACtE,OAAO,EAGL,KAAK,0BAA0B,EAChC,MAAM,aAAa,CAAC;AAWrB,MAAM,MAAM,wBAAwB,GAAG;IACrC,GAAG,EAAE,OAAO,CAAC;IACb,GAAG,EAAE,OAAO,CAAC;IACb,YAAY,EAAE,OAAO,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAC3B,UAAU,EAAE,oBAAoB,GAAG,IAAI,CAAC;IACxC,UAAU,EAAE,oBAAoB,GAAG,IAAI,CAAC;IACxC,mBAAmB,EAAE,oBAAoB,GAAG,IAAI,CAAC;CAClD,CAAC;AAaF,wBAAgB,2BAA2B,CACzC,YAAY,EAAE,0BAA0B,GAAG,SAAS,GACnD,wBAAwB,CAO1B;AAED,wBAAgB,oCAAoC,CAAC,MAAM,EAAE;IAC3D,SAAS,EAAE,wBAAwB,CAAC;IACpC,YAAY,EAAE,0BAA0B,GAAG,SAAS,CAAC;IACrD,MAAM,EAAE,MAAM,CAAC;CAChB,GAAG,IAAI,CAkCP;AAED,wBAAgB,8BAA8B,CAAC,MAAM,EAAE;IACrD,SAAS,EAAE,wBAAwB,CAAC;IACpC,YAAY,EAAE,0BAA0B,GAAG,SAAS,CAAC;IACrD,QAAQ,EAAE,cAAc,CAAC;IACzB,MAAM,EAAE,MAAM,CAAC;CAChB,GAAG,cAAc,CAwEjB"}

package/dist/server/server/speech/providers/openai/runtime.js

@@ -0,0 +1,103 @@
+import { DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL, DEFAULT_OPENAI_TTS_MODEL, } from "./config.js";
+import { OpenAIRealtimeTranscriptionSession } from "./realtime-transcription-session.js";
+import { OpenAISTT } from "./stt.js";
+import { OpenAITTS } from "./tts.js";
+function resolveOpenAiCredentials(openaiConfig) {
+    const openaiApiKey = openaiConfig?.apiKey;
+    return {
+        openaiSttApiKey: openaiConfig?.stt?.apiKey ?? openaiApiKey,
+        openaiTtsApiKey: openaiConfig?.tts?.apiKey ?? openaiApiKey,
+        openaiDictationApiKey: openaiApiKey,
+    };
+}
+export function getOpenAiSpeechAvailability(openaiConfig) {
+    const credentials = resolveOpenAiCredentials(openaiConfig);
+    return {
+        stt: Boolean(credentials.openaiSttApiKey),
+        tts: Boolean(credentials.openaiTtsApiKey),
+        dictationStt: Boolean(credentials.openaiDictationApiKey),
+    };
+}
+export function validateOpenAiCredentialRequirements(params) {
+    const { providers, logger, openaiConfig } = params;
+    const openAiCredentials = resolveOpenAiCredentials(openaiConfig);
+    const missingOpenAiCredentialsFor = [];
+    if (providers.voiceStt.provider === "openai" && !openAiCredentials.openaiSttApiKey) {
+        missingOpenAiCredentialsFor.push("voice.stt");
+    }
+    if (providers.voiceTts.provider === "openai" && !openAiCredentials.openaiTtsApiKey) {
+        missingOpenAiCredentialsFor.push("voice.tts");
+    }
+    if (providers.dictationStt.provider === "openai" &&
+        !openAiCredentials.openaiDictationApiKey) {
+        missingOpenAiCredentialsFor.push("dictation.stt");
+    }
+    if (missingOpenAiCredentialsFor.length > 0) {
+        logger.error({
+            requestedProviders: {
+                dictationStt: providers.dictationStt.provider,
+                voiceStt: providers.voiceStt.provider,
+                voiceTts: providers.voiceTts.provider,
+            },
+            missingOpenAiCredentialsFor,
+        }, "Invalid speech configuration: OpenAI provider selected but credentials are missing");
+        throw new Error(`Missing OpenAI credentials for configured speech features: ${missingOpenAiCredentialsFor.join(", ")}`);
+    }
+}
+export function initializeOpenAiSpeechServices(params) {
+    const { providers, openaiConfig, existing, logger } = params;
+    const openAiCredentials = resolveOpenAiCredentials(openaiConfig);
+    let sttService = existing.sttService;
+    let ttsService = existing.ttsService;
+    let dictationSttService = existing.dictationSttService;
+    const needsOpenAiStt = !sttService && providers.voiceStt.provider === "openai";
+    const needsOpenAiTts = !ttsService && providers.voiceTts.provider === "openai";
+    const needsOpenAiDictation = !dictationSttService && providers.dictationStt.provider === "openai";
+    if ((needsOpenAiStt || needsOpenAiTts || needsOpenAiDictation) &&
+        (openAiCredentials.openaiSttApiKey ||
+            openAiCredentials.openaiTtsApiKey ||
+            openAiCredentials.openaiDictationApiKey)) {
+        logger.info("OpenAI speech provider initialized");
+        if (needsOpenAiStt && openAiCredentials.openaiSttApiKey) {
+            const { apiKey: _sttApiKey, ...sttConfig } = openaiConfig?.stt ?? {};
+            sttService = new OpenAISTT({
+                apiKey: openAiCredentials.openaiSttApiKey,
+                ...sttConfig,
+            }, logger);
+        }
+        if (needsOpenAiTts && openAiCredentials.openaiTtsApiKey) {
+            const { apiKey: _ttsApiKey, ...ttsConfig } = openaiConfig?.tts ?? {};
+            ttsService = new OpenAITTS({
+                apiKey: openAiCredentials.openaiTtsApiKey,
+                voice: "alloy",
+                model: DEFAULT_OPENAI_TTS_MODEL,
+                responseFormat: "pcm",
+                ...ttsConfig,
+            }, logger);
+        }
+        const dictationApiKey = openAiCredentials.openaiDictationApiKey;
+        if (needsOpenAiDictation && dictationApiKey) {
+            dictationSttService = {
+                id: "openai",
+                createSession: ({ logger: sessionLogger, language, prompt }) => new OpenAIRealtimeTranscriptionSession({
+                    apiKey: dictationApiKey,
+                    logger: sessionLogger,
+                    transcriptionModel: openaiConfig?.realtimeTranscriptionModel ??
+                        DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL,
+                    ...(language ? { language } : {}),
+                    ...(prompt ? { prompt } : {}),
+                    turnDetection: null,
+                }),
+            };
+        }
+    }
+    else if (needsOpenAiStt || needsOpenAiTts || needsOpenAiDictation) {
+        logger.warn("OpenAI speech providers are configured but credentials are missing");
+    }
+    return {
+        sttService,
+        ttsService,
+        dictationSttService,
+    };
+}
+//# sourceMappingURL=runtime.js.map

package/dist/server/server/speech/providers/openai/runtime.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"runtime.js","sourceRoot":"","sources":["../../../../../../src/server/speech/providers/openai/runtime.ts"],"names":[],"mappings":"AAIA,OAAO,EACL,2CAA2C,EAC3C,wBAAwB,GAEzB,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,kCAAkC,EAAE,MAAM,qCAAqC,CAAC;AACzF,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAoBrC,SAAS,wBAAwB,CAC/B,YAAoD;IAEpD,MAAM,YAAY,GAAG,YAAY,EAAE,MAAM,CAAC;IAC1C,OAAO;QACL,eAAe,EAAE,YAAY,EAAE,GAAG,EAAE,MAAM,IAAI,YAAY;QAC1D,eAAe,EAAE,YAAY,EAAE,GAAG,EAAE,MAAM,IAAI,YAAY;QAC1D,qBAAqB,EAAE,YAAY;KACpC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,2BAA2B,CACzC,YAAoD;IAEpD,MAAM,WAAW,GAAG,wBAAwB,CAAC,YAAY,CAAC,CAAC;IAC3D,OAAO;QACL,GAAG,EAAE,OAAO,CAAC,WAAW,CAAC,eAAe,CAAC;QACzC,GAAG,EAAE,OAAO,CAAC,WAAW,CAAC,eAAe,CAAC;QACzC,YAAY,EAAE,OAAO,CAAC,WAAW,CAAC,qBAAqB,CAAC;KACzD,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,oCAAoC,CAAC,MAIpD;IACC,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,CAAC;IACnD,MAAM,iBAAiB,GAAG,wBAAwB,CAAC,YAAY,CAAC,CAAC;IAEjE,MAAM,2BAA2B,GAAa,EAAE,CAAC;IACjD,IAAI,SAAS,CAAC,QAAQ,CAAC,QAAQ,KAAK,QAAQ,IAAI,CAAC,iBAAiB,CAAC,eAAe,EAAE,CAAC;QACnF,2BAA2B,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAChD,CAAC;IACD,IAAI,SAAS,CAAC,QAAQ,CAAC,QAAQ,KAAK,QAAQ,IAAI,CAAC,iBAAiB,CAAC,eAAe,EAAE,CAAC;QACnF,2BAA2B,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAChD,CAAC;IACD,IACE,SAAS,CAAC,YAAY,CAAC,QAAQ,KAAK,QAAQ;QAC5C,CAAC,iBAAiB,CAAC,qBAAqB,EACxC,CAAC;QACD,2BAA2B,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IACpD,CAAC;IAED,IAAI,2BAA2B,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3C,MAAM,CAAC,KAAK,CACV;YACE,kBAAkB,EAAE;gBAClB,YAAY,EAAE,SAAS,CAAC,YAAY,CAAC,QAAQ;gBAC7C,QAAQ,EAAE,SAAS,CAAC,QAAQ,CAAC,QAAQ;gBACrC,QAAQ,EAAE,SAAS,CAAC,QAAQ,CAAC,QAAQ;aACtC;YACD,2BAA2B;SAC5B,EACD,oFAAoF,CACrF,CAAC;QACF,MAAM,IAAI,KAAK,CACb,8DAA8D,2BAA2B,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACvG,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,8BAA8B,CAAC,MAK9C;IACC,MAAM,EAAE,SAAS,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC;IAC7D,MAAM,iBAAiB,GAAG,wBAAwB,CAAC,YAAY,CAAC,CAAC;IAEjE,IAAI,UAAU,GAAG,QAAQ,CAAC,UAAU,CAAC;IACrC,IAAI,UAAU,GAAG,QAAQ,CAAC,UAAU,CAAC;IACrC,IAAI,mBAAmB,GAAG,QAAQ,CAAC,mBAAmB,CAAC;IAEvD,MAAM,cAAc,GAAG,CAAC,UAAU,IAAI,SAAS,CAAC,QAAQ,CAAC,QAAQ,KAAK,QAAQ,CAAC;IAC/E,MAAM,cAAc,GAAG,CAAC,UAAU,IAAI,SAAS,CAAC,QAAQ,CAAC,QAAQ,KAAK,QAAQ,CAAC;IAC/E,MAAM,oBAAoB,GACxB,CAAC,mBAAmB,IAAI,SAAS,CAAC,YAAY,CAAC,QAAQ,KAAK,QAAQ,CAAC;IAEvE,IACE,CAAC,cAAc,IAAI,cAAc,IAAI,oBAAoB,CAAC;QAC1D,CAAC,iBAAiB,CAAC,eAAe;YAChC,iBAAiB,CAAC,eAAe;YACjC,iBAAiB,CAAC,qBAAqB,CAAC,EAC1C,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAC;QAElD,IAAI,cAAc,IAAI,iBAAiB,CAAC,eAAe,EAAE,CAAC;YACxD,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,SAAS,EAAE,GAAG,YAAY,EAAE,GAAG,IAAI,EAAE,CAAC;YACrE,UAAU,GAAG,IAAI,SAAS,CACxB;gBACE,MAAM,EAAE,iBAAiB,CAAC,eAAe;gBACzC,GAAG,SAAS;aACb,EACD,MAAM,CACP,CAAC;QACJ,CAAC;QAED,IAAI,cAAc,IAAI,iBAAiB,CAAC,eAAe,EAAE,CAAC;YACxD,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,SAAS,EAAE,GAAG,YAAY,EAAE,GAAG,IAAI,EAAE,CAAC;YACrE,UAAU,GAAG,IAAI,SAAS,CACxB;gBACE,MAAM,EAAE,iBAAiB,CAAC,eAAe;gBACzC,KAAK,EAAE,OAAO;gBACd,KAAK,EAAE,wBAAwB;gBAC/B,cAAc,EAAE,KAAK;gBACrB,GAAG,SAAS;aACb,EACD,MAAM,CACP,CAAC;QACJ,CAAC;QAED,MAAM,eAAe,GAAG,iBAAiB,CAAC,qBAAqB,CAAC;QAChE,IAAI,oBAAoB,IAAI,eAAe,EAAE,CAAC;YAC5C,mBAAmB,GAAG;gBACpB,EAAE,EAAE,QAAQ;gBACZ,aAAa,EAAE,CAAC,EAAE,MAAM,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,CAC7D,IAAI,kCAAkC,CAAC;oBACrC,MAAM,EAAE,eAAe;oBACvB,MAAM,EAAE,aAAa;oBACrB,kBAAkB,EAChB,YAAY,EAAE,0BAA0B;wBACxC,2CAA2C;oBAC7C,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBACjC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBAC7B,aAAa,EAAE,IAAI;iBACpB,CAAC;aACL,CAAC;QACJ,CAAC;IACH,CAAC;SAAM,IAAI,cAAc,IAAI,cAAc,IAAI,oBAAoB,EAAE,CAAC;QACpE,MAAM,CAAC,IAAI,CAAC,oEAAoE,CAAC,CAAC;IACpF,CAAC;IAED,OAAO;QACL,UAAU;QACV,UAAU;QACV,mBAAmB;KACpB,CAAC;AACJ,CAAC"}