@getpaseo/server 0.1.16 → 0.1.17

package/dist/src/server/speech/providers/local/sherpa/sherpa-realtime-session.js

@@ -0,0 +1,112 @@
+import { EventEmitter } from "node:events";
+import { v4 as uuidv4 } from "uuid";
+import { pcm16lePeakAbs, pcm16leToFloat32 } from "../../../audio.js";
+export class SherpaRealtimeTranscriptionSession extends EventEmitter {
+    constructor(params) {
+        super();
+        this.stream = null;
+        this.connected = false;
+        this.currentSegmentId = null;
+        this.previousSegmentId = null;
+        this.lastPartialText = "";
+        this.engine = params.engine;
+        this.requiredSampleRate = this.engine.sampleRate;
+        this.tailPaddingMs = params.tailPaddingMs ?? 500;
+    }
+    async connect() {
+        if (this.connected) {
+            return;
+        }
+        this.stream = this.engine.createStream();
+        this.currentSegmentId = uuidv4();
+        this.connected = true;
+    }
+    appendPcm16(pcm16le) {
+        if (!this.connected || !this.stream || !this.currentSegmentId) {
+            this.emit("error", new Error("Sherpa realtime session not connected"));
+            return;
+        }
+        try {
+            const peak = pcm16lePeakAbs(pcm16le);
+            const peakFloat = peak / 32768.0;
+            const targetPeak = 0.6;
+            const maxGain = 50;
+            const gain = peakFloat > 0 && peakFloat < targetPeak
+                ? Math.min(maxGain, targetPeak / peakFloat)
+                : 1;
+            const floatSamples = pcm16leToFloat32(pcm16le, gain);
+            this.stream.acceptWaveform(this.engine.sampleRate, floatSamples);
+            while (this.engine.recognizer.isReady(this.stream)) {
+                this.engine.recognizer.decode(this.stream);
+            }
+            const text = String(this.engine.recognizer.getResult(this.stream)?.text ?? "").trim();
+            if (text !== this.lastPartialText) {
+                this.lastPartialText = text;
+                this.emit("transcript", {
+                    segmentId: this.currentSegmentId,
+                    transcript: text,
+                    isFinal: false,
+                });
+            }
+        }
+        catch (err) {
+            this.emit("error", err instanceof Error ? err : new Error(String(err)));
+        }
+    }
+    commit() {
+        if (!this.connected || !this.stream || !this.currentSegmentId) {
+            this.emit("error", new Error("Sherpa realtime session not connected"));
+            return;
+        }
+        try {
+            const padSamples = Math.max(0, Math.round((this.engine.sampleRate * this.tailPaddingMs) / 1000));
+            if (padSamples > 0) {
+                this.stream.acceptWaveform(this.engine.sampleRate, new Float32Array(padSamples));
+            }
+            while (this.engine.recognizer.isReady(this.stream)) {
+                this.engine.recognizer.decode(this.stream);
+            }
+            const finalText = String(this.engine.recognizer.getResult(this.stream)?.text ?? "").trim();
+            const segmentId = this.currentSegmentId;
+            const previousSegmentId = this.previousSegmentId;
+            this.emit("committed", { segmentId, previousSegmentId });
+            this.emit("transcript", { segmentId, transcript: finalText, isFinal: true });
+            this.previousSegmentId = segmentId;
+            this.currentSegmentId = uuidv4();
+            this.lastPartialText = "";
+            this.engine.recognizer.reset(this.stream);
+        }
+        catch (err) {
+            this.emit("error", err instanceof Error ? err : new Error(String(err)));
+        }
+    }
+    clear() {
+        if (!this.connected || !this.stream) {
+            return;
+        }
+        try {
+            this.engine.recognizer.reset(this.stream);
+            this.currentSegmentId = uuidv4();
+            this.lastPartialText = "";
+        }
+        catch (err) {
+            this.emit("error", err instanceof Error ? err : new Error(String(err)));
+        }
+    }
+    close() {
+        if (!this.stream) {
+            return;
+        }
+        try {
+            this.stream.free?.();
+        }
+        catch {
+            // ignore
+        }
+        finally {
+            this.stream = null;
+            this.connected = false;
+        }
+    }
+}
+//# sourceMappingURL=sherpa-realtime-session.js.map

package/dist/src/server/speech/providers/local/sherpa/sherpa-realtime-session.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sherpa-realtime-session.js","sourceRoot":"","sources":["../../../../../../../src/server/speech/providers/local/sherpa/sherpa-realtime-session.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,MAAM,CAAC;AAGpC,OAAO,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAGrE,MAAM,OAAO,kCACX,SAAQ,YAAY;IAapB,YAAY,MAAwE;QAClF,KAAK,EAAE,CAAC;QAVF,WAAM,GAAe,IAAI,CAAC;QAC1B,cAAS,GAAG,KAAK,CAAC;QAGlB,qBAAgB,GAAkB,IAAI,CAAC;QACvC,sBAAiB,GAAkB,IAAI,CAAC;QACxC,oBAAe,GAAG,EAAE,CAAC;QAK3B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QAC5B,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC;QACjD,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,IAAI,GAAG,CAAC;IACnD,CAAC;IAED,KAAK,CAAC,OAAO;QACX,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,OAAO;QACT,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;QACzC,IAAI,CAAC,gBAAgB,GAAG,MAAM,EAAE,CAAC;QACjC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;IACxB,CAAC;IAED,WAAW,CAAC,OAAe;QACzB,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC9D,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC,CAAC;YACvE,OAAO;QACT,CAAC;QAED,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;YACrC,MAAM,SAAS,GAAG,IAAI,GAAG,OAAO,CAAC;YACjC,MAAM,UAAU,GAAG,GAAG,CAAC;YACvB,MAAM,OAAO,GAAG,EAAE,CAAC;YACnB,MAAM,IAAI,GACR,SAAS,GAAG,CAAC,IAAI,SAAS,GAAG,UAAU;gBACrC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,UAAU,GAAG,SAAS,CAAC;gBAC3C,CAAC,CAAC,CAAC,CAAC;YACR,MAAM,YAAY,GAAG,gBAAgB,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;YACrD,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;YAEjE,OAAO,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;gBACnD,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC7C,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;YACtF,IAAI,IAAI,KAAK,IAAI,CAAC,eAAe,EAAE,CAAC;gBAClC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;gBAC5B,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE;oBACtB,SAAS,EAAE,IAAI,CAAC,gBAAgB;oBAChC,UAAU,EAAE,IAAI;oBAChB,OAAO,EAAE,KAAK;iBACf,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC1E,CAAC;IACH,CAAC;IAED,MAAM;QACJ,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC9D,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC,CAAC;YACvE,OAAO;QACT,CAAC;QAED,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;YACjG,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;gBACnB,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,IAAI,YAAY,CAAC,UAAU,CAAC,CAAC,CAAC;YACnF,CAAC;YAED,OAAO,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;gBACnD,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC7C,CAAC;YAED,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;YAC3F,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,CAAC;YACxC,MAAM,iBAAiB,GAAG,IAAI,CAAC,iBAAiB,CAAC;YAEjD,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,iBAAiB,EAAE,CAAC,CAAC;YACzD,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;YAE7E,IAAI,CAAC,iBAAiB,GAAG,SAAS,CAAC;YACnC,IAAI,CAAC,gBAAgB,GAAG,MAAM,EAAE,CAAC;YACjC,IAAI,CAAC,eAAe,GAAG,EAAE,CAAC;YAC1B,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC5C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC1E,CAAC;IACH,CAAC;IAED,KAAK;QACH,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACpC,OAAO;QACT,CAAC;QACD,IAAI,CAAC;YACH,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC1C,IAAI,CAAC,gBAAgB,GAAG,MAAM,EAAE,CAAC;YACjC,IAAI,CAAC,eAAe,GAAG,EAAE,CAAC;QAC5B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC1E,CAAC;IACH,CAAC;IAED,KAAK;QACH,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,OAAO;QACT,CAAC;QACD,IAAI,CAAC;YACH,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC;QACvB,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;YACnB,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACzB,CAAC;IACH,CAAC;CACF"}

package/dist/src/server/speech/providers/local/sherpa/sherpa-stt.js

@@ -0,0 +1,140 @@
+import { EventEmitter } from "node:events";
+import { v4 as uuidv4 } from "uuid";
+import { Pcm16MonoResampler } from "../../../../agent/pcm16-resampler.js";
+import { parsePcm16MonoWav, parsePcmRateFromFormat, pcm16lePeakAbs, pcm16leToFloat32 } from "../../../audio.js";
+export class SherpaOnnxSTT {
+    constructor(config, logger) {
+        this.id = "local";
+        this.engine = config.engine;
+        this.silencePeakThreshold = config.silencePeakThreshold ?? 300;
+        this.tailPaddingMs = config.tailPaddingMs ?? 500;
+        this.logger = logger.child({ module: "speech", provider: "local", component: "stt" });
+    }
+    createSession(params) {
+        const emitter = new EventEmitter();
+        void params;
+        const requiredSampleRate = this.engine.sampleRate;
+        let connected = false;
+        let segmentId = uuidv4();
+        let previousSegmentId = null;
+        let pcm16 = Buffer.alloc(0);
+        return {
+            requiredSampleRate,
+            async connect() {
+                connected = true;
+            },
+            appendPcm16(chunk) {
+                if (!connected) {
+                    emitter.emit("error", new Error("STT session not connected"));
+                    return;
+                }
+                pcm16 = pcm16.length === 0 ? chunk : Buffer.concat([pcm16, chunk]);
+            },
+            commit: () => {
+                if (!connected) {
+                    emitter.emit("error", new Error("STT session not connected"));
+                    return;
+                }
+                const committedId = segmentId;
+                const prev = previousSegmentId;
+                emitter.emit("committed", { segmentId: committedId, previousSegmentId: prev });
+                void (async () => {
+                    try {
+                        const rt = await this.transcribeAudio(pcm16, `audio/pcm;rate=${requiredSampleRate}`);
+                        emitter.emit("transcript", {
+                            segmentId: committedId,
+                            transcript: rt.text,
+                            isFinal: true,
+                            language: rt.language,
+                            logprobs: rt.logprobs,
+                            avgLogprob: rt.avgLogprob,
+                            isLowConfidence: rt.isLowConfidence,
+                        });
+                    }
+                    catch (err) {
+                        emitter.emit("error", err);
+                    }
+                    finally {
+                        previousSegmentId = committedId;
+                        segmentId = uuidv4();
+                        pcm16 = Buffer.alloc(0);
+                    }
+                })();
+            },
+            clear() {
+                pcm16 = Buffer.alloc(0);
+                segmentId = uuidv4();
+            },
+            close() {
+                connected = false;
+                pcm16 = Buffer.alloc(0);
+            },
+            on(event, handler) {
+                emitter.on(event, handler);
+                return undefined;
+            },
+        };
+    }
+    async transcribeAudio(audioBuffer, format) {
+        const start = Date.now();
+        let inputRate;
+        let pcm16;
+        if (format.toLowerCase().includes("audio/wav")) {
+            const parsed = parsePcm16MonoWav(audioBuffer);
+            inputRate = parsed.sampleRate;
+            pcm16 = parsed.pcm16;
+        }
+        else if (format.toLowerCase().includes("audio/pcm")) {
+            inputRate = parsePcmRateFromFormat(format, this.engine.sampleRate) ?? this.engine.sampleRate;
+            pcm16 = audioBuffer;
+        }
+        else {
+            throw new Error(`Unsupported audio format for sherpa STT: ${format}`);
+        }
+        const peak = pcm16lePeakAbs(pcm16);
+        if (peak < this.silencePeakThreshold) {
+            return { text: "", duration: Date.now() - start, isLowConfidence: true };
+        }
+        let pcmForModel = pcm16;
+        if (inputRate !== this.engine.sampleRate) {
+            const resampler = new Pcm16MonoResampler({ inputRate, outputRate: this.engine.sampleRate });
+            pcmForModel = resampler.processChunk(pcm16);
+            inputRate = this.engine.sampleRate;
+        }
+        const peakForModel = pcm16lePeakAbs(pcmForModel);
+        const peakFloat = peakForModel / 32768.0;
+        const targetPeak = 0.6;
+        const maxGain = 50;
+        const gain = peakFloat > 0 && peakFloat < targetPeak
+            ? Math.min(maxGain, targetPeak / peakFloat)
+            : 1;
+        const stream = this.engine.createStream();
+        try {
+            const floatSamples = pcm16leToFloat32(pcmForModel, gain);
+            stream.acceptWaveform(inputRate, floatSamples);
+            while (this.engine.recognizer.isReady(stream)) {
+                this.engine.recognizer.decode(stream);
+            }
+            const padSamples = Math.max(0, Math.round((this.engine.sampleRate * this.tailPaddingMs) / 1000));
+            if (padSamples > 0) {
+                stream.acceptWaveform(this.engine.sampleRate, new Float32Array(padSamples));
+            }
+            while (this.engine.recognizer.isReady(stream)) {
+                this.engine.recognizer.decode(stream);
+            }
+            const text = String(this.engine.recognizer.getResult(stream)?.text ?? "").trim();
+            const duration = Date.now() - start;
+            this.logger.debug({ duration, textLength: text.length }, "Sherpa transcription complete");
+            return { text, duration, ...(text.length === 0 ? { isLowConfidence: true } : {}) };
+        }
+        finally {
+            try {
+                stream.free?.();
+            }
+            catch {
+                // ignore
+            }
+        }
+    }
+}
+//# sourceMappingURL=sherpa-stt.js.map

package/dist/src/server/speech/providers/local/sherpa/sherpa-stt.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sherpa-stt.js","sourceRoot":"","sources":["../../../../../../../src/server/speech/providers/local/sherpa/sherpa-stt.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,EAAE,IAAI,MAAM,EAAE,MAAM,MAAM,CAAC;AAQpC,OAAO,EAAE,kBAAkB,EAAE,MAAM,sCAAsC,CAAC;AAC1E,OAAO,EAAE,iBAAiB,EAAE,sBAAsB,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAShH,MAAM,OAAO,aAAa;IAOxB,YAAY,MAAuB,EAAE,MAAmB;QAFxC,OAAE,GAAG,OAAgB,CAAC;QAGpC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QAC5B,IAAI,CAAC,oBAAoB,GAAG,MAAM,CAAC,oBAAoB,IAAI,GAAG,CAAC;QAC/D,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,IAAI,GAAG,CAAC;QACjD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;IACxF,CAAC;IAEM,aAAa,CAAC,MAIpB;QACC,MAAM,OAAO,GAAG,IAAI,YAAY,EAAE,CAAC;QACnC,KAAK,MAAM,CAAC;QACZ,MAAM,kBAAkB,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC;QAClD,IAAI,SAAS,GAAG,KAAK,CAAC;QACtB,IAAI,SAAS,GAAG,MAAM,EAAE,CAAC;QACzB,IAAI,iBAAiB,GAAkB,IAAI,CAAC;QAC5C,IAAI,KAAK,GAAW,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAEpC,OAAO;YACL,kBAAkB;YAClB,KAAK,CAAC,OAAO;gBACX,SAAS,GAAG,IAAI,CAAC;YACnB,CAAC;YACD,WAAW,CAAC,KAAa;gBACvB,IAAI,CAAC,SAAS,EAAE,CAAC;oBACd,OAAe,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC,CAAC;oBACvE,OAAO;gBACT,CAAC;gBACD,KAAK,GAAG,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;YACrE,CAAC;YACD,MAAM,EAAE,GAAG,EAAE;gBACX,IAAI,CAAC,SAAS,EAAE,CAAC;oBACd,OAAe,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC,CAAC;oBACvE,OAAO;gBACT,CAAC;gBAED,MAAM,WAAW,GAAG,SAAS,CAAC;gBAC9B,MAAM,IAAI,GAAG,iBAAiB,CAAC;gBAC9B,OAAe,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,WAAW,EAAE,iBAAiB,EAAE,IAAI,EAAE,CAAC,CAAC;gBAExF,KAAK,CAAC,KAAK,IAAI,EAAE;oBACf,IAAI,CAAC;wBACH,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,kBAAkB,kBAAkB,EAAE,CAAC,CAAC;wBACpF,OAAe,CAAC,IAAI,CAAC,YAAY,EAAE;4BAClC,SAAS,EAAE,WAAW;4BACtB,UAAU,EAAE,EAAE,CAAC,IAAI;4BACnB,OAAO,EAAE,IAAI;4BACb,QAAQ,EAAE,EAAE,CAAC,QAAQ;4BACrB,QAAQ,EAAE,EAAE,CAAC,QAAQ;4BACrB,UAAU,EAAE,EAAE,CAAC,UAAU;4BACzB,eAAe,EAAE,EAAE,CAAC,eAAe;yBACpC,CAAC,CAAC;oBACL,CAAC;oBAAC,OAAO,GAAG,EAAE,CAAC;wBACZ,OAAe,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;oBACtC,CAAC;4BAAS,CAAC;wBACT,iBAAiB,GAAG,WAAW,CAAC;wBAChC,SAAS,GAAG,MAAM,EAAE,CAAC;wBACrB,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;oBAC1B,CAAC;gBACH,CAAC,CAAC,EAAE,CAAC;YACP,CAAC;YACD,KAAK;gBACH,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBACxB,SAAS,GAAG,MAAM,EAAE,CAAC;YACvB,CAAC;YACD,KAAK;gBACH,SAAS,GAAG,KAAK,CAAC;gBAClB,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAC1B,CAAC;YACD,EAAE,CAAC,KAAU,EAAE,OAAY;gBACzB,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;gBAC3B,OAAO,SAAS,CAAC;YACnB,CAAC;SACF,CAAC;IACJ,CAAC;IAEM,KAAK,CAAC,eAAe,CAAC,WAAmB,EAAE,MAAc;QAC9D,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEzB,IAAI,SAAiB,CAAC;QACtB,IAAI,KAAa,CAAC;QAElB,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;YAC/C,MAAM,MAAM,GAAG,iBAAiB,CAAC,WAAW,CAAC,CAAC;YAC9C,SAAS,GAAG,MAAM,CAAC,UAAU,CAAC;YAC9B,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;QACvB,CAAC;aAAM,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;YACtD,SAAS,GAAG,sBAAsB,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC;YAC7F,KAAK,GAAG,WAAW,CAAC;QACtB,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,4CAA4C,MAAM,EAAE,CAAC,CAAC;QACxE,CAAC;QAED,MAAM,IAAI,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;QACnC,IAAI,IAAI,GAAG,IAAI,CAAC,oBAAoB,EAAE,CAAC;YACrC,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC;QAC3E,CAAC;QAED,IAAI,WAAW,GAAG,KAAK,CAAC;QACxB,IAAI,SAAS,KAAK,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC;YACzC,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,EAAE,SAAS,EAAE,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC;YAC5F,WAAW,GAAG,SAAS,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;YAC5C,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC;QACrC,CAAC;QAED,MAAM,YAAY,GAAG,cAAc,CAAC,WAAW,CAAC,CAAC;QACjD,MAAM,SAAS,GAAG,YAAY,GAAG,OAAO,CAAC;QACzC,MAAM,UAAU,GAAG,GAAG,CAAC;QACvB,MAAM,OAAO,GAAG,EAAE,CAAC;QACnB,MAAM,IAAI,GACR,SAAS,GAAG,CAAC,IAAI,SAAS,GAAG,UAAU;YACrC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,UAAU,GAAG,SAAS,CAAC;YAC3C,CAAC,CAAC,CAAC,CAAC;QAER,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;QAC1C,IAAI,CAAC;YACH,MAAM,YAAY,GAAG,gBAAgB,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;YACzD,MAAM,CAAC,cAAc,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;YAE/C,OAAO,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC9C,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACxC,CAAC;YAED,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;YACjG,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;gBACnB,MAAM,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,IAAI,YAAY,CAAC,UAAU,CAAC,CAAC,CAAC;YAC9E,CAAC;YAED,OAAO,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC9C,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACxC,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;YACjF,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;YACpC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,QAAQ,EAAE,UAAU,EAAE,IAAI,CAAC,MAAM,EAAE,EAAE,+BAA+B,CAAC,CAAC;YAC1F,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;QACrF,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC;gBACH,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC;YAClB,CAAC;YAAC,MAAM,CAAC;gBACP,SAAS;YACX,CAAC;QACH,CAAC;IACH,CAAC;CACF"}

package/dist/src/server/speech/providers/local/sherpa/sherpa-tts.js

@@ -0,0 +1,95 @@
+import { Readable } from "node:stream";
+import { existsSync } from "node:fs";
+import { chunkBuffer, float32ToPcm16le } from "../../../audio.js";
+import { loadSherpaOnnxNode } from "./sherpa-onnx-node-loader.js";
+function assertFileExists(filePath, label) {
+    if (!existsSync(filePath)) {
+        throw new Error(`Missing ${label}: ${filePath}`);
+    }
+}
+export class SherpaOnnxTTS {
+    constructor(config, logger) {
+        if (config.preset !== "kokoro-en-v0_19" && config.preset !== "kitten-nano-en-v0_1-fp16") {
+            throw new Error(`Unsupported Sherpa TTS preset: ${config.preset}`);
+        }
+        this.logger = logger.child({ module: "speech", provider: "local", component: "tts" });
+        this.speakerId = config.speakerId ?? 0;
+        this.speed = config.speed ?? 1.0;
+        const sherpa = loadSherpaOnnxNode();
+        if (typeof sherpa.OfflineTts !== "function") {
+            throw new Error("sherpa-onnx-node OfflineTts is unavailable");
+        }
+        const modelFile = config.preset === "kokoro-en-v0_19" ? "model.onnx" : "model.fp16.onnx";
+        const modelPath = `${config.modelDir}/${modelFile}`;
+        const voicesPath = `${config.modelDir}/voices.bin`;
+        const tokensPath = `${config.modelDir}/tokens.txt`;
+        const dataDir = `${config.modelDir}/espeak-ng-data`;
+        assertFileExists(modelPath, "TTS model");
+        assertFileExists(voicesPath, "TTS voices");
+        assertFileExists(tokensPath, "TTS tokens");
+        assertFileExists(dataDir, "TTS espeak-ng dataDir");
+        const modelConfig = config.preset === "kokoro-en-v0_19"
+            ? {
+                kokoro: {
+                    model: modelPath,
+                    voices: voicesPath,
+                    tokens: tokensPath,
+                    dataDir,
+                    lengthScale: config.lengthScale ?? 1.0,
+                },
+            }
+            : {
+                kitten: {
+                    model: modelPath,
+                    voices: voicesPath,
+                    tokens: tokensPath,
+                    dataDir,
+                    lengthScale: config.lengthScale ?? 1.0,
+                },
+            };
+        const offlineTtsConfig = {
+            model: modelConfig,
+            numThreads: config.numThreads ?? 2,
+            provider: "cpu",
+            maxNumSentences: 1,
+        };
+        this.tts = new sherpa.OfflineTts(offlineTtsConfig);
+        this.logger.info({ preset: config.preset, modelDir: config.modelDir }, "Sherpa offline TTS initialized");
+    }
+    async synthesizeSpeech(text) {
+        const trimmed = text.trim();
+        if (!trimmed) {
+            throw new Error("Cannot synthesize empty text");
+        }
+        const audio = this.tts.generate({ text: trimmed, sid: this.speakerId, speed: this.speed });
+        const samples = audio && audio.samples instanceof Float32Array
+            ? audio.samples
+            : audio && Array.isArray(audio.samples)
+                ? Float32Array.from(audio.samples)
+                : null;
+        const sampleRate = audio && typeof audio.sampleRate === "number" && Number.isFinite(audio.sampleRate) && audio.sampleRate > 0
+            ? audio.sampleRate
+            : typeof this.tts.sampleRate === "number"
+                ? this.tts.sampleRate
+                : 24000;
+        if (!samples) {
+            throw new Error("Unexpected sherpa TTS output: missing Float32 samples");
+        }
+        const pcm16 = float32ToPcm16le(samples);
+        const chunkBytes = Math.max(2, Math.round(sampleRate * 0.05) * 2); // ~50ms
+        const chunks = chunkBuffer(pcm16, chunkBytes);
+        return {
+            stream: Readable.from(chunks),
+            format: `pcm;rate=${sampleRate}`,
+        };
+    }
+    free() {
+        try {
+            this.tts?.free?.();
+        }
+        catch {
+            // ignore
+        }
+    }
+}
+//# sourceMappingURL=sherpa-tts.js.map

package/dist/src/server/speech/providers/local/sherpa/sherpa-tts.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sherpa-tts.js","sourceRoot":"","sources":["../../../../../../../src/server/speech/providers/local/sherpa/sherpa-tts.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAGrC,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAClE,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AAalE,SAAS,gBAAgB,CAAC,QAAgB,EAAE,KAAa;IACvD,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,WAAW,KAAK,KAAK,QAAQ,EAAE,CAAC,CAAC;IACnD,CAAC;AACH,CAAC;AAED,MAAM,OAAO,aAAa;IAMxB,YAAY,MAAuB,EAAE,MAAmB;QACtD,IAAI,MAAM,CAAC,MAAM,KAAK,iBAAiB,IAAI,MAAM,CAAC,MAAM,KAAK,0BAA0B,EAAE,CAAC;YACxF,MAAM,IAAI,KAAK,CAAC,kCAAkC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QACrE,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;QACtF,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,CAAC,CAAC;QACvC,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,GAAG,CAAC;QAEjC,MAAM,MAAM,GAAG,kBAAkB,EAAE,CAAC;QACpC,IAAI,OAAO,MAAM,CAAC,UAAU,KAAK,UAAU,EAAE,CAAC;YAC5C,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;QAChE,CAAC;QAED,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,KAAK,iBAAiB,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,iBAAiB,CAAC;QACzF,MAAM,SAAS,GAAG,GAAG,MAAM,CAAC,QAAQ,IAAI,SAAS,EAAE,CAAC;QACpD,MAAM,UAAU,GAAG,GAAG,MAAM,CAAC,QAAQ,aAAa,CAAC;QACnD,MAAM,UAAU,GAAG,GAAG,MAAM,CAAC,QAAQ,aAAa,CAAC;QACnD,MAAM,OAAO,GAAG,GAAG,MAAM,CAAC,QAAQ,iBAAiB,CAAC;QAEpD,gBAAgB,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;QACzC,gBAAgB,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;QAC3C,gBAAgB,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;QAC3C,gBAAgB,CAAC,OAAO,EAAE,uBAAuB,CAAC,CAAC;QAEnD,MAAM,WAAW,GACf,MAAM,CAAC,MAAM,KAAK,iBAAiB;YACjC,CAAC,CAAC;gBACE,MAAM,EAAE;oBACN,KAAK,EAAE,SAAS;oBAChB,MAAM,EAAE,UAAU;oBAClB,MAAM,EAAE,UAAU;oBAClB,OAAO;oBACP,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,GAAG;iBACvC;aACF;YACH,CAAC,CAAC;gBACE,MAAM,EAAE;oBACN,KAAK,EAAE,SAAS;oBAChB,MAAM,EAAE,UAAU;oBAClB,MAAM,EAAE,UAAU;oBAClB,OAAO;oBACP,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,GAAG;iBACvC;aACF,CAAC;QAER,MAAM,gBAAgB,GAAG;YACvB,KAAK,EAAE,WAAW;YAClB,UAAU,EAAE,MAAM,CAAC,UAAU,IAAI,CAAC;YAClC,QAAQ,EAAE,KAAK;YACf,eAAe,EAAE,CAAC;SACnB,CAAC;QAEF,IAAI,CAAC,GAAG,GAAG,IAAI,MAAM,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC;QACnD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,EAAE,gCAAgC,CAAC,CAAC;IAC3G,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,IAAY;QACjC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAC5B,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;QAClD,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,SAAS,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;QAC3F,MAAM,OAAO,GACX,KAAK,IAAI,KAAK,CAAC,OAAO,YAAY,YAAY;YAC5C,CAAC,CAAC,KAAK,CAAC,OAAO;YACf,CAAC,CAAC,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC;gBACrC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,OAAmB,CAAC;gBAC9C,CAAC,CAAC,IAAI,CAAC;QACb,MAAM,UAAU,GACd,KAAK,IAAI,OAAO,KAAK,CAAC,UAAU,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,UAAU,GAAG,CAAC;YACxG,CAAC,CAAC,KAAK,CAAC,UAAU;YAClB,CAAC,CAAC,OAAO,IAAI,CAAC,GAAG,CAAC,UAAU,KAAK,QAAQ;gBACvC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU;gBACrB,CAAC,CAAC,KAAK,CAAC;QAEd,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAC;QAC3E,CAAC;QAED,MAAM,KAAK,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;QACxC,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ;QAC3E,MAAM,MAAM,GAAG,WAAW,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QAE9C,OAAO;YACL,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC;YAC7B,MAAM,EAAE,YAAY,UAAU,EAAE;SACjC,CAAC;IACJ,CAAC;IAED,IAAI;QACF,IAAI,CAAC;YACH,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC;QACrB,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;IACH,CAAC;CACF"}

package/dist/src/server/speech/providers/openai/config.js

@@ -0,0 +1,99 @@
+import { z } from "zod";
+export const DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL = "gpt-4o-transcribe";
+export const DEFAULT_OPENAI_TTS_MODEL = "tts-1";
+const OpenAiTtsVoiceSchema = z.enum([
+    "alloy",
+    "echo",
+    "fable",
+    "onyx",
+    "nova",
+    "shimmer",
+]);
+const OpenAiTtsModelSchema = z.enum(["tts-1", "tts-1-hd"]);
+const NumberLikeSchema = z.union([
+    z.number(),
+    z.string().trim().min(1),
+]);
+const OptionalFiniteNumberSchema = NumberLikeSchema
+    .pipe(z.coerce.number().finite())
+    .optional();
+const OptionalTrimmedStringSchema = z
+    .string()
+    .trim()
+    .optional()
+    .transform((value) => (value && value.length > 0 ? value : undefined));
+const OpenAiSpeechResolutionSchema = z.object({
+    apiKey: OptionalTrimmedStringSchema,
+    sttConfidenceThreshold: OptionalFiniteNumberSchema,
+    sttModel: OptionalTrimmedStringSchema,
+    ttsVoice: z
+        .string()
+        .trim()
+        .toLowerCase()
+        .pipe(OpenAiTtsVoiceSchema)
+        .default("alloy"),
+    ttsModel: z
+        .string()
+        .trim()
+        .toLowerCase()
+        .pipe(OpenAiTtsModelSchema)
+        .default(DEFAULT_OPENAI_TTS_MODEL),
+    realtimeTranscriptionModel: OptionalTrimmedStringSchema.default(DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL),
+});
+export function resolveOpenAiSpeechConfig(params) {
+    const parsed = OpenAiSpeechResolutionSchema.parse({
+        apiKey: params.env.OPENAI_API_KEY ?? params.persisted.providers?.openai?.apiKey,
+        sttConfidenceThreshold: params.env.STT_CONFIDENCE_THRESHOLD ??
+            params.persisted.features?.dictation?.stt?.confidenceThreshold,
+        sttModel: params.env.STT_MODEL ??
+            (params.providers.voiceStt.enabled !== false &&
+                params.providers.voiceStt.provider === "openai"
+                ? params.persisted.features?.voiceMode?.stt?.model
+                : undefined) ??
+            (params.providers.dictationStt.enabled !== false &&
+                params.providers.dictationStt.provider === "openai"
+                ? params.persisted.features?.dictation?.stt?.model
+                : undefined),
+        ttsVoice: params.env.TTS_VOICE ??
+            (params.providers.voiceTts.enabled !== false &&
+                params.providers.voiceTts.provider === "openai"
+                ? params.persisted.features?.voiceMode?.tts?.voice
+                : undefined) ??
+            "alloy",
+        ttsModel: params.env.TTS_MODEL ??
+            (params.providers.voiceTts.enabled !== false &&
+                params.providers.voiceTts.provider === "openai"
+                ? params.persisted.features?.voiceMode?.tts?.model
+                : undefined) ??
+            DEFAULT_OPENAI_TTS_MODEL,
+        realtimeTranscriptionModel: params.env.OPENAI_REALTIME_TRANSCRIPTION_MODEL ??
+            (params.providers.dictationStt.enabled !== false &&
+                params.providers.dictationStt.provider === "openai"
+                ? params.persisted.features?.dictation?.stt?.model
+                : undefined) ??
+            DEFAULT_OPENAI_REALTIME_TRANSCRIPTION_MODEL,
+    });
+    if (!parsed.apiKey) {
+        return undefined;
+    }
+    return {
+        apiKey: parsed.apiKey,
+        stt: {
+            apiKey: parsed.apiKey,
+            ...(parsed.sttConfidenceThreshold !== undefined
+                ? { confidenceThreshold: parsed.sttConfidenceThreshold }
+                : {}),
+            ...(parsed.sttModel
+                ? { model: parsed.sttModel }
+                : {}),
+        },
+        tts: {
+            apiKey: parsed.apiKey,
+            voice: parsed.ttsVoice,
+            model: parsed.ttsModel,
+            responseFormat: "pcm",
+        },
+        realtimeTranscriptionModel: parsed.realtimeTranscriptionModel,
+    };
+}
+//# sourceMappingURL=config.js.map

package/dist/src/server/speech/providers/openai/config.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.js","sourceRoot":"","sources":["../../../../../../src/server/speech/providers/openai/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAOxB,MAAM,CAAC,MAAM,2CAA2C,GAAG,mBAAmB,CAAC;AAC/E,MAAM,CAAC,MAAM,wBAAwB,GAAG,OAAO,CAAC;AAShD,MAAM,oBAAoB,GAAG,CAAC,CAAC,IAAI,CAAC;IAClC,OAAO;IACP,MAAM;IACN,OAAO;IACP,MAAM;IACN,MAAM;IACN,SAAS;CACV,CAAC,CAAC;AAEH,MAAM,oBAAoB,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC;AAE3D,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC;IAC/B,CAAC,CAAC,MAAM,EAAE;IACV,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;CACzB,CAAC,CAAC;AAEH,MAAM,0BAA0B,GAAG,gBAAgB;KAChD,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC;KAChC,QAAQ,EAAE,CAAC;AAEd,MAAM,2BAA2B,GAAG,CAAC;KAClC,MAAM,EAAE;KACR,IAAI,EAAE;KACN,QAAQ,EAAE;KACV,SAAS,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;AAEzE,MAAM,4BAA4B,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,MAAM,EAAE,2BAA2B;IACnC,sBAAsB,EAAE,0BAA0B;IAClD,QAAQ,EAAE,2BAA2B;IACrC,QAAQ,EAAE,CAAC;SACR,MAAM,EAAE;SACR,IAAI,EAAE;SACN,WAAW,EAAE;SACb,IAAI,CAAC,oBAAoB,CAAC;SAC1B,OAAO,CAAC,OAAO,CAAC;IACnB,QAAQ,EAAE,CAAC;SACR,MAAM,EAAE;SACR,IAAI,EAAE;SACN,WAAW,EAAE;SACb,IAAI,CAAC,oBAAoB,CAAC;SAC1B,OAAO,CAAC,wBAAwB,CAAC;IACpC,0BAA0B,EAAE,2BAA2B,CAAC,OAAO,CAC7D,2CAA2C,CAC5C;CACF,CAAC,CAAC;AAEH,MAAM,UAAU,yBAAyB,CAAC,MAIzC;IACC,MAAM,MAAM,GAAG,4BAA4B,CAAC,KAAK,CAAC;QAChD,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,cAAc,IAAI,MAAM,CAAC,SAAS,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM;QAC/E,sBAAsB,EACpB,MAAM,CAAC,GAAG,CAAC,wBAAwB;YACnC,MAAM,CAAC,SAAS,CAAC,QAAQ,EAAE,SAAS,EAAE,GAAG,EAAE,mBAAmB;QAChE,QAAQ,EACN,MAAM,CAAC,GAAG,CAAC,SAAS;YACpB,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,OAAO,KAAK,KAAK;gBAC5C,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,QAAQ,KAAK,QAAQ;gBAC7C,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,EAAE,SAAS,EAAE,GAAG,EAAE,KAAK;gBAClD,CAAC,CAAC,SAAS,CAAC;YACd,CAAC,MAAM,CAAC,SAAS,CAAC,YAAY,CAAC,OAAO,KAAK,KAAK;gBAChD,MAAM,CAAC,SAAS,CAAC,YAAY,CAAC,QAAQ,KAAK,QAAQ;gBACjD,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,EAAE,SAAS,EAAE,GAAG,EAAE,KAAK;gBAClD,CAAC,CAAC,SAAS,CAAC;QAChB,QAAQ,EACN,MAAM,CAAC,GAAG,CAAC,SAAS;YACpB,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,OAAO,KAAK,KAAK;gBAC5C,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,QAAQ,KAAK,QAAQ;gBAC7C,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,EAAE,SAAS,EAAE,GAAG,EAAE,KAAK;gBAClD,CAAC,CAAC,SAAS,CAAC;YACd,OAAO;QACT,QAAQ,EACN,MAAM,CAAC,GAAG,CAAC,SAAS;YACpB,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,OAAO,KAAK,KAAK;gBAC5C,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,QAAQ,KAAK,QAAQ;gBAC7C,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,EAAE,SAAS,EAAE,GAAG,EAAE,KAAK;gBAClD,CAAC,CAAC,SAAS,CAAC;YACd,wBAAwB;QAC1B,0BAA0B,EACxB,MAAM,CAAC,GAAG,CAAC,mCAAmC;YAC9C,CAAC,MAAM,CAAC,SAAS,CAAC,YAAY,CAAC,OAAO,KAAK,KAAK;gBAChD,MAAM,CAAC,SAAS,CAAC,YAAY,CAAC,QAAQ,KAAK,QAAQ;gBACjD,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,EAAE,SAAS,EAAE,GAAG,EAAE,KAAK;gBAClD,CAAC,CAAC,SAAS,CAAC;YACd,2CAA2C;KAC9C,CAAC,CAAC;IAEH,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QACnB,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,OAAO;QACL,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,GAAG,EAAE;YACH,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,GAAG,CAAC,MAAM,CAAC,sBAAsB,KAAK,SAAS;gBAC7C,CAAC,CAAC,EAAE,mBAAmB,EAAE,MAAM,CAAC,sBAAsB,EAAE;gBACxD,CAAC,CAAC,EAAE,CAAC;YACP,GAAG,CAAC,MAAM,CAAC,QAAQ;gBACjB,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,QAAQ,EAAE;gBAC5B,CAAC,CAAC,EAAE,CAAC;SACR;QACD,GAAG,EAAE;YACH,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,KAAK,EAAE,MAAM,CAAC,QAAQ;YACtB,KAAK,EAAE,MAAM,CAAC,QAAQ;YACtB,cAAc,EAAE,KAAK;SACtB;QACD,0BAA0B,EAAE,MAAM,CAAC,0BAA0B;KAC9D,CAAC;AACJ,CAAC"}

package/dist/src/server/speech/providers/openai/realtime-transcription-session.js

@@ -0,0 +1,165 @@
+import WebSocket from "ws";
+import { EventEmitter } from "node:events";
+export class OpenAIRealtimeTranscriptionSession extends EventEmitter {
+    constructor(params) {
+        super();
+        this.requiredSampleRate = 24000;
+        this.ws = null;
+        this.ready = null;
+        this.closing = false;
+        this.partialByItemId = new Map();
+        this.apiKey = params.apiKey;
+        this.logger = params.logger.child({ provider: "openai", component: "realtime-transcription" });
+        this.transcriptionModel = params.transcriptionModel;
+        this.language = params.language;
+        this.prompt = params.prompt;
+        this.turnDetection = params.turnDetection ?? null;
+    }
+    async connect() {
+        if (this.ready) {
+            return this.ready;
+        }
+        this.closing = false;
+        this.ready = new Promise((resolve, reject) => {
+            const url = "wss://api.openai.com/v1/realtime?intent=transcription";
+            const ws = new WebSocket(url, {
+                headers: {
+                    Authorization: `Bearer ${this.apiKey}`,
+                },
+            });
+            this.ws = ws;
+            let resolved = false;
+            const fail = (error) => {
+                if (resolved) {
+                    this.emit("error", error);
+                    return;
+                }
+                resolved = true;
+                reject(error);
+            };
+            ws.on("open", () => {
+                this.logger.debug("OpenAI realtime transcription websocket connected");
+                const update = {
+                    type: "session.update",
+                    session: {
+                        type: "transcription",
+                        audio: {
+                            input: {
+                                format: { type: "audio/pcm", rate: 24000 },
+                                transcription: {
+                                    model: this.transcriptionModel,
+                                    ...(this.language ? { language: this.language } : {}),
+                                    ...(this.prompt ? { prompt: this.prompt } : {}),
+                                },
+                                turn_detection: this.turnDetection,
+                            },
+                        },
+                    },
+                };
+                ws.send(JSON.stringify(update));
+            });
+            ws.on("message", (data) => {
+                const text = typeof data === "string" ? data : data.toString("utf-8");
+                let parsed;
+                try {
+                    parsed = JSON.parse(text);
+                }
+                catch {
+                    return;
+                }
+                const event = parsed;
+                if (event.type === "session.created" ||
+                    event.type === "session.updated") {
+                    if (!resolved) {
+                        resolved = true;
+                        resolve();
+                    }
+                    return;
+                }
+                if (event.type === "input_audio_buffer.committed") {
+                    this.emit("committed", {
+                        segmentId: event.item_id,
+                        previousSegmentId: event.previous_item_id,
+                    });
+                    return;
+                }
+                if (event.type === "input_audio_buffer.speech_started") {
+                    this.emit("speech_started");
+                    return;
+                }
+                if (event.type === "input_audio_buffer.speech_stopped") {
+                    this.emit("speech_stopped");
+                    return;
+                }
+                if (event.type === "conversation.item.input_audio_transcription.delta") {
+                    const replaceDelta = this.transcriptionModel === "whisper-1";
+                    const prev = this.partialByItemId.get(event.item_id) ?? "";
+                    const next = replaceDelta ? event.delta : prev + event.delta;
+                    this.partialByItemId.set(event.item_id, next);
+                    this.emit("transcript", { segmentId: event.item_id, transcript: next, isFinal: false });
+                    return;
+                }
+                if (event.type === "conversation.item.input_audio_transcription.completed") {
+                    this.partialByItemId.set(event.item_id, event.transcript);
+                    this.emit("transcript", { segmentId: event.item_id, transcript: event.transcript, isFinal: true });
+                    return;
+                }
+                if (event.type === "error") {
+                    const message = event.error?.message ?? "OpenAI realtime error";
+                    fail(new Error(message));
+                }
+            });
+            ws.on("error", (err) => {
+                fail(err instanceof Error ? err : new Error(String(err)));
+            });
+            ws.on("close", () => {
+                this.logger.debug("OpenAI realtime websocket closed");
+                if (this.closing) {
+                    return;
+                }
+                if (!resolved) {
+                    fail(new Error("OpenAI realtime websocket closed before ready"));
+                    return;
+                }
+                fail(new Error("OpenAI realtime websocket closed"));
+            });
+        });
+        return this.ready;
+    }
+    appendPcm16(pcm16le) {
+        if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
+            throw new Error("OpenAI realtime websocket not connected");
+        }
+        const base64Audio = pcm16le.toString("base64");
+        const event = { type: "input_audio_buffer.append", audio: base64Audio };
+        this.ws.send(JSON.stringify(event));
+    }
+    commit() {
+        if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
+            throw new Error("OpenAI realtime websocket not connected");
+        }
+        const event = { type: "input_audio_buffer.commit" };
+        this.ws.send(JSON.stringify(event));
+    }
+    clear() {
+        if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
+            return;
+        }
+        const event = { type: "input_audio_buffer.clear" };
+        this.ws.send(JSON.stringify(event));
+    }
+    close() {
+        try {
+            this.closing = true;
+            this.ws?.close();
+        }
+        catch {
+            // no-op
+        }
+        finally {
+            this.ws = null;
+            this.ready = null;
+        }
+    }
+}
+//# sourceMappingURL=realtime-transcription-session.js.map