@camerontaylor/paseo-server 0.8.0-fork.1

package/dist/server/server/speech/providers/openai/stt.js

@@ -0,0 +1,209 @@
+import { EventEmitter } from "node:events";
+import { OpenAI } from "openai";
+import { writeFile, unlink } from "fs/promises";
+import { join } from "path";
+import { tmpdir } from "os";
+import { v4 } from "uuid";
+import { inferAudioExtension } from "../../../agent/audio-utils.js";
+function isObject(value) {
+    return typeof value === "object" && value !== null;
+}
+function isLogprobToken(value) {
+    if (!isObject(value)) {
+        return false;
+    }
+    if (typeof value.token !== "string") {
+        return false;
+    }
+    if (typeof value.logprob !== "number") {
+        return false;
+    }
+    if (value.bytes === undefined) {
+        return true;
+    }
+    return Array.isArray(value.bytes) && value.bytes.every((entry) => typeof entry === "number");
+}
+function isLogprobTokenArray(value) {
+    return Array.isArray(value) && value.every((entry) => isLogprobToken(entry));
+}
+export class OpenAISTT {
+    constructor(sttConfig, parentLogger) {
+        this.id = "openai";
+        this.config = sttConfig;
+        this.logger = parentLogger.child({ module: "agent", provider: "openai", component: "stt" });
+        this.openaiClient = new OpenAI({
+            apiKey: sttConfig.apiKey,
+            ...(sttConfig.baseUrl ? { baseURL: sttConfig.baseUrl } : {}),
+        });
+        this.logger.info({ model: sttConfig.model || "whisper-1" }, "STT (OpenAI Whisper) initialized");
+    }
+    createSession(params) {
+        const emitter = new EventEmitter();
+        const logger = params.logger.child({ provider: "openai", component: "stt-session" });
+        const requiredSampleRate = 24000;
+        let connected = false;
+        let segmentId = v4();
+        let previousSegmentId = null;
+        let pcm16 = Buffer.alloc(0);
+        const transcribeAudio = this.transcribeAudioInternal.bind(this);
+        const convertPCMToWavBuffer = (pcmBuffer) => {
+            const headerSize = 44;
+            const channels = 1;
+            const bitsPerSample = 16;
+            const sampleRate = requiredSampleRate;
+            const wavBuffer = Buffer.alloc(headerSize + pcmBuffer.length);
+            const byteRate = (sampleRate * channels * bitsPerSample) / 8;
+            const blockAlign = (channels * bitsPerSample) / 8;
+            wavBuffer.write("RIFF", 0);
+            wavBuffer.writeUInt32LE(36 + pcmBuffer.length, 4);
+            wavBuffer.write("WAVE", 8);
+            wavBuffer.write("fmt ", 12);
+            wavBuffer.writeUInt32LE(16, 16);
+            wavBuffer.writeUInt16LE(1, 20);
+            wavBuffer.writeUInt16LE(channels, 22);
+            wavBuffer.writeUInt32LE(sampleRate, 24);
+            wavBuffer.writeUInt32LE(byteRate, 28);
+            wavBuffer.writeUInt16LE(blockAlign, 32);
+            wavBuffer.writeUInt16LE(bitsPerSample, 34);
+            wavBuffer.write("data", 36);
+            wavBuffer.writeUInt32LE(pcmBuffer.length, 40);
+            pcmBuffer.copy(wavBuffer, 44);
+            return wavBuffer;
+        };
+        return {
+            requiredSampleRate,
+            async connect() {
+                connected = true;
+            },
+            appendPcm16(chunk) {
+                if (!connected) {
+                    emitter.emit("error", new Error("STT session not connected"));
+                    return;
+                }
+                pcm16 = pcm16.length === 0 ? chunk : Buffer.concat([pcm16, chunk]);
+            },
+            commit() {
+                if (!connected) {
+                    emitter.emit("error", new Error("STT session not connected"));
+                    return;
+                }
+                const committedId = segmentId;
+                const prev = previousSegmentId;
+                emitter.emit("committed", { segmentId: committedId, previousSegmentId: prev });
+                void (async () => {
+                    try {
+                        if (pcm16.length === 0) {
+                            emitter.emit("transcript", {
+                                segmentId: committedId,
+                                transcript: "",
+                                isFinal: true,
+                                language: params.language,
+                                isLowConfidence: true,
+                            });
+                            return;
+                        }
+                        const wav = convertPCMToWavBuffer(pcm16);
+                        const result = await transcribeAudio(wav, "audio/wav", params.language ?? "en", logger, params.prompt);
+                        emitter.emit("transcript", {
+                            segmentId: committedId,
+                            transcript: result.text,
+                            isFinal: true,
+                            language: result.language,
+                            logprobs: result.logprobs,
+                            avgLogprob: result.avgLogprob,
+                            isLowConfidence: result.isLowConfidence,
+                        });
+                    }
+                    catch (err) {
+                        emitter.emit("error", err);
+                    }
+                    finally {
+                        previousSegmentId = committedId;
+                        segmentId = v4();
+                        pcm16 = Buffer.alloc(0);
+                    }
+                })();
+            },
+            clear() {
+                pcm16 = Buffer.alloc(0);
+                segmentId = v4();
+            },
+            close() {
+                connected = false;
+                pcm16 = Buffer.alloc(0);
+            },
+            on(event, handler) {
+                emitter.on(event, handler);
+                return undefined;
+            },
+        };
+    }
+    async transcribeAudioInternal(audioBuffer, format, language, logger, prompt) {
+        const startTime = Date.now();
+        let tempFilePath = null;
+        try {
+            const ext = inferAudioExtension(format);
+            tempFilePath = join(tmpdir(), `audio-${v4()}.${ext}`);
+            await writeFile(tempFilePath, audioBuffer);
+            logger.debug({ tempFilePath, bytes: audioBuffer.length }, "Transcribing audio file");
+            const modelToUse = this.config.model ?? "whisper-1";
+            const supportsLogprobs = modelToUse === "gpt-4o-transcribe" || modelToUse === "gpt-4o-mini-transcribe";
+            const includeLogprobs = ["logprobs"];
+            const response = await this.openaiClient.audio.transcriptions.create({
+                file: await import("fs").then((fs) => fs.createReadStream(tempFilePath)),
+                language,
+                model: modelToUse,
+                ...(prompt ? { prompt } : {}),
+                ...(supportsLogprobs ? { include: includeLogprobs } : {}),
+                response_format: "json",
+            });
+            const duration = Date.now() - startTime;
+            const confidenceThreshold = this.config.confidenceThreshold ?? -3.0;
+            let avgLogprob;
+            let isLowConfidence = false;
+            const logprobs = supportsLogprobs && isObject(response) && isLogprobTokenArray(response.logprobs)
+                ? response.logprobs
+                : undefined;
+            if (logprobs && logprobs.length > 0) {
+                const totalLogprob = logprobs.reduce((sum, token) => sum + token.logprob, 0);
+                avgLogprob = totalLogprob / logprobs.length;
+                isLowConfidence = avgLogprob < confidenceThreshold;
+                if (isLowConfidence) {
+                    logger.debug({
+                        avgLogprob,
+                        threshold: confidenceThreshold,
+                        text: response.text,
+                        tokenLogprobs: logprobs.map((t) => `${t.token}:${t.logprob.toFixed(2)}`).join(", "),
+                    }, "Low confidence transcription detected");
+                }
+            }
+            logger.debug({ duration, text: response.text, avgLogprob }, "Transcription complete");
+            return {
+                text: response.text,
+                duration: duration,
+                logprobs: logprobs,
+                avgLogprob: avgLogprob,
+                isLowConfidence: isLowConfidence,
+                language: isObject(response) && typeof response.language === "string"
+                    ? response.language
+                    : undefined,
+            };
+        }
+        catch (error) {
+            logger.error({ err: error }, "Transcription error");
+            const message = error instanceof Error ? error.message : String(error);
+            throw new Error(`STT transcription failed: ${message}`, { cause: error });
+        }
+        finally {
+            if (tempFilePath) {
+                try {
+                    await unlink(tempFilePath);
+                }
+                catch {
+                    logger.warn({ tempFilePath }, "Failed to clean up temp file");
+                }
+            }
+        }
+    }
+}
+//# sourceMappingURL=stt.js.map

package/dist/server/server/speech/providers/openai/tts.d.ts

@@ -0,0 +1,19 @@
+import type pino from "pino";
+import type { SpeechStreamResult, TextToSpeechProvider } from "../../speech-provider.js";
+export type { SpeechStreamResult };
+export interface TTSConfig {
+    apiKey: string;
+    baseUrl?: string;
+    model?: "tts-1" | "tts-1-hd";
+    voice?: "alloy" | "echo" | "fable" | "onyx" | "nova" | "shimmer";
+    responseFormat?: "mp3" | "opus" | "aac" | "flac" | "wav" | "pcm";
+}
+export declare class OpenAITTS implements TextToSpeechProvider {
+    private readonly openaiClient;
+    private readonly config;
+    private readonly logger;
+    constructor(ttsConfig: TTSConfig, parentLogger: pino.Logger);
+    getConfig(): TTSConfig;
+    synthesizeSpeech(text: string): Promise<SpeechStreamResult>;
+}
+//# sourceMappingURL=tts.d.ts.map

package/dist/server/server/speech/providers/openai/tts.js

@@ -0,0 +1,48 @@
+import { OpenAI } from "openai";
+export class OpenAITTS {
+    constructor(ttsConfig, parentLogger) {
+        this.config = {
+            model: "tts-1",
+            voice: "alloy",
+            responseFormat: "pcm",
+            ...ttsConfig,
+        };
+        this.logger = parentLogger.child({ module: "agent", provider: "openai", component: "tts" });
+        this.openaiClient = new OpenAI({
+            apiKey: ttsConfig.apiKey,
+            ...(ttsConfig.baseUrl ? { baseURL: ttsConfig.baseUrl } : {}),
+        });
+        this.logger.info({ voice: this.config.voice, model: this.config.model, format: this.config.responseFormat }, "TTS (OpenAI) initialized");
+    }
+    getConfig() {
+        return this.config;
+    }
+    async synthesizeSpeech(text) {
+        if (!text || text.trim().length === 0) {
+            throw new Error("Cannot synthesize empty text");
+        }
+        const startTime = Date.now();
+        try {
+            this.logger.debug({ textLength: text.length, preview: text.substring(0, 50) }, "Synthesizing speech");
+            const response = await this.openaiClient.audio.speech.create({
+                model: this.config.model,
+                voice: this.config.voice,
+                input: text,
+                response_format: this.config.responseFormat,
+            });
+            const audioStream = response.body;
+            const duration = Date.now() - startTime;
+            this.logger.debug({ duration }, "Speech synthesis stream ready");
+            return {
+                stream: audioStream,
+                format: this.config.responseFormat || "mp3",
+            };
+        }
+        catch (error) {
+            this.logger.error({ err: error }, "Speech synthesis error");
+            const message = error instanceof Error ? error.message : String(error);
+            throw new Error(`TTS synthesis failed: ${message}`, { cause: error });
+        }
+    }
+}
+//# sourceMappingURL=tts.js.map

package/dist/server/server/speech/speech-config-resolver.d.ts

@@ -0,0 +1,11 @@
+import type { PersistedConfig } from "../persisted-config.js";
+import type { PaseoOpenAIConfig, PaseoSpeechConfig } from "../bootstrap.js";
+export declare function resolveSpeechConfig(params: {
+    paseoHome: string;
+    env: NodeJS.ProcessEnv;
+    persisted: PersistedConfig;
+}): {
+    openai: PaseoOpenAIConfig | undefined;
+    speech: PaseoSpeechConfig;
+};
+//# sourceMappingURL=speech-config-resolver.d.ts.map

package/dist/server/server/speech/speech-config-resolver.js

@@ -0,0 +1,132 @@
+import { z } from "zod";
+import { resolveLocalSpeechConfig } from "./providers/local/config.js";
+import { resolveOpenAiSpeechConfig } from "./providers/openai/config.js";
+import { SpeechProviderIdSchema, } from "./speech-types.js";
+const OptionalSpeechProviderSchema = z
+    .string()
+    .trim()
+    .toLowerCase()
+    .pipe(SpeechProviderIdSchema)
+    .optional();
+const OptionalBooleanFlagSchema = z
+    .union([z.boolean(), z.string().trim().toLowerCase()])
+    .optional()
+    .transform((value) => {
+    if (typeof value === "boolean") {
+        return value;
+    }
+    if (value === undefined) {
+        return undefined;
+    }
+    if (["1", "true", "yes", "y", "on"].includes(value)) {
+        return true;
+    }
+    if (["0", "false", "no", "n", "off"].includes(value)) {
+        return false;
+    }
+    return undefined;
+});
+const RequestedSpeechProvidersSchema = z.object({
+    dictationStt: OptionalSpeechProviderSchema.default("local"),
+    voiceTurnDetection: OptionalSpeechProviderSchema.default("local"),
+    voiceStt: OptionalSpeechProviderSchema.default("local"),
+    voiceTts: OptionalSpeechProviderSchema.default("local"),
+});
+function resolveOptionalBooleanFlag(value) {
+    return OptionalBooleanFlagSchema.parse(value) ?? true;
+}
+function firstSpeechDefinedValue(values) {
+    for (const value of values) {
+        if (value !== undefined && value !== null) {
+            return value;
+        }
+    }
+    return undefined;
+}
+function buildFeatureProviderInputs(params) {
+    const voiceModeEnabled = resolveOptionalBooleanFlag(firstSpeechDefinedValue([
+        params.env.PASEO_VOICE_MODE_ENABLED,
+        params.persisted.features?.voiceMode?.enabled,
+    ]));
+    return {
+        dictationStt: {
+            configuredValue: firstSpeechDefinedValue([
+                params.env.PASEO_DICTATION_STT_PROVIDER,
+                params.persisted.features?.dictation?.stt?.provider,
+            ]),
+            enabled: resolveOptionalBooleanFlag(firstSpeechDefinedValue([
+                params.env.PASEO_DICTATION_ENABLED,
+                params.persisted.features?.dictation?.enabled,
+            ])),
+        },
+        voiceTurnDetection: {
+            configuredValue: firstSpeechDefinedValue([
+                params.env.PASEO_VOICE_TURN_DETECTION_PROVIDER,
+                params.persisted.features?.voiceMode?.turnDetection?.provider,
+            ]),
+            enabled: voiceModeEnabled,
+        },
+        voiceStt: {
+            configuredValue: firstSpeechDefinedValue([
+                params.env.PASEO_VOICE_STT_PROVIDER,
+                params.persisted.features?.voiceMode?.stt?.provider,
+            ]),
+            enabled: voiceModeEnabled,
+        },
+        voiceTts: {
+            configuredValue: firstSpeechDefinedValue([
+                params.env.PASEO_VOICE_TTS_PROVIDER,
+                params.persisted.features?.voiceMode?.tts?.provider,
+            ]),
+            enabled: voiceModeEnabled,
+        },
+    };
+}
+function buildRequestedFeatureProvider(inputs, parsedValue) {
+    return {
+        provider: parsedValue,
+        explicit: inputs.configuredValue !== undefined,
+        enabled: inputs.enabled,
+    };
+}
+function resolveRequestedSpeechProviders(params) {
+    const featureProviders = buildFeatureProviderInputs(params);
+    const parsed = RequestedSpeechProvidersSchema.parse({
+        dictationStt: featureProviders.dictationStt.configuredValue ?? "local",
+        voiceTurnDetection: featureProviders.voiceTurnDetection.configuredValue ?? "local",
+        voiceStt: featureProviders.voiceStt.configuredValue ?? "local",
+        voiceTts: featureProviders.voiceTts.configuredValue ?? "local",
+    });
+    return {
+        dictationStt: buildRequestedFeatureProvider(featureProviders.dictationStt, parsed.dictationStt),
+        voiceTurnDetection: buildRequestedFeatureProvider(featureProviders.voiceTurnDetection, parsed.voiceTurnDetection),
+        voiceStt: buildRequestedFeatureProvider(featureProviders.voiceStt, parsed.voiceStt),
+        voiceTts: buildRequestedFeatureProvider(featureProviders.voiceTts, parsed.voiceTts),
+    };
+}
+export function resolveSpeechConfig(params) {
+    const providers = resolveRequestedSpeechProviders({
+        env: params.env,
+        persisted: params.persisted,
+    });
+    const local = resolveLocalSpeechConfig({
+        paseoHome: params.paseoHome,
+        env: params.env,
+        persisted: params.persisted,
+        providers,
+    });
+    const openai = resolveOpenAiSpeechConfig({
+        env: params.env,
+        persisted: params.persisted,
+        providers,
+    });
+    return {
+        openai,
+        speech: {
+            providers,
+            sttLanguages: local.sttLanguages,
+            ...(local.local ? { local: local.local } : {}),
+        },
+    };
+}
+//# sourceMappingURL=speech-config-resolver.js.map

package/dist/server/server/speech/speech-provider.d.ts

@@ -0,0 +1,59 @@
+import type pino from "pino";
+import type { Readable } from "node:stream";
+export interface LogprobToken {
+    token: string;
+    logprob: number;
+    bytes?: number[];
+}
+export interface TranscriptionResult {
+    text: string;
+    language?: string;
+    duration?: number;
+    logprobs?: LogprobToken[];
+    avgLogprob?: number;
+    isLowConfidence?: boolean;
+}
+export interface StreamingTranscriptionCommittedEvent {
+    segmentId: string;
+    previousSegmentId: string | null;
+}
+export interface StreamingTranscriptionEvent {
+    segmentId: string;
+    transcript: string;
+    isFinal: boolean;
+    language?: string;
+    logprobs?: LogprobToken[];
+    avgLogprob?: number;
+    isLowConfidence?: boolean;
+}
+export interface StreamingTranscriptionSession {
+    /**
+     * Required PCM16LE sample rate for `appendPcm16()`.
+     * Callers are responsible for resampling before appending.
+     */
+    requiredSampleRate: number;
+    connect(): Promise<void>;
+    appendPcm16(pcm16le: Buffer): void;
+    commit(): void;
+    clear(): void;
+    close(): void;
+    on(event: "committed", handler: (payload: StreamingTranscriptionCommittedEvent) => void): unknown;
+    on(event: "transcript", handler: (payload: StreamingTranscriptionEvent) => void): unknown;
+    on(event: "error", handler: (err: unknown) => void): unknown;
+}
+export interface SpeechToTextProvider {
+    id: "openai" | "local" | (string & {});
+    createSession(params: {
+        logger: pino.Logger;
+        language?: string;
+        prompt?: string;
+    }): StreamingTranscriptionSession;
+}
+export interface SpeechStreamResult {
+    stream: Readable;
+    format: string;
+}
+export interface TextToSpeechProvider {
+    synthesizeSpeech(text: string): Promise<SpeechStreamResult>;
+}
+//# sourceMappingURL=speech-provider.d.ts.map

package/dist/server/server/speech/speech-provider.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=speech-provider.js.map

package/dist/server/server/speech/speech-runtime.d.ts

@@ -0,0 +1,45 @@
+import type { Logger } from "pino";
+import type { PaseoOpenAIConfig, PaseoSpeechConfig } from "../bootstrap.js";
+import type { LocalSpeechModelId } from "./providers/local/config.js";
+import type { SpeechToTextProvider, TextToSpeechProvider } from "./speech-provider.js";
+import type { TurnDetectionProvider } from "./turn-detection-provider.js";
+export type SpeechReadinessReasonCode = "ready" | "disabled" | "model_download_in_progress" | "models_missing" | "model_download_failed" | "turn_detection_unavailable" | "stt_unavailable" | "tts_unavailable";
+export interface SpeechReadinessState {
+    enabled: boolean;
+    available: boolean;
+    reasonCode: SpeechReadinessReasonCode;
+    message: string;
+    retryable: boolean;
+    missingModelIds: LocalSpeechModelId[];
+}
+export interface SpeechReadinessSnapshot {
+    generatedAt: string;
+    requiredLocalModelIds: LocalSpeechModelId[];
+    missingLocalModelIds: LocalSpeechModelId[];
+    download: {
+        inProgress: boolean;
+        error: string | null;
+    };
+    realtimeVoice: SpeechReadinessState;
+    dictation: SpeechReadinessState;
+    voiceFeature: SpeechReadinessState;
+}
+export interface SpeechService {
+    resolveStt: () => SpeechToTextProvider | null;
+    resolveSttLanguage: () => string;
+    resolveTts: () => TextToSpeechProvider | null;
+    resolveTurnDetection: () => TurnDetectionProvider | null;
+    resolveDictationStt: () => SpeechToTextProvider | null;
+    resolveDictationSttLanguage: () => string;
+    getReadiness: () => SpeechReadinessSnapshot;
+    onReadinessChange: (listener: (snapshot: SpeechReadinessSnapshot) => void) => () => void;
+    start: () => void;
+    stop: () => Promise<void>;
+    ready: Promise<void>;
+}
+export declare function createSpeechService(params: {
+    logger: Logger;
+    openaiConfig?: PaseoOpenAIConfig;
+    speechConfig?: PaseoSpeechConfig;
+}): SpeechService;
+//# sourceMappingURL=speech-runtime.d.ts.map