@getpaseo/server 0.1.84 → 0.1.86

package/dist/server/server/speech/providers/local/runtime.js

@@ -1,12 +1,5 @@
-import { getLocalSpeechModelDir, DEFAULT_LOCAL_STT_MODEL, DEFAULT_LOCAL_TTS_MODEL, LocalSttModelIdSchema, LocalTtsModelIdSchema, } from "./models.js";
-import { SherpaOfflineRecognizerEngine } from "./sherpa/sherpa-offline-recognizer.js";
-import { SherpaOnnxParakeetSTT } from "./sherpa/sherpa-parakeet-stt.js";
-import { SherpaParakeetRealtimeTranscriptionSession } from "./sherpa/sherpa-parakeet-realtime-session.js";
-import { SherpaOnnxTTS } from "./sherpa/sherpa-tts.js";
-import { ensureSileroVadModel, SherpaSileroTurnDetectionProvider, } from "./sherpa/silero-vad-provider.js";
-function buildModelDownloadHint(modelId) {
-    return `Use 'paseo speech download --model ${modelId}' to download this model.`;
-}
+import { DEFAULT_LOCAL_STT_MODEL, DEFAULT_LOCAL_TTS_MODEL, LocalSttModelIdSchema, LocalTtsModelIdSchema, } from "./models.js";
+import { LocalSpeechWorkerClient, WorkerBackedSpeechToTextProvider, WorkerBackedTextToSpeechProvider, WorkerBackedTurnDetectionProvider, } from "./worker-client.js";
 function resolveConfiguredLocalModels(speechConfig) {
     return {
         dictationLocalSttModel: LocalSttModelIdSchema.parse(speechConfig?.local?.models.dictationStt ?? DEFAULT_LOCAL_STT_MODEL),
@@ -37,84 +30,27 @@ function computeRequiredLocalModelIds(params) {
     }
     return Array.from(ids);
 }
-async function createLocalSttEngine(params) {
-    const { modelId, modelsDir, logger } = params;
-    const modelDir = getLocalSpeechModelDir(modelsDir, modelId);
-    return new SherpaOfflineRecognizerEngine({
-        model: {
-            kind: "nemo_transducer",
-            encoder: `${modelDir}/encoder.int8.onnx`,
-            decoder: `${modelDir}/decoder.int8.onnx`,
-            joiner: `${modelDir}/joiner.int8.onnx`,
-            tokens: `${modelDir}/tokens.txt`,
-        },
-        numThreads: 2,
-        debug: 0,
-    }, logger);
-}
 function isLocalProviderEnabled(provider) {
     return provider.enabled !== false && provider.provider === "local";
 }
-async function initializeLocalTurnDetection(localConfig, logger) {
-    let vadModelPath;
-    if (localConfig) {
-        try {
-            vadModelPath = await ensureSileroVadModel(localConfig.modelsDir, logger);
-        }
-        catch (err) {
-            logger.warn({ err }, "Failed to provision Silero VAD model, falling back to bundled");
-        }
-    }
-    return new SherpaSileroTurnDetectionProvider({ modelPath: vadModelPath }, logger);
+function warnLocalConfigMissing(logger, feature) {
+    logger.warn({ configured: false }, `Local ${feature} selected but local provider config is missing; ${feature} will be unavailable`);
 }
-async function initializeLocalVoiceStt(params) {
-    const { localConfig, modelId, logger, getLocalSttEngine } = params;
-    if (!localConfig) {
-        logger.warn({ configured: false }, "Local STT selected for voice but local provider config is missing; STT will be unavailable");
-        return null;
-    }
-    const voiceEngine = await getLocalSttEngine(modelId);
-    return voiceEngine ? new SherpaOnnxParakeetSTT({ engine: voiceEngine }, logger) : null;
+function initializeLocalTurnDetection(params) {
+    const { client } = params;
+    return new WorkerBackedTurnDetectionProvider(client);
 }
-async function initializeLocalDictationStt(params) {
-    const { localConfig, modelId, logger, getLocalSttEngine } = params;
-    if (!localConfig) {
-        logger.warn({ configured: false }, "Local STT selected for dictation but local provider config is missing; dictation STT will be unavailable");
-        return null;
-    }
-    const dictationEngine = await getLocalSttEngine(modelId);
-    if (dictationEngine) {
-        return {
-            id: "local",
-            createSession: () => new SherpaParakeetRealtimeTranscriptionSession({ engine: dictationEngine }),
-        };
-    }
-    return null;
+function initializeLocalVoiceStt(params) {
+    const { client } = params;
+    return new WorkerBackedSpeechToTextProvider(client, "voiceStt");
 }
-async function initializeLocalVoiceTts(params) {
-    const { localConfig, speechConfig, localModels, logger } = params;
-    if (!localConfig) {
-        logger.warn({ configured: false }, "Local TTS selected for voice but local provider config is missing; TTS will be unavailable");
-        return null;
-    }
-    try {
-        const modelDir = getLocalSpeechModelDir(localConfig.modelsDir, localModels.voiceLocalTtsModel);
-        return new SherpaOnnxTTS({
-            preset: localModels.voiceLocalTtsModel,
-            modelDir,
-            speakerId: speechConfig?.local?.models.voiceTtsSpeakerId,
-            speed: speechConfig?.local?.models.voiceTtsSpeed,
-        }, logger);
-    }
-    catch (err) {
-        logger.warn({
-            err,
-            modelsDir: localConfig.modelsDir,
-            modelId: localModels.voiceLocalTtsModel,
-            hint: buildModelDownloadHint(localModels.voiceLocalTtsModel),
-        }, "Local TTS engine unavailable");
-        return null;
-    }
+function initializeLocalDictationStt(params) {
+    const { client } = params;
+    return new WorkerBackedSpeechToTextProvider(client, "dictationStt");
+}
+function initializeLocalVoiceTts(params) {
+    const { client } = params;
+    return new WorkerBackedTextToSpeechProvider(client);
 }
 export async function initializeLocalSpeechServices(params) {
     const { providers, logger, speechConfig } = params;
@@ -129,72 +65,55 @@ export async function initializeLocalSpeechServices(params) {
         providers,
         models: localModels,
     });
-    const localSttEngines = new Map();
-    const getLocalSttEngine = async (modelId) => {
-        const existing = localSttEngines.get(modelId);
-        if (existing) {
-            return existing;
-        }
-        if (!localConfig) {
-            return null;
-        }
-        try {
-            const created = await createLocalSttEngine({
-                modelId,
+    const workerClient = localConfig
+        ? new LocalSpeechWorkerClient({
+            config: {
                 modelsDir: localConfig.modelsDir,
-                logger,
-            });
-            localSttEngines.set(modelId, created);
-            return created;
+                voiceSttModel: localModels.voiceLocalSttModel,
+                dictationSttModel: localModels.dictationLocalSttModel,
+                voiceTtsModel: localModels.voiceLocalTtsModel,
+                voiceTtsSpeakerId: speechConfig?.local?.models.voiceTtsSpeakerId,
+                voiceTtsSpeed: speechConfig?.local?.models.voiceTtsSpeed,
+            },
+        })
+        : null;
+    if (isLocalProviderEnabled(providers.voiceTurnDetection)) {
+        if (workerClient) {
+            turnDetectionService = initializeLocalTurnDetection({ client: workerClient });
         }
-        catch (err) {
-            logger.warn({
-                err,
-                modelsDir: localConfig.modelsDir,
-                modelId,
-                hint: buildModelDownloadHint(modelId),
-            }, "Local STT engine unavailable");
-            return null;
+        else {
+            warnLocalConfigMissing(logger, "turn detection");
         }
-    };
-    if (isLocalProviderEnabled(providers.voiceTurnDetection)) {
-        turnDetectionService = await initializeLocalTurnDetection(localConfig, logger);
     }
     if (isLocalProviderEnabled(providers.voiceStt)) {
-        sttService = await initializeLocalVoiceStt({
-            localConfig,
-            modelId: localModels.voiceLocalSttModel,
-            logger,
-            getLocalSttEngine,
-        });
+        if (workerClient) {
+            sttService = initializeLocalVoiceStt({ client: workerClient });
+        }
+        else {
+            warnLocalConfigMissing(logger, "voice STT");
+        }
     }
     if (isLocalProviderEnabled(providers.dictationStt)) {
-        dictationSttService = await initializeLocalDictationStt({
-            localConfig,
-            modelId: localModels.dictationLocalSttModel,
-            logger,
-            getLocalSttEngine,
-        });
+        if (workerClient) {
+            dictationSttService = initializeLocalDictationStt({ client: workerClient });
+        }
+        else {
+            warnLocalConfigMissing(logger, "dictation STT");
+        }
     }
     if (isLocalProviderEnabled(providers.voiceTts)) {
-        localVoiceTtsProvider = await initializeLocalVoiceTts({
-            localConfig,
-            speechConfig,
-            localModels,
-            logger,
-        });
+        if (workerClient) {
+            localVoiceTtsProvider = initializeLocalVoiceTts({ client: workerClient });
+        }
+        else {
+            warnLocalConfigMissing(logger, "voice TTS");
+        }
         if (localVoiceTtsProvider) {
             ttsService = localVoiceTtsProvider;
         }
     }
     const cleanup = () => {
-        const maybeFreeable = localVoiceTtsProvider;
-        if (typeof maybeFreeable?.free === "function") {
-            maybeFreeable.free();
-        }
-        for (const engine of localSttEngines.values()) {
-            engine.free();
-        }
+        workerClient?.shutdown();
     };
     return {
         turnDetectionService,

package/dist/server/server/speech/providers/local/sherpa/model-catalog.d.ts

@@ -18,6 +18,13 @@ export declare const SHERPA_ONNX_MODEL_CATALOG: {
         readonly description: "NVIDIA Parakeet TDT v2 (offline NeMo transducer, English).";
         readonly defaultFor: "stt";
     };
+    readonly "parakeet-tdt-0.6b-v3-int8": {
+        readonly kind: "stt-offline";
+        readonly archiveUrl: "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8.tar.bz2";
+        readonly extractedDir: "sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8";
+        readonly requiredFiles: ["encoder.int8.onnx", "decoder.int8.onnx", "joiner.int8.onnx", "tokens.txt"];
+        readonly description: "NVIDIA Parakeet TDT v3 (offline NeMo transducer, 25 European languages, auto-detected).";
+    };
     readonly "kokoro-en-v0_19": {
         readonly kind: "tts";
         readonly archiveUrl: "https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2";
@@ -36,9 +43,9 @@ export type LocalSttModelId = ModelIdByKind<"stt-offline">;
 export type LocalTtsModelId = ModelIdByKind<"tts">;
 export declare const LOCAL_STT_MODEL_IDS: LocalSttModelId[];
 export declare const LOCAL_TTS_MODEL_IDS: LocalTtsModelId[];
-export declare const DEFAULT_LOCAL_STT_MODEL: "parakeet-tdt-0.6b-v2-int8";
+export declare const DEFAULT_LOCAL_STT_MODEL: LocalSttModelId;
 export declare const DEFAULT_LOCAL_TTS_MODEL: "kokoro-en-v0_19";
-export declare const LocalSttModelIdSchema: z.ZodType<"parakeet-tdt-0.6b-v2-int8", z.ZodTypeDef, string>;
+export declare const LocalSttModelIdSchema: z.ZodType<LocalSttModelId, z.ZodTypeDef, string>;
 export declare const LocalTtsModelIdSchema: z.ZodType<"kokoro-en-v0_19", z.ZodTypeDef, string>;
 export type SherpaOnnxModelSpec = SherpaOnnxCatalogEntry & {
     id: SherpaOnnxModelId;

package/dist/server/server/speech/providers/local/sherpa/model-catalog.js

@@ -8,6 +8,13 @@ export const SHERPA_ONNX_MODEL_CATALOG = {
         description: "NVIDIA Parakeet TDT v2 (offline NeMo transducer, English).",
         defaultFor: "stt",
     },
+    "parakeet-tdt-0.6b-v3-int8": {
+        kind: "stt-offline",
+        archiveUrl: "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8.tar.bz2",
+        extractedDir: "sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8",
+        requiredFiles: ["encoder.int8.onnx", "decoder.int8.onnx", "joiner.int8.onnx", "tokens.txt"],
+        description: "NVIDIA Parakeet TDT v3 (offline NeMo transducer, 25 European languages, auto-detected).",
+    },
     "kokoro-en-v0_19": {
         kind: "tts",
         archiveUrl: "https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2",

package/dist/server/server/speech/providers/local/worker-bytes.d.ts

@@ -0,0 +1,4 @@
+export type LocalSpeechWorkerBytes = ArrayBuffer;
+export declare function bufferToWorkerBytes(buffer: Buffer): LocalSpeechWorkerBytes;
+export declare function workerBytesToBuffer(bytes: LocalSpeechWorkerBytes): Buffer;
+//# sourceMappingURL=worker-bytes.d.ts.map

package/dist/server/server/speech/providers/local/worker-bytes.js

@@ -0,0 +1,9 @@
+export function bufferToWorkerBytes(buffer) {
+    const bytes = new Uint8Array(buffer.byteLength);
+    bytes.set(buffer);
+    return bytes.buffer;
+}
+export function workerBytesToBuffer(bytes) {
+    return Buffer.from(bytes);
+}
+//# sourceMappingURL=worker-bytes.js.map

package/dist/server/server/speech/providers/local/worker-client.d.ts

@@ -0,0 +1,80 @@
+import { EventEmitter } from "node:events";
+import type pino from "pino";
+import type { SpeechStreamResult, SpeechToTextProvider, StreamingTranscriptionSession, TextToSpeechProvider } from "../../speech-provider.js";
+import type { TurnDetectionProvider, TurnDetectionSession } from "../../turn-detection-provider.js";
+import type { LocalSpeechSessionKind, LocalSpeechTranscriptionResult, LocalSpeechWorkerConfig, LocalSpeechWorkerRequest, LocalSpeechWorkerToParentMessage } from "./worker-protocol.js";
+interface LocalSpeechWorkerProcess {
+    connected: boolean;
+    killed: boolean;
+    send(message: LocalSpeechWorkerRequest, callback: (error: Error | null) => void): boolean;
+    disconnect(): void;
+    kill(): boolean;
+    on(event: "message", listener: (message: LocalSpeechWorkerToParentMessage) => void): this;
+    on(event: "exit", listener: (code: number | null, signal: NodeJS.Signals | null) => void): this;
+}
+interface LocalSpeechWorkerClientOptions {
+    config: LocalSpeechWorkerConfig;
+    requestTimeoutMs?: number;
+    idleTtlMs?: number;
+    forkWorker?: () => LocalSpeechWorkerProcess;
+}
+export declare class LocalSpeechWorkerClient {
+    private readonly config;
+    private readonly requestTimeoutMs;
+    private readonly idleTtlMs;
+    private readonly forkWorker;
+    private readonly pendingRequests;
+    private readonly activeSessionIds;
+    private readonly sessionEmitters;
+    private worker;
+    private inFlightRequests;
+    private idleTimer;
+    constructor(options: LocalSpeechWorkerClientOptions);
+    synthesizeSpeech(text: string): Promise<SpeechStreamResult>;
+    transcribeVoice(audio: Buffer, format: string): Promise<LocalSpeechTranscriptionResult>;
+    createSession(kind: LocalSpeechSessionKind, emitter: EventEmitter): Promise<{
+        sessionId: string;
+        requiredSampleRate: number;
+    }>;
+    appendSessionAudio(sessionId: string, audio: Buffer): void;
+    commitSession(sessionId: string): void;
+    clearSession(sessionId: string): void;
+    flushSession(sessionId: string): void;
+    resetSession(sessionId: string): void;
+    closeSession(sessionId: string): void;
+    shutdown(): void;
+    private sendRequest;
+    private ensureWorker;
+    private handleWorkerMessage;
+    private handleWorkerExit;
+    private rejectAllPending;
+    private emitSessionError;
+    private scheduleIdleShutdownIfReady;
+    private clearIdleTimer;
+}
+export declare class WorkerBackedTextToSpeechProvider implements TextToSpeechProvider {
+    private readonly client;
+    constructor(client: LocalSpeechWorkerClient);
+    synthesizeSpeech(text: string): Promise<SpeechStreamResult>;
+}
+export declare class WorkerBackedSpeechToTextProvider implements SpeechToTextProvider {
+    private readonly client;
+    private readonly kind;
+    readonly id: "local";
+    constructor(client: LocalSpeechWorkerClient, kind: Extract<LocalSpeechSessionKind, "voiceStt" | "dictationStt">);
+    createSession(_params: {
+        logger: pino.Logger;
+        language?: string;
+        prompt?: string;
+    }): StreamingTranscriptionSession;
+}
+export declare class WorkerBackedTurnDetectionProvider implements TurnDetectionProvider {
+    private readonly client;
+    readonly id: "local";
+    constructor(client: LocalSpeechWorkerClient);
+    createSession(_params: {
+        logger: pino.Logger;
+    }): TurnDetectionSession;
+}
+export {};
+//# sourceMappingURL=worker-client.d.ts.map