@getpaseo/server 0.1.84 → 0.1.86

package/dist/server/server/speech/providers/local/worker-client.js

@@ -0,0 +1,438 @@
+import { fork } from "node:child_process";
+import { randomUUID } from "node:crypto";
+import { EventEmitter } from "node:events";
+import { Readable } from "node:stream";
+import { fileURLToPath } from "node:url";
+import { applySherpaLoaderEnv } from "./sherpa/sherpa-runtime-env.js";
+import { bufferToWorkerBytes, workerBytesToBuffer } from "./worker-bytes.js";
+const DEFAULT_REQUEST_TIMEOUT_MS = 30000;
+const DEFAULT_IDLE_TTL_MS = 5 * 60 * 1000;
+const DEFAULT_LOCAL_SAMPLE_RATE = 16000;
+function resolveWorkerUrl() {
+    const currentUrl = import.meta.url;
+    if (currentUrl.endsWith(".ts")) {
+        return new URL("./worker-process.ts", currentUrl);
+    }
+    return new URL("./worker-process.js", currentUrl);
+}
+function resolveWorkerExecArgv() {
+    if (!import.meta.url.endsWith(".ts")) {
+        return [];
+    }
+    const loaderUrl = new URL("../../../../terminal/terminal-ts-loader.mjs", import.meta.url).href;
+    const importSource = [
+        'import { register } from "node:module";',
+        'import { pathToFileURL } from "node:url";',
+        `register(${JSON.stringify(loaderUrl)}, pathToFileURL("./"));`,
+    ].join(" ");
+    return [
+        "--experimental-strip-types",
+        "--import",
+        `data:text/javascript,${encodeURIComponent(importSource)}`,
+    ];
+}
+function forkLocalSpeechWorker() {
+    const env = { ...process.env };
+    applySherpaLoaderEnv(env);
+    return fork(fileURLToPath(resolveWorkerUrl()), [], {
+        env,
+        execArgv: resolveWorkerExecArgv(),
+        serialization: "advanced",
+        stdio: ["ignore", "ignore", "inherit", "ipc"],
+    });
+}
+function isResponse(message) {
+    return message.type === "response";
+}
+export class LocalSpeechWorkerClient {
+    constructor(options) {
+        this.pendingRequests = new Map();
+        this.activeSessionIds = new Set();
+        this.sessionEmitters = new Map();
+        this.worker = null;
+        this.inFlightRequests = 0;
+        this.idleTimer = null;
+        this.config = options.config;
+        this.requestTimeoutMs = options.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS;
+        this.idleTtlMs = options.idleTtlMs ?? DEFAULT_IDLE_TTL_MS;
+        this.forkWorker = options.forkWorker ?? forkLocalSpeechWorker;
+    }
+    async synthesizeSpeech(text) {
+        const result = await this.sendRequest({
+            type: "tts.synthesize",
+            config: this.config,
+            text,
+        });
+        return {
+            stream: Readable.from([workerBytesToBuffer(result.audio)]),
+            format: result.format,
+        };
+    }
+    transcribeVoice(audio, format) {
+        return this.sendRequest({
+            type: "stt.transcribe",
+            config: this.config,
+            model: "voice",
+            audio: bufferToWorkerBytes(audio),
+            format,
+        });
+    }
+    async createSession(kind, emitter) {
+        const sessionId = randomUUID();
+        this.activeSessionIds.add(sessionId);
+        this.sessionEmitters.set(sessionId, emitter);
+        try {
+            const result = await this.sendRequest({
+                type: "session.create",
+                config: this.config,
+                sessionId,
+                kind,
+            });
+            return { sessionId, requiredSampleRate: result.requiredSampleRate };
+        }
+        catch (err) {
+            this.activeSessionIds.delete(sessionId);
+            this.sessionEmitters.delete(sessionId);
+            this.scheduleIdleShutdownIfReady();
+            throw err;
+        }
+    }
+    appendSessionAudio(sessionId, audio) {
+        void this.sendRequest({
+            type: "session.append",
+            sessionId,
+            audio: bufferToWorkerBytes(audio),
+        }).catch((err) => {
+            this.emitSessionError(sessionId, err);
+        });
+    }
+    commitSession(sessionId) {
+        void this.sendRequest({ type: "session.commit", sessionId }).catch((err) => {
+            this.emitSessionError(sessionId, err);
+        });
+    }
+    clearSession(sessionId) {
+        void this.sendRequest({ type: "session.clear", sessionId }).catch((err) => {
+            this.emitSessionError(sessionId, err);
+        });
+    }
+    flushSession(sessionId) {
+        void this.sendRequest({ type: "session.flush", sessionId }).catch((err) => {
+            this.emitSessionError(sessionId, err);
+        });
+    }
+    resetSession(sessionId) {
+        void this.sendRequest({ type: "session.reset", sessionId }).catch((err) => {
+            this.emitSessionError(sessionId, err);
+        });
+    }
+    closeSession(sessionId) {
+        this.activeSessionIds.delete(sessionId);
+        this.sessionEmitters.delete(sessionId);
+        void this.sendRequest({ type: "session.close", sessionId }).catch(() => {
+            // Closing is best-effort; the parent already dropped the session.
+        });
+        this.scheduleIdleShutdownIfReady();
+    }
+    shutdown() {
+        this.clearIdleTimer();
+        this.rejectAllPending(new Error("Local speech worker shut down"));
+        this.activeSessionIds.clear();
+        this.sessionEmitters.clear();
+        const worker = this.worker;
+        this.worker = null;
+        if (worker && !worker.killed) {
+            try {
+                worker.disconnect();
+            }
+            catch {
+                // ignore
+            }
+            try {
+                worker.kill();
+            }
+            catch {
+                // ignore
+            }
+        }
+    }
+    sendRequest(input) {
+        const worker = this.ensureWorker();
+        const requestId = randomUUID();
+        const message = { ...input, requestId };
+        this.inFlightRequests++;
+        this.clearIdleTimer();
+        return new Promise((resolve, reject) => {
+            const timeout = setTimeout(() => {
+                this.pendingRequests.delete(requestId);
+                this.inFlightRequests = Math.max(0, this.inFlightRequests - 1);
+                this.scheduleIdleShutdownIfReady();
+                reject(new Error(`Local speech worker request timed out: ${input.type}`));
+            }, this.requestTimeoutMs);
+            this.pendingRequests.set(requestId, {
+                resolve: (value) => resolve(value),
+                reject,
+                timeout,
+            });
+            const sent = worker.send(message, (error) => {
+                if (!error) {
+                    return;
+                }
+                const pending = this.pendingRequests.get(requestId);
+                if (!pending) {
+                    return;
+                }
+                clearTimeout(pending.timeout);
+                this.pendingRequests.delete(requestId);
+                this.inFlightRequests = Math.max(0, this.inFlightRequests - 1);
+                this.scheduleIdleShutdownIfReady();
+                pending.reject(error);
+            });
+            if (!sent) {
+                const pending = this.pendingRequests.get(requestId);
+                if (pending) {
+                    clearTimeout(pending.timeout);
+                    this.pendingRequests.delete(requestId);
+                    this.inFlightRequests = Math.max(0, this.inFlightRequests - 1);
+                    this.scheduleIdleShutdownIfReady();
+                    pending.reject(new Error("Local speech worker IPC channel is not writable"));
+                }
+            }
+        });
+    }
+    ensureWorker() {
+        if (this.worker && !this.worker.killed && this.worker.connected) {
+            return this.worker;
+        }
+        const worker = this.forkWorker();
+        this.worker = worker;
+        worker.on("message", (message) => this.handleWorkerMessage(message));
+        worker.on("exit", () => this.handleWorkerExit());
+        return worker;
+    }
+    handleWorkerMessage(message) {
+        if (isResponse(message)) {
+            const pending = this.pendingRequests.get(message.requestId);
+            if (!pending) {
+                return;
+            }
+            clearTimeout(pending.timeout);
+            this.pendingRequests.delete(message.requestId);
+            this.inFlightRequests = Math.max(0, this.inFlightRequests - 1);
+            this.scheduleIdleShutdownIfReady();
+            if (message.ok) {
+                pending.resolve(message.result);
+            }
+            else {
+                pending.reject(new Error(message.error));
+            }
+            return;
+        }
+        const emitter = this.sessionEmitters.get(message.sessionId);
+        if (!emitter) {
+            return;
+        }
+        switch (message.type) {
+            case "session.committed":
+                emitter.emit("committed", message.payload);
+                return;
+            case "session.transcript":
+                emitter.emit("transcript", message.payload);
+                return;
+            case "session.speech_started":
+                emitter.emit("speech_started");
+                return;
+            case "session.speech_stopped":
+                emitter.emit("speech_stopped");
+                return;
+            case "session.error":
+                emitter.emit("error", new Error(message.error));
+                return;
+        }
+    }
+    handleWorkerExit() {
+        this.worker = null;
+        this.clearIdleTimer();
+        this.rejectAllPending(new Error("Local speech worker exited"));
+        for (const [sessionId, emitter] of this.sessionEmitters) {
+            if (this.activeSessionIds.has(sessionId)) {
+                emitter.emit("error", new Error("Local speech worker exited"));
+            }
+        }
+        this.activeSessionIds.clear();
+        this.sessionEmitters.clear();
+        this.inFlightRequests = 0;
+    }
+    rejectAllPending(error) {
+        for (const [requestId, pending] of this.pendingRequests) {
+            clearTimeout(pending.timeout);
+            pending.reject(error);
+            this.pendingRequests.delete(requestId);
+        }
+    }
+    emitSessionError(sessionId, error) {
+        const emitter = this.sessionEmitters.get(sessionId);
+        if (!emitter) {
+            return;
+        }
+        emitter.emit("error", error instanceof Error ? error : new Error(String(error)));
+    }
+    scheduleIdleShutdownIfReady() {
+        if (!this.worker || this.inFlightRequests > 0 || this.activeSessionIds.size > 0) {
+            return;
+        }
+        this.clearIdleTimer();
+        this.idleTimer = setTimeout(() => {
+            if (this.inFlightRequests === 0 && this.activeSessionIds.size === 0) {
+                this.shutdown();
+            }
+        }, this.idleTtlMs);
+    }
+    clearIdleTimer() {
+        if (this.idleTimer) {
+            clearTimeout(this.idleTimer);
+            this.idleTimer = null;
+        }
+    }
+}
+export class WorkerBackedTextToSpeechProvider {
+    constructor(client) {
+        this.client = client;
+    }
+    synthesizeSpeech(text) {
+        return this.client.synthesizeSpeech(text);
+    }
+}
+export class WorkerBackedSpeechToTextProvider {
+    constructor(client, kind) {
+        this.client = client;
+        this.kind = kind;
+        this.id = "local";
+    }
+    createSession(_params) {
+        return new WorkerBackedTranscriptionSession(this.client, this.kind);
+    }
+}
+export class WorkerBackedTurnDetectionProvider {
+    constructor(client) {
+        this.client = client;
+        this.id = "local";
+    }
+    createSession(_params) {
+        return new WorkerBackedTurnDetectionSession(this.client);
+    }
+}
+class WorkerBackedTranscriptionSession extends EventEmitter {
+    constructor(client, kind) {
+        super();
+        this.client = client;
+        this.kind = kind;
+        this.requiredSampleRate = DEFAULT_LOCAL_SAMPLE_RATE;
+        this.connectedSessionId = null;
+        this.connecting = null;
+    }
+    async connect() {
+        if (this.connectedSessionId) {
+            return;
+        }
+        if (!this.connecting) {
+            this.connecting = this.connectRemoteSession();
+        }
+        await this.connecting;
+    }
+    async connectRemoteSession() {
+        try {
+            const result = await this.client.createSession(this.kind, this);
+            this.connectedSessionId = result.sessionId;
+            this.requiredSampleRate = result.requiredSampleRate;
+        }
+        finally {
+            this.connecting = null;
+        }
+    }
+    appendPcm16(pcm16le) {
+        const sessionId = this.connectedSessionId;
+        if (!sessionId) {
+            this.emit("error", new Error("Local STT session not connected"));
+            return;
+        }
+        this.client.appendSessionAudio(sessionId, pcm16le);
+    }
+    commit() {
+        const sessionId = this.connectedSessionId;
+        if (!sessionId) {
+            this.emit("error", new Error("Local STT session not connected"));
+            return;
+        }
+        this.client.commitSession(sessionId);
+    }
+    clear() {
+        const sessionId = this.connectedSessionId;
+        if (sessionId) {
+            this.client.clearSession(sessionId);
+        }
+    }
+    close() {
+        const sessionId = this.connectedSessionId;
+        this.connectedSessionId = null;
+        if (sessionId) {
+            this.client.closeSession(sessionId);
+        }
+    }
+}
+class WorkerBackedTurnDetectionSession extends EventEmitter {
+    constructor(client) {
+        super();
+        this.client = client;
+        this.requiredSampleRate = DEFAULT_LOCAL_SAMPLE_RATE;
+        this.connectedSessionId = null;
+        this.connecting = null;
+    }
+    async connect() {
+        if (this.connectedSessionId) {
+            return;
+        }
+        if (!this.connecting) {
+            this.connecting = this.connectRemoteSession();
+        }
+        await this.connecting;
+    }
+    async connectRemoteSession() {
+        try {
+            const result = await this.client.createSession("vad", this);
+            this.connectedSessionId = result.sessionId;
+            this.requiredSampleRate = result.requiredSampleRate;
+        }
+        finally {
+            this.connecting = null;
+        }
+    }
+    appendPcm16(pcm16le) {
+        const sessionId = this.connectedSessionId;
+        if (!sessionId) {
+            this.emit("error", new Error("Local turn-detection session not connected"));
+            return;
+        }
+        this.client.appendSessionAudio(sessionId, pcm16le);
+    }
+    flush() {
+        const sessionId = this.connectedSessionId;
+        if (sessionId) {
+            this.client.flushSession(sessionId);
+        }
+    }
+    reset() {
+        const sessionId = this.connectedSessionId;
+        if (sessionId) {
+            this.client.resetSession(sessionId);
+        }
+    }
+    close() {
+        const sessionId = this.connectedSessionId;
+        this.connectedSessionId = null;
+        if (sessionId) {
+            this.client.closeSession(sessionId);
+        }
+    }
+}
+//# sourceMappingURL=worker-client.js.map

package/dist/server/server/speech/providers/local/worker-process.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=worker-process.d.ts.map

package/dist/server/server/speech/providers/local/worker-process.js

@@ -0,0 +1,270 @@
+import pino from "pino";
+import { getLocalSpeechModelDir } from "./models.js";
+import { SherpaOfflineRecognizerEngine } from "./sherpa/sherpa-offline-recognizer.js";
+import { SherpaOnnxParakeetSTT } from "./sherpa/sherpa-parakeet-stt.js";
+import { SherpaParakeetRealtimeTranscriptionSession } from "./sherpa/sherpa-parakeet-realtime-session.js";
+import { SherpaOnnxTTS } from "./sherpa/sherpa-tts.js";
+import { ensureSileroVadModel, SherpaSileroTurnDetectionProvider, } from "./sherpa/silero-vad-provider.js";
+import { bufferToWorkerBytes, workerBytesToBuffer } from "./worker-bytes.js";
+process.title = "Paseo Voice";
+const logger = pino({
+    level: process.env.PASEO_LOG_LEVEL ?? "info",
+}).child({ module: "speech", component: "local-worker" });
+const sttEngines = new Map();
+const sttProviders = new Map();
+const ttsProviders = new Map();
+const sessions = new Map();
+const unsubscribeBySessionId = new Map();
+let ipcClosing = false;
+function sendToParent(message) {
+    if (ipcClosing || !process.connected || !process.send) {
+        return;
+    }
+    try {
+        process.send(message, (error) => {
+            if (error) {
+                ipcClosing = true;
+            }
+        });
+    }
+    catch {
+        ipcClosing = true;
+    }
+}
+function sttModelId(config, model) {
+    return (model === "voice" ? config.voiceSttModel : config.dictationSttModel);
+}
+function ttsModelId(config) {
+    return config.voiceTtsModel;
+}
+function sttEngineKey(config, modelId) {
+    return `${config.modelsDir}:${modelId}`;
+}
+function ttsKey(config) {
+    return [
+        config.modelsDir,
+        config.voiceTtsModel,
+        config.voiceTtsSpeakerId ?? 0,
+        config.voiceTtsSpeed ?? 1,
+    ].join(":");
+}
+function getSttEngine(config, model) {
+    const modelId = sttModelId(config, model);
+    const key = sttEngineKey(config, modelId);
+    const existing = sttEngines.get(key);
+    if (existing) {
+        return existing;
+    }
+    const modelDir = getLocalSpeechModelDir(config.modelsDir, modelId);
+    const created = new SherpaOfflineRecognizerEngine({
+        model: {
+            kind: "nemo_transducer",
+            encoder: `${modelDir}/encoder.int8.onnx`,
+            decoder: `${modelDir}/decoder.int8.onnx`,
+            joiner: `${modelDir}/joiner.int8.onnx`,
+            tokens: `${modelDir}/tokens.txt`,
+        },
+        numThreads: 2,
+        debug: 0,
+    }, logger);
+    sttEngines.set(key, created);
+    return created;
+}
+function getSttProvider(config, model) {
+    const modelId = sttModelId(config, model);
+    const key = sttEngineKey(config, modelId);
+    const existing = sttProviders.get(key);
+    if (existing) {
+        return existing;
+    }
+    const created = new SherpaOnnxParakeetSTT({ engine: getSttEngine(config, model) }, logger);
+    sttProviders.set(key, created);
+    return created;
+}
+function getTtsProvider(config) {
+    const key = ttsKey(config);
+    const existing = ttsProviders.get(key);
+    if (existing) {
+        return existing;
+    }
+    const modelDir = getLocalSpeechModelDir(config.modelsDir, ttsModelId(config));
+    const created = new SherpaOnnxTTS({
+        preset: ttsModelId(config),
+        modelDir,
+        speakerId: config.voiceTtsSpeakerId,
+        speed: config.voiceTtsSpeed,
+    }, logger);
+    ttsProviders.set(key, created);
+    return created;
+}
+function cleanupSession(sessionId) {
+    const unsubscribe = unsubscribeBySessionId.get(sessionId);
+    if (unsubscribe) {
+        for (const fn of unsubscribe) {
+            try {
+                fn();
+            }
+            catch {
+                // ignore
+            }
+        }
+    }
+    unsubscribeBySessionId.delete(sessionId);
+    const session = sessions.get(sessionId);
+    sessions.delete(sessionId);
+    try {
+        session?.close();
+    }
+    catch {
+        // ignore
+    }
+}
+function trackTranscriptionSession(sessionId, session) {
+    session.on("committed", (payload) => {
+        sendToParent({ type: "session.committed", sessionId, payload });
+    });
+    session.on("transcript", (payload) => {
+        sendToParent({ type: "session.transcript", sessionId, payload });
+    });
+    session.on("error", (err) => {
+        sendToParent({
+            type: "session.error",
+            sessionId,
+            error: err instanceof Error ? err.message : String(err),
+        });
+    });
+    unsubscribeBySessionId.set(sessionId, []);
+}
+function trackTurnDetectionSession(sessionId, session) {
+    session.on("speech_started", () => {
+        sendToParent({ type: "session.speech_started", sessionId });
+    });
+    session.on("speech_stopped", () => {
+        sendToParent({ type: "session.speech_stopped", sessionId });
+    });
+    session.on("error", (err) => {
+        sendToParent({
+            type: "session.error",
+            sessionId,
+            error: err instanceof Error ? err.message : String(err),
+        });
+    });
+    unsubscribeBySessionId.set(sessionId, []);
+}
+async function createSession(message) {
+    cleanupSession(message.sessionId);
+    if (message.kind === "vad") {
+        let vadModelPath;
+        try {
+            vadModelPath = await ensureSileroVadModel(message.config.modelsDir, logger);
+        }
+        catch (err) {
+            logger.warn({ err }, "Failed to provision Silero VAD model, falling back to bundled");
+        }
+        const provider = new SherpaSileroTurnDetectionProvider({ modelPath: vadModelPath }, logger);
+        const session = provider.createSession({ logger });
+        trackTurnDetectionSession(message.sessionId, session);
+        await session.connect();
+        sessions.set(message.sessionId, session);
+        return { requiredSampleRate: session.requiredSampleRate };
+    }
+    const model = message.kind === "voiceStt" ? "voice" : "dictation";
+    const engine = getSttEngine(message.config, model);
+    const session = message.kind === "voiceStt"
+        ? getSttProvider(message.config, "voice").createSession({ logger })
+        : new SherpaParakeetRealtimeTranscriptionSession({ engine });
+    trackTranscriptionSession(message.sessionId, session);
+    await session.connect();
+    sessions.set(message.sessionId, session);
+    return { requiredSampleRate: session.requiredSampleRate };
+}
+function sendOk(requestId, result) {
+    sendToParent({ type: "response", requestId, ok: true, result });
+}
+function handleSessionRequest(message) {
+    if (message.type === "session.close") {
+        cleanupSession(message.sessionId);
+        sendOk(message.requestId);
+        return;
+    }
+    const session = sessions.get(message.sessionId);
+    switch (message.type) {
+        case "session.append":
+            session?.appendPcm16(workerBytesToBuffer(message.audio));
+            break;
+        case "session.commit":
+            if (session && "commit" in session) {
+                session.commit();
+            }
+            break;
+        case "session.clear":
+            if (session && "clear" in session) {
+                session.clear();
+            }
+            break;
+        case "session.flush":
+            if (session && "flush" in session) {
+                session.flush();
+            }
+            break;
+        case "session.reset":
+            if (session && "reset" in session) {
+                session.reset();
+            }
+            break;
+    }
+    sendOk(message.requestId);
+}
+async function handleRequest(message) {
+    if (message.type === "tts.synthesize") {
+        const result = await getTtsProvider(message.config).synthesizeSpeech(message.text);
+        const chunks = [];
+        for await (const chunk of result.stream) {
+            chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
+        }
+        sendOk(message.requestId, {
+            audio: bufferToWorkerBytes(Buffer.concat(chunks)),
+            format: result.format,
+        });
+        return;
+    }
+    if (message.type === "stt.transcribe") {
+        const result = await getSttProvider(message.config, message.model).transcribeAudio(workerBytesToBuffer(message.audio), message.format);
+        sendOk(message.requestId, result);
+        return;
+    }
+    if (message.type === "session.create") {
+        const result = await createSession(message);
+        sendOk(message.requestId, result);
+        return;
+    }
+    handleSessionRequest(message);
+}
+process.on("message", (message) => {
+    void handleRequest(message).catch((error) => {
+        sendToParent({
+            type: "response",
+            requestId: message.requestId,
+            ok: false,
+            error: error instanceof Error ? error.message : "Local speech worker request failed",
+        });
+    });
+});
+process.once("disconnect", () => {
+    ipcClosing = true;
+    for (const sessionId of Array.from(sessions.keys())) {
+        cleanupSession(sessionId);
+    }
+    for (const tts of ttsProviders.values()) {
+        try {
+            tts.free();
+        }
+        catch {
+            // ignore
+        }
+    }
+    for (const engine of sttEngines.values()) {
+        engine.free();
+    }
+});
+//# sourceMappingURL=worker-process.js.map