@flowingspring/dsh-voco 0.2.0

package/lib/plugins/voice-local.js

@@ -0,0 +1,593 @@
+import { VoiceCommandCallId, VoiceResponseId, VoiceUtteranceId } from "./voice.js";
+import { randomUUID } from "node:crypto";
+import z from "@deepseek-ai/schemastery";
+import { existsSync } from "node:fs";
+import { dirname, join, resolve } from "node:path";
+import { loadEnvFile } from "node:process";
+import { fileURLToPath } from "node:url";
+import { mkdtemp, readFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { EdgeTTS } from "node-edge-tts";
+//#region ../voice-local/src/edge-tts.ts
+const EDGE_TTS_VOICE = "zh-CN-XiaoxiaoNeural";
+const DEFAULT_EDGE_TTS_RATE = "+20%";
+/** Edge TTS adapter. Returns the service's MP3 bytes for browser decoding. */
+async function synthesizeEdgeSpeech(text, rate = DEFAULT_EDGE_TTS_RATE) {
+	const normalized = normalizeSpeechText(text);
+	if (normalized === "") return /* @__PURE__ */ new Uint8Array();
+	const directory = await mkdtemp(join(tmpdir(), "dsh-voco-edge-"));
+	const output = join(directory, "speech.mp3");
+	try {
+		await new EdgeTTS({
+			voice: EDGE_TTS_VOICE,
+			lang: "zh-CN",
+			outputFormat: "audio-24khz-48kbitrate-mono-mp3",
+			rate,
+			pitch: "default",
+			volume: "default",
+			timeout: 2e4
+		}).ttsPromise(normalized, output);
+		return new Uint8Array(await readFile(output));
+	} finally {
+		await rm(directory, {
+			recursive: true,
+			force: true
+		});
+	}
+}
+function normalizeSpeechText(text) {
+	return text.normalize("NFKC").replace(/[`*_#>|]/g, " ").replace(/\s+/g, " ").replace(/\s+([，。！？；：,.!?;:])/g, "$1").trim();
+}
+/** Split prose into short requests so the first sentence can play immediately. */
+function splitSpeechText(text) {
+	const normalized = normalizeSpeechText(text);
+	if (normalized === "") return [];
+	return (normalized.match(/[^。！？!?；;：:\n]+[。！？!?；;：:]?|[^。！？!?；;：:\n]+$/g) ?? [normalized]).map((part) => part.trim()).filter(Boolean);
+}
+//#endregion
+//#region ../voice-local/src/session.ts
+/** Provider session translating local speech events to the shared voice seam. */
+var LocalSession = class {
+	backend;
+	emit;
+	voiceSessionId;
+	interactionMode;
+	audio;
+	closed = false;
+	pending = [];
+	pendingSpeech = [];
+	pendingCommands = /* @__PURE__ */ new Map();
+	pendingOutputs = [];
+	activeSpeech;
+	activeTaskId;
+	constructor(backend, emit, voiceSessionId, interactionMode = "speech-shell") {
+		this.backend = backend;
+		this.emit = emit;
+		this.voiceSessionId = voiceSessionId;
+		this.interactionMode = interactionMode;
+		this.audio = backend.audio;
+	}
+	async start() {
+		await this.backend.start((event) => this.receive(event));
+	}
+	appendAudio(audio) {
+		this.backend.appendAudio(audio);
+	}
+	commitAudio() {
+		this.backend.commitAudio();
+	}
+	interruptResponse() {
+		this.backend.interrupt();
+		this.pendingSpeech.splice(0);
+		this.pendingOutputs.splice(0);
+		this.activeSpeech = void 0;
+		this.emit({ type: "response.interrupted" });
+	}
+	playbackEnded() {
+		for (const output of this.pendingOutputs.splice(0)) this.emit({
+			type: "output_text.done",
+			utteranceId: output.utteranceId,
+			responseId: output.responseId,
+			text: output.text
+		});
+	}
+	appendTaskObservation(event) {
+		const text = event.voiceMessage?.text.trim() || event.announcement?.trim();
+		if (text !== void 0 && text !== "") this.pending.push(text);
+		if (event.taskId === this.activeTaskId && isTerminalTaskStatus(event.status)) this.activeTaskId = void 0;
+	}
+	appendSpeechText(text) {
+		if (text.trim() === "") return;
+		if (this.activeSpeech === void 0) {
+			const responseId = VoiceResponseId(String(this.voiceSessionId) + ":response:" + randomUUID());
+			const utteranceId = VoiceUtteranceId(String(responseId) + ":text");
+			this.activeSpeech = {
+				responseId,
+				utteranceId,
+				text: ""
+			};
+			this.pendingOutputs.push(this.activeSpeech);
+			this.emit({
+				type: "output_text.started",
+				utteranceId,
+				responseId
+			});
+		}
+		this.activeSpeech.text += text;
+		this.emit({
+			type: "output_text.delta",
+			utteranceId: this.activeSpeech.utteranceId,
+			responseId: this.activeSpeech.responseId,
+			text
+		});
+		this.backend.synthesize(String(this.activeSpeech.responseId), text);
+	}
+	requestResponse(_policy) {
+		const text = [...this.pending.splice(0), ...this.pendingSpeech.splice(0)].join("\n");
+		if (text !== "") this.appendSpeechText(text);
+		const responseId = this.activeSpeech?.responseId;
+		this.activeSpeech = void 0;
+		if (responseId !== void 0) this.backend.finishSynthesis?.(String(responseId));
+	}
+	completeTaskCommand(callId, result) {
+		if (this.interactionMode === "speech-shell") throw new Error("local speech-shell sessions do not accept task commands");
+		const command = this.pendingCommands.get(callId);
+		if (command === void 0) return;
+		this.pendingCommands.delete(callId);
+		if (result.kind === "accepted" && (command.type === "route_transcription" || command.type === "realtime_delegation")) this.activeTaskId = result.taskId;
+		else if (result.kind === "rejected" && command.type === "send_task_message" && (result.code === "task_not_active" || result.code === "task_not_found")) this.activeTaskId = void 0;
+	}
+	async close() {
+		if (this.closed) return;
+		this.closed = true;
+		await this.backend.close();
+	}
+	receive(event) {
+		switch (event.type) {
+			case "ready": return;
+			case "transcription.started":
+				this.emit({
+					type: "transcription.started",
+					utteranceId: VoiceUtteranceId(String(this.voiceSessionId) + ":input:" + event.utteranceId)
+				});
+				return;
+			case "transcription.updated":
+				this.emit({
+					type: "transcription.updated",
+					utteranceId: VoiceUtteranceId(String(this.voiceSessionId) + ":input:" + event.utteranceId),
+					text: event.text
+				});
+				return;
+			case "transcription.completed": {
+				const utteranceId = VoiceUtteranceId(String(this.voiceSessionId) + ":input:" + event.utteranceId);
+				this.emit({
+					type: "transcription.completed",
+					utteranceId,
+					text: event.text
+				});
+				if (this.interactionMode === "frontend-agent" && event.text.trim() !== "") {
+					const text = event.text.trim();
+					this.emitTaskCommand(this.activeTaskId === void 0 ? {
+						type: "route_transcription",
+						input: text
+					} : {
+						type: "send_task_message",
+						taskId: this.activeTaskId,
+						message: text
+					});
+				}
+				return;
+			}
+			case "transcription.failed":
+				this.emit({
+					type: "transcription.failed",
+					utteranceId: VoiceUtteranceId(String(this.voiceSessionId) + ":input:" + event.utteranceId),
+					message: event.message
+				});
+				return;
+			case "tts.started": {
+				const responseId = this.responseId(event.responseId);
+				this.emit({
+					type: "output_audio.started",
+					responseId
+				});
+				return;
+			}
+			case "tts.delta":
+				this.emit({
+					type: "output_audio.delta",
+					responseId: this.responseId(event.responseId),
+					audio: event.audio
+				});
+				return;
+			case "tts.done": {
+				const responseId = this.responseId(event.responseId);
+				this.emit({
+					type: "output_audio.done",
+					responseId
+				});
+				return;
+			}
+			case "error":
+				this.emit({
+					type: "error",
+					message: event.message
+				});
+				return;
+			case "closed":
+				this.emit({
+					type: "closed",
+					...event.reason === void 0 ? {} : { reason: event.reason }
+				});
+				return;
+		}
+	}
+	responseId(value) {
+		return VoiceResponseId(value.startsWith(String(this.voiceSessionId) + ":response:") ? value : String(this.voiceSessionId) + ":response:" + value);
+	}
+	emitTaskCommand(command) {
+		const id = VoiceCommandCallId(String(this.voiceSessionId) + ":task:" + randomUUID());
+		this.pendingCommands.set(id, command);
+		this.emit({
+			type: "task.command",
+			call: {
+				id,
+				command
+			}
+		});
+	}
+};
+function isTerminalTaskStatus(status) {
+	return status === "completed" || status === "failed" || status === "cancelled" || status === "interrupted";
+}
+//#endregion
+//#region ../voice-local/src/siliconflow-asr.ts
+const WAV_HEADER_BYTES = 44;
+/** Upload one completed PCM utterance to SiliconFlow and return its final transcript. */
+var SiliconFlowAsr = class {
+	config;
+	constructor(config) {
+		this.config = config;
+	}
+	async transcribe(pcm, sampleRate) {
+		if (pcm.byteLength === 0) return "";
+		const wav = pcm16MonoWav(pcm, sampleRate);
+		const form = new FormData();
+		form.append("file", new Blob([wav.buffer], { type: "audio/wav" }), "utterance.wav");
+		form.append("model", this.config.model);
+		const response = await (this.config.fetch ?? fetch)(this.config.endpoint, {
+			method: "POST",
+			headers: { Authorization: `Bearer ${this.config.apiKey}` },
+			body: form,
+			signal: AbortSignal.timeout(this.config.timeoutMs)
+		});
+		const body = await response.text();
+		if (!response.ok) throw new Error(`SiliconFlow ASR returned ${response.status}: ${body.slice(0, 500)}`);
+		let parsed;
+		try {
+			parsed = JSON.parse(body);
+		} catch {
+			throw new Error("SiliconFlow ASR returned invalid JSON");
+		}
+		if (!isRecord(parsed) || typeof parsed.text !== "string") throw new Error("SiliconFlow ASR response has no text field");
+		return parsed.text.trim();
+	}
+};
+/** Wrap signed 16-bit little-endian mono PCM in a standard WAV container. */
+function pcm16MonoWav(pcm, sampleRate) {
+	if (pcm.byteLength % 2 !== 0) throw new Error("PCM16 input has an odd byte length");
+	const wav = new Uint8Array(WAV_HEADER_BYTES + pcm.byteLength);
+	const view = new DataView(wav.buffer);
+	writeAscii(wav, 0, "RIFF");
+	view.setUint32(4, 36 + pcm.byteLength, true);
+	writeAscii(wav, 8, "WAVE");
+	writeAscii(wav, 12, "fmt ");
+	view.setUint32(16, 16, true);
+	view.setUint16(20, 1, true);
+	view.setUint16(22, 1, true);
+	view.setUint32(24, sampleRate, true);
+	view.setUint32(28, sampleRate * 2, true);
+	view.setUint16(32, 2, true);
+	view.setUint16(34, 16, true);
+	writeAscii(wav, 36, "data");
+	view.setUint32(40, pcm.byteLength, true);
+	wav.set(pcm, WAV_HEADER_BYTES);
+	return wav;
+}
+function writeAscii(target, offset, value) {
+	for (const [index, character] of [...value].entries()) target[offset + index] = character.charCodeAt(0);
+}
+function isRecord(value) {
+	return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+//#endregion
+//#region ../voice-local/src/node-backend.ts
+const INPUT_SAMPLE_RATE = 16e3;
+/** Lightweight PCM silence detector plus SiliconFlow cloud ASR and Edge TTS. */
+var NodeSpeechBackend = class {
+	config;
+	audio;
+	asr;
+	emit;
+	preRoll = [];
+	preRollBytes = 0;
+	active;
+	recognitionQueue = Promise.resolve();
+	synthesisQueue = Promise.resolve();
+	synthesisGeneration = 0;
+	synthesisResponses = /* @__PURE__ */ new Map();
+	closed = false;
+	constructor(config) {
+		this.config = config;
+		if (config.inputSampleRate !== INPUT_SAMPLE_RATE) throw new Error("SiliconFlow speech input requires 16000 Hz microphone audio");
+		if (config.apiKey.trim() === "") throw new Error("SILICONFLOW_API_KEY is required for cloud speech recognition");
+		if (config.speechThreshold <= 0 || config.speechThreshold >= 1) throw new Error("speechThreshold must be between 0 and 1");
+		if ((config.minSpeechDurationMs ?? 250) <= 0) throw new Error("minSpeechDurationMs must be greater than 0");
+		if (!/^[+-]\d{1,3}%$/.test(config.ttsRate)) throw new Error("ttsRate must use an Edge TTS relative percentage such as +20%");
+		this.audio = {
+			inputSampleRate: config.inputSampleRate,
+			outputSampleRate: config.outputSampleRate,
+			format: "audio_mpeg"
+		};
+		this.asr = new SiliconFlowAsr({
+			apiKey: config.apiKey,
+			endpoint: config.endpoint,
+			model: config.model,
+			timeoutMs: config.requestTimeoutMs,
+			...config.fetch === void 0 ? {} : { fetch: config.fetch }
+		});
+	}
+	start(emit) {
+		if (this.emit !== void 0) return Promise.reject(/* @__PURE__ */ new Error("speech backend is already started"));
+		this.emit = emit;
+		emit({ type: "ready" });
+		return Promise.resolve();
+	}
+	appendAudio(audio) {
+		if (this.closed || audio.byteLength === 0) return;
+		if (audio.byteLength % 2 !== 0) {
+			this.emit?.({
+				type: "error",
+				message: "microphone PCM16 frame has an odd byte length"
+			});
+			return;
+		}
+		const frame = audio.slice();
+		const voiced = pcm16Rms(frame) >= this.config.speechThreshold;
+		if (this.active === void 0) {
+			this.rememberPreRoll(frame);
+			if (!voiced) return;
+			this.beginUtterance();
+			return;
+		}
+		this.active.frames.push(frame);
+		this.active.totalBytes += frame.byteLength;
+		if (voiced) {
+			this.active.voicedBytes += frame.byteLength;
+			this.active.silenceBytes = 0;
+			if (!this.active.confirmed && this.active.voicedBytes >= this.bytesFor(this.config.minSpeechDurationMs ?? 250)) this.confirmUtterance();
+		} else this.active.silenceBytes += frame.byteLength;
+		if (!this.active.confirmed && this.active.silenceBytes >= this.bytesFor(this.config.minSpeechDurationMs ?? 250)) {
+			this.active = void 0;
+			return;
+		}
+		if (this.active.silenceBytes >= this.bytesFor(this.config.silenceDurationMs) || this.active.totalBytes >= this.bytesFor(this.config.maxUtteranceMs)) this.finishUtterance();
+	}
+	commitAudio() {
+		this.finishUtterance();
+	}
+	synthesize(responseId, text) {
+		const generation = this.synthesisGeneration;
+		const response = this.synthesisResponses.get(responseId) ?? {
+			finished: false,
+			started: false
+		};
+		this.synthesisResponses.set(responseId, response);
+		this.synthesisQueue = this.synthesisQueue.catch(() => {}).then(async () => {
+			if (generation !== this.synthesisGeneration || this.closed) return;
+			if (!response.started) {
+				response.started = true;
+				this.emit?.({
+					type: "tts.started",
+					responseId
+				});
+			}
+			try {
+				for (const sentence of splitSpeechText(text)) {
+					if (generation !== this.synthesisGeneration || this.closed) return;
+					const audio = await synthesizeEdgeSpeech(sentence, this.config.ttsRate);
+					if (generation !== this.synthesisGeneration || this.closed) return;
+					this.emit?.({
+						type: "tts.delta",
+						responseId,
+						audio
+					});
+				}
+				if (generation !== this.synthesisGeneration || this.closed) return;
+				if (response.finished) {
+					this.synthesisResponses.delete(responseId);
+					this.emit?.({
+						type: "tts.done",
+						responseId
+					});
+				}
+			} catch (error) {
+				if (generation === this.synthesisGeneration && !this.closed) this.emit?.({
+					type: "error",
+					message: "Edge TTS failed: " + errorMessage(error)
+				});
+			}
+		});
+	}
+	finishSynthesis(responseId) {
+		const response = this.synthesisResponses.get(responseId);
+		if (response !== void 0) response.finished = true;
+	}
+	interrupt() {
+		this.synthesisGeneration += 1;
+		this.synthesisResponses.clear();
+	}
+	async close() {
+		if (this.closed) return;
+		this.closed = true;
+		this.synthesisGeneration += 1;
+		this.synthesisResponses.clear();
+		this.active = void 0;
+		this.preRoll.splice(0);
+		await Promise.allSettled([this.recognitionQueue, this.synthesisQueue]);
+		this.emit?.({
+			type: "closed",
+			reason: "SiliconFlow speech backend closed"
+		});
+		this.emit = void 0;
+	}
+	beginUtterance() {
+		const frames = this.preRoll.splice(0);
+		const totalBytes = this.preRollBytes;
+		this.preRollBytes = 0;
+		const last = frames.at(-1);
+		this.active = {
+			id: randomUUID(),
+			frames,
+			confirmed: false,
+			totalBytes,
+			voicedBytes: last?.byteLength ?? 0,
+			silenceBytes: 0
+		};
+		if (this.active.voicedBytes >= this.bytesFor(this.config.minSpeechDurationMs ?? 250)) this.confirmUtterance();
+	}
+	confirmUtterance() {
+		const utterance = this.active;
+		if (utterance === void 0 || utterance.confirmed) return;
+		utterance.confirmed = true;
+		this.interrupt();
+		this.emit?.({
+			type: "transcription.started",
+			utteranceId: utterance.id
+		});
+	}
+	finishUtterance() {
+		const utterance = this.active;
+		if (utterance === void 0) return;
+		this.active = void 0;
+		const keepSilence = this.bytesFor(this.config.trailingSilenceMs);
+		const trimBytes = Math.max(0, utterance.silenceBytes - keepSilence);
+		const pcm = concatFrames(utterance.frames, Math.max(0, utterance.totalBytes - trimBytes));
+		if (!utterance.confirmed || utterance.voicedBytes < this.bytesFor(this.config.minSpeechDurationMs ?? 250) || pcm.byteLength === 0) return;
+		this.recognitionQueue = this.recognitionQueue.catch(() => {}).then(async () => {
+			if (this.closed) return;
+			try {
+				const text = await this.asr.transcribe(pcm, this.config.inputSampleRate);
+				if (this.closed) return;
+				if (text === "") this.emit?.({
+					type: "transcription.failed",
+					utteranceId: utterance.id,
+					message: "云端语音识别没有返回文字。"
+				});
+				else this.emit?.({
+					type: "transcription.completed",
+					utteranceId: utterance.id,
+					text
+				});
+			} catch (error) {
+				if (!this.closed) this.emit?.({
+					type: "transcription.failed",
+					utteranceId: utterance.id,
+					message: errorMessage(error)
+				});
+			}
+		});
+	}
+	rememberPreRoll(frame) {
+		this.preRoll.push(frame);
+		this.preRollBytes += frame.byteLength;
+		const limit = this.bytesFor(this.config.preRollMs);
+		while (this.preRollBytes > limit && this.preRoll.length > 1) {
+			const removed = this.preRoll.shift();
+			if (removed !== void 0) this.preRollBytes -= removed.byteLength;
+		}
+	}
+	bytesFor(milliseconds) {
+		return Math.round(this.config.inputSampleRate * 2 * milliseconds / 1e3);
+	}
+};
+function pcm16Rms(bytes) {
+	const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+	let squares = 0;
+	const samples = bytes.byteLength / 2;
+	for (let offset = 0; offset < bytes.byteLength; offset += 2) {
+		const value = view.getInt16(offset, true) / 32768;
+		squares += value * value;
+	}
+	return samples === 0 ? 0 : Math.sqrt(squares / samples);
+}
+function concatFrames(frames, length) {
+	const output = new Uint8Array(length);
+	let offset = 0;
+	for (const frame of frames) {
+		if (offset >= length) break;
+		const part = frame.subarray(0, Math.min(frame.byteLength, length - offset));
+		output.set(part, offset);
+		offset += part.byteLength;
+	}
+	return output;
+}
+function errorMessage(error) {
+	return error instanceof Error ? error.message : String(error);
+}
+//#endregion
+//#region ../voice-local/src/index.ts
+/** Silence-gated SiliconFlow cloud ASR with Edge TTS. */
+const name = "voice-local";
+const inject = ["voice"];
+const Config = z.object({
+	apiKey: z.string(),
+	endpoint: z.string().default("https://api.siliconflow.cn/v1/audio/transcriptions"),
+	model: z.string().default("XingChenAGI/XingChenASR-V3.2-Ultra"),
+	interactionMode: z.union(["speech-shell", "frontend-agent"]).default("speech-shell"),
+	requestTimeoutMs: z.natural().min(1).default(6e4),
+	inputSampleRate: z.natural().min(1).default(16e3),
+	outputSampleRate: z.natural().min(1).default(48e3),
+	ttsRate: z.string().default(DEFAULT_EDGE_TTS_RATE),
+	silenceDurationMs: z.natural().min(100).default(1500),
+	speechThreshold: z.number().min(.001).max(.5).default(.015),
+	minSpeechDurationMs: z.natural().min(20).default(250),
+	preRollMs: z.natural().default(400),
+	trailingSilenceMs: z.natural().default(200),
+	maxUtteranceMs: z.natural().min(1e3).default(6e4)
+});
+const PACKAGE_ROOT = dirname(fileURLToPath(import.meta.url));
+const PROJECT_ENV = resolve(PACKAGE_ROOT, "../../../.env");
+function apply(ctx, config = {}) {
+	if (existsSync(PROJECT_ENV)) loadEnvFile(PROJECT_ENV);
+	return ctx.voice.registerProvider({
+		id: "local",
+		available: () => true,
+		connect: async ({ voiceSessionId, emit }) => {
+			const session = new LocalSession(new NodeSpeechBackend({
+				apiKey: config.apiKey ?? process.env.SILICONFLOW_API_KEY ?? "",
+				endpoint: config.endpoint ?? "https://api.siliconflow.cn/v1/audio/transcriptions",
+				model: config.model ?? "XingChenAGI/XingChenASR-V3.2-Ultra",
+				requestTimeoutMs: config.requestTimeoutMs ?? 6e4,
+				inputSampleRate: config.inputSampleRate ?? 16e3,
+				outputSampleRate: config.outputSampleRate ?? 48e3,
+				ttsRate: config.ttsRate ?? "+20%",
+				silenceDurationMs: config.silenceDurationMs ?? 1500,
+				speechThreshold: config.speechThreshold ?? .015,
+				minSpeechDurationMs: config.minSpeechDurationMs ?? 250,
+				preRollMs: config.preRollMs ?? 400,
+				trailingSilenceMs: config.trailingSilenceMs ?? 200,
+				maxUtteranceMs: config.maxUtteranceMs ?? 6e4
+			}), emit, voiceSessionId, config.interactionMode ?? "speech-shell");
+			try {
+				await session.start();
+				return session;
+			} catch (error) {
+				await session.close().catch(() => {});
+				throw error;
+			}
+		}
+	});
+}
+//#endregion
+export { Config, LocalSession, NodeSpeechBackend, SiliconFlowAsr, apply, inject, name, pcm16MonoWav };