agent-relay-server 0.13.0 → 0.15.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-relay-server",
-  "version": "0.13.0",
+  "version": "0.15.0",
   "description": "Lightweight HTTP message relay for inter-agent communication across machines",
   "module": "src/index.ts",
   "type": "module",

package/public/index.html

@@ -10300,6 +10300,21 @@ async function apiPostAudio(path, blob) {
 	}
 	return response.json();
 }
+/** POST JSON and get an audio (or other) Blob back — used for server-side TTS. */
+async function apiPostJsonForBlob(path, body) {
+	const headers = { "Content-Type": "application/json" };
+	if (authToken) headers["X-Agent-Relay-Token"] = authToken;
+	const response = await fetch(new URL("api" + path, baseUrl()), {
+		method: "POST",
+		headers,
+		body: JSON.stringify(body)
+	});
+	if (!response.ok) {
+		if (response.status === 401) throw makeError(401, "Authentication required");
+		throw makeError(response.status, await responseErrorMessage(response));
+	}
+	return response.blob();
+}
 async function apiBlob(path) {
 	const opts = {
 		method: "GET",
@@ -10377,13 +10392,18 @@ function chunkForSpeech(text) {
 var synthAvailable = typeof window !== "undefined" && "speechSynthesis" in window;
 var VoiceTts = class {
 	enabled = false;
+	lang = "en-US";
+	mode = "kokoro";
+	kokoroVoice = "am_michael";
 	active = null;
 	queue = [];
 	currentChat = null;
 	speaking = false;
 	gen = 0;
+	audioEl = null;
+	audioUrl = null;
 	get available() {
-		return synthAvailable;
+		return synthAvailable || typeof Audio !== "undefined";
 	}
 	isEnabled() {
 		return this.enabled;
@@ -10393,6 +10413,18 @@ var VoiceTts = class {
 		this.enabled = on;
 		if (!on) this.reset();
 	}
+	/** Set the spoken-voice language (BCP-47, e.g. "en-US"). Empty = browser locale. */
+	setLang(lang) {
+		this.lang = lang;
+	}
+	setMode(mode) {
+		if (mode === this.mode) return;
+		this.mode = mode;
+		this.reset();
+	}
+	setKokoroVoice(voice) {
+		this.kokoroVoice = voice;
+	}
 	setActiveChat(chatId) {
 		if (chatId === this.active) return;
 		this.active = chatId;
@@ -10400,7 +10432,7 @@ var VoiceTts = class {
 	}
 	/** A captured agent response turn arrived for `chatId`. */
 	onResponse(chatId, rawText) {
-		if (!this.enabled || !synthAvailable || !chatId || chatId !== this.active) return;
+		if (!this.enabled || !this.available || !chatId || chatId !== this.active) return;
 		const text = speechify(rawText);
 		if (!text) return;
 		if (this.speaking && this.currentChat && this.currentChat !== chatId) {
@@ -10428,6 +10460,19 @@ var VoiceTts = class {
 		try {
 			window.speechSynthesis.cancel();
 		} catch {}
+		this.stopAudio();
+	}
+	stopAudio() {
+		if (this.audioEl) try {
+			this.audioEl.pause();
+			this.audioEl.src = "";
+		} catch {}
+		if (this.audioUrl) {
+			try {
+				URL.revokeObjectURL(this.audioUrl);
+			} catch {}
+			this.audioUrl = null;
+		}
 	}
 	pump() {
 		if (this.speaking) return;
@@ -10437,24 +10482,78 @@ var VoiceTts = class {
 		this.currentChat = item.chatId;
 		const gen = ++this.gen;
 		const chunks = chunkForSpeech(item.text);
-		const speakAt = (i) => {
+		const done = () => {
 			if (gen !== this.gen) return;
-			if (i >= chunks.length) {
-				this.speaking = false;
-				this.currentChat = null;
-				this.pump();
-				return;
-			}
-			const u = new SpeechSynthesisUtterance(chunks[i]);
-			u.lang = navigator.language || "en-US";
-			u.onend = () => speakAt(i + 1);
-			u.onerror = () => speakAt(i + 1);
-			window.speechSynthesis.speak(u);
+			this.speaking = false;
+			this.currentChat = null;
+			this.pump();
 		};
-		speakAt(0);
+		if (this.mode === "kokoro") this.speakKokoro(chunks, 0, gen, done);
+		else this.speakBrowser(chunks, 0, gen, done);
+	}
+	speakBrowser(chunks, i, gen, done) {
+		if (gen !== this.gen) return;
+		if (!synthAvailable || i >= chunks.length) return done();
+		const u = new SpeechSynthesisUtterance(chunks[i]);
+		u.lang = this.lang || navigator.language || "en-US";
+		u.onend = () => this.speakBrowser(chunks, i + 1, gen, done);
+		u.onerror = () => this.speakBrowser(chunks, i + 1, gen, done);
+		window.speechSynthesis.speak(u);
+	}
+	speakKokoro(chunks, i, gen, done, prefetched) {
+		if (gen !== this.gen) return;
+		const text = chunks[i];
+		if (text === void 0) return done();
+		const cur = prefetched ?? this.fetchSpeech(text);
+		const nextText = chunks[i + 1];
+		const next = nextText !== void 0 ? this.fetchSpeech(nextText) : void 0;
+		cur.then((blob) => {
+			if (gen !== this.gen) return;
+			this.playBlob(blob, gen, () => this.speakKokoro(chunks, i + 1, gen, done, next), () => {
+				this.speakBrowser(chunks, i, gen, done);
+			});
+		}).catch(() => {
+			if (gen !== this.gen) return;
+			this.speakBrowser(chunks, i, gen, done);
+		});
+	}
+	fetchSpeech(text) {
+		return apiPostJsonForBlob("/connectors/voice/call/speak", {
+			text,
+			voice: this.kokoroVoice
+		});
+	}
+	playBlob(blob, gen, onend, onerror) {
+		if (gen !== this.gen) return;
+		if (typeof Audio === "undefined") return onerror();
+		this.stopAudio();
+		if (!this.audioEl) this.audioEl = new Audio();
+		const url = URL.createObjectURL(blob);
+		this.audioUrl = url;
+		const el = this.audioEl;
+		el.src = url;
+		el.onended = () => {
+			if (gen === this.gen) onend();
+		};
+		el.onerror = () => {
+			if (gen === this.gen) onerror();
+		};
+		el.play().catch(() => {
+			if (gen === this.gen) onerror();
+		});
 	}
 };
 var voiceTts = new VoiceTts();
+/** Sorted unique BCP-47 languages the browser's speech engine can speak. May be empty
+*  until the engine finishes loading voices (listen for `voiceschanged` and re-read). */
+function availableSpeechLangs() {
+	if (!synthAvailable) return [];
+	try {
+		return [...new Set(window.speechSynthesis.getVoices().map((v) => v.lang).filter(Boolean))].sort();
+	} catch {
+		return [];
+	}
+}
 var micAvailable = typeof navigator !== "undefined" && !!navigator.mediaDevices?.getUserMedia && typeof window !== "undefined" && "MediaRecorder" in window;
 function pickMimeType() {
 	for (const m of [
@@ -11861,6 +11960,10 @@ var useRelayStore = create$1()(persist((set, get) => ({
 	showBuiltIns: false,
 	autoRefresh: true,
 	voiceTtsEnabled: false,
+	voiceTtsLang: "en-US",
+	voiceTtsMode: "kokoro",
+	voiceTtsKokoroVoice: "am_michael",
+	voiceInputMode: "compose",
 	agentSort: "status",
 	agentSortDir: "asc",
 	agentPresetFilter: "",
@@ -12052,6 +12155,21 @@ var useRelayStore = create$1()(persist((set, get) => ({
 		voiceTts.setEnabled(on);
 		set({ voiceTtsEnabled: on });
 	},
+	setVoiceTtsLang(lang) {
+		voiceTts.setLang(lang);
+		set({ voiceTtsLang: lang });
+	},
+	setVoiceTtsMode(mode) {
+		voiceTts.setMode(mode);
+		set({ voiceTtsMode: mode });
+	},
+	setVoiceTtsKokoroVoice(voice) {
+		voiceTts.setKokoroVoice(voice);
+		set({ voiceTtsKokoroVoice: voice });
+	},
+	setVoiceInputMode(mode) {
+		set({ voiceInputMode: mode });
+	},
 	async init() {
 		if (!useRelayStore.persist.hasHydrated()) await new Promise((resolve) => {
 			const unsub = useRelayStore.persist.onFinishHydration(() => {
@@ -12062,6 +12180,9 @@ var useRelayStore = create$1()(persist((set, get) => ({
 		const token = get().authToken;
 		if (token) setAuthToken(token);
 		voiceTts.setEnabled(get().voiceTtsEnabled);
+		voiceTts.setLang(get().voiceTtsLang);
+		voiceTts.setMode(get().voiceTtsMode);
+		voiceTts.setKokoroVoice(get().voiceTtsKokoroVoice);
 		syncVoiceActiveChat(get());
 		setUnauthorizedHandler(() => {
 			if (!get().authNeeded) set({
@@ -12576,9 +12697,9 @@ var useRelayStore = create$1()(persist((set, get) => ({
 					const msgs = [...s.messages, msg];
 					if (msgs.length > 500) msgs.splice(0, msgs.length - 500);
 					set({ messages: msgs });
-					if (msg.kind === "session") {
+					if (msg.kind === "session" && msg.from !== "user") {
 						const sess = msg.payload?.session;
-						if ((sess?.type ?? "response") === "response" && (sess?.origin ?? "provider") === "provider") voiceTts.onResponse(inboxPeer(msg), msg.body);
+						if (sess?.type === "response" && sess?.origin === "provider") voiceTts.onResponse(inboxPeer(msg), msg.body);
 					}
 					const peer = inboxPeer(msg);
 					if (isHumanInboundMessage(msg) && peer && s.view === "chat" && s.selectedInboxThread === peer && !isDashboardHidden()) get().markInboxThreadReadTo(peer, msg.id);
@@ -14150,6 +14271,10 @@ var useRelayStore = create$1()(persist((set, get) => ({
 		showBuiltIns: state.showBuiltIns,
 		autoRefresh: state.autoRefresh,
 		voiceTtsEnabled: state.voiceTtsEnabled,
+		voiceTtsLang: state.voiceTtsLang,
+		voiceTtsMode: state.voiceTtsMode,
+		voiceTtsKokoroVoice: state.voiceTtsKokoroVoice,
+		voiceInputMode: state.voiceInputMode,
 		agentSort: state.agentSort,
 		agentSortDir: state.agentSortDir,
 		agentPresetFilter: state.agentPresetFilter,
@@ -125564,6 +125689,40 @@ var TIMELINE_STATUS_LABELS = {
 var TIMELINE_STATUSES = new Set(Object.keys(TIMELINE_STATUS_LABELS));
 var STATUS_DEDUPE_WINDOW_MS = 3e3;
 var CHAT_BOTTOM_THRESHOLD_PX = 96;
+var KOKORO_VOICES = [
+	{
+		id: "am_michael",
+		label: "Michael (US ♂)"
+	},
+	{
+		id: "am_adam",
+		label: "Adam (US ♂)"
+	},
+	{
+		id: "af_heart",
+		label: "Heart (US ♀)"
+	},
+	{
+		id: "af_bella",
+		label: "Bella (US ♀)"
+	},
+	{
+		id: "af_nicole",
+		label: "Nicole (US ♀)"
+	},
+	{
+		id: "af_sarah",
+		label: "Sarah (US ♀)"
+	},
+	{
+		id: "bm_george",
+		label: "George (UK ♂)"
+	},
+	{
+		id: "bf_emma",
+		label: "Emma (UK ♀)"
+	}
+];
 function StatusMarker({ event }) {
 	return /* @__PURE__ */ (0, import_jsx_runtime.jsxs)("div", {
 		className: "flex items-center justify-center gap-2 py-2 my-1",
@@ -126599,6 +126758,21 @@ function ChatPanel({ threads, onBack, showBackButton }) {
 	const fetchOrchestrators = useRelayStore((s) => s.fetchOrchestrators);
 	const voiceTtsEnabled = useRelayStore((s) => s.voiceTtsEnabled);
 	const setVoiceTtsEnabled = useRelayStore((s) => s.setVoiceTtsEnabled);
+	const voiceTtsLang = useRelayStore((s) => s.voiceTtsLang);
+	const setVoiceTtsLang = useRelayStore((s) => s.setVoiceTtsLang);
+	const voiceTtsMode = useRelayStore((s) => s.voiceTtsMode);
+	const setVoiceTtsMode = useRelayStore((s) => s.setVoiceTtsMode);
+	const voiceTtsKokoroVoice = useRelayStore((s) => s.voiceTtsKokoroVoice);
+	const setVoiceTtsKokoroVoice = useRelayStore((s) => s.setVoiceTtsKokoroVoice);
+	const voiceInputMode = useRelayStore((s) => s.voiceInputMode);
+	const [speechLangs, setSpeechLangs] = (0, import_react.useState)(() => availableSpeechLangs());
+	(0, import_react.useEffect)(() => {
+		if (!voiceTts.available) return;
+		const refresh = () => setSpeechLangs(availableSpeechLangs());
+		refresh();
+		window.speechSynthesis.addEventListener?.("voiceschanged", refresh);
+		return () => window.speechSynthesis.removeEventListener?.("voiceschanged", refresh);
+	}, []);
 	const fileInputRef = (0, import_react.useRef)(null);
 	const pttRecorderRef = (0, import_react.useRef)(null);
 	const [micState, setMicState] = (0, import_react.useState)("idle");
@@ -126915,7 +127089,21 @@ function ChatPanel({ threads, onBack, showBackButton }) {
 			const clip = await recorder.stop();
 			if (!clip) return;
 			const text = await transcribeClip(clip);
-			if (text) setReplyDraft(selectedInboxThread, draft ? `${draft} ${text}` : text);
+			if (!text) return;
+			const combined = draft ? `${draft} ${text}` : text;
+			if (voiceInputMode === "autosend" && !chatSending && !hasPendingUploads) {
+				const attachments = readyAttachments.map((item) => ({
+					artifactId: item.artifact.id,
+					kind: item.artifact.kind,
+					role: "media",
+					title: item.artifact.filename || item.fileName
+				}));
+				sendChatMessage(combined, {
+					peer: selectedInboxThread,
+					lastMessage: thread?.lastMessage || null
+				}, attachments);
+				clearPendingAttachments();
+			} else setReplyDraft(selectedInboxThread, combined);
 		} catch (e) {
 			showError("Transcription failed", e?.message || "Could not transcribe audio.");
 		} finally {
@@ -126996,6 +127184,47 @@ function ChatPanel({ threads, onBack, showBackButton }) {
 									onClick: () => setVoiceTtsEnabled(!voiceTtsEnabled),
 									children: voiceTtsEnabled ? /* @__PURE__ */ (0, import_jsx_runtime.jsx)(Volume2, { className: "w-3.5 h-3.5" }) : /* @__PURE__ */ (0, import_jsx_runtime.jsx)(VolumeX, { className: "w-3.5 h-3.5" })
 								}),
+								voiceTts.available && voiceTtsEnabled && /* @__PURE__ */ (0, import_jsx_runtime.jsxs)(import_jsx_runtime.Fragment, { children: [/* @__PURE__ */ (0, import_jsx_runtime.jsxs)("select", {
+									value: voiceTtsMode,
+									onChange: (e) => setVoiceTtsMode(e.target.value),
+									title: "Voice engine — Kokoro (server, natural) falls back to browser automatically",
+									className: "h-7 rounded border border-border bg-background px-1 text-xs",
+									children: [/* @__PURE__ */ (0, import_jsx_runtime.jsx)("option", {
+										value: "kokoro",
+										children: "Kokoro"
+									}), /* @__PURE__ */ (0, import_jsx_runtime.jsx)("option", {
+										value: "browser",
+										children: "Browser"
+									})]
+								}), voiceTtsMode === "kokoro" ? /* @__PURE__ */ (0, import_jsx_runtime.jsx)("select", {
+									value: voiceTtsKokoroVoice,
+									onChange: (e) => setVoiceTtsKokoroVoice(e.target.value),
+									title: "Kokoro voice",
+									className: "h-7 rounded border border-border bg-background px-1 text-xs",
+									children: [...KOKORO_VOICES, ...KOKORO_VOICES.some((v) => v.id === voiceTtsKokoroVoice) ? [] : [{
+										id: voiceTtsKokoroVoice,
+										label: voiceTtsKokoroVoice
+									}]].map((v) => /* @__PURE__ */ (0, import_jsx_runtime.jsx)("option", {
+										value: v.id,
+										children: v.label
+									}, v.id))
+								}) : /* @__PURE__ */ (0, import_jsx_runtime.jsxs)("select", {
+									value: voiceTtsLang,
+									onChange: (e) => setVoiceTtsLang(e.target.value),
+									title: "Voice language",
+									className: "h-7 rounded border border-border bg-background px-1 text-xs",
+									children: [[...new Set([
+										"en-US",
+										...speechLangs,
+										...voiceTtsLang ? [voiceTtsLang] : []
+									])].sort().map((l) => /* @__PURE__ */ (0, import_jsx_runtime.jsx)("option", {
+										value: l,
+										children: l
+									}, l)), /* @__PURE__ */ (0, import_jsx_runtime.jsx)("option", {
+										value: "",
+										children: "Browser default"
+									})]
+								})] }),
 								canOpenTerminal && /* @__PURE__ */ (0, import_jsx_runtime.jsx)(Button, {
 									variant: "ghost",
 									size: "icon-sm",
@@ -153453,10 +153682,38 @@ function SettingsView() {
 					]
 				})]
 			}),
-			/* @__PURE__ */ (0, import_jsx_runtime.jsx)(StewardSettings, {})
+			/* @__PURE__ */ (0, import_jsx_runtime.jsx)(StewardSettings, {}),
+			/* @__PURE__ */ (0, import_jsx_runtime.jsx)(VoiceSettings, {})
 		]
 	});
 }
+function VoiceSettings() {
+	const voiceInputMode = useRelayStore((s) => s.voiceInputMode);
+	const setVoiceInputMode = useRelayStore((s) => s.setVoiceInputMode);
+	return /* @__PURE__ */ (0, import_jsx_runtime.jsxs)("section", {
+		className: "space-y-3 rounded-lg border p-4",
+		children: [/* @__PURE__ */ (0, import_jsx_runtime.jsxs)("div", { children: [/* @__PURE__ */ (0, import_jsx_runtime.jsx)("h2", {
+			className: "text-sm font-semibold",
+			children: "Voice"
+		}), /* @__PURE__ */ (0, import_jsx_runtime.jsx)("p", {
+			className: "text-xs text-muted-foreground",
+			children: "How push-to-talk behaves in chat."
+		})] }), /* @__PURE__ */ (0, import_jsx_runtime.jsx)(Field, {
+			label: "Push-to-talk input",
+			children: /* @__PURE__ */ (0, import_jsx_runtime.jsxs)(Select$1, {
+				value: voiceInputMode,
+				onValueChange: (v) => setVoiceInputMode(v),
+				children: [/* @__PURE__ */ (0, import_jsx_runtime.jsx)(SelectTrigger, { children: /* @__PURE__ */ (0, import_jsx_runtime.jsx)(SelectValue, {}) }), /* @__PURE__ */ (0, import_jsx_runtime.jsxs)(SelectContent, { children: [/* @__PURE__ */ (0, import_jsx_runtime.jsx)(SelectItem, {
+					value: "compose",
+					children: "Fill the message box (review, then Enter)"
+				}), /* @__PURE__ */ (0, import_jsx_runtime.jsx)(SelectItem, {
+					value: "autosend",
+					children: "Send immediately (speak-and-send)"
+				})] })]
+			})
+		})]
+	});
+}
 var EFFORTS = [
 	"low",
 	"medium",

package/src/routes.ts

@@ -5679,7 +5679,7 @@ const putConnectorConfig: Handler = async (req, params) => {
 // Endpoints a connector daemon may expose for the dashboard to call through the
 // relay (single-origin — no CORS, no extra port exposure). Kept to a small,
 // non-mutating allowlist; the connector advertises its HTTP base via its status.
-const PROXYABLE_CONNECTOR_CALLS = new Set(["transcribe", "utterance"]);
+const PROXYABLE_CONNECTOR_CALLS = new Set(["transcribe", "utterance", "speak"]);
 
 function connectorAdvertisedEndpoint(connector: NonNullable<ReturnType<typeof getConnector>>): string | null {
   const raw = connector.state?.raw;