bosun 0.40.21 → 0.41.1

package/ui/modules/voice-client.js

@@ -249,12 +249,18 @@ let _traceTtsFirstAudioMarked = false;
 
 const RECONNECT_AT_MS = 28 * 60 * 1000; // 28 minutes
 const MAX_RECONNECT_ATTEMPTS = 3;
-const AUTO_BARGE_IN_COOLDOWN_MS = 700;
+const AUTO_BARGE_IN_COOLDOWN_MS = 1200;
 const AUTO_BARGE_IN_MIC_LEVEL_THRESHOLD = 0.08;
 const AUTO_BARGE_IN_FADE_MS = 220;
-// Noise-control default: disable user-side live ASR transcript output/persistence.
-// Assistant response text remains enabled.
-const ENABLE_USER_TRANSCRIPT = false;
+// Minimum speech duration (ms) before an interrupt is allowed — filters keyboard/click noise
+let _speechStartedAt = 0;
+const MIN_SPEECH_DURATION_FOR_INTERRUPT_MS = 400;
+// Delayed response clear — keep response visible in center after turn ends
+let _responseClearTimer = null;
+const RESPONSE_DISPLAY_HOLD_MS = 8000;
+// User transcript is always enabled — transcription is surfaced from the API's
+// input_audio_transcription feature (primary) or browser SpeechRecognition (backup).
+const ENABLE_USER_TRANSCRIPT = true;
 let _reconnectAttempts = 0;
 let _pendingResponseCreateTimer = null;
 let _awaitingAutoResponse = false;
@@ -266,6 +272,64 @@ const SpeechRecognition = typeof globalThis !== "undefined"
   ? (globalThis.SpeechRecognition || globalThis.webkitSpeechRecognition)
   : null;
 
+// ── Browser SpeechRecognition (parallel backup for user transcription) ──────
+
+let _browserRecognition = null;
+let _browserTranscriptActive = false;
+let _apiTranscriptDelivered = false;
+
+function _startBrowserTranscription() {
+  if (!SpeechRecognition || _browserRecognition) return;
+  try {
+    const recognition = new SpeechRecognition();
+    recognition.continuous = true;
+    recognition.interimResults = true;
+    recognition.maxAlternatives = 1;
+    recognition.lang = navigator?.language || "en-US";
+
+    recognition.onresult = (event) => {
+      if (_apiTranscriptDelivered) return;
+      let transcript = "";
+      for (let i = event.resultIndex; i < event.results.length; i++) {
+        transcript += event.results[i][0].transcript;
+      }
+      const text = transcript.trim();
+      if (!text) return;
+      voiceTranscript.value = text;
+      emit("transcript", { text, final: event.results[event.resultIndex]?.isFinal || false, source: "browser" });
+      if (event.results[event.resultIndex]?.isFinal) {
+        _recordVoiceTranscriptIfNew("user", text, "browser.speech_recognition.final");
+      }
+    };
+
+    recognition.onerror = (e) => {
+      if (e.error !== "no-speech" && e.error !== "aborted") {
+        console.warn("[voice-client] Browser SpeechRecognition error:", e.error);
+      }
+    };
+
+    recognition.onend = () => {
+      if (_browserTranscriptActive && (_dc || _ws)) {
+        try { recognition.start(); } catch { /* already running or stopped */ }
+      }
+    };
+
+    recognition.start();
+    _browserRecognition = recognition;
+    _browserTranscriptActive = true;
+  } catch (err) {
+    console.warn("[voice-client] Browser SpeechRecognition unavailable:", err?.message);
+  }
+}
+
+function _stopBrowserTranscription() {
+  _browserTranscriptActive = false;
+  if (_browserRecognition) {
+    try { _browserRecognition.stop(); } catch { /* ignore */ }
+    _browserRecognition = null;
+  }
+}
+
 function _normalizeCallContext(options = {}) {
   const sessionId = String(options?.sessionId || "").trim() || null;
   const executor = String(options?.executor || "").trim() || null;
@@ -448,12 +512,8 @@ async function _processResponsesAudioTurn(text) {
   });
 
   voiceState.value = "thinking";
-  if (ENABLE_USER_TRANSCRIPT) {
-    voiceTranscript.value = inputText;
-    emit("transcript", { text: inputText, final: true });
-  } else {
-    voiceTranscript.value = "";
-  }
+  voiceTranscript.value = inputText;
+  emit("transcript", { text: inputText, final: true, source: "api" });
   _recordVoiceTranscriptIfNew("user", inputText, "responses-audio.user_input");
 
   if (_responsesAbortController) {
@@ -503,7 +563,7 @@ async function _processResponsesAudioTurn(text) {
   _traceEndTurn("turn_end", {
     reason: "responses-audio.turn_completed",
   });
-  voiceResponse.value = "";
+  _scheduleResponseClear();
   voiceState.value = "listening";
 }
 
@@ -580,6 +640,9 @@ async function _startResponsesAudioSession(tokenData) {
   _sessionStartTime = Date.now();
   startDurationTimer();
   voiceState.value = "connected";
+  // Start browser SpeechRecognition as parallel/backup transcription
+  _apiTranscriptDelivered = false;
+  _startBrowserTranscription();
   emit("connected", {
     provider: tokenData?.provider || "openai",
     sessionId: voiceSessionId.value,
@@ -668,6 +731,26 @@ function _markAssistantToolResponseObserved() {
   _clearToolCompletionAckTimer();
 }
 
+// ── Response display hold ──────────────────────────────────────────────────
+// Keep assistant response visible in center for RESPONSE_DISPLAY_HOLD_MS
+// after the turn ends, instead of clearing immediately.
+
+function _scheduleResponseClear() {
+  if (_responseClearTimer) clearTimeout(_responseClearTimer);
+  _responseClearTimer = setTimeout(() => {
+    _responseClearTimer = null;
+    voiceResponse.value = "";
+  }, RESPONSE_DISPLAY_HOLD_MS);
+}
+
+function _clearResponseForNewTurn() {
+  if (_responseClearTimer) {
+    clearTimeout(_responseClearTimer);
+    _responseClearTimer = null;
+  }
+  voiceResponse.value = "";
+}
+
 // ── Event System ────────────────────────────────────────────────────────────
 
 export function onVoiceEvent(event, handler) {
@@ -747,18 +830,18 @@ function sendSessionUpdate(tokenData = {}) {
     type: turnDetection,
     ...(turnDetection === "server_vad"
       ? {
-          threshold: 0.7,
-          prefix_padding_ms: 400,
-          silence_duration_ms: 1200,
+          threshold: 0.82,
+          prefix_padding_ms: 500,
+          silence_duration_ms: 1600,
           create_response: true,
-          interrupt_response: true,
+          interrupt_response: false,
         }
       : {}),
     ...(turnDetection === "semantic_vad"
       ? {
-          eagerness: "medium",
+          eagerness: "low",
           create_response: true,
-          interrupt_response: true,
+          interrupt_response: false,
         }
       : {}),
   };
@@ -769,6 +852,18 @@ function sendSessionUpdate(tokenData = {}) {
   const transcriptionEnabled =
     sessionConfig?.input_audio_transcription !== undefined;
 
+  // Include instructions from the server session config so the voice agent
+  // receives its system prompt (persona, tools, behaviour rules).
+  const instructions = sessionConfig?.instructions || tokenData?.instructions || undefined;
+
+  // Include tool definitions so the realtime model can invoke them.
+  const tools = Array.isArray(sessionConfig?.tools) && sessionConfig.tools.length
+    ? sessionConfig.tools
+    : Array.isArray(tokenData?.tools) && tokenData.tools.length
+      ? tokenData.tools
+      : undefined;
+  const toolChoice = sessionConfig?.tool_choice || undefined;
+
   sendRealtimeEvent({
     type: "session.update",
     session: {
@@ -776,6 +871,8 @@ function sendSessionUpdate(tokenData = {}) {
       voice: voiceId,
       input_audio_format: "pcm16",
       output_audio_format: "pcm16",
+      ...(instructions ? { instructions } : {}),
+      ...(tools ? { tools, tool_choice: toolChoice || "auto" } : {}),
       ...(transcriptionEnabled
         ? { input_audio_transcription: { model: transcriptionModel } }
         : {}),
@@ -964,6 +1061,10 @@ async function _startWebSocketTransport(tokenData, mediaStream) {
       _sessionStartTime = Date.now();
       startDurationTimer();
 
+      // Start browser SpeechRecognition as parallel/backup transcription
+      _apiTranscriptDelivered = false;
+      _startBrowserTranscription();
+
       emit("connected", {
         provider: tokenData.provider || "azure",
         sessionId: voiceSessionId.value,
@@ -1222,6 +1323,9 @@ export async function startVoiceSession(options = {}) {
       voiceSessionId.value = _callContext.sessionId || `voice-${Date.now()}`;
       startDurationTimer();
       startReconnectTimer();
+      // Start browser SpeechRecognition as parallel/backup transcription
+      _apiTranscriptDelivered = false;
+      _startBrowserTranscription();
       emit("connected", {
         provider: tokenData.provider,
         sessionId: voiceSessionId.value,
@@ -1340,6 +1444,7 @@ export function stopVoiceSession() {
   _explicitStop = true;
   emit("session-ending", { sessionId: voiceSessionId.value });
   _stopMicLevelMonitor();
+  _stopBrowserTranscription();
   cleanup();
   voiceState.value = "idle";
   voiceTranscript.value = "";
@@ -1348,6 +1453,8 @@ export function stopVoiceSession() {
   voiceSessionId.value = null;
   voiceBoundSessionId.value = null;
   voiceDuration.value = 0;
+  _speechStartedAt = 0;
+  if (_responseClearTimer) { clearTimeout(_responseClearTimer); _responseClearTimer = null; }
   _webrtcUnavailableForProvider = false;
   _lastTokenData = null;
   _callContext = {
@@ -1372,31 +1479,40 @@ function handleServerEvent(event) {
       break;
 
     case "input_audio_buffer.speech_started":
+      _speechStartedAt = Date.now();
       _traceBeginTurn("turn_start", { reason: type });
-      triggerAutoBargeIn("speech-started");
+      // Clear lingering response so center shows user's new transcript
+      _clearResponseForNewTurn();
+      // Don't interrupt immediately — wait for MIN_SPEECH_DURATION_FOR_INTERRUPT_MS
+      setTimeout(() => {
+        if (_speechStartedAt > 0 && (Date.now() - _speechStartedAt) >= MIN_SPEECH_DURATION_FOR_INTERRUPT_MS) {
+          triggerAutoBargeIn("speech-started-confirmed");
+        }
+      }, MIN_SPEECH_DURATION_FOR_INTERRUPT_MS);
       voiceState.value = "listening";
       emit("speech-started", {});
       break;
 
     case "input_audio_buffer.speech_stopped":
+      _speechStartedAt = 0;
       voiceState.value = "thinking";
       scheduleManualResponseCreate("speech-stopped");
       emit("speech-stopped", {});
       break;
 
     case "conversation.item.input_audio_transcription.completed":
-      if (ENABLE_USER_TRANSCRIPT) {
-        voiceTranscript.value = event.transcript || "";
-        emit("transcript", { text: event.transcript, final: true });
-      } else {
-        voiceTranscript.value = "";
-      }
+      // API-level transcript delivered — prefer over browser SpeechRecognition
+      _apiTranscriptDelivered = true;
+      voiceTranscript.value = event.transcript || "";
+      emit("transcript", { text: event.transcript, final: true, source: "api" });
       _recordVoiceTranscriptIfNew(
         "user",
         event.transcript || "",
         "conversation.item.input_audio_transcription.completed",
       );
       scheduleManualResponseCreate("transcription-completed");
+      // Reset for next utterance
+      setTimeout(() => { _apiTranscriptDelivered = false; }, 500);
       break;
 
     case "conversation.item.created": {
@@ -1407,11 +1523,11 @@ function handleServerEvent(event) {
           .map((part) => String(part?.transcript || part?.text || ""))
           .join("")
           .trim();
-        if (transcript && ENABLE_USER_TRANSCRIPT) {
+        if (transcript) {
+          _apiTranscriptDelivered = true;
           voiceTranscript.value = transcript;
-          emit("transcript", { text: transcript, final: true });
-        } else if (!ENABLE_USER_TRANSCRIPT) {
-          voiceTranscript.value = "";
+          emit("transcript", { text: transcript, final: true, source: "api" });
+          setTimeout(() => { _apiTranscriptDelivered = false; }, 500);
         }
         _recordVoiceTranscriptIfNew(
           "user",
@@ -1463,7 +1579,7 @@ function handleServerEvent(event) {
         "response.audio_transcript.done",
       );
       _traceEndTurn("turn_end", { reason: type });
-      voiceResponse.value = "";
+      _scheduleResponseClear();
       break;
 
     case "response.text.done":
@@ -1475,7 +1591,7 @@ function handleServerEvent(event) {
         "response.text.done",
       );
       _traceEndTurn("turn_end", { reason: type });
-      voiceResponse.value = "";
+      _scheduleResponseClear();
       break;
 
     case "response.output_text.done":
@@ -1487,7 +1603,7 @@ function handleServerEvent(event) {
         "response.output_text.done",
       );
       _traceEndTurn("turn_end", { reason: type });
-      voiceResponse.value = "";
+      _scheduleResponseClear();
       break;
 
     case "response.audio.delta":
@@ -1539,7 +1655,7 @@ function handleServerEvent(event) {
           voiceResponse.value,
           "response.done.fallback",
         );
-        voiceResponse.value = "";
+        _scheduleResponseClear();
       }
       if (voiceState.value !== "listening") {
         voiceState.value = "connected";
@@ -1606,6 +1722,23 @@ async function handleToolCall(event) {
     });
     const result = await res.json();
 
+    // Normalize tool output — handle empty strings, objects, and large payloads
+    let toolOutput = "";
+    if (result.error) {
+      toolOutput = `Tool error: ${result.error}`;
+    } else if (result.result != null && result.result !== "") {
+      toolOutput = typeof result.result === "string"
+        ? result.result
+        : JSON.stringify(result.result);
+    } else {
+      toolOutput = "Tool completed with no output";
+    }
+    // Truncate very large outputs to avoid overwhelming the Realtime API context
+    const VOICE_TOOL_OUTPUT_MAX = 6000;
+    if (toolOutput.length > VOICE_TOOL_OUTPUT_MAX) {
+      toolOutput = toolOutput.slice(0, VOICE_TOOL_OUTPUT_MAX) + "\n... (truncated for voice — full result available in chat)";
+    }
+
     // Update tool call status
     voiceToolCalls.value = voiceToolCalls.value.map(tc =>
       tc.callId === callId ? { ...tc, status: "complete", result: result.result } : tc
@@ -1617,7 +1750,7 @@ async function handleToolCall(event) {
       item: {
         type: "function_call_output",
         call_id: callId,
-        output: result.result || result.error || "No output",
+        output: toolOutput,
       },
     });
     // Trigger response generation
@@ -1678,6 +1811,13 @@ function fadeElementVolumeTo(el, targetVolume, durationMs) {
 
 function triggerAutoBargeIn(reason = "speech-started") {
   const now = Date.now();
+  // Only interrupt if speech has been ongoing long enough to be real speech
+  if (_speechStartedAt > 0) {
+    const speechDuration = now - _speechStartedAt;
+    if (speechDuration < MIN_SPEECH_DURATION_FOR_INTERRUPT_MS) {
+      return false;
+    }
+  }
   const audioActive = isAssistantPlaybackActive();
   if (!shouldAutoBargeIn({
     muted: isVoiceMicMuted.value,

package/ui/setup.html

@@ -909,6 +909,9 @@ function App() {
   const [kanbanBackend, setKanbanBackend] = useState("internal");
   const [telegramToken, setTelegramToken] = useState("");
   const [telegramChatId, setTelegramChatId] = useState("");
+  const [telegramDiscoveredChats, setTelegramDiscoveredChats] = useState([]);
+  const [telegramChatLookupLoading, setTelegramChatLookupLoading] = useState(false);
+  const [telegramChatLookupMessage, setTelegramChatLookupMessage] = useState("");
   const [maxParallel, setMaxParallel] = useState(4);
   const [maxRetries, setMaxRetries] = useState(3);
   const [failoverStrategy, setFailoverStrategy] = useState("next-in-line");
@@ -2270,6 +2273,42 @@ function App() {
     if (idx <= step || completedSteps.has(idx)) setStep(idx);
   };
 
+  const discoverTelegramChatIds = async () => {
+    const token = String(telegramToken || "").trim();
+    if (!token) {
+      setTelegramDiscoveredChats([]);
+      setTelegramChatLookupMessage("Enter a Telegram bot token first.");
+      return;
+    }
+
+    setTelegramChatLookupLoading(true);
+    setTelegramChatLookupMessage("");
+    try {
+      const result = await apiPost("telegram-chat-id", { token });
+      if (!result?.ok) {
+        setTelegramDiscoveredChats([]);
+        setTelegramChatLookupMessage(result?.error || "Failed to discover Telegram chats.");
+        return;
+      }
+
+      const chats = Array.isArray(result.chats) ? result.chats : [];
+      setTelegramDiscoveredChats(chats);
+      if (chats.length === 1) {
+        setTelegramChatId(String(chats[0].id));
+        setTelegramChatLookupMessage(`Found 1 chat: ${chats[0].id}`);
+      } else if (chats.length > 1) {
+        setTelegramChatLookupMessage(`Found ${chats.length} chats. Choose the one Bosun should use.`);
+      } else {
+        setTelegramChatLookupMessage(result.message || "No chats found yet. Send a message to your bot, then try again.");
+      }
+    } catch (err) {
+      setTelegramDiscoveredChats([]);
+      setTelegramChatLookupMessage(err.message || "Failed to discover Telegram chats.");
+    } finally {
+      setTelegramChatLookupLoading(false);
+    }
+  };
+
   // ── Build EXECUTORS env string ─────────────────────────────────────────────
 
   const buildExecutorsEnv = () =>
@@ -3645,7 +3684,11 @@ function App() {
     ${telegramEnabled && html`
       <div class="form-group">
         <label>Telegram Bot Token</label>
-        <input type="password" value=${telegramToken} oninput=${(e) => setTelegramToken(e.target.value)}
+        <input type="password" value=${telegramToken} oninput=${(e) => {
+          setTelegramToken(e.target.value);
+          setTelegramDiscoveredChats([]);
+          setTelegramChatLookupMessage("");
+        }}
           placeholder="123456:ABCdefGHIjklMNO..." />
         <div class="hint">Create a bot via <a href="https://t.me/botfather" target="_blank">@BotFather</a>.</div>
       </div>
@@ -3653,7 +3696,30 @@ function App() {
         <label>Telegram Chat ID</label>
         <input type="text" value=${telegramChatId} oninput=${(e) => setTelegramChatId(e.target.value)}
           placeholder="-1001234567890" />
-        <div class="hint">Use <code style="font-family:var(--font-mono)">bosun --get-chat-id</code> to find your chat ID after the bot is running.</div>
+        <div style="display:flex;gap:10px;align-items:center;flex-wrap:wrap;margin-top:8px">
+          <button class="btn" type="button" onclick=${discoverTelegramChatIds}
+            disabled=${telegramChatLookupLoading || !String(telegramToken || "").trim()}>
+            ${telegramChatLookupLoading ? "Finding Chats..." : "Discover Chats"}
+          </button>
+          ${telegramDiscoveredChats.length > 1 && html`
+            <select
+              value=${telegramDiscoveredChats.some((chat) => String(chat.id) === String(telegramChatId)) ? String(telegramChatId) : ""}
+              onchange=${(e) => setTelegramChatId(e.target.value)}
+              style="min-width:260px;flex:1"
+            >
+              <option value="">Choose a discovered chat</option>
+              ${telegramDiscoveredChats.map((chat) => html`
+                <option value=${String(chat.id)}>
+                  ${`${chat.id}${chat.username ? ` · @${chat.username}` : ""}${chat.title ? ` · ${chat.title}` : ""}${chat.type ? ` · ${chat.type}` : ""}`}
+                </option>
+              `)}
+            </select>
+          `}
+        </div>
+        ${telegramChatLookupMessage && html`
+          <div class="hint" style="margin-top:8px">${telegramChatLookupMessage}</div>
+        `}
+        <div class="hint">Use Discover Chats after sending your bot a message, or run <code style="font-family:var(--font-mono)">bosun --get-chat-id</code>.</div>
       </div>
     `}
     <div class="nav-buttons">