bosun 0.36.0 → 0.36.2

package/ui/modules/streaming.js

@@ -609,8 +609,10 @@ export function startAgentStatusTracking() {
     if (!payload) return;
 
     const message = payload.message || payload;
-    const role = message.role;
-    const type = message.type;
+    const role = String(message.role || "").toLowerCase();
+    const type = String(message.type || "").toLowerCase();
+    const content = String(message.content || "").toLowerCase();
+    const lifecycle = String(message?.meta?.lifecycle || "").toLowerCase();
     const adapter = payload.session?.type || "";
     const sessionId = payload.sessionId || payload.taskId || payload.session?.id || "";
     const sessionStatus = payload.session?.status || "active";
@@ -622,12 +624,38 @@ export function startAgentStatusTracking() {
       return;
     }
 
+    const isCompletionEvent =
+      type === "turn.completed" ||
+      type === "session.completed" ||
+      lifecycle === "turn_completed" ||
+      lifecycle === "session_completed" ||
+      content.includes("turn completed") ||
+      content.includes("session completed") ||
+      content.includes("evt[turn.completed]") ||
+      content.includes("evt[session.completed]") ||
+      content.includes("session.idle");
+
+    if (isCompletionEvent) {
+      _setAgentState("idle", "", "");
+      return;
+    }
+
     if (role === "assistant" || type === "agent_message") {
       _setAgentState("streaming", adapter, sessionId);
     } else if (type === "tool_call") {
       _setAgentState("executing", adapter, sessionId);
     } else if (type === "tool_result") {
       _setAgentState("streaming", adapter, sessionId);
+    } else if (type === "system") {
+      if (
+        content.includes("running:") ||
+        content.includes("command done:") ||
+        content.includes("command_execution")
+      ) {
+        _setAgentState("executing", adapter, sessionId);
+      } else {
+        _setAgentState("thinking", adapter, sessionId);
+      }
     } else if (type === "error" || type === "stream_error") {
       _setAgentState("idle", "", "");
     }

package/ui/modules/vision-stream.js

@@ -0,0 +1,275 @@
+/**
+ * vision-stream.js — Live camera/screen frame streaming for voice calls.
+ *
+ * Captures compressed JPEG frames at a fixed interval and sends them to
+ * /api/vision/frame with the active chat session context.
+ */
+
+import { signal } from "@preact/signals";
+
+export const visionShareState = signal("off"); // off | starting | streaming | error
+export const visionShareSource = signal(null); // screen | camera | null
+export const visionShareError = signal(null);
+export const visionLastSummary = signal("");
+export const visionLastAnalyzedAt = signal(0);
+
+let _stream = null;
+let _video = null;
+let _canvas = null;
+let _captureTimer = null;
+let _sendInFlight = false;
+let _context = {
+  sessionId: null,
+  executor: null,
+  mode: null,
+  model: null,
+  source: null,
+  intervalMs: 1000,
+  maxWidth: 1280,
+  jpegQuality: 0.65,
+};
+
+function normalizeSource(source) {
+  const value = String(source || "").trim().toLowerCase();
+  if (value === "camera") return "camera";
+  return "screen";
+}
+
+function normalizeNumber(value, fallback, min, max) {
+  const n = Number(value);
+  if (!Number.isFinite(n)) return fallback;
+  return Math.min(max, Math.max(min, n));
+}
+
+function resetContext() {
+  _context = {
+    sessionId: null,
+    executor: null,
+    mode: null,
+    model: null,
+    source: null,
+    intervalMs: 1000,
+    maxWidth: 1280,
+    jpegQuality: 0.65,
+  };
+}
+
+function stopTracks(stream) {
+  if (!stream) return;
+  for (const track of stream.getTracks()) {
+    try {
+      track.stop();
+    } catch {
+      // no-op
+    }
+  }
+}
+
+async function waitForVideoReady(video) {
+  if (!video) return;
+  if (video.readyState >= 2 && video.videoWidth > 0 && video.videoHeight > 0) return;
+  await new Promise((resolve) => {
+    const onReady = () => {
+      cleanup();
+      resolve();
+    };
+    const cleanup = () => {
+      try {
+        video.removeEventListener("loadedmetadata", onReady);
+        video.removeEventListener("loadeddata", onReady);
+      } catch {
+        // no-op
+      }
+    };
+    video.addEventListener("loadedmetadata", onReady, { once: true });
+    video.addEventListener("loadeddata", onReady, { once: true });
+    setTimeout(() => {
+      cleanup();
+      resolve();
+    }, 1500);
+  });
+}
+
+async function captureAndSendFrame() {
+  if (_sendInFlight) return;
+  if (!_video || !_canvas) return;
+  if (!_context.sessionId) return;
+  const vw = Number(_video.videoWidth) || 0;
+  const vh = Number(_video.videoHeight) || 0;
+  if (vw <= 0 || vh <= 0) return;
+
+  const targetWidth = Math.min(vw, Number(_context.maxWidth) || 1280);
+  const targetHeight = Math.max(1, Math.round((vh * targetWidth) / vw));
+  _canvas.width = targetWidth;
+  _canvas.height = targetHeight;
+
+  const ctx = _canvas.getContext("2d", { alpha: false, desynchronized: true });
+  if (!ctx) return;
+  ctx.drawImage(_video, 0, 0, targetWidth, targetHeight);
+  const frameDataUrl = _canvas.toDataURL("image/jpeg", _context.jpegQuality);
+
+  _sendInFlight = true;
+  try {
+    const res = await fetch("/api/vision/frame", {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({
+        sessionId: _context.sessionId,
+        executor: _context.executor || undefined,
+        mode: _context.mode || undefined,
+        model: _context.model || undefined,
+        source: _context.source || "screen",
+        frameDataUrl,
+        width: targetWidth,
+        height: targetHeight,
+      }),
+    });
+    const data = await res.json().catch(() => ({}));
+    if (!res.ok) {
+      throw new Error(data?.error || `Vision upload failed (${res.status})`);
+    }
+    if (data?.analyzed && typeof data?.summary === "string" && data.summary.trim()) {
+      visionLastSummary.value = data.summary.trim();
+      visionLastAnalyzedAt.value = Date.now();
+    }
+  } catch (err) {
+    visionShareState.value = "error";
+    visionShareError.value = err?.message || "Vision stream failed";
+  } finally {
+    _sendInFlight = false;
+  }
+}
+
+function cleanupDomNodes() {
+  if (_video) {
+    try {
+      _video.pause();
+      _video.srcObject = null;
+    } catch {
+      // no-op
+    }
+    _video = null;
+  }
+  _canvas = null;
+}
+
+export async function startVisionShare(options = {}) {
+  const sessionId = String(options?.sessionId || "").trim();
+  if (!sessionId) {
+    throw new Error("sessionId required to start vision share");
+  }
+
+  const source = normalizeSource(options?.source);
+  const intervalMs = normalizeNumber(options?.intervalMs, 1000, 300, 10_000);
+  const maxWidth = normalizeNumber(options?.maxWidth, 1280, 320, 1920);
+  const jpegQuality = normalizeNumber(options?.jpegQuality, 0.65, 0.35, 0.92);
+
+  await stopVisionShare();
+  visionShareState.value = "starting";
+  visionShareError.value = null;
+  visionShareSource.value = source;
+
+  try {
+    if (!navigator?.mediaDevices) {
+      throw new Error("Media devices API unavailable");
+    }
+
+    if (source === "screen") {
+      if (typeof navigator.mediaDevices.getDisplayMedia !== "function") {
+        throw new Error("Screen sharing is not supported in this browser");
+      }
+      _stream = await navigator.mediaDevices.getDisplayMedia({
+        video: {
+          frameRate: { ideal: 6, max: 12 },
+        },
+        audio: false,
+      });
+    } else {
+      _stream = await navigator.mediaDevices.getUserMedia({
+        video: {
+          width: { ideal: 1280 },
+          height: { ideal: 720 },
+          frameRate: { ideal: 8, max: 12 },
+        },
+        audio: false,
+      });
+    }
+
+    const videoTrack = _stream.getVideoTracks()[0];
+    if (!videoTrack) {
+      throw new Error("No video track available");
+    }
+    videoTrack.addEventListener("ended", () => {
+      stopVisionShare().catch(() => {});
+    });
+
+    _video = document.createElement("video");
+    _video.autoplay = true;
+    _video.muted = true;
+    _video.playsInline = true;
+    _video.srcObject = _stream;
+    _canvas = document.createElement("canvas");
+
+    await waitForVideoReady(_video);
+    try {
+      await _video.play();
+    } catch {
+      // Some browsers auto-play once frames are requested.
+    }
+
+    _context = {
+      sessionId,
+      executor: String(options?.executor || "").trim() || null,
+      mode: String(options?.mode || "").trim() || null,
+      model: String(options?.model || "").trim() || null,
+      source,
+      intervalMs,
+      maxWidth,
+      jpegQuality,
+    };
+
+    visionShareState.value = "streaming";
+    _captureTimer = setInterval(() => {
+      captureAndSendFrame().catch((err) => {
+        visionShareState.value = "error";
+        visionShareError.value = err?.message || "Vision capture failed";
+      });
+    }, intervalMs);
+    await captureAndSendFrame();
+  } catch (err) {
+    stopTracks(_stream);
+    _stream = null;
+    cleanupDomNodes();
+    resetContext();
+    visionShareState.value = "error";
+    visionShareError.value = err?.message || "Could not start vision share";
+    throw err;
+  }
+}
+
+export async function stopVisionShare() {
+  if (_captureTimer) {
+    clearInterval(_captureTimer);
+    _captureTimer = null;
+  }
+  stopTracks(_stream);
+  _stream = null;
+  cleanupDomNodes();
+  resetContext();
+  _sendInFlight = false;
+  visionShareState.value = "off";
+  visionShareSource.value = null;
+  visionShareError.value = null;
+  visionLastSummary.value = "";
+  visionLastAnalyzedAt.value = 0;
+}
+
+export async function toggleVisionShare(source, options = {}) {
+  const nextSource = normalizeSource(source);
+  if (visionShareState.value === "streaming" && visionShareSource.value === nextSource) {
+    await stopVisionShare();
+    return false;
+  }
+  await startVisionShare({ ...options, source: nextSource });
+  return true;
+}

package/ui/modules/voice-client.js

@@ -17,6 +17,7 @@ export const voiceResponse = signal(""); // current assistant response text
 export const voiceError = signal(null);
 export const voiceToolCalls = signal([]); // active tool calls
 export const voiceSessionId = signal(null);
+export const voiceBoundSessionId = signal(null);
 export const voiceDuration = signal(0); // seconds connected
 
 export const isVoiceActive = computed(() =>
@@ -33,11 +34,48 @@ let _reconnectTimer = null;    // 28-min reconnect timer
 let _durationTimer = null;     // Duration counter
 let _sessionStartTime = 0;
 let _eventHandlers = new Map();
+let _callContext = {
+  sessionId: null,
+  executor: null,
+  mode: null,
+  model: null,
+};
 
 const RECONNECT_AT_MS = 28 * 60 * 1000; // 28 minutes
 const MAX_RECONNECT_ATTEMPTS = 3;
 let _reconnectAttempts = 0;
 
+function _normalizeCallContext(options = {}) {
+  const sessionId = String(options?.sessionId || "").trim() || null;
+  const executor = String(options?.executor || "").trim() || null;
+  const mode = String(options?.mode || "").trim() || null;
+  const model = String(options?.model || "").trim() || null;
+  return { sessionId, executor, mode, model };
+}
+
+async function _recordVoiceTranscript(role, content, eventType = "") {
+  const sessionId = String(_callContext?.sessionId || voiceSessionId.value || "").trim();
+  const text = String(content || "").trim();
+  if (!sessionId || !text) return;
+  try {
+    await fetch("/api/voice/transcript", {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({
+        sessionId,
+        role,
+        content: text,
+        eventType,
+        executor: _callContext?.executor || undefined,
+        mode: _callContext?.mode || undefined,
+        model: _callContext?.model || undefined,
+      }),
+    });
+  } catch (err) {
+    console.warn("[voice-client] transcript persistence failed:", err?.message || err);
+  }
+}
+
 // ── Event System ────────────────────────────────────────────────────────────
 
 export function onVoiceEvent(event, handler) {
@@ -67,12 +105,14 @@ function emit(event, data) {
  * 4. Set up data channel for events
  * 5. Create offer, set remote answer
  */
-export async function startVoiceSession() {
+export async function startVoiceSession(options = {}) {
   if (_pc) {
     console.warn("[voice-client] Session already active");
     return;
   }
 
+  _callContext = _normalizeCallContext(options);
+  voiceBoundSessionId.value = _callContext.sessionId;
   voiceState.value = "connecting";
   voiceError.value = null;
   voiceTranscript.value = "";
@@ -82,7 +122,17 @@ export async function startVoiceSession() {
 
   try {
     // 1. Fetch ephemeral token
-    const tokenRes = await fetch("/api/voice/token", { method: "POST" });
+    const tokenRes = await fetch("/api/voice/token", {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({
+        sessionId: _callContext.sessionId || undefined,
+        executor: _callContext.executor || undefined,
+        mode: _callContext.mode || undefined,
+        model: _callContext.model || undefined,
+        delegateOnly: Boolean(_callContext.sessionId),
+      }),
+    });
     if (!tokenRes.ok) {
       const err = await tokenRes.json().catch(() => ({ error: "Token fetch failed" }));
       throw new Error(err.error || `Token fetch failed (${tokenRes.status})`);
@@ -90,6 +140,19 @@ export async function startVoiceSession() {
     const tokenData = await tokenRes.json();
 
     // 2. Get microphone
+    const mediaDevices = navigator?.mediaDevices;
+    if (!mediaDevices?.getUserMedia) {
+      const host = String(globalThis.location?.hostname || "").toLowerCase();
+      const localhostLike =
+        host === "localhost" || host === "127.0.0.1" || host === "::1";
+      if (!globalThis.isSecureContext && !localhostLike) {
+        throw new Error(
+          "Microphone access requires HTTPS (or localhost). Open the UI via the Cloudflare HTTPS URL or localhost.",
+        );
+      }
+      throw new Error("Microphone API unavailable in this browser/runtime.");
+    }
+
     _mediaStream = await navigator.mediaDevices.getUserMedia({
       audio: {
         echoCancellation: true,
@@ -119,10 +182,14 @@ export async function startVoiceSession() {
     _dc.onopen = () => {
       voiceState.value = "connected";
       _sessionStartTime = Date.now();
-      voiceSessionId.value = `voice-${Date.now()}`;
+      voiceSessionId.value = _callContext.sessionId || `voice-${Date.now()}`;
       startDurationTimer();
       startReconnectTimer();
-      emit("connected", { provider: tokenData.provider });
+      emit("connected", {
+        provider: tokenData.provider,
+        sessionId: voiceSessionId.value,
+        callContext: { ..._callContext },
+      });
     };
     _dc.onclose = () => {
       if (voiceState.value !== "reconnecting") {
@@ -162,7 +229,10 @@ export async function startVoiceSession() {
     const answerSdp = await sdpResponse.text();
     await _pc.setRemoteDescription({ type: "answer", sdp: answerSdp });
 
-    emit("session-started", { sessionId: voiceSessionId.value });
+    emit("session-started", {
+      sessionId: voiceSessionId.value,
+      callContext: { ..._callContext },
+    });
   } catch (err) {
     console.error("[voice-client] Failed to start voice session:", err);
     voiceState.value = "error";
@@ -183,7 +253,9 @@ export function stopVoiceSession() {
   voiceResponse.value = "";
   voiceToolCalls.value = [];
   voiceSessionId.value = null;
+  voiceBoundSessionId.value = null;
   voiceDuration.value = 0;
+  _callContext = { sessionId: null, executor: null, mode: null, model: null };
   emit("session-ended", {});
 }
 
@@ -211,6 +283,11 @@ function handleServerEvent(event) {
     case "conversation.item.input_audio_transcription.completed":
       voiceTranscript.value = event.transcript || "";
       emit("transcript", { text: event.transcript, final: true });
+      _recordVoiceTranscript(
+        "user",
+        event.transcript || "",
+        "conversation.item.input_audio_transcription.completed",
+      );
       break;
 
     case "response.audio_transcript.delta":
@@ -220,6 +297,11 @@ function handleServerEvent(event) {
 
     case "response.audio_transcript.done":
       emit("response-complete", { text: voiceResponse.value });
+      _recordVoiceTranscript(
+        "assistant",
+        voiceResponse.value,
+        "response.audio_transcript.done",
+      );
       voiceResponse.value = "";
       break;
 
@@ -238,7 +320,9 @@ function handleServerEvent(event) {
       break;
 
     case "response.function_call_arguments.done":
-      handleToolCall(event);
+      handleToolCall(event).catch((err) => {
+        console.error("[voice-client] Tool call handling failed:", err);
+      });
       break;
 
     case "response.done":
@@ -284,7 +368,14 @@ async function handleToolCall(event) {
     const res = await fetch("/api/voice/tool", {
       method: "POST",
       headers: { "Content-Type": "application/json" },
-      body: JSON.stringify({ toolName: name, args, sessionId: voiceSessionId.value }),
+      body: JSON.stringify({
+        toolName: name,
+        args,
+        sessionId: voiceSessionId.value,
+        executor: _callContext.executor || undefined,
+        mode: _callContext.mode || undefined,
+        model: _callContext.model || undefined,
+      }),
     });
     const result = await res.json();
 
@@ -364,7 +455,9 @@ export function sendTextMessage(text) {
 function startReconnectTimer() {
   clearTimeout(_reconnectTimer);
   _reconnectTimer = setTimeout(() => {
-    reconnect();
+    reconnect().catch((err) => {
+      console.error("[voice-client] Reconnect timer error:", err);
+    });
   }, RECONNECT_AT_MS);
 }
 
@@ -385,7 +478,7 @@ async function reconnect() {
   _mediaStream = stream;
 
   try {
-    await startVoiceSession();
+    await startVoiceSession(_callContext);
   } catch (err) {
     console.error("[voice-client] Reconnect failed:", err);
     if (_reconnectAttempts < MAX_RECONNECT_ATTEMPTS) {

package/ui/modules/voice-fallback.js

@@ -1,7 +1,7 @@
 /**
  * voice-fallback.js — Tier 2 voice using browser Web Speech API + bosun chat.
  *
- * Flow: SpeechRecognition → text → POST /api/sessions/{id}/message → response → SpeechSynthesis
+ * Flow: SpeechRecognition → text → POST /api/voice/tool(delegate_to_agent) → SpeechSynthesis
  *
  * @module voice-fallback
  */
@@ -17,6 +17,11 @@ let _recognition = null;
 let _synthesis = null;
 let _sessionId = null;
 let _isSpeaking = false;
+let _callContext = {
+  executor: null,
+  mode: null,
+  model: null,
+};
 
 const SpeechRecognition = typeof globalThis !== "undefined"
   ? (globalThis.SpeechRecognition || globalThis.webkitSpeechRecognition)
@@ -24,18 +29,48 @@ const SpeechRecognition = typeof globalThis !== "undefined"
 
 export const fallbackSupported = Boolean(SpeechRecognition) && typeof globalThis.speechSynthesis !== "undefined";
 
+async function recordFallbackTranscript(role, content, eventType = "") {
+  const sessionId = String(_sessionId || "").trim();
+  const text = String(content || "").trim();
+  if (!sessionId || !text) return;
+  try {
+    await fetch("/api/voice/transcript", {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({
+        sessionId,
+        role,
+        content: text,
+        eventType,
+        executor: _callContext.executor || undefined,
+        mode: _callContext.mode || undefined,
+        model: _callContext.model || undefined,
+        provider: "fallback",
+      }),
+    });
+  } catch {
+    // best effort persistence
+  }
+}
+
 /**
  * Start a fallback voice session.
  * @param {string} sessionId — existing chat session ID to use
+ * @param {{ executor?: string, mode?: string, model?: string }} [options]
  */
-export function startFallbackSession(sessionId) {
+export function startFallbackSession(sessionId, options = {}) {
   if (!fallbackSupported) {
     fallbackError.value = "Speech APIs not supported in this browser";
     fallbackState.value = "error";
     return;
   }
 
-  _sessionId = sessionId;
+  _sessionId = sessionId || null;
+  _callContext = {
+    executor: String(options?.executor || "").trim() || null,
+    mode: String(options?.mode || "").trim() || null,
+    model: String(options?.model || "").trim() || null,
+  };
   _synthesis = globalThis.speechSynthesis;
   fallbackState.value = "idle";
   fallbackError.value = null;
@@ -52,6 +87,7 @@ export function stopFallbackSession() {
   fallbackTranscript.value = "";
   fallbackResponse.value = "";
   _sessionId = null;
+  _callContext = { executor: null, mode: null, model: null };
 }
 
 function startListening() {
@@ -119,12 +155,26 @@ function stopListening() {
 async function processUserInput(text) {
   fallbackState.value = "processing";
   stopListening();
+  await recordFallbackTranscript("user", text, "fallback.user_input");
 
   try {
-    const res = await fetch(`/api/sessions/${_sessionId}/message`, {
+    // Use the same server-side voice tool pipeline as Tier 1 for consistency.
+    const res = await fetch("/api/voice/tool", {
       method: "POST",
       headers: { "Content-Type": "application/json" },
-      body: JSON.stringify({ message: text }),
+      body: JSON.stringify({
+        toolName: "delegate_to_agent",
+        args: {
+          message: text,
+          mode: _callContext.mode || "ask",
+          executor: _callContext.executor || undefined,
+          model: _callContext.model || undefined,
+        },
+        sessionId: _sessionId || undefined,
+        executor: _callContext.executor || undefined,
+        mode: _callContext.mode || undefined,
+        model: _callContext.model || undefined,
+      }),
     });
 
     if (!res.ok) {
@@ -132,9 +182,15 @@ async function processUserInput(text) {
     }
 
     const data = await res.json();
-    const responseText = data.response || data.text || data.message || data.content || JSON.stringify(data);
+    const responseText =
+      data?.result ||
+      data?.text ||
+      data?.message ||
+      data?.content ||
+      (data?.error ? `Error: ${data.error}` : JSON.stringify(data));
 
     fallbackResponse.value = responseText;
+    await recordFallbackTranscript("assistant", responseText, "fallback.assistant_output");
     await speak(responseText);
   } catch (err) {
     fallbackError.value = `Processing error: ${err.message}`;