@oneciel-ai/ciel-runtime 0.2.9 → 0.2.10

package/ciel_runtime_support/channel_mcp_tools.py

@@ -46,7 +46,25 @@ def channel_mcp_tool_schemas() -> list[dict[str, Any]]:
                 "type": "object",
                 "properties": {
                     "channel": {"type": "string", "description": "Destination channel id from the incoming message."},
-                    "message": {"type": "string", "description": "Message body to send."},
+                    "message": {"type": "string", "description": "Legacy unstructured message body."},
+                    "response": {
+                        "type": "object",
+                        "description": "Structured Web Chat response. Prefer this for browser acknowledgements and replies.",
+                        "properties": {
+                            "spoken": {
+                                "type": "string",
+                                "description": "Short conversational text intended for speech synthesis.",
+                            },
+                            "overview": {
+                                "type": "string",
+                                "description": "Concise on-screen summary.",
+                            },
+                            "details": {
+                                "type": "string",
+                                "description": "Optional detailed supporting Markdown.",
+                            },
+                        },
+                    },
                     "recipients": {
                         "description": "Recipient id, 'all', or an array of recipients. Use 'web' for /ca/web/chat replies."
                     },
@@ -59,7 +77,11 @@ def channel_mcp_tool_schemas() -> list[dict[str, Any]]:
                     },
                     "kind": {"type": "string", "description": "Optional message kind, for example 'reply' or 'status'."},
                 },
-                "required": ["channel", "message"],
+                "required": ["channel"],
+                "anyOf": [
+                    {"required": ["message"]},
+                    {"required": ["response"]},
+                ],
             },
         },
         {
@@ -169,13 +191,41 @@ def _send_message(
     services: ChannelMcpToolServices,
 ) -> dict[str, Any]:
     channel = str(args.get("channel") or "").strip()
+    structured = _structured_response(args.get("response"))
     message = str(args.get("message") or args.get("text") or "").strip()
+    if structured and not message:
+        message = _structured_message_text(structured)
     if not channel or not message:
-        return channel_mcp_tool_response(request_id, "send_message requires channel and message.", True)
-    saved = services.append_message(_message_payload(args, channel, message, "reply"))
+        return channel_mcp_tool_response(
+            request_id,
+            "send_message requires channel and either message or a non-empty response object.",
+            True,
+        )
+    payload = _message_payload(args, channel, message, "reply")
+    if structured:
+        payload["meta"]["web_response"] = structured
+    saved = services.append_message(payload)
     return _json_response(request_id, {"ok": True, "message": saved})
 
 
+def _structured_response(value: Any) -> dict[str, str]:
+    if not isinstance(value, dict):
+        return {}
+    response = {
+        key: str(value.get(key) or "").strip()
+        for key in ("spoken", "overview", "details")
+    }
+    return response if any(response.values()) else {}
+
+
+def _structured_message_text(response: dict[str, str]) -> str:
+    overview = response.get("overview", "")
+    details = response.get("details", "")
+    if overview and details:
+        return f"{overview}\n\n{details}"
+    return overview or details or response.get("spoken", "")
+
+
 def _send_file(
     request_id: Any,
     args: dict[str, Any],

package/ciel_runtime_support/channel_message_prompt.py

@@ -156,7 +156,8 @@ def _web_chat_reply_routes(messages: list[dict[str, Any]]) -> list[dict[str, str
         if route in seen:
             continue
         seen.add(route)
-        routes.append({"channel": channel, "thread_id": thread})
+        input_mode = str(meta.get("input_mode") or "text").strip().lower()
+        routes.append({"channel": channel, "thread_id": thread, "input_mode": input_mode})
     return routes
 
 
@@ -166,12 +167,21 @@ def _web_chat_reply_instruction(messages: list[dict[str, Any]]) -> str:
         return ""
     encoded_routes = json.dumps(routes, ensure_ascii=False, separators=(",", ":"))
     return (
-        "[ciel-runtime web reply required] Do not leave the answer only in the terminal. "
-        "After completing the answer, call MCP server `ciel-runtime-router` tool "
-        "`send_message` once for each route in "
-        f"{encoded_routes}, with that channel and thread_id, recipients=[\"web\"], "
-        "delivery=[\"web\"], kind=\"reply\", and message set to the complete answer. "
-        "Use `send_file` on the same route for files."
+        "[ciel-runtime web reply required] Do not leave responses only in the terminal. "
+        "For each route in "
+        f"{encoded_routes}: (1) immediately acknowledge the request with one short, honest sentence by calling "
+        "MCP server `ciel-runtime-router` tool `send_message` using that channel and thread_id, "
+        "recipients=[\"web\"], delivery=[\"web\"], kind=\"ack\", and "
+        "response={\"spoken\":\"brief conversational acknowledgement\","
+        "\"overview\":\"brief acknowledgement\",\"details\":\"\"}. "
+        "Do not claim completion in the acknowledgement. (2) Perform the requested work and then call "
+        "`send_message` again with kind=\"reply\" and a structured response object: "
+        "response={\"spoken\":\"one to three short conversational sentences suitable for TTS\","
+        "\"overview\":\"concise screen summary\","
+        "\"details\":\"supporting Markdown only when useful, otherwise empty\"}. "
+        "For input_mode=voice, spoken is required and must avoid Markdown, URLs, code, tables, and long lists; "
+        "the browser speaks only this field. Keep overview compact and put evidence, commands, links, and "
+        "technical detail in details. Use `send_file` on the same route for files."
     )
 
 

package/ciel_runtime_support/runtime_constants.py

@@ -93,7 +93,7 @@ OFFICIAL_CHANNEL_PLUGINS = {
 }
 
 APP_NAME = "Ciel Runtime"
-VERSION = "0.2.9"
+VERSION = "0.2.10"
 CREDITS = "Credits: One Ciel LLC"
 PRELAUNCH_CANCEL = 10
 PRELAUNCH_LAUNCH_CODEX = 11

package/ciel_runtime_support/web_ui.py

@@ -115,6 +115,13 @@ def render_web_chat_page(
     .recording {{ border-color: #ef4444 !important; color: #fecaca !important; }}
     .message-actions {{ display: flex; align-items: flex-start; padding: 4px; }}
     .message-actions button {{ border: 1px solid var(--line); border-radius: 999px; background: #0b111b; color: var(--muted); cursor: pointer; padding: 4px 8px; }}
+    .structured-response {{ display: grid; gap: 10px; }}
+    .response-section {{ display: grid; gap: 4px; }}
+    .response-section + .response-section {{ border-top: 1px solid rgba(255,255,255,.1); padding-top: 9px; }}
+    .response-label {{ color: #9fb1c8; font-size: 10px; font-weight: 700; letter-spacing: .08em; text-transform: uppercase; }}
+    .response-spoken {{ color: #f0fdfa; }}
+    .live-transcript {{ display: none; width: 100%; border: 1px solid #315b66; border-radius: 6px; background: #0d1d25; color: #bcecf3; padding: 7px 9px; font-size: 13px; }}
+    .live-transcript.active {{ display: block; }}
     dialog {{ width: min(720px, calc(100vw - 28px)); max-height: calc(100vh - 28px); overflow: auto; border: 1px solid var(--line); border-radius: 10px; background: var(--panel); color: var(--text); padding: 0; }}
     dialog::backdrop {{ background: rgba(0,0,0,.72); }}
     .settings-head {{ position: sticky; top: 0; display: flex; justify-content: space-between; align-items: center; gap: 12px; padding: 14px 16px; border-bottom: 1px solid var(--line); background: var(--panel); }}
@@ -197,6 +204,7 @@ def render_web_chat_page(
           <input id="fileInput" type="file" multiple>
           <div class="attachment-tray" id="attachmentTray" aria-live="polite"></div>
         </div>
+        <div class="live-transcript" id="liveTranscript" aria-live="polite"></div>
         <div class="hint">Enter sends. Shift+Enter inserts a new line. Live voice detects the end of each utterance, transcribes and sends it automatically, and supports interruption while TTS is speaking. The active coding-agent session handles the message, so its configured tools and MCP servers remain available.</div>
       </form>
     </main>
@@ -269,6 +277,7 @@ def render_web_chat_page(
     const micButton = document.getElementById('micButton');
     const fileInput = document.getElementById('fileInput');
     const attachmentTray = document.getElementById('attachmentTray');
+    const liveTranscript = document.getElementById('liveTranscript');
     const shareButton = document.getElementById('shareButton');
     const clearButton = document.getElementById('clearButton');
     const speechSettingsButton = document.getElementById('speechSettingsButton');
@@ -316,6 +325,9 @@ def render_web_chat_page(
     let vadVoicedSamples = 0;
     let vadNoiseFloor = 0.006;
     let liveTranscriptionQueue = Promise.resolve();
+    let livePartialInFlight = false;
+    let livePartialLastAt = 0;
+    let liveUtteranceSerial = 0;
     let activeSpeechAudio = null;
     let activeSpeechUrl = '';
     let speechGenerationController = null;
@@ -521,6 +533,58 @@ def render_web_chat_page(
       }}
       return bubble;
     }}
+    function structuredWebResponse(message) {{
+      const value = message && message.meta && message.meta.web_response;
+      if (!value || typeof value !== 'object') return null;
+      const response = {{
+        spoken: String(value.spoken || '').trim(),
+        overview: String(value.overview || '').trim(),
+        details: String(value.details || '').trim(),
+      }};
+      return response.spoken || response.overview || response.details ? response : null;
+    }}
+    function addStructuredBubble(response, mode = 'append', id = null) {{
+      if (id !== null && id !== undefined) {{
+        const key = String(id);
+        if (renderedIds.has(key)) return null;
+        renderedIds.add(key);
+      }}
+      const row = document.createElement('div');
+      row.className = 'row assistant';
+      const bubble = document.createElement('div');
+      bubble.className = 'bubble structured-response';
+      const sections = [
+        ['Voice', response.spoken, 'response-spoken'],
+        ['Overview', response.overview, 'markdown'],
+        ['Details', response.details, 'markdown'],
+      ];
+      sections.forEach(([labelText, value, className]) => {{
+        if (!value) return;
+        const section = document.createElement('section');
+        section.className = 'response-section ' + className;
+        const label = document.createElement('div');
+        label.className = 'response-label';
+        label.textContent = labelText;
+        const content = document.createElement('div');
+        if (className === 'markdown') content.innerHTML = renderMarkdown(value);
+        else content.textContent = value;
+        section.appendChild(label);
+        section.appendChild(content);
+        bubble.appendChild(section);
+      }});
+      row.appendChild(bubble);
+      const actions = document.createElement('div');
+      actions.className = 'message-actions';
+      const speak = document.createElement('button');
+      speak.type = 'button';
+      speak.textContent = 'Speak';
+      speak.addEventListener('click', () => speakText(response.spoken || response.overview));
+      actions.appendChild(speak);
+      row.appendChild(actions);
+      if (mode === 'prepend') transcript.insertBefore(row, transcript.firstChild);
+      else {{ transcript.appendChild(row); transcript.scrollTop = transcript.scrollHeight; }}
+      return bubble;
+    }}
     function rememberLastId(id) {{
       const numeric = Number(id || 0) || 0;
       if (numeric > lastId) {{
@@ -534,11 +598,14 @@ def render_web_chat_page(
     function renderIncomingMessage(message, mode = 'append') {{
       if (mode !== 'prepend') rememberLastId(message.id);
       const text = message.message || '';
-      if (!text.trim()) return;
-      addBubble(roleForMessage(message), text, mode, message.id);
+      const structured = structuredWebResponse(message);
+      if (!text.trim() && !structured) return;
+      if (structured && roleForMessage(message) === 'assistant') addStructuredBubble(structured, mode, message.id);
+      else addBubble(roleForMessage(message), text, mode, message.id);
       if (mode !== 'prepend' && message.sender_id !== 'web-user') {{
         setState('reply received', 'ok');
-        if (speechConfig.tts && speechConfig.tts.enabled && (speechConfig.tts.auto_speak || liveVoiceEnabled)) speakText(text);
+        const speechText = structured ? (structured.spoken || structured.overview) : text;
+        if (speechConfig.tts && speechConfig.tts.enabled && speechText && (speechConfig.tts.auto_speak || liveVoiceEnabled)) speakText(speechText);
       }}
     }}
     function formatBytes(bytes) {{
@@ -712,17 +779,21 @@ def render_web_chat_page(
         setState('generating speech');
         const response = await fetch('/v1/audio/speech', {{
           method: 'POST',
-          headers: {{'content-type': 'application/json'}},
+          headers: {{'content-type': 'application/json', 'accept': 'audio/*'}},
           body: JSON.stringify({{input: String(text || ''), model: speechConfig.tts.model, voice: speechConfig.tts.voice, language: speechConfig.tts.language, response_format: speechConfig.tts.response_format || 'wav'}}),
           signal: controller.signal,
         }});
         if (!response.ok) throw new Error(await response.text() || `HTTP ${{response.status}}`);
         const blob = await response.blob();
+        if (!blob.size) throw new Error('TTS returned empty audio');
+        if (blob.type && !blob.type.startsWith('audio/')) throw new Error(`TTS returned ${{blob.type}} instead of audio`);
         if (controller.signal.aborted) return;
         speechGenerationController = null;
         activeSpeechUrl = URL.createObjectURL(blob);
         activeSpeechAudio = new Audio(activeSpeechUrl);
         const audio = activeSpeechAudio;
+        audio.preload = 'auto';
+        audio.playsInline = true;
         const cleanup = () => {{
           if (activeSpeechAudio === audio) {{
             activeSpeechAudio = null;
@@ -742,13 +813,14 @@ def render_web_chat_page(
         addBubble('system', 'TTS failed: ' + String(err && err.message ? err.message : err));
       }}
     }}
-    async function transcribeRecording(blob, populatePrompt = true) {{
-      setState('transcribing');
+    async function transcribeRecording(blob, populatePrompt = true, options = {{}}) {{
+      if (!options.quiet) setState('transcribing');
       const audio_base64 = await fileToBase64(blob);
       const response = await fetch('/v1/audio/transcriptions', {{
         method: 'POST',
         headers: {{'content-type': 'application/json', 'accept': 'application/json'}},
-        body: JSON.stringify({{audio_base64, filename: 'web-chat-recording.wav', content_type: blob.type || 'audio/wav', model: speechConfig.asr.model, language: speechConfig.asr.language}})
+        body: JSON.stringify({{audio_base64, filename: 'web-chat-recording.wav', content_type: blob.type || 'audio/wav', model: speechConfig.asr.model, language: speechConfig.asr.language}}),
+        signal: options.signal,
       }});
       const text = await response.text();
       let data = {{}};
@@ -760,7 +832,7 @@ def render_web_chat_page(
         prompt.value = prompt.value ? prompt.value + ' ' + transcriptText : transcriptText;
         prompt.focus();
       }}
-      setState('transcribed', 'ok');
+      if (!options.quiet) setState('transcribed', 'ok');
       return transcriptText;
     }}
     function encodePcmWav(chunks, sampleRate) {{
@@ -798,13 +870,35 @@ def render_web_chat_page(
       vadLastVoiceAt = 0;
       vadVoicedSamples = 0;
     }}
-    function queueLiveUtterance(chunks, sampleRate) {{
+    function setLiveTranscript(text, active = true) {{
+      liveTranscript.textContent = text;
+      liveTranscript.classList.toggle('active', Boolean(active && text));
+    }}
+    function requestLivePartial(now) {{
+      if (!vadSpeechActive || livePartialInFlight || !audioContext) return;
+      if (now - livePartialLastAt < 1200 || now - vadSpeechStartedAt < 900) return;
+      const serial = liveUtteranceSerial;
+      const blob = encodePcmWav(vadSpeechChunks.slice(), audioContext.sampleRate);
+      livePartialInFlight = true;
+      livePartialLastAt = now;
+      transcribeRecording(blob, false, {{quiet: true}}).then(text => {{
+        if (liveVoiceEnabled && vadSpeechActive && serial === liveUtteranceSerial) {{
+          setLiveTranscript('Live: ' + text);
+        }}
+      }}).catch(() => {{
+        // Partial transcription is best-effort; final transcription reports actionable errors.
+      }}).finally(() => {{
+        livePartialInFlight = false;
+      }});
+    }}
+    function queueLiveUtterance(chunks, sampleRate, serial) {{
       if (!chunks.length) return;
       const blob = encodePcmWav(chunks, sampleRate);
       liveTranscriptionQueue = liveTranscriptionQueue.then(async () => {{
         const transcriptText = await transcribeRecording(blob, false);
+        if (serial === liveUtteranceSerial) setLiveTranscript('Heard: ' + transcriptText);
         setState('sending voice');
-        await sendMessage(transcriptText, []);
+        await sendMessage(transcriptText, [], {{inputMode: 'voice'}});
         if (liveVoiceEnabled) setState('listening', 'ok');
       }}).catch(err => {{
         setState('STT error', 'error');
@@ -814,9 +908,12 @@ def render_web_chat_page(
     function finishVadUtterance() {{
       const chunks = vadSpeechChunks.slice();
       const sampleRate = audioContext ? audioContext.sampleRate : 48000;
+      const serial = liveUtteranceSerial;
+      const partialText = liveTranscript.textContent.replace(/^Live:\\s*/, '');
+      setLiveTranscript(partialText ? 'Finalizing: ' + partialText : 'Finalizing speech...');
       resetVadUtterance();
       vadPreRollChunks = [];
-      queueLiveUtterance(chunks, sampleRate);
+      queueLiveUtterance(chunks, sampleRate, serial);
     }}
     function processVadFrame(event) {{
       if (!liveVoiceEnabled || !audioContext) return;
@@ -838,12 +935,15 @@ def render_web_chat_page(
           return;
         }}
         vadSpeechActive = true;
+        liveUtteranceSerial += 1;
+        livePartialLastAt = now;
         vadSpeechStartedAt = now;
         vadLastVoiceAt = now;
         vadVoicedSamples = chunk.length;
         vadSpeechChunks = vadPreRollChunks.concat([chunk]);
         vadPreRollChunks = [];
         stopActiveSpeech();
+        setLiveTranscript('Listening to speech...');
         setState('hearing speech', 'ok');
         return;
       }}
@@ -856,9 +956,10 @@ def render_web_chat_page(
       const minSpeechMs = Number((speechConfig.asr && speechConfig.asr.min_speech_ms) || 300);
       const voicedMs = vadVoicedSamples * 1000 / audioContext.sampleRate;
       const utteranceMs = now - vadSpeechStartedAt;
+      requestLivePartial(now);
       if (now - vadLastVoiceAt >= silenceMs) {{
         if (voicedMs >= minSpeechMs) finishVadUtterance();
-        else resetVadUtterance();
+        else {{ resetVadUtterance(); setLiveTranscript('Listening...', liveVoiceEnabled); }}
       }} else if (utteranceMs >= 30000) {{
         finishVadUtterance();
       }}
@@ -881,6 +982,7 @@ def render_web_chat_page(
       audioProcessor.connect(audioContext.destination);
       micButton.textContent = 'Stop live voice';
       micButton.classList.add('recording');
+      setLiveTranscript('Listening...');
       setState('listening', 'ok');
     }}
     async function stopVoiceInput() {{
@@ -901,6 +1003,7 @@ def render_web_chat_page(
       audioContext = null;
       mediaStream = null;
       stopActiveSpeech();
+      setLiveTranscript('', false);
       micButton.textContent = 'Start live voice';
       micButton.classList.remove('recording');
       setState('ready');
@@ -1033,7 +1136,7 @@ def render_web_chat_page(
         setTimeout(startChannelStream, 1200);
       }};
     }}
-    async function sendMessage(text, files = []) {{
+    async function sendMessage(text, files = [], options = {{}}) {{
       setState('queued');
       sendButton.disabled = true;
       attachButton.disabled = true;
@@ -1054,9 +1157,11 @@ def render_web_chat_page(
             meta: {{
               source: 'ciel-runtime-web-chat',
               web_chat_session: sessionId,
+              input_mode: options.inputMode || 'text',
               reply_channel: channel,
               reply_recipient: 'web',
-              reply_instruction: 'Use the ciel-runtime-router send_message tool to answer this browser chat on the same channel/thread_id with recipients web and delivery web. Use send_file when returning a file attachment to this browser chat.',
+              response_contract: {{version: 1, fields: ['spoken', 'overview', 'details'], tts_field: 'spoken'}},
+              reply_instruction: 'Acknowledge briefly first, then use the ciel-runtime-router send_message tool with response.spoken, response.overview, and optional response.details. The browser speaks only response.spoken. Use send_file when returning a file attachment.',
               attachments: uploads
             }}
           }})

package/docs/COLAB_SPEECH.md

@@ -26,7 +26,11 @@ Colab sessions are ephemeral. Re-run the bootstrap after a runtime reset. The wo
 
 ## Live voice
 
-Web Chat's **Start live voice** button keeps the microphone open and uses browser-side voice activity detection (VAD). A completed utterance is encoded as PCM WAV, transcribed, and sent to the active coding-agent session automatically. While MOSS TTS is generating or playing a reply, new speech stops it immediately and starts a new utterance (barge-in). Tune end-of-speech silence, minimum speech duration, and the VAD threshold in Speech Settings. This is turn-based low-latency voice for the Colab HTTP workers; token-level streaming ASR/TTS requires workers with streaming protocols.
+Web Chat's **Start live voice** button keeps the microphone open and uses browser-side voice activity detection (VAD). While the user is speaking, the browser sends rate-limited snapshots of the growing utterance to the Qwen worker and displays the latest best-effort partial transcript. A completed utterance is encoded as PCM WAV, transcribed once more for the final text, and sent to the active coding-agent session automatically. While MOSS TTS is generating or playing a reply, new speech stops it immediately and starts a new utterance (barge-in). Tune end-of-speech silence, minimum speech duration, and the VAD threshold in Speech Settings.
+
+The Colab Qwen endpoint remains a batch HTTP API, so the live caption is progressive re-transcription rather than token-level server streaming. A future WebSocket or streaming HTTP worker can replace this transport without changing the final-turn behavior.
+
+Web Chat requests carry an input mode and a structured response contract. The active agent first sends a short acknowledgement and then a final response containing `spoken`, `overview`, and optional `details` fields. The browser renders the fields separately and sends only `spoken` to TTS, avoiding long Markdown, URLs, code, and tables in synthesized speech. Legacy plain `message` replies remain supported.
 
 ## API surface
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@oneciel-ai/ciel-runtime",
-  "version": "0.2.9",
+  "version": "0.2.10",
   "description": "Universal AI coding-agent runtime and model-routing layer for Claude, Codex, AGY, and compatible runtimes.",
   "license": "MIT",
   "author": "One Ciel LLC",