@oneciel-ai/ciel-runtime 0.2.13 → 0.2.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ciel_runtime_support/config_repository.py +1 -0
- package/ciel_runtime_support/runtime_constants.py +1 -1
- package/ciel_runtime_support/speech_http_controller.py +5 -0
- package/ciel_runtime_support/web_ui.py +217 -68
- package/docs/COLAB_SPEECH.md +3 -1
- package/package.json +1 -1
- package/scripts/colab/bootstrap_qwen_asr.py +48 -23
- package/scripts/configure_speech_workers.py +10 -1
- package/scripts/deploy_colab_speech.ps1 +6 -1
|
@@ -35,6 +35,7 @@ def build_default_config(provider_defaults: dict[str, Any]) -> dict[str, Any]:
|
|
|
35
35
|
"profile": "default",
|
|
36
36
|
"asr_session": "ciel-asr",
|
|
37
37
|
"tts_session": "ciel-tts",
|
|
38
|
+
"asr_model": "Qwen/Qwen3-ASR-0.6B",
|
|
38
39
|
"asr_accelerator": "T4",
|
|
39
40
|
"tts_accelerator": "T4",
|
|
40
41
|
"tts_backend": "moss",
|
|
@@ -256,6 +256,7 @@ class SpeechHttpController:
|
|
|
256
256
|
"profile",
|
|
257
257
|
"asr_session",
|
|
258
258
|
"tts_session",
|
|
259
|
+
"asr_model",
|
|
259
260
|
"asr_accelerator",
|
|
260
261
|
"tts_accelerator",
|
|
261
262
|
"tts_backend",
|
|
@@ -280,6 +281,10 @@ class SpeechHttpController:
|
|
|
280
281
|
if backend not in {"moss", "cosyvoice3"}:
|
|
281
282
|
raise ValueError("unsupported Colab TTS backend")
|
|
282
283
|
return backend
|
|
284
|
+
if key == "asr_model":
|
|
285
|
+
if text not in {"Qwen/Qwen3-ASR-0.6B", "Qwen/Qwen3-ASR-1.7B"}:
|
|
286
|
+
raise ValueError("unsupported Colab ASR model")
|
|
287
|
+
return text
|
|
283
288
|
if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]{0,127}", text):
|
|
284
289
|
raise ValueError(f"invalid Colab {key}")
|
|
285
290
|
return text
|
|
@@ -45,9 +45,10 @@ def render_web_chat_page(
|
|
|
45
45
|
--ok: #86efac;
|
|
46
46
|
}}
|
|
47
47
|
* {{ box-sizing: border-box; }}
|
|
48
|
-
body {{
|
|
49
|
-
|
|
50
|
-
|
|
48
|
+
html, body {{ width: 100%; height: 100%; overflow: hidden; }}
|
|
49
|
+
body {{ margin: 0; background: var(--bg); color: var(--text); }}
|
|
50
|
+
.shell {{ display: grid; grid-template-columns: 280px minmax(0, 1fr); width: 100%; height: 100dvh; min-height: 0; overflow: hidden; }}
|
|
51
|
+
aside {{ height: 100%; overflow-y: auto; border-right: 1px solid var(--line); background: #0e1521; padding: 18px; }}
|
|
51
52
|
.brand {{ font-size: 19px; font-weight: 700; letter-spacing: 0; margin: 0 0 12px; }}
|
|
52
53
|
.status-card {{ border: 1px solid var(--line); border-radius: 8px; background: var(--panel); padding: 12px; display: grid; gap: 10px; }}
|
|
53
54
|
.meta-label {{ color: var(--muted); font-size: 11px; text-transform: uppercase; }}
|
|
@@ -59,12 +60,12 @@ def render_web_chat_page(
|
|
|
59
60
|
background: #0b111b; color: var(--text); text-decoration: none; cursor: pointer;
|
|
60
61
|
}}
|
|
61
62
|
.nav a:hover, .ghost:hover {{ border-color: var(--accent); }}
|
|
62
|
-
main {{ display: grid; grid-template-rows: auto minmax(0, 1fr) auto; min-width: 0; }}
|
|
63
|
+
main {{ display: grid; grid-template-rows: auto minmax(0, 1fr) auto; width: 100%; height: 100%; min-width: 0; min-height: 0; overflow: hidden; }}
|
|
63
64
|
header {{ min-height: 66px; display: flex; align-items: center; justify-content: space-between; gap: 12px; padding: 14px 18px; border-bottom: 1px solid var(--line); background: #0d1420; }}
|
|
64
65
|
h1 {{ margin: 0; font-size: 18px; letter-spacing: 0; }}
|
|
65
66
|
.sub {{ color: var(--muted); font-size: 12px; margin-top: 4px; }}
|
|
66
67
|
.pill {{ border: 1px solid var(--line); border-radius: 999px; padding: 5px 9px; color: var(--muted); font-size: 12px; white-space: nowrap; }}
|
|
67
|
-
#transcript {{ overflow-y: auto; padding: 18px; display: flex; flex-direction: column; gap: 12px; }}
|
|
68
|
+
#transcript {{ min-height: 0; overflow-y: auto; overflow-x: hidden; overscroll-behavior: contain; scrollbar-gutter: stable; padding: 18px; display: flex; flex-direction: column; gap: 12px; }}
|
|
68
69
|
.row {{ display: flex; width: 100%; }}
|
|
69
70
|
.row.user {{ justify-content: flex-end; }}
|
|
70
71
|
.bubble {{
|
|
@@ -98,10 +99,10 @@ def render_web_chat_page(
|
|
|
98
99
|
.markdown th, .markdown td {{ border: 1px solid #3a4b63; padding: 6px 8px; text-align: left; vertical-align: top; }}
|
|
99
100
|
.markdown th {{ background: rgba(255,255,255,.06); font-weight: 700; }}
|
|
100
101
|
.markdown hr {{ border: 0; border-top: 1px solid var(--line); margin: 12px 0; }}
|
|
101
|
-
.composer {{ border-top: 1px solid var(--line); padding: 12px 18px; background: #0d1420; }}
|
|
102
|
+
.composer {{ max-height: 46dvh; overflow-y: auto; border-top: 1px solid var(--line); padding: 12px 18px; background: #0d1420; }}
|
|
102
103
|
.composer-inner {{ display: grid; grid-template-columns: minmax(0, 1fr) auto; gap: 10px; align-items: end; }}
|
|
103
104
|
textarea {{
|
|
104
|
-
width: 100%; min-height: 54px; max-height:
|
|
105
|
+
width: 100%; height: 54px; min-height: 54px; max-height: 54px; resize: none;
|
|
105
106
|
border: 1px solid var(--line); border-radius: 8px; background: #080d14; color: var(--text);
|
|
106
107
|
padding: 10px 12px; line-height: 1.4; font: inherit;
|
|
107
108
|
}}
|
|
@@ -111,6 +112,9 @@ def render_web_chat_page(
|
|
|
111
112
|
}}
|
|
112
113
|
button.primary:disabled {{ opacity: .55; cursor: not-allowed; }}
|
|
113
114
|
.composer-actions {{ display: flex; gap: 8px; align-items: center; margin-top: 8px; flex-wrap: wrap; }}
|
|
115
|
+
.voice-option {{ display: inline-flex; align-items: center; gap: 6px; color: var(--muted); font-size: 11px; }}
|
|
116
|
+
.voice-option select, .voice-option input {{ height: 32px; border: 1px solid var(--line); border-radius: 6px; background: #080d14; color: var(--text); padding: 0 7px; }}
|
|
117
|
+
.voice-option input {{ width: 58px; }}
|
|
114
118
|
.attach-button {{
|
|
115
119
|
min-height: 34px; border: 1px solid var(--line); border-radius: 6px;
|
|
116
120
|
background: #0b111b; color: var(--text); padding: 0 12px; cursor: pointer;
|
|
@@ -125,7 +129,8 @@ def render_web_chat_page(
|
|
|
125
129
|
.response-section + .response-section {{ border-top: 1px solid rgba(255,255,255,.1); padding-top: 9px; }}
|
|
126
130
|
.response-label {{ color: #9fb1c8; font-size: 10px; font-weight: 700; letter-spacing: .08em; text-transform: uppercase; }}
|
|
127
131
|
.response-spoken {{ color: #f0fdfa; }}
|
|
128
|
-
.live-transcript {{ display: none; width:
|
|
132
|
+
.live-transcript {{ display: none; width: min(760px, 86%); margin: 0 0 9px auto; border: 1px solid #276a8d; border-radius: 8px; background: rgba(23,76,107,.72); color: #d9f5ff; padding: 9px 11px; font-size: 13px; box-shadow: 0 1px 0 rgba(255,255,255,.03) inset; }}
|
|
133
|
+
.live-transcript::before {{ content: 'VOICE PREVIEW'; display: block; margin-bottom: 3px; color: #9fd8ef; font-size: 10px; font-weight: 700; letter-spacing: .08em; }}
|
|
129
134
|
.live-transcript.active {{ display: block; }}
|
|
130
135
|
dialog {{ width: min(720px, calc(100vw - 28px)); max-height: calc(100vh - 28px); overflow: auto; border: 1px solid var(--line); border-radius: 10px; background: var(--panel); color: var(--text); padding: 0; }}
|
|
131
136
|
dialog::backdrop {{ background: rgba(0,0,0,.72); }}
|
|
@@ -201,6 +206,7 @@ def render_web_chat_page(
|
|
|
201
206
|
</header>
|
|
202
207
|
<section id="transcript" aria-live="polite"></section>
|
|
203
208
|
<form class="composer" id="composer">
|
|
209
|
+
<div class="live-transcript" id="liveTranscript" aria-live="polite"></div>
|
|
204
210
|
<div class="composer-inner">
|
|
205
211
|
<textarea id="prompt" placeholder="Type a message..." autocomplete="off"></textarea>
|
|
206
212
|
<button class="primary" id="sendButton" type="submit">Send</button>
|
|
@@ -208,10 +214,11 @@ def render_web_chat_page(
|
|
|
208
214
|
<div class="composer-actions">
|
|
209
215
|
<button class="attach-button" id="micButton" type="button">Start live voice</button>
|
|
210
216
|
<button class="attach-button" id="attachButton" type="button">Attach files</button>
|
|
217
|
+
<label class="voice-option">Mic sensitivity<select id="voiceSensitivity"><option value="low">Low (typing resistant)</option><option value="normal">Normal</option><option value="high">High</option></select></label>
|
|
218
|
+
<label class="voice-option">Min chars<input id="minimumTranscriptChars" type="number" min="1" max="20" value="3"></label>
|
|
211
219
|
<input id="fileInput" type="file" multiple>
|
|
212
220
|
<div class="attachment-tray" id="attachmentTray" aria-live="polite"></div>
|
|
213
221
|
</div>
|
|
214
|
-
<div class="live-transcript" id="liveTranscript" aria-live="polite"></div>
|
|
215
222
|
<div class="hint">Enter sends. Shift+Enter inserts a new line. Live voice detects the end of each utterance, transcribes and sends it automatically, and supports interruption while TTS is speaking. The active coding-agent session handles the message, so its configured tools and MCP servers remain available.</div>
|
|
216
223
|
</form>
|
|
217
224
|
</main>
|
|
@@ -229,12 +236,12 @@ def render_web_chat_page(
|
|
|
229
236
|
<label>Minimum speech (ms)<input id="asrMinSpeechMs" type="number" min="100" max="2000" step="50"></label>
|
|
230
237
|
<label>VAD threshold<input id="asrVadThreshold" type="number" min="0.005" max="0.2" step="0.001"></label>
|
|
231
238
|
<label class="wide">Tailscale base URL<input id="asrBaseUrl" placeholder="http://ciel-asr:8000"></label>
|
|
232
|
-
<label class="wide">
|
|
239
|
+
<label class="wide">ASR model<select id="asrModel"><option value="Qwen/Qwen3-ASR-0.6B">Qwen3-ASR 0.6B (faster)</option><option value="Qwen/Qwen3-ASR-1.7B">Qwen3-ASR 1.7B (higher accuracy)</option></select></label>
|
|
233
240
|
<label class="wide">Remote bearer token<input id="asrApiKey" type="password" autocomplete="new-password" placeholder="Leave blank to keep current token"></label>
|
|
234
241
|
</div>
|
|
235
242
|
</section>
|
|
236
243
|
<section class="settings-section">
|
|
237
|
-
<h3>TTS
|
|
244
|
+
<h3>TTS model and playback</h3>
|
|
238
245
|
<div class="settings-grid">
|
|
239
246
|
<label class="check"><input id="ttsEnabled" type="checkbox"> Enable TTS</label>
|
|
240
247
|
<label class="check"><input id="ttsAutoSpeak" type="checkbox"> Speak replies automatically</label>
|
|
@@ -243,12 +250,13 @@ def render_web_chat_page(
|
|
|
243
250
|
<label>Voice<input id="ttsVoice" placeholder="default"></label>
|
|
244
251
|
<label>Language<input id="ttsLanguage" placeholder="ko"></label>
|
|
245
252
|
<label>PCM sample rate<input id="ttsSampleRate" type="number" min="8000" max="192000" step="1000"></label>
|
|
246
|
-
<label class="wide">
|
|
247
|
-
<label class="wide">Reference voice (required
|
|
253
|
+
<label class="wide">TTS model<select id="ttsModel"><option value="OpenMOSS-Team/MOSS-TTS-Nano">MOSS-TTS-Nano</option><option value="FunAudioLLM/Fun-CosyVoice3-0.5B-2512">Fun-CosyVoice 3</option></select></label>
|
|
254
|
+
<label class="wide">Reference voice (required for voice cloning)<input id="ttsReferenceAudio" type="file" accept="audio/*"><span class="hint" id="ttsReferenceAudioStatus">No reference voice configured</span></label>
|
|
248
255
|
<label class="wide">Reference transcript (required by CosyVoice 3)<input id="ttsReferenceText" placeholder="Exact transcript of the reference clip"></label>
|
|
249
256
|
<label class="check wide"><input id="ttsClearReferenceAudio" type="checkbox"> Remove the saved reference voice</label>
|
|
250
257
|
<label class="wide">Remote bearer token<input id="ttsApiKey" type="password" autocomplete="new-password" placeholder="Leave blank to keep current token"></label>
|
|
251
258
|
</div>
|
|
259
|
+
<div class="hint">Choose the target model, then run <strong>Recover & deploy</strong> before using it. Deployment updates the Colab worker and active routing together.</div>
|
|
252
260
|
</section>
|
|
253
261
|
<section class="settings-section">
|
|
254
262
|
<h3>Colab CLI connection</h3>
|
|
@@ -257,7 +265,6 @@ def render_web_chat_page(
|
|
|
257
265
|
<label>WSL distribution<input id="colabDistribution" placeholder="Ubuntu-26.04"></label>
|
|
258
266
|
<label>Authentication<select id="colabAuth"><option value="adc">ADC</option><option value="oauth2">OAuth2</option></select></label>
|
|
259
267
|
<label>Account profile<input id="colabProfile" placeholder="default"></label>
|
|
260
|
-
<label>TTS engine<select id="colabTtsBackend"><option value="moss">MOSS-TTS-Nano</option><option value="cosyvoice3">Fun-CosyVoice 3</option></select></label>
|
|
261
268
|
<label>ASR session<input id="colabAsrSession" placeholder="ciel-asr"></label>
|
|
262
269
|
<label>TTS session<input id="colabTtsSession" placeholder="ciel-tts"></label>
|
|
263
270
|
<label>ASR GPU<select id="colabAsrAccelerator"><option>T4</option><option>L4</option><option>G4</option><option>A100</option><option>H100</option></select></label>
|
|
@@ -308,6 +315,8 @@ def render_web_chat_page(
|
|
|
308
315
|
const fileInput = document.getElementById('fileInput');
|
|
309
316
|
const attachmentTray = document.getElementById('attachmentTray');
|
|
310
317
|
const liveTranscript = document.getElementById('liveTranscript');
|
|
318
|
+
const voiceSensitivity = document.getElementById('voiceSensitivity');
|
|
319
|
+
const minimumTranscriptChars = document.getElementById('minimumTranscriptChars');
|
|
311
320
|
const shareButton = document.getElementById('shareButton');
|
|
312
321
|
const clearButton = document.getElementById('clearButton');
|
|
313
322
|
const speechSettingsButton = document.getElementById('speechSettingsButton');
|
|
@@ -319,6 +328,8 @@ def render_web_chat_page(
|
|
|
319
328
|
const statePill = document.getElementById('statePill');
|
|
320
329
|
const SESSION_KEY = 'ciel-runtime-web-chat-session';
|
|
321
330
|
const LAST_ID_KEY = 'ciel-runtime-web-chat-last-id';
|
|
331
|
+
const VOICE_SENSITIVITY_KEY = 'ciel-runtime-web-chat-voice-sensitivity';
|
|
332
|
+
const MIN_TRANSCRIPT_CHARS_KEY = 'ciel-runtime-web-chat-min-transcript-chars';
|
|
322
333
|
const HISTORY_PAGE_SIZE = 80;
|
|
323
334
|
const renderedIds = new Set();
|
|
324
335
|
let oldestId = 0;
|
|
@@ -354,6 +365,12 @@ def render_web_chat_page(
|
|
|
354
365
|
let vadSpeechStartedAt = 0;
|
|
355
366
|
let vadLastVoiceAt = 0;
|
|
356
367
|
let vadVoicedSamples = 0;
|
|
368
|
+
let vadPeakRms = 0;
|
|
369
|
+
let vadBargeInPending = false;
|
|
370
|
+
let vadCandidateChunks = [];
|
|
371
|
+
let vadCandidateVoicedSamples = 0;
|
|
372
|
+
let vadCandidateStartedAt = 0;
|
|
373
|
+
let vadCandidatePeakRms = 0;
|
|
357
374
|
let vadNoiseFloor = 0.006;
|
|
358
375
|
let liveTranscriptionQueue = Promise.resolve();
|
|
359
376
|
let livePartialInFlight = false;
|
|
@@ -365,7 +382,12 @@ def render_web_chat_page(
|
|
|
365
382
|
let activeSpeechSource = null;
|
|
366
383
|
const activeSpeechSources = new Set();
|
|
367
384
|
let speechGenerationController = null;
|
|
385
|
+
let speechQueue = Promise.resolve();
|
|
386
|
+
let speechQueueEpoch = 0;
|
|
387
|
+
let autoSpeechReplySeen = false;
|
|
368
388
|
let pendingTtsReferenceAudio = '';
|
|
389
|
+
voiceSensitivity.value = localStorage.getItem(VOICE_SENSITIVITY_KEY) || 'low';
|
|
390
|
+
minimumTranscriptChars.value = localStorage.getItem(MIN_TRANSCRIPT_CHARS_KEY) || '3';
|
|
369
391
|
function setState(text, cls = '') {{
|
|
370
392
|
statePill.textContent = text;
|
|
371
393
|
statePill.className = 'pill ' + cls;
|
|
@@ -563,10 +585,17 @@ def render_web_chat_page(
|
|
|
563
585
|
transcript.insertBefore(row, transcript.firstChild);
|
|
564
586
|
}} else {{
|
|
565
587
|
transcript.appendChild(row);
|
|
566
|
-
|
|
588
|
+
scrollTranscriptToBottom();
|
|
567
589
|
}}
|
|
568
590
|
return bubble;
|
|
569
591
|
}}
|
|
592
|
+
function scrollTranscriptToBottom() {{
|
|
593
|
+
transcript.scrollTop = transcript.scrollHeight;
|
|
594
|
+
requestAnimationFrame(() => {{
|
|
595
|
+
transcript.scrollTop = transcript.scrollHeight;
|
|
596
|
+
requestAnimationFrame(() => {{ transcript.scrollTop = transcript.scrollHeight; }});
|
|
597
|
+
}});
|
|
598
|
+
}}
|
|
570
599
|
function blockRuntimeIdentity(reason) {{
|
|
571
600
|
const detail = String(reason || 'Runtime identity changed.');
|
|
572
601
|
if (instanceIdentityBlocked === detail && sendButton.disabled) return;
|
|
@@ -651,7 +680,7 @@ def render_web_chat_page(
|
|
|
651
680
|
actions.appendChild(speak);
|
|
652
681
|
row.appendChild(actions);
|
|
653
682
|
if (mode === 'prepend') transcript.insertBefore(row, transcript.firstChild);
|
|
654
|
-
else {{ transcript.appendChild(row);
|
|
683
|
+
else {{ transcript.appendChild(row); scrollTranscriptToBottom(); }}
|
|
655
684
|
return bubble;
|
|
656
685
|
}}
|
|
657
686
|
function rememberLastId(id) {{
|
|
@@ -669,12 +698,17 @@ def render_web_chat_page(
|
|
|
669
698
|
const text = message.message || '';
|
|
670
699
|
const structured = structuredWebResponse(message);
|
|
671
700
|
if (!text.trim() && !structured) return;
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
if (
|
|
701
|
+
const role = roleForMessage(message);
|
|
702
|
+
const kind = String(message.kind || '').toLowerCase();
|
|
703
|
+
if (role === 'user') autoSpeechReplySeen = false;
|
|
704
|
+
const duplicateTurnSpeech = role === 'assistant' && ((kind === 'ack' && autoSpeechReplySeen) || (kind === 'reply' && autoSpeechReplySeen));
|
|
705
|
+
if (kind === 'reply' && role === 'assistant') autoSpeechReplySeen = true;
|
|
706
|
+
if (structured && role === 'assistant') addStructuredBubble(structured, mode, message.id);
|
|
707
|
+
else addBubble(role, text, mode, message.id);
|
|
708
|
+
if (mode === 'append' && role === 'assistant') {{
|
|
675
709
|
setState('reply received', 'ok');
|
|
676
710
|
const speechText = structured ? (structured.spoken || structured.overview) : text;
|
|
677
|
-
if (speechConfig.tts && speechConfig.tts.enabled && speechText && (speechConfig.tts.auto_speak || liveVoiceEnabled))
|
|
711
|
+
if (!duplicateTurnSpeech && speechConfig.tts && speechConfig.tts.enabled && speechText && (speechConfig.tts.auto_speak || liveVoiceEnabled)) enqueueSpeech(speechText);
|
|
678
712
|
}}
|
|
679
713
|
}}
|
|
680
714
|
function formatBytes(bytes) {{
|
|
@@ -763,7 +797,6 @@ def render_web_chat_page(
|
|
|
763
797
|
document.getElementById('colabDistribution').value = colab.distribution || 'Ubuntu-26.04';
|
|
764
798
|
document.getElementById('colabAuth').value = colab.auth || 'adc';
|
|
765
799
|
document.getElementById('colabProfile').value = colab.profile || 'default';
|
|
766
|
-
document.getElementById('colabTtsBackend').value = colab.tts_backend || (String(tts.model || '').includes('CosyVoice3') ? 'cosyvoice3' : 'moss');
|
|
767
800
|
document.getElementById('colabAsrSession').value = colab.asr_session || 'ciel-asr';
|
|
768
801
|
document.getElementById('colabTtsSession').value = colab.tts_session || 'ciel-tts';
|
|
769
802
|
document.getElementById('colabAsrAccelerator').value = colab.asr_accelerator || 'T4';
|
|
@@ -816,7 +849,8 @@ def render_web_chat_page(
|
|
|
816
849
|
distribution: document.getElementById('colabDistribution').value,
|
|
817
850
|
auth: document.getElementById('colabAuth').value,
|
|
818
851
|
profile: document.getElementById('colabProfile').value,
|
|
819
|
-
|
|
852
|
+
asr_model: document.getElementById('asrModel').value,
|
|
853
|
+
tts_backend: String(document.getElementById('ttsModel').value || '').includes('CosyVoice3') ? 'cosyvoice3' : 'moss',
|
|
820
854
|
asr_session: document.getElementById('colabAsrSession').value,
|
|
821
855
|
tts_session: document.getElementById('colabTtsSession').value,
|
|
822
856
|
asr_accelerator: document.getElementById('colabAsrAccelerator').value,
|
|
@@ -901,7 +935,23 @@ def render_web_chat_page(
|
|
|
901
935
|
if (activeSpeechUrl) URL.revokeObjectURL(activeSpeechUrl);
|
|
902
936
|
activeSpeechUrl = '';
|
|
903
937
|
}}
|
|
904
|
-
|
|
938
|
+
function speechPlaybackActive() {{
|
|
939
|
+
return Boolean(speechGenerationController || activeSpeechSource || activeSpeechAudio || activeSpeechSources.size);
|
|
940
|
+
}}
|
|
941
|
+
function cancelSpeechQueue() {{
|
|
942
|
+
speechQueueEpoch += 1;
|
|
943
|
+
speechQueue = Promise.resolve();
|
|
944
|
+
stopActiveSpeech();
|
|
945
|
+
}}
|
|
946
|
+
function enqueueSpeech(text) {{
|
|
947
|
+
const epoch = speechQueueEpoch;
|
|
948
|
+
speechQueue = speechQueue.catch(() => {{}}).then(() => {{
|
|
949
|
+
if (epoch !== speechQueueEpoch) return;
|
|
950
|
+
return speakText(text, {{interrupt: false}});
|
|
951
|
+
}});
|
|
952
|
+
return speechQueue;
|
|
953
|
+
}}
|
|
954
|
+
async function playSpeechBlob(blob, controller) {{
|
|
905
955
|
const context = speechPlaybackContext;
|
|
906
956
|
if (context && context.state === 'running') {{
|
|
907
957
|
try {{
|
|
@@ -911,14 +961,20 @@ def render_web_chat_page(
|
|
|
911
961
|
source.connect(context.destination);
|
|
912
962
|
activeSpeechSource = source;
|
|
913
963
|
activeSpeechSources.add(source);
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
964
|
+
await new Promise(resolve => {{
|
|
965
|
+
let settled = false;
|
|
966
|
+
const finish = () => {{
|
|
967
|
+
if (settled) return;
|
|
968
|
+
settled = true;
|
|
969
|
+
activeSpeechSources.delete(source);
|
|
970
|
+
if (activeSpeechSource === source) activeSpeechSource = null;
|
|
971
|
+
resolve();
|
|
972
|
+
}};
|
|
973
|
+
source.addEventListener('ended', finish, {{once: true}});
|
|
974
|
+
controller.signal.addEventListener('abort', finish, {{once: true}});
|
|
975
|
+
source.start();
|
|
976
|
+
setState('speaking', 'ok');
|
|
977
|
+
}});
|
|
922
978
|
return;
|
|
923
979
|
}} catch {{}}
|
|
924
980
|
}}
|
|
@@ -927,17 +983,24 @@ def render_web_chat_page(
|
|
|
927
983
|
const audio = activeSpeechAudio;
|
|
928
984
|
audio.preload = 'auto';
|
|
929
985
|
audio.playsInline = true;
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
if (
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
986
|
+
await new Promise((resolve, reject) => {{
|
|
987
|
+
let settled = false;
|
|
988
|
+
const cleanup = error => {{
|
|
989
|
+
if (settled) return;
|
|
990
|
+
settled = true;
|
|
991
|
+
if (activeSpeechAudio === audio) {{
|
|
992
|
+
activeSpeechAudio = null;
|
|
993
|
+
if (activeSpeechUrl) URL.revokeObjectURL(activeSpeechUrl);
|
|
994
|
+
activeSpeechUrl = '';
|
|
995
|
+
}}
|
|
996
|
+
if (error) reject(error);
|
|
997
|
+
else resolve();
|
|
998
|
+
}};
|
|
999
|
+
audio.addEventListener('ended', () => cleanup(), {{once: true}});
|
|
1000
|
+
audio.addEventListener('error', () => cleanup(new Error('Browser could not play the TTS audio')), {{once: true}});
|
|
1001
|
+
controller.signal.addEventListener('abort', () => cleanup(), {{once: true}});
|
|
1002
|
+
audio.play().then(() => setState('speaking', 'ok')).catch(cleanup);
|
|
1003
|
+
}});
|
|
941
1004
|
}}
|
|
942
1005
|
async function playPcmSpeechStream(response, controller, sampleRate) {{
|
|
943
1006
|
const context = unlockSpeechPlayback();
|
|
@@ -947,6 +1010,7 @@ def render_web_chat_page(
|
|
|
947
1010
|
let remainder = new Uint8Array(0);
|
|
948
1011
|
let nextStart = context.currentTime + 0.06;
|
|
949
1012
|
let received = false;
|
|
1013
|
+
let lastSource = null;
|
|
950
1014
|
while (true) {{
|
|
951
1015
|
const part = await reader.read();
|
|
952
1016
|
if (part.done) break;
|
|
@@ -970,6 +1034,7 @@ def render_web_chat_page(
|
|
|
970
1034
|
source.buffer = audioBuffer;
|
|
971
1035
|
source.connect(context.destination);
|
|
972
1036
|
activeSpeechSources.add(source);
|
|
1037
|
+
lastSource = source;
|
|
973
1038
|
source.addEventListener('ended', () => {{
|
|
974
1039
|
activeSpeechSources.delete(source);
|
|
975
1040
|
if (!activeSpeechSources.size && !speechGenerationController) setState(liveVoiceEnabled ? 'listening' : 'ready', 'ok');
|
|
@@ -981,14 +1046,20 @@ def render_web_chat_page(
|
|
|
981
1046
|
received = true;
|
|
982
1047
|
}}
|
|
983
1048
|
if (!received) throw new Error('TTS returned an empty PCM stream');
|
|
1049
|
+
if (lastSource && activeSpeechSources.has(lastSource) && !controller.signal.aborted) {{
|
|
1050
|
+
await new Promise(resolve => {{
|
|
1051
|
+
lastSource.addEventListener('ended', resolve, {{once: true}});
|
|
1052
|
+
controller.signal.addEventListener('abort', resolve, {{once: true}});
|
|
1053
|
+
}});
|
|
1054
|
+
}}
|
|
984
1055
|
}}
|
|
985
|
-
async function speakText(text) {{
|
|
1056
|
+
async function speakText(text, options = {{}}) {{
|
|
986
1057
|
if (!speechConfig.tts || !speechConfig.tts.enabled) {{
|
|
987
1058
|
setState('TTS disabled', 'error');
|
|
988
1059
|
return;
|
|
989
1060
|
}}
|
|
990
1061
|
if (liveVoiceEnabled && vadSpeechActive) return;
|
|
991
|
-
|
|
1062
|
+
if (options.interrupt !== false) cancelSpeechQueue();
|
|
992
1063
|
const controller = new AbortController();
|
|
993
1064
|
speechGenerationController = controller;
|
|
994
1065
|
try {{
|
|
@@ -1005,24 +1076,40 @@ def render_web_chat_page(
|
|
|
1005
1076
|
if (!response.ok) throw new Error(await response.text() || `HTTP ${{response.status}}`);
|
|
1006
1077
|
if (streamAudio) {{
|
|
1007
1078
|
await playPcmSpeechStream(response, controller, Number(speechConfig.tts.sample_rate || 24000));
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1079
|
+
}} else {{
|
|
1080
|
+
const blob = await response.blob();
|
|
1081
|
+
if (!blob.size) throw new Error('TTS returned empty audio');
|
|
1082
|
+
if (blob.type && !blob.type.startsWith('audio/')) throw new Error(`TTS returned ${{blob.type}} instead of audio`);
|
|
1083
|
+
if (controller.signal.aborted) return;
|
|
1084
|
+
await playSpeechBlob(blob, controller);
|
|
1011
1085
|
}}
|
|
1012
|
-
|
|
1013
|
-
if (!blob.size) throw new Error('TTS returned empty audio');
|
|
1014
|
-
if (blob.type && !blob.type.startsWith('audio/')) throw new Error(`TTS returned ${{blob.type}} instead of audio`);
|
|
1015
|
-
if (controller.signal.aborted) return;
|
|
1016
|
-
speechGenerationController = null;
|
|
1017
|
-
await playSpeechBlob(blob);
|
|
1018
|
-
setState('speaking', 'ok');
|
|
1086
|
+
if (!controller.signal.aborted) setState(liveVoiceEnabled ? 'listening' : 'ready', 'ok');
|
|
1019
1087
|
}} catch (err) {{
|
|
1020
1088
|
if (err && err.name === 'AbortError') return;
|
|
1021
|
-
speechGenerationController = null;
|
|
1022
1089
|
setState('TTS error', 'error');
|
|
1023
1090
|
addBubble('system', 'TTS failed: ' + String(err && err.message ? err.message : err));
|
|
1091
|
+
}} finally {{
|
|
1092
|
+
if (speechGenerationController === controller) speechGenerationController = null;
|
|
1024
1093
|
}}
|
|
1025
1094
|
}}
|
|
1095
|
+
function normalizeAsrTranscript(value) {{
|
|
1096
|
+
let text = String(value || '').trim();
|
|
1097
|
+
const marker = text.indexOf('<asr_text>');
|
|
1098
|
+
if (marker >= 0) text = text.slice(marker + '<asr_text>'.length);
|
|
1099
|
+
text = text.replace(/<\\/asr_text>[\\s\\S]*$/i, '').trim();
|
|
1100
|
+
return text;
|
|
1101
|
+
}}
|
|
1102
|
+
function voiceSensitivityPolicy() {{
|
|
1103
|
+
const policies = {{
|
|
1104
|
+
low: {{thresholdFloor: 0.045, onsetMs: 280, bargeInMs: 360, strongMultiplier: 2.1}},
|
|
1105
|
+
normal: {{thresholdFloor: 0.028, onsetMs: 160, bargeInMs: 260, strongMultiplier: 1.8}},
|
|
1106
|
+
high: {{thresholdFloor: 0.012, onsetMs: 80, bargeInMs: 180, strongMultiplier: 1.5}},
|
|
1107
|
+
}};
|
|
1108
|
+
return policies[voiceSensitivity.value] || policies.low;
|
|
1109
|
+
}}
|
|
1110
|
+
function minimumVoiceTranscriptLength() {{
|
|
1111
|
+
return Math.max(1, Math.min(20, Number(minimumTranscriptChars.value || 3)));
|
|
1112
|
+
}}
|
|
1026
1113
|
async function transcribeRecording(blob, populatePrompt = true, options = {{}}) {{
|
|
1027
1114
|
if (!options.quiet) setState('transcribing');
|
|
1028
1115
|
const audio_base64 = await fileToBase64(blob);
|
|
@@ -1036,8 +1123,12 @@ def render_web_chat_page(
|
|
|
1036
1123
|
let data = {{}};
|
|
1037
1124
|
try {{ data = text ? JSON.parse(text) : {{}}; }} catch {{}}
|
|
1038
1125
|
if (!response.ok) throw new Error((data.error && (data.error.message || data.error)) || text || `HTTP ${{response.status}}`);
|
|
1039
|
-
const transcriptText =
|
|
1040
|
-
if (!transcriptText)
|
|
1126
|
+
const transcriptText = normalizeAsrTranscript(data.text || data.transcript || '');
|
|
1127
|
+
if (!transcriptText) {{
|
|
1128
|
+
const error = new Error('ASR returned no transcript');
|
|
1129
|
+
error.name = 'EmptyTranscriptError';
|
|
1130
|
+
throw error;
|
|
1131
|
+
}}
|
|
1041
1132
|
if (populatePrompt) {{
|
|
1042
1133
|
prompt.value = prompt.value ? prompt.value + ' ' + transcriptText : transcriptText;
|
|
1043
1134
|
prompt.focus();
|
|
@@ -1079,6 +1170,12 @@ def render_web_chat_page(
|
|
|
1079
1170
|
vadSpeechStartedAt = 0;
|
|
1080
1171
|
vadLastVoiceAt = 0;
|
|
1081
1172
|
vadVoicedSamples = 0;
|
|
1173
|
+
vadPeakRms = 0;
|
|
1174
|
+
vadBargeInPending = false;
|
|
1175
|
+
vadCandidateChunks = [];
|
|
1176
|
+
vadCandidateVoicedSamples = 0;
|
|
1177
|
+
vadCandidateStartedAt = 0;
|
|
1178
|
+
vadCandidatePeakRms = 0;
|
|
1082
1179
|
}}
|
|
1083
1180
|
function setLiveTranscript(text, active = true) {{
|
|
1084
1181
|
liveTranscript.textContent = text;
|
|
@@ -1106,11 +1203,22 @@ def render_web_chat_page(
|
|
|
1106
1203
|
const blob = encodePcmWav(chunks, sampleRate);
|
|
1107
1204
|
liveTranscriptionQueue = liveTranscriptionQueue.then(async () => {{
|
|
1108
1205
|
const transcriptText = await transcribeRecording(blob, false);
|
|
1206
|
+
const transcriptLength = Array.from(transcriptText.replace(/\\s+/g, '')).length;
|
|
1207
|
+
if (transcriptLength < minimumVoiceTranscriptLength()) {{
|
|
1208
|
+
setLiveTranscript(`Ignored short transcript (${{transcriptLength}} chars).`, true);
|
|
1209
|
+
if (liveVoiceEnabled) setState('listening', 'ok');
|
|
1210
|
+
return;
|
|
1211
|
+
}}
|
|
1109
1212
|
if (serial === liveUtteranceSerial) setLiveTranscript('Heard: ' + transcriptText);
|
|
1110
1213
|
setState('sending voice');
|
|
1111
1214
|
await sendMessage(transcriptText, [], {{inputMode: 'voice'}});
|
|
1112
1215
|
if (liveVoiceEnabled) setState('listening', 'ok');
|
|
1113
1216
|
}}).catch(err => {{
|
|
1217
|
+
if (err && err.name === 'EmptyTranscriptError') {{
|
|
1218
|
+
setLiveTranscript('Listening...', liveVoiceEnabled);
|
|
1219
|
+
if (liveVoiceEnabled) setState('listening', 'ok');
|
|
1220
|
+
return;
|
|
1221
|
+
}}
|
|
1114
1222
|
setState('STT error', 'error');
|
|
1115
1223
|
addBubble('system', 'STT failed: ' + String(err && err.message ? err.message : err));
|
|
1116
1224
|
}});
|
|
@@ -1132,11 +1240,16 @@ def render_web_chat_page(
|
|
|
1132
1240
|
for (let index = 0; index < chunk.length; index += 1) sumSquares += chunk[index] * chunk[index];
|
|
1133
1241
|
const rms = Math.sqrt(sumSquares / Math.max(1, chunk.length));
|
|
1134
1242
|
const configuredThreshold = Number((speechConfig.asr && speechConfig.asr.vad_threshold) || 0.018);
|
|
1135
|
-
const
|
|
1243
|
+
const sensitivity = voiceSensitivityPolicy();
|
|
1244
|
+
const threshold = Math.max(configuredThreshold, sensitivity.thresholdFloor, Math.min(0.08, vadNoiseFloor * 2.8));
|
|
1136
1245
|
const voiceDetected = rms >= threshold;
|
|
1137
1246
|
const now = performance.now();
|
|
1138
1247
|
if (!vadSpeechActive) {{
|
|
1139
1248
|
if (!voiceDetected) {{
|
|
1249
|
+
vadCandidateChunks = [];
|
|
1250
|
+
vadCandidateVoicedSamples = 0;
|
|
1251
|
+
vadCandidateStartedAt = 0;
|
|
1252
|
+
vadCandidatePeakRms = 0;
|
|
1140
1253
|
vadNoiseFloor = Math.max(0.002, Math.min(0.03, vadNoiseFloor * 0.98 + rms * 0.02));
|
|
1141
1254
|
vadPreRollChunks.push(chunk);
|
|
1142
1255
|
const maxPreRollSamples = audioContext.sampleRate * 0.25;
|
|
@@ -1144,31 +1257,52 @@ def render_web_chat_page(
|
|
|
1144
1257
|
while (preRollSamples > maxPreRollSamples && vadPreRollChunks.length > 1) preRollSamples -= vadPreRollChunks.shift().length;
|
|
1145
1258
|
return;
|
|
1146
1259
|
}}
|
|
1260
|
+
if (!vadCandidateStartedAt) vadCandidateStartedAt = now;
|
|
1261
|
+
vadCandidateChunks.push(chunk);
|
|
1262
|
+
vadCandidateVoicedSamples += chunk.length;
|
|
1263
|
+
vadCandidatePeakRms = Math.max(vadCandidatePeakRms, rms);
|
|
1264
|
+
const candidateMs = vadCandidateVoicedSamples * 1000 / audioContext.sampleRate;
|
|
1265
|
+
if (candidateMs < sensitivity.onsetMs) return;
|
|
1147
1266
|
vadSpeechActive = true;
|
|
1148
1267
|
liveUtteranceSerial += 1;
|
|
1149
1268
|
livePartialLastAt = now;
|
|
1150
|
-
vadSpeechStartedAt =
|
|
1269
|
+
vadSpeechStartedAt = vadCandidateStartedAt;
|
|
1151
1270
|
vadLastVoiceAt = now;
|
|
1152
|
-
vadVoicedSamples =
|
|
1153
|
-
|
|
1271
|
+
vadVoicedSamples = vadCandidateVoicedSamples;
|
|
1272
|
+
vadPeakRms = vadCandidatePeakRms;
|
|
1273
|
+
vadBargeInPending = speechPlaybackActive();
|
|
1274
|
+
vadSpeechChunks = vadPreRollChunks.concat(vadCandidateChunks);
|
|
1154
1275
|
vadPreRollChunks = [];
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1276
|
+
vadCandidateChunks = [];
|
|
1277
|
+
vadCandidateVoicedSamples = 0;
|
|
1278
|
+
vadCandidateStartedAt = 0;
|
|
1279
|
+
vadCandidatePeakRms = 0;
|
|
1280
|
+
if (vadBargeInPending) setLiveTranscript('Possible interruption...');
|
|
1281
|
+
else {{ setLiveTranscript('Listening to speech...'); setState('hearing speech', 'ok'); }}
|
|
1158
1282
|
return;
|
|
1159
1283
|
}}
|
|
1160
1284
|
vadSpeechChunks.push(chunk);
|
|
1161
1285
|
if (voiceDetected) {{
|
|
1162
1286
|
vadLastVoiceAt = now;
|
|
1163
1287
|
vadVoicedSamples += chunk.length;
|
|
1288
|
+
vadPeakRms = Math.max(vadPeakRms, rms);
|
|
1164
1289
|
}}
|
|
1165
1290
|
const silenceMs = Number((speechConfig.asr && speechConfig.asr.silence_ms) || 900);
|
|
1166
1291
|
const minSpeechMs = Number((speechConfig.asr && speechConfig.asr.min_speech_ms) || 300);
|
|
1167
1292
|
const voicedMs = vadVoicedSamples * 1000 / audioContext.sampleRate;
|
|
1168
1293
|
const utteranceMs = now - vadSpeechStartedAt;
|
|
1169
|
-
|
|
1294
|
+
if (vadBargeInPending) {{
|
|
1295
|
+
const strongVoice = vadPeakRms >= Math.max(threshold * sensitivity.strongMultiplier, configuredThreshold + 0.015);
|
|
1296
|
+
if (voicedMs >= sensitivity.bargeInMs && strongVoice) {{
|
|
1297
|
+
vadBargeInPending = false;
|
|
1298
|
+
cancelSpeechQueue();
|
|
1299
|
+
setLiveTranscript('Listening to interruption...');
|
|
1300
|
+
setState('hearing speech', 'ok');
|
|
1301
|
+
}}
|
|
1302
|
+
}} else requestLivePartial(now);
|
|
1170
1303
|
if (now - vadLastVoiceAt >= silenceMs) {{
|
|
1171
|
-
if (
|
|
1304
|
+
if (vadBargeInPending) {{ resetVadUtterance(); setLiveTranscript('Listening...', liveVoiceEnabled); }}
|
|
1305
|
+
else if (voicedMs >= minSpeechMs) finishVadUtterance();
|
|
1172
1306
|
else {{ resetVadUtterance(); setLiveTranscript('Listening...', liveVoiceEnabled); }}
|
|
1173
1307
|
}} else if (utteranceMs >= 30000) {{
|
|
1174
1308
|
finishVadUtterance();
|
|
@@ -1213,7 +1347,7 @@ def render_web_chat_page(
|
|
|
1213
1347
|
audioInput = null;
|
|
1214
1348
|
audioContext = null;
|
|
1215
1349
|
mediaStream = null;
|
|
1216
|
-
|
|
1350
|
+
cancelSpeechQueue();
|
|
1217
1351
|
setLiveTranscript('', false);
|
|
1218
1352
|
micButton.textContent = 'Start live voice';
|
|
1219
1353
|
micButton.classList.remove('recording');
|
|
@@ -1300,7 +1434,7 @@ def render_web_chat_page(
|
|
|
1300
1434
|
try {{
|
|
1301
1435
|
const json = await fetchMessagePage({{latest: '1'}});
|
|
1302
1436
|
const messages = Array.isArray(json.messages) ? json.messages : [];
|
|
1303
|
-
messages.forEach(message => renderIncomingMessage(message, '
|
|
1437
|
+
messages.forEach(message => renderIncomingMessage(message, 'history'));
|
|
1304
1438
|
updateHistoryBounds(messages);
|
|
1305
1439
|
historyExhausted = messages.length < HISTORY_PAGE_SIZE;
|
|
1306
1440
|
}} catch (err) {{
|
|
@@ -1350,6 +1484,7 @@ def render_web_chat_page(
|
|
|
1350
1484
|
}}
|
|
1351
1485
|
async function sendMessage(text, files = [], options = {{}}) {{
|
|
1352
1486
|
if (!await verifyRuntimeIdentity()) return;
|
|
1487
|
+
cancelSpeechQueue();
|
|
1353
1488
|
setState('queued');
|
|
1354
1489
|
sendButton.disabled = true;
|
|
1355
1490
|
attachButton.disabled = true;
|
|
@@ -1480,6 +1615,20 @@ def render_web_chat_page(
|
|
|
1480
1615
|
await speakText('음성 재생 테스트입니다.');
|
|
1481
1616
|
}} catch (err) {{ addBubble('system', 'Voice playback test failed: ' + String(err && err.message ? err.message : err)); }}
|
|
1482
1617
|
}});
|
|
1618
|
+
voiceSensitivity.addEventListener('change', () => {{
|
|
1619
|
+
localStorage.setItem(VOICE_SENSITIVITY_KEY, voiceSensitivity.value);
|
|
1620
|
+
resetVadUtterance();
|
|
1621
|
+
setLiveTranscript(liveVoiceEnabled ? 'Listening...' : '', liveVoiceEnabled);
|
|
1622
|
+
}});
|
|
1623
|
+
minimumTranscriptChars.addEventListener('change', () => {{
|
|
1624
|
+
minimumTranscriptChars.value = String(minimumVoiceTranscriptLength());
|
|
1625
|
+
localStorage.setItem(MIN_TRANSCRIPT_CHARS_KEY, minimumTranscriptChars.value);
|
|
1626
|
+
}});
|
|
1627
|
+
document.getElementById('ttsModel').addEventListener('change', event => {{
|
|
1628
|
+
const cosyVoice = String(event.target.value || '').includes('CosyVoice3');
|
|
1629
|
+
document.getElementById('ttsStreaming').checked = cosyVoice;
|
|
1630
|
+
document.getElementById('ttsSampleRate').value = cosyVoice ? '24000' : '48000';
|
|
1631
|
+
}});
|
|
1483
1632
|
fileInput.addEventListener('change', () => {{
|
|
1484
1633
|
addSelectedFiles(fileInput.files);
|
|
1485
1634
|
fileInput.value = '';
|
package/docs/COLAB_SPEECH.md
CHANGED
|
@@ -18,7 +18,7 @@ From PowerShell at the repository root:
|
|
|
18
18
|
|
|
19
19
|
Set the WSL distribution, authentication mode, ASR/TTS session names, and accelerators in **Web Chat > Speech Settings > Colab CLI connection**. These values are available through `GET|POST /ca/speech/config`; Ciel does not store Colab credentials. The deployment script reads the saved values automatically. Command-line parameters such as `-Distribution`, `-ColabAuth`, `-AsrSession`, and `-AsrAccelerator` override them for one run.
|
|
20
20
|
|
|
21
|
-
The script reuses matching active sessions when possible, otherwise creates them, installs Qwen3-ASR
|
|
21
|
+
The script reuses matching active sessions when possible, otherwise creates them, installs the selected Qwen3-ASR model (0.6B or 1.7B) plus the selected TTS engine (MOSS-TTS-Nano or Fun-CosyVoice3-0.5B-2512), starts Tailscale in userspace networking mode, publishes each localhost model server with Tailscale Serve, and saves both returned `base_url` values into Web Chat > Speech Settings automatically. Choose both models in their STT/TTS sections before **Recover & deploy**, or pass `-AsrModel Qwen/Qwen3-ASR-1.7B` and `-TtsBackend moss|cosyvoice3` to the script.
|
|
22
22
|
|
|
23
23
|
### Session recovery and account profiles
|
|
24
24
|
|
|
@@ -50,6 +50,8 @@ Colab sessions are ephemeral. Re-run the bootstrap after a runtime reset. The wo
|
|
|
50
50
|
|
|
51
51
|
Web Chat's **Start live voice** button keeps the microphone open and uses browser-side voice activity detection (VAD). While the user is speaking, the browser sends rate-limited snapshots of the growing utterance to the Qwen worker and displays the latest best-effort partial transcript. A completed utterance is encoded as PCM WAV, transcribed once more for the final text, and sent to the active coding-agent session automatically. While MOSS TTS is generating or playing a reply, new speech stops it immediately and starts a new utterance (barge-in). Tune end-of-speech silence, minimum speech duration, and the VAD threshold in Speech Settings.
|
|
52
52
|
|
|
53
|
+
The composer also exposes a local microphone-sensitivity preset and a minimum transcript character count. Low sensitivity requires sustained audio before opening an utterance, which filters keyboard clicks and protects TTS from false barge-in. Empty Qwen wrapper output and transcripts shorter than the selected character count are never sent to the coding agent. Live partial text is shown above the input as an outgoing-message preview.
|
|
54
|
+
|
|
53
55
|
The Colab Qwen endpoint remains a batch HTTP API, so the live caption is progressive re-transcription rather than token-level server streaming. A future WebSocket or streaming HTTP worker can replace this transport without changing the final-turn behavior.
|
|
54
56
|
|
|
55
57
|
Web Chat requests carry an input mode and a structured response contract. The active agent first sends a short acknowledgement and then a final response containing `spoken`, `overview`, and optional `details` fields. The browser renders the fields separately and sends only `spoken` to TTS, avoiding long Markdown, URLs, code, and tables in synthesized speech. Legacy plain `message` replies remain supported.
|
package/package.json
CHANGED
|
@@ -17,10 +17,13 @@ import urllib.request
|
|
|
17
17
|
|
|
18
18
|
|
|
19
19
|
HOSTNAME = os.environ.get("CIEL_ASR_HOSTNAME", "ciel-asr")
|
|
20
|
+
SUPPORTED_MODELS = {"Qwen/Qwen3-ASR-0.6B", "Qwen/Qwen3-ASR-1.7B"}
|
|
21
|
+
MODEL = os.environ.get("CIEL_ASR_MODEL", "Qwen/Qwen3-ASR-0.6B").strip()
|
|
20
22
|
PORT = 8000
|
|
21
23
|
SOCKET = "/tmp/ciel-asr-tailscaled.sock"
|
|
22
24
|
STATE = "/tmp/ciel-asr-tailscaled.state"
|
|
23
25
|
LOG_DIR = Path("/content/ciel-speech-logs")
|
|
26
|
+
MODEL_MARKER = Path("/content/ciel-speech-asr-model")
|
|
24
27
|
|
|
25
28
|
|
|
26
29
|
def secret(name: str, *, required: bool = False) -> str:
|
|
@@ -63,12 +66,13 @@ def start_tailscale(auth_key: str) -> tuple[str, str]:
|
|
|
63
66
|
install_tailscale()
|
|
64
67
|
LOG_DIR.mkdir(parents=True, exist_ok=True)
|
|
65
68
|
tail_log = (LOG_DIR / "tailscale-asr.log").open("ab")
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
69
|
+
if not Path(SOCKET).exists():
|
|
70
|
+
subprocess.Popen(
|
|
71
|
+
["tailscaled", f"--socket={SOCKET}", f"--state={STATE}", "--tun=userspace-networking"],
|
|
72
|
+
stdout=tail_log,
|
|
73
|
+
stderr=subprocess.STDOUT,
|
|
74
|
+
start_new_session=True,
|
|
75
|
+
)
|
|
72
76
|
for _ in range(60):
|
|
73
77
|
if Path(SOCKET).exists():
|
|
74
78
|
break
|
|
@@ -82,34 +86,55 @@ def start_tailscale(auth_key: str) -> tuple[str, str]:
|
|
|
82
86
|
return dns_name, f"http://{dns_name}"
|
|
83
87
|
|
|
84
88
|
|
|
85
|
-
def
|
|
89
|
+
def server_is_healthy(api_key: str) -> bool:
|
|
86
90
|
headers = {"authorization": f"Bearer {api_key}"} if api_key else {}
|
|
91
|
+
try:
|
|
92
|
+
with urllib.request.urlopen(urllib.request.Request(f"http://127.0.0.1:{PORT}/health", headers=headers), timeout=3) as response:
|
|
93
|
+
return response.status < 500
|
|
94
|
+
except Exception:
|
|
95
|
+
return False
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def wait_for_server(api_key: str, process: subprocess.Popen[bytes]) -> None:
|
|
87
99
|
for _ in range(180):
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
100
|
+
if process.poll() is not None:
|
|
101
|
+
log_path = LOG_DIR / "qwen-asr.log"
|
|
102
|
+
log_tail = log_path.read_text(encoding="utf-8", errors="replace")[-12000:] if log_path.exists() else "log unavailable"
|
|
103
|
+
raise RuntimeError(f"Qwen ASR exited with status {process.returncode}:\n{log_tail}")
|
|
104
|
+
if server_is_healthy(api_key):
|
|
105
|
+
return
|
|
106
|
+
time.sleep(2)
|
|
94
107
|
raise RuntimeError("Qwen ASR did not become healthy; inspect /content/ciel-speech-logs/qwen-asr.log")
|
|
95
108
|
|
|
96
109
|
|
|
110
|
+
def prepare_model() -> None:
|
|
111
|
+
current = MODEL_MARKER.read_text(encoding="utf-8", errors="replace").strip() if MODEL_MARKER.exists() else ""
|
|
112
|
+
if current != MODEL:
|
|
113
|
+
run("bash", "-lc", f"fuser -k {PORT}/tcp >/dev/null 2>&1 || true", check=False)
|
|
114
|
+
time.sleep(2)
|
|
115
|
+
|
|
116
|
+
|
|
97
117
|
def main() -> None:
|
|
118
|
+
if MODEL not in SUPPORTED_MODELS:
|
|
119
|
+
raise RuntimeError(f"Unsupported CIEL_ASR_MODEL: {MODEL}")
|
|
98
120
|
auth_key = secret("TAILSCALE_AUTHKEY", required=True)
|
|
99
121
|
api_key = secret("CIEL_SPEECH_API_KEY")
|
|
100
122
|
run(sys.executable, "-m", "pip", "install", "-U", "qwen-asr[vllm]", "vllm[audio]")
|
|
101
123
|
LOG_DIR.mkdir(parents=True, exist_ok=True)
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
124
|
+
prepare_model()
|
|
125
|
+
if not server_is_healthy(api_key):
|
|
126
|
+
command = [
|
|
127
|
+
"qwen-asr-serve", MODEL, "--host", "127.0.0.1", "--port", str(PORT),
|
|
128
|
+
"--gpu-memory-utilization", "0.78", "--max-model-len", "8192",
|
|
129
|
+
]
|
|
130
|
+
if api_key:
|
|
131
|
+
command.extend(["--api-key", api_key])
|
|
132
|
+
server_log = (LOG_DIR / "qwen-asr.log").open("ab")
|
|
133
|
+
process = subprocess.Popen(command, stdout=server_log, stderr=subprocess.STDOUT, start_new_session=True)
|
|
134
|
+
wait_for_server(api_key, process)
|
|
135
|
+
MODEL_MARKER.write_text(MODEL, encoding="utf-8")
|
|
111
136
|
dns_name, base_url = start_tailscale(auth_key)
|
|
112
|
-
print(json.dumps({"ok": True, "role": "asr", "hostname": dns_name, "base_url": base_url, "model":
|
|
137
|
+
print(json.dumps({"ok": True, "role": "asr", "hostname": dns_name, "base_url": base_url, "model": MODEL, "api_key_set": bool(api_key)}, indent=2))
|
|
113
138
|
|
|
114
139
|
|
|
115
140
|
if __name__ == "__main__":
|
|
@@ -18,6 +18,7 @@ TTS_BACKENDS = {
|
|
|
18
18
|
"moss": {"model": "OpenMOSS-Team/MOSS-TTS-Nano", "sample_rate": 48000, "streaming": False},
|
|
19
19
|
"cosyvoice3": {"model": "FunAudioLLM/Fun-CosyVoice3-0.5B-2512", "sample_rate": 24000, "streaming": True},
|
|
20
20
|
}
|
|
21
|
+
ASR_MODELS = {"Qwen/Qwen3-ASR-0.6B", "Qwen/Qwen3-ASR-1.7B"}
|
|
21
22
|
DEFAULT_COLAB_SETTINGS: dict[str, Any] = {
|
|
22
23
|
"enabled": True,
|
|
23
24
|
"distribution": "Ubuntu-26.04",
|
|
@@ -25,6 +26,7 @@ DEFAULT_COLAB_SETTINGS: dict[str, Any] = {
|
|
|
25
26
|
"profile": "default",
|
|
26
27
|
"asr_session": "ciel-asr",
|
|
27
28
|
"tts_session": "ciel-tts",
|
|
29
|
+
"asr_model": "Qwen/Qwen3-ASR-0.6B",
|
|
28
30
|
"asr_accelerator": "T4",
|
|
29
31
|
"tts_accelerator": "T4",
|
|
30
32
|
"tts_backend": "moss",
|
|
@@ -50,6 +52,7 @@ def configure(
|
|
|
50
52
|
profile: str | None = None,
|
|
51
53
|
asr_session: str | None = None,
|
|
52
54
|
tts_session: str | None = None,
|
|
55
|
+
asr_model: str | None = None,
|
|
53
56
|
asr_accelerator: str | None = None,
|
|
54
57
|
tts_accelerator: str | None = None,
|
|
55
58
|
tts_backend: str | None = None,
|
|
@@ -67,6 +70,7 @@ def configure(
|
|
|
67
70
|
"profile": profile,
|
|
68
71
|
"asr_session": asr_session,
|
|
69
72
|
"tts_session": tts_session,
|
|
73
|
+
"asr_model": asr_model,
|
|
70
74
|
"asr_accelerator": asr_accelerator,
|
|
71
75
|
"tts_accelerator": tts_accelerator,
|
|
72
76
|
"tts_backend": tts_backend,
|
|
@@ -77,8 +81,11 @@ def configure(
|
|
|
77
81
|
if backend not in TTS_BACKENDS:
|
|
78
82
|
raise ValueError(f"unsupported TTS backend: {backend}")
|
|
79
83
|
backend_settings = TTS_BACKENDS[backend]
|
|
84
|
+
selected_asr_model = str(colab.get("asr_model") or "Qwen/Qwen3-ASR-0.6B").strip()
|
|
85
|
+
if selected_asr_model not in ASR_MODELS:
|
|
86
|
+
raise ValueError(f"unsupported ASR model: {selected_asr_model}")
|
|
80
87
|
speech["colab"] = colab
|
|
81
|
-
asr.update({"enabled": True, "base_url": asr_base_url.rstrip("/"), "model":
|
|
88
|
+
asr.update({"enabled": True, "base_url": asr_base_url.rstrip("/"), "model": selected_asr_model})
|
|
82
89
|
tts.update({"enabled": True, "base_url": tts_base_url.rstrip("/"), **backend_settings})
|
|
83
90
|
known_defaults = {DEFAULT_TTS_REFERENCE_AUDIO, DEFAULT_COSYVOICE_REFERENCE_AUDIO, ""}
|
|
84
91
|
current_reference = str(tts.get("ref_audio") or "").strip()
|
|
@@ -101,6 +108,7 @@ def main() -> int:
|
|
|
101
108
|
parser.add_argument("--profile")
|
|
102
109
|
parser.add_argument("--asr-session")
|
|
103
110
|
parser.add_argument("--tts-session")
|
|
111
|
+
parser.add_argument("--asr-model", choices=tuple(sorted(ASR_MODELS)))
|
|
104
112
|
parser.add_argument("--asr-accelerator")
|
|
105
113
|
parser.add_argument("--tts-accelerator")
|
|
106
114
|
parser.add_argument("--tts-backend", choices=tuple(TTS_BACKENDS))
|
|
@@ -120,6 +128,7 @@ def main() -> int:
|
|
|
120
128
|
profile=args.profile,
|
|
121
129
|
asr_session=args.asr_session,
|
|
122
130
|
tts_session=args.tts_session,
|
|
131
|
+
asr_model=args.asr_model,
|
|
123
132
|
asr_accelerator=args.asr_accelerator,
|
|
124
133
|
tts_accelerator=args.tts_accelerator,
|
|
125
134
|
tts_backend=args.tts_backend,
|
|
@@ -7,6 +7,7 @@ param(
|
|
|
7
7
|
[string]$ColabAuth,
|
|
8
8
|
[string]$AsrSession,
|
|
9
9
|
[string]$TtsSession,
|
|
10
|
+
[string]$AsrModel,
|
|
10
11
|
[string]$AsrAccelerator,
|
|
11
12
|
[string]$TtsAccelerator,
|
|
12
13
|
[string]$TtsBackend
|
|
@@ -24,6 +25,8 @@ if ([string]::IsNullOrWhiteSpace($Profile)) { $Profile = [string]$settings.profi
|
|
|
24
25
|
if ([string]::IsNullOrWhiteSpace($Profile)) { $Profile = "default" }
|
|
25
26
|
if ([string]::IsNullOrWhiteSpace($AsrSession)) { $AsrSession = [string]$settings.asr_session }
|
|
26
27
|
if ([string]::IsNullOrWhiteSpace($TtsSession)) { $TtsSession = [string]$settings.tts_session }
|
|
28
|
+
if ([string]::IsNullOrWhiteSpace($AsrModel)) { $AsrModel = [string]$settings.asr_model }
|
|
29
|
+
if ([string]::IsNullOrWhiteSpace($AsrModel)) { $AsrModel = "Qwen/Qwen3-ASR-0.6B" }
|
|
27
30
|
if ([string]::IsNullOrWhiteSpace($AsrAccelerator)) { $AsrAccelerator = [string]$settings.asr_accelerator }
|
|
28
31
|
if ([string]::IsNullOrWhiteSpace($TtsAccelerator)) { $TtsAccelerator = [string]$settings.tts_accelerator }
|
|
29
32
|
if ([string]::IsNullOrWhiteSpace($TtsBackend)) { $TtsBackend = [string]$settings.tts_backend }
|
|
@@ -38,6 +41,7 @@ foreach ($accelerator in @($AsrAccelerator, $TtsAccelerator)) {
|
|
|
38
41
|
if ($accelerator -notin @('T4', 'L4', 'G4', 'A100', 'H100')) { throw "Unsupported Colab accelerator: $accelerator" }
|
|
39
42
|
}
|
|
40
43
|
if ($TtsBackend -notin @('moss', 'cosyvoice3')) { throw "TtsBackend must be moss or cosyvoice3." }
|
|
44
|
+
if ($AsrModel -notin @('Qwen/Qwen3-ASR-0.6B', 'Qwen/Qwen3-ASR-1.7B')) { throw "Unsupported Qwen3-ASR model: $AsrModel" }
|
|
41
45
|
$wslRepo = (& wsl -d $Distribution -- wslpath -a ($repo -replace '\\', '/')).Trim()
|
|
42
46
|
if (-not $wslRepo) { throw "Could not resolve the repository path in WSL." }
|
|
43
47
|
$wslHome = (& wsl -d $Distribution -- bash -lc 'printf %s "$HOME"').Trim()
|
|
@@ -120,6 +124,7 @@ if ($Action -eq 'Start') {
|
|
|
120
124
|
|
|
121
125
|
Write-Host "Installing Qwen3-ASR and its Tailscale service..."
|
|
122
126
|
$asrArguments = @('exec', '--session', $AsrSession)
|
|
127
|
+
$asrArguments += @('--env', "CIEL_ASR_MODEL=$AsrModel")
|
|
123
128
|
if ($env:TAILSCALE_AUTHKEY) { $asrArguments += @('--env', "TAILSCALE_AUTHKEY=$($env:TAILSCALE_AUTHKEY)") }
|
|
124
129
|
if ($env:CIEL_SPEECH_API_KEY) { $asrArguments += @('--env', "CIEL_SPEECH_API_KEY=$($env:CIEL_SPEECH_API_KEY)") }
|
|
125
130
|
$asrArguments += @('--file', "$wslRepo/scripts/colab/bootstrap_qwen_asr.py")
|
|
@@ -145,7 +150,7 @@ function Read-BootstrapResult([string]$Text, [string]$Role) {
|
|
|
145
150
|
|
|
146
151
|
$asr = Read-BootstrapResult $asrOutput "asr"
|
|
147
152
|
$tts = Read-BootstrapResult $ttsOutput "tts"
|
|
148
|
-
& python (Join-Path $PSScriptRoot "configure_speech_workers.py") --asr-base-url $asr.base_url --tts-base-url $tts.base_url --distribution $Distribution --auth $ColabAuth --profile $Profile --asr-session $AsrSession --tts-session $TtsSession --asr-accelerator $AsrAccelerator --tts-accelerator $TtsAccelerator --tts-backend $TtsBackend
|
|
153
|
+
& python (Join-Path $PSScriptRoot "configure_speech_workers.py") --asr-base-url $asr.base_url --tts-base-url $tts.base_url --distribution $Distribution --auth $ColabAuth --profile $Profile --asr-session $AsrSession --tts-session $TtsSession --asr-model $AsrModel --asr-accelerator $AsrAccelerator --tts-accelerator $TtsAccelerator --tts-backend $TtsBackend
|
|
149
154
|
if ($LASTEXITCODE -ne 0) { throw "Workers started, but Ciel speech configuration failed." }
|
|
150
155
|
|
|
151
156
|
Write-Host "Both services are running and connected to Web Chat > Speech Settings."
|