@firstpick/pi-package-webui 0.5.9 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,466 @@
1
+ // Browser voice loop for Natural Conversation Mode (Phase 3).
2
+ // The controller is dependency-injected: speech recognition, speech synthesis,
3
+ // timers, and delivery callbacks all arrive through options so Node tests can
4
+ // drive the full listen -> prompt -> speak -> listen loop without a browser.
5
+ //
6
+ // Turn-taking contract (see docs/webui/natural-conversation/NATURAL_CONVERSATION_MODE_PLAN.md):
7
+ // - Final transcripts while Pi is idle are sent as normal prompts.
8
+ // - Transcripts while the assistant streams final text are interruptions.
9
+ // - Transcripts during a tool call/execution are queued until the tool phase
10
+ // ends; they are never injected into an active tool phase.
11
+ // - While TTS is speaking, the microphone is paused unless barge-in is enabled.
12
+ // - After the assistant asks a question, silence past the timeout emits a
13
+ // single structured silence event instead of guessing user intent.
14
+ // - The browser voice loop can be manually paused/resumed without disabling the
15
+ // package-owned conversation-mode safety constraints.
16
+
17
+ export const VOICE_UI_STATES = [
18
+ "off",
19
+ "listening",
20
+ "transcribing",
21
+ "answering",
22
+ "speaking",
23
+ "interrupting",
24
+ "silence",
25
+ "paused",
26
+ "error",
27
+ ];
28
+
29
+ export const DEFAULT_SILENCE_TIMEOUT_MS = 8000;
30
+
31
+ export function voiceSilenceEventMessage(silentSeconds = Math.round(DEFAULT_SILENCE_TIMEOUT_MS / 1000)) {
32
+ const seconds = Number.isFinite(silentSeconds) && silentSeconds > 0 ? Math.round(silentSeconds) : 8;
33
+ return `[Conversation mode: the user stayed silent for ${seconds}s after your question. Treat the silence as possible confusion, discomfort, missing context, or an unneeded question; reframe, explain why you asked, or continue without pressuring the user. Do not invent intent from the silence.]`;
34
+ }
35
+
36
+ export function createVoiceConversationController(options = {}) {
37
+ const {
38
+ recognitionFactory = null,
39
+ synthesis = null,
40
+ utteranceFactory = null,
41
+ sendTranscript = async () => {},
42
+ sendSilenceEvent = async () => {},
43
+ onPartialTranscript = () => {},
44
+ onStateChange = () => {},
45
+ onEvent = () => {},
46
+ bargeInEnabled = false,
47
+ silenceTimeoutMs = DEFAULT_SILENCE_TIMEOUT_MS,
48
+ setTimeoutFn = (handler, delayMs) => setTimeout(handler, delayMs),
49
+ clearTimeoutFn = (id) => clearTimeout(id),
50
+ } = options;
51
+
52
+ const state = {
53
+ active: false,
54
+ uiState: "off",
55
+ sttAvailable: null,
56
+ ttsAvailable: !!(synthesis && typeof utteranceFactory === "function"),
57
+ assistantStreaming: false,
58
+ toolRunning: false,
59
+ speaking: false,
60
+ userPaused: false,
61
+ partialTranscript: "",
62
+ queuedInterrupt: "",
63
+ lastError: "",
64
+ };
65
+
66
+ let recognition = null;
67
+ let recognitionRunning = false;
68
+ let recognitionPausedForSpeech = false;
69
+ let silenceTimer = null;
70
+ let currentUtterance = null;
71
+
72
+ function snapshot() {
73
+ return { ...state };
74
+ }
75
+
76
+ function setUiState(uiState, detail = {}) {
77
+ state.uiState = uiState;
78
+ try {
79
+ onStateChange(uiState, { ...snapshot(), ...detail });
80
+ } catch {
81
+ // State listeners must never break the voice loop.
82
+ }
83
+ }
84
+
85
+ function emitEvent(message, level = "info") {
86
+ try {
87
+ onEvent(message, level);
88
+ } catch {
89
+ // Log listeners must never break the voice loop.
90
+ }
91
+ }
92
+
93
+ function setPartialTranscript(text = "") {
94
+ const partialTranscript = String(text || "").trim();
95
+ if (state.partialTranscript === partialTranscript) return;
96
+ state.partialTranscript = partialTranscript;
97
+ try {
98
+ onPartialTranscript(partialTranscript, snapshot());
99
+ } catch {
100
+ // Partial transcript listeners must never break the voice loop.
101
+ }
102
+ }
103
+
104
+ function clearPartialTranscript() {
105
+ setPartialTranscript("");
106
+ }
107
+
108
+ function clearSilenceTimer() {
109
+ if (silenceTimer === null) return;
110
+ clearTimeoutFn(silenceTimer);
111
+ silenceTimer = null;
112
+ }
113
+
114
+ function startSilenceTimer() {
115
+ clearSilenceTimer();
116
+ if (!state.active || !silenceTimeoutMs || silenceTimeoutMs <= 0) return;
117
+ silenceTimer = setTimeoutFn(() => {
118
+ silenceTimer = null;
119
+ if (!state.active || state.speaking || state.assistantStreaming) return;
120
+ if (state.uiState !== "listening" && state.uiState !== "transcribing") return;
121
+ setUiState("silence");
122
+ const silentSeconds = Math.round(silenceTimeoutMs / 1000);
123
+ Promise.resolve(sendSilenceEvent({ silentSeconds }))
124
+ .then(() => {
125
+ if (state.active && state.uiState === "silence") setUiState("answering", { reason: "silence" });
126
+ })
127
+ .catch((error) => {
128
+ emitEvent(`Failed to send voice silence event: ${error?.message || error}`, "warn");
129
+ if (state.active && state.uiState === "silence") setUiState("listening");
130
+ });
131
+ }, silenceTimeoutMs);
132
+ }
133
+
134
+ function ensureRecognition() {
135
+ if (recognition) return recognition;
136
+ if (typeof recognitionFactory !== "function") {
137
+ state.sttAvailable = false;
138
+ return null;
139
+ }
140
+ let created = null;
141
+ try {
142
+ created = recognitionFactory();
143
+ } catch {
144
+ created = null;
145
+ }
146
+ if (!created) {
147
+ state.sttAvailable = false;
148
+ return null;
149
+ }
150
+ recognition = created;
151
+ try {
152
+ recognition.continuous = true;
153
+ recognition.interimResults = true;
154
+ } catch {
155
+ // Some engines expose read-only configuration; keep their defaults.
156
+ }
157
+ recognition.onresult = handleRecognitionResult;
158
+ recognition.onerror = handleRecognitionError;
159
+ recognition.onend = handleRecognitionEnd;
160
+ return recognition;
161
+ }
162
+
163
+ function startRecognition() {
164
+ const instance = ensureRecognition();
165
+ if (!instance) return false;
166
+ recognitionPausedForSpeech = false;
167
+ if (recognitionRunning) return true;
168
+ try {
169
+ instance.start();
170
+ recognitionRunning = true;
171
+ } catch {
172
+ // start() throws InvalidStateError when already started; treat as running.
173
+ recognitionRunning = true;
174
+ }
175
+ state.sttAvailable = true;
176
+ return true;
177
+ }
178
+
179
+ function stopRecognition({ forSpeech = false } = {}) {
180
+ recognitionPausedForSpeech = forSpeech;
181
+ if (!recognition || !recognitionRunning) return;
182
+ recognitionRunning = false;
183
+ try {
184
+ if (typeof recognition.abort === "function") recognition.abort();
185
+ else recognition.stop();
186
+ } catch {
187
+ // Recognition may already be stopped by the engine.
188
+ }
189
+ }
190
+
191
+ function handleRecognitionEnd() {
192
+ recognitionRunning = false;
193
+ if (!state.active || state.userPaused || recognitionPausedForSpeech || state.uiState === "error") return;
194
+ // Browsers stop recognition spontaneously after silence; keep listening.
195
+ try {
196
+ recognition.start();
197
+ recognitionRunning = true;
198
+ } catch {
199
+ // Engine refused an immediate restart; the next turn end restarts it.
200
+ }
201
+ }
202
+
203
+ function handleRecognitionError(event) {
204
+ const code = String(event?.error || "");
205
+ if (code === "not-allowed" || code === "service-not-allowed") {
206
+ state.lastError = code;
207
+ // Enter the error state before stopping so onend does not auto-restart.
208
+ setUiState("error", { reason: code });
209
+ stopRecognition();
210
+ emitEvent("Microphone access was denied, so Natural Conversation Mode cannot listen. Allow microphone access for this site, then end and restart the conversation.", "error");
211
+ return;
212
+ }
213
+ if (code === "no-speech" || code === "aborted") return;
214
+ emitEvent(`Speech recognition error: ${code || "unknown"}`, "warn");
215
+ }
216
+
217
+ function handleRecognitionResult(event) {
218
+ if (!state.active) return;
219
+ const results = event?.results || [];
220
+ let finalText = "";
221
+ let interimText = "";
222
+ for (let index = event?.resultIndex || 0; index < results.length; index += 1) {
223
+ const result = results[index];
224
+ if (!result) continue;
225
+ const alternative = result[0];
226
+ const text = alternative && typeof alternative.transcript === "string" ? alternative.transcript : "";
227
+ if (result.isFinal) finalText += text;
228
+ else if (text.trim()) interimText += text;
229
+ }
230
+ if (finalText.trim()) {
231
+ clearPartialTranscript();
232
+ handleFinalTranscript(finalText.trim());
233
+ return;
234
+ }
235
+ if (interimText.trim()) {
236
+ setPartialTranscript(interimText.trim());
237
+ if (state.uiState === "listening") setUiState("transcribing");
238
+ }
239
+ }
240
+
241
+ function deliverTranscript(text, reason) {
242
+ Promise.resolve(sendTranscript(text, { reason }))
243
+ .then(() => {
244
+ if (state.active && state.uiState !== "error" && state.uiState !== "off") setUiState("answering", { reason });
245
+ })
246
+ .catch((error) => {
247
+ emitEvent(`Failed to send voice transcript: ${error?.message || error}`, "error");
248
+ if (state.active && state.uiState !== "error") setUiState("listening");
249
+ });
250
+ }
251
+
252
+ function handleFinalTranscript(text) {
253
+ const transcript = String(text || "").trim();
254
+ if (!transcript || !state.active || state.userPaused) return;
255
+ clearPartialTranscript();
256
+ clearSilenceTimer();
257
+ if (state.speaking) {
258
+ // Barge-in: only reachable when bargeInEnabled keeps the mic open during TTS.
259
+ stopSpeaking();
260
+ setUiState("interrupting", { transcript });
261
+ deliverTranscript(transcript, "interrupt");
262
+ return;
263
+ }
264
+ if (state.assistantStreaming && state.toolRunning) {
265
+ state.queuedInterrupt = state.queuedInterrupt ? `${state.queuedInterrupt} ${transcript}` : transcript;
266
+ setUiState("interrupting", { queued: true, transcript });
267
+ emitEvent("Voice interruption queued; it will be delivered after the current tool phase ends.", "info");
268
+ return;
269
+ }
270
+ if (state.assistantStreaming) {
271
+ setUiState("interrupting", { transcript });
272
+ deliverTranscript(transcript, "interrupt");
273
+ return;
274
+ }
275
+ setUiState("answering", { transcript });
276
+ deliverTranscript(transcript, "prompt");
277
+ }
278
+
279
+ function flushQueuedInterrupt() {
280
+ const text = state.queuedInterrupt;
281
+ state.queuedInterrupt = "";
282
+ if (!text || !state.active) return;
283
+ setUiState("interrupting", { queued: true });
284
+ deliverTranscript(text, state.assistantStreaming ? "interrupt" : "prompt");
285
+ }
286
+
287
+ function setAssistantActivity(patch = {}) {
288
+ if (!state.active) return snapshot();
289
+ const wasToolRunning = state.toolRunning;
290
+ const wasStreaming = state.assistantStreaming;
291
+ if (typeof patch.streaming === "boolean") state.assistantStreaming = patch.streaming;
292
+ if (typeof patch.toolRunning === "boolean") state.toolRunning = patch.toolRunning;
293
+ if (state.assistantStreaming && !wasStreaming) {
294
+ clearSilenceTimer();
295
+ if (!state.userPaused && !state.speaking && state.uiState !== "interrupting") setUiState("answering");
296
+ }
297
+ if (wasToolRunning && !state.toolRunning && state.queuedInterrupt) flushQueuedInterrupt();
298
+ else if (wasStreaming && !state.assistantStreaming && state.queuedInterrupt) flushQueuedInterrupt();
299
+ return snapshot();
300
+ }
301
+
302
+ function stopSpeaking() {
303
+ const utterance = currentUtterance;
304
+ currentUtterance = null;
305
+ state.speaking = false;
306
+ recognitionPausedForSpeech = false;
307
+ if (utterance) {
308
+ utterance.onend = null;
309
+ utterance.onerror = null;
310
+ }
311
+ try {
312
+ synthesis?.cancel?.();
313
+ } catch {
314
+ // Synthesis may already be idle.
315
+ }
316
+ }
317
+
318
+ function resumeListeningAfterTurn({ askedQuestion = false } = {}) {
319
+ if (!state.active) return;
320
+ if (state.userPaused) {
321
+ setUiState("paused", { reason: "manual" });
322
+ return;
323
+ }
324
+ const listening = startRecognition();
325
+ if (listening) setUiState("listening");
326
+ else if (state.uiState !== "error") setUiState("paused", { reason: "stt-unavailable" });
327
+ if (askedQuestion && listening) startSilenceTimer();
328
+ }
329
+
330
+ function speak(text, { askedQuestion = false } = {}) {
331
+ const spoken = String(text || "").trim();
332
+ if (state.userPaused) return;
333
+ if (!spoken || !state.ttsAvailable) {
334
+ resumeListeningAfterTurn({ askedQuestion: askedQuestion && !!spoken });
335
+ return;
336
+ }
337
+ stopSpeaking();
338
+ if (!bargeInEnabled) stopRecognition({ forSpeech: true });
339
+ let utterance = null;
340
+ try {
341
+ utterance = utteranceFactory(spoken);
342
+ } catch {
343
+ utterance = null;
344
+ }
345
+ if (!utterance) {
346
+ resumeListeningAfterTurn({ askedQuestion });
347
+ return;
348
+ }
349
+ state.speaking = true;
350
+ setUiState("speaking");
351
+ currentUtterance = utterance;
352
+ const finish = () => {
353
+ if (currentUtterance !== utterance) return;
354
+ currentUtterance = null;
355
+ state.speaking = false;
356
+ recognitionPausedForSpeech = false;
357
+ if (!state.active) return;
358
+ resumeListeningAfterTurn({ askedQuestion });
359
+ };
360
+ utterance.onend = finish;
361
+ utterance.onerror = finish;
362
+ try {
363
+ synthesis.speak(utterance);
364
+ } catch (error) {
365
+ emitEvent(`Speech synthesis failed: ${error?.message || error}`, "warn");
366
+ finish();
367
+ }
368
+ }
369
+
370
+ function handleAssistantTurnEnd(finalText, { askedQuestion = false } = {}) {
371
+ if (!state.active) return snapshot();
372
+ state.assistantStreaming = false;
373
+ state.toolRunning = false;
374
+ if (state.userPaused) return snapshot();
375
+ if (state.queuedInterrupt) {
376
+ flushQueuedInterrupt();
377
+ return snapshot();
378
+ }
379
+ speak(finalText, { askedQuestion });
380
+ return snapshot();
381
+ }
382
+
383
+ function handleUserActivity() {
384
+ clearSilenceTimer();
385
+ }
386
+
387
+ function start() {
388
+ if (state.active) return snapshot();
389
+ state.active = true;
390
+ state.userPaused = false;
391
+ state.queuedInterrupt = "";
392
+ state.assistantStreaming = false;
393
+ state.toolRunning = false;
394
+ state.lastError = "";
395
+ const listening = startRecognition();
396
+ if (listening) {
397
+ setUiState("listening");
398
+ } else if (state.ttsAvailable) {
399
+ setUiState("paused", { reason: "stt-unavailable" });
400
+ emitEvent("Browser speech recognition is unavailable here; Pi answers will still be spoken, but the microphone stays off. Type prompts normally or configure another STT provider in a later phase.", "warn");
401
+ } else {
402
+ setUiState("error", { reason: "speech-unsupported" });
403
+ emitEvent("This browser exposes neither speech recognition nor speech synthesis, so the Natural Conversation voice loop cannot run.", "error");
404
+ }
405
+ return snapshot();
406
+ }
407
+
408
+ function pause({ reason = "manual" } = {}) {
409
+ if (!state.active) return snapshot();
410
+ state.userPaused = true;
411
+ clearSilenceTimer();
412
+ clearPartialTranscript();
413
+ stopSpeaking();
414
+ stopRecognition({ forSpeech: true });
415
+ setUiState("paused", { reason });
416
+ return snapshot();
417
+ }
418
+
419
+ function resume() {
420
+ if (!state.active) return snapshot();
421
+ state.userPaused = false;
422
+ recognitionPausedForSpeech = false;
423
+ if (state.assistantStreaming) {
424
+ if (!state.toolRunning) startRecognition();
425
+ setUiState("answering", { reason: "manual-resume" });
426
+ return snapshot();
427
+ }
428
+ resumeListeningAfterTurn();
429
+ return snapshot();
430
+ }
431
+
432
+ function togglePaused() {
433
+ return state.userPaused ? resume() : pause();
434
+ }
435
+
436
+ function stop() {
437
+ if (!state.active) {
438
+ state.uiState = "off";
439
+ return snapshot();
440
+ }
441
+ state.active = false;
442
+ state.userPaused = false;
443
+ clearSilenceTimer();
444
+ clearPartialTranscript();
445
+ stopSpeaking();
446
+ stopRecognition();
447
+ state.assistantStreaming = false;
448
+ state.toolRunning = false;
449
+ state.queuedInterrupt = "";
450
+ setUiState("off");
451
+ return snapshot();
452
+ }
453
+
454
+ return {
455
+ start,
456
+ stop,
457
+ pause,
458
+ resume,
459
+ togglePaused,
460
+ getState: snapshot,
461
+ isActive: () => state.active,
462
+ setAssistantActivity,
463
+ handleAssistantTurnEnd,
464
+ handleUserActivity,
465
+ };
466
+ }
File without changes
@@ -1,8 +1,23 @@
1
1
  #!/usr/bin/env node
2
2
  // Minimal JSONL RPC stub standing in for the pi coding agent so HTTP endpoint
3
3
  // tests can boot the real pi-webui server without a model provider.
4
+ //
5
+ // Optional env-gated features (default off; without them the voice scripting
6
+ // and logging additions are inert and command responses are unchanged, except
7
+ // that "steer" now gets an explicit success payload instead of the generic
8
+ // default-case response — no test asserts on either shape):
9
+ // - FAKE_PI_LOG_FILE=<path>: append one JSON line per received RPC command
10
+ // ({direction:"command", type, message}) plus one per scripted event emitted
11
+ // ({direction:"event", type}), so browser-driver tests can assert ordering.
12
+ // - FAKE_PI_VOICE_SCRIPTS=1: prompts containing "voice test say|question|tool|slow"
13
+ // respond success and then asynchronously emit a scripted agent event flow
14
+ // (agent_start/message_*/tool_execution_*/agent_end) over stdout; the scripted
15
+ // assistant turns are appended to a dynamic transcript returned by
16
+ // get_messages AFTER the three baseline messages, and get_state reports
17
+ // isStreaming=true while a scripted flow runs.
4
18
  import { createInterface } from "node:readline";
5
19
  import { randomUUID } from "node:crypto";
20
+ import { appendFileSync } from "node:fs";
6
21
 
7
22
  const sessionIndex = process.argv.indexOf("--session");
8
23
  const sessionFile = sessionIndex !== -1 ? process.argv[sessionIndex + 1] : undefined;
@@ -12,6 +27,20 @@ let peakBash = 0;
12
27
  let thinkingLevel = "off";
13
28
  let conversationEnabled = false;
14
29
 
30
+ const voiceScriptsEnabled = process.env.FAKE_PI_VOICE_SCRIPTS === "1";
31
+ const commandLogFile = process.env.FAKE_PI_LOG_FILE || "";
32
+ const dynamicMessages = [];
33
+ let scriptedStreaming = false;
34
+
35
+ function logJsonLine(entry) {
36
+ if (!commandLogFile) return;
37
+ try {
38
+ appendFileSync(commandLogFile, `${JSON.stringify({ at: Date.now(), ...entry })}\n`);
39
+ } catch {
40
+ // Logging must never break the fixture.
41
+ }
42
+ }
43
+
15
44
  function respond(payload) {
16
45
  process.stdout.write(`${JSON.stringify(payload)}\n`);
17
46
  }
@@ -20,6 +49,85 @@ function emitEvent(payload) {
20
49
  process.stdout.write(`${JSON.stringify(payload)}\n`);
21
50
  }
22
51
 
52
+ function emitScriptedEvent(payload) {
53
+ logJsonLine({ direction: "event", type: payload.type, ...(payload.toolName ? { toolName: payload.toolName } : {}) });
54
+ emitEvent(payload);
55
+ }
56
+
57
+ const scriptedTimers = new Set();
58
+
59
+ function runScriptedSteps(steps) {
60
+ let at = 0;
61
+ for (const step of steps) {
62
+ at += step.afterMs ?? 0;
63
+ const timer = setTimeout(() => {
64
+ scriptedTimers.delete(timer);
65
+ step.run();
66
+ }, at);
67
+ scriptedTimers.add(timer);
68
+ }
69
+ }
70
+
71
+ function cancelScriptedSteps() {
72
+ for (const timer of scriptedTimers) clearTimeout(timer);
73
+ scriptedTimers.clear();
74
+ }
75
+
76
+ // Emits a full scripted assistant turn: agent_start, optional tool phase,
77
+ // streamed assistant text, message_end, agent_end. The final assistant text is
78
+ // appended to the dynamic transcript so get_messages returns it afterwards.
79
+ function runVoiceScriptFlow({ text, chunks, chunkSpacingMs = 60, toolPhase = false, toolDurationMs = 1500 }) {
80
+ scriptedStreaming = true;
81
+ const steps = [{ afterMs: 30, run: () => emitScriptedEvent({ type: "agent_start" }) }];
82
+ if (toolPhase) {
83
+ const toolCallId = `voice-tool-${randomUUID()}`;
84
+ steps.push({ afterMs: 50, run: () => emitScriptedEvent({ type: "tool_execution_start", toolCallId, toolName: "read", args: { path: "README.md" } }) });
85
+ steps.push({ afterMs: toolDurationMs, run: () => emitScriptedEvent({ type: "tool_execution_end", toolCallId, toolName: "read", isError: false, result: { content: [{ type: "text", text: "fake tool output" }] } }) });
86
+ }
87
+ steps.push({ afterMs: toolPhase ? 120 : 50, run: () => emitScriptedEvent({ type: "message_start", message: { role: "assistant" } }) });
88
+ for (const delta of chunks || [text.slice(0, Math.ceil(text.length / 2)), text.slice(Math.ceil(text.length / 2))]) {
89
+ steps.push({ afterMs: chunkSpacingMs, run: () => emitScriptedEvent({ type: "message_update", assistantMessageEvent: { type: "text_delta", delta } }) });
90
+ }
91
+ steps.push({
92
+ afterMs: chunkSpacingMs,
93
+ run: () => {
94
+ dynamicMessages.push({ role: "assistant", content: [{ type: "text", text }], timestamp: Date.now() });
95
+ emitScriptedEvent({ type: "message_end", message: { role: "assistant", content: [{ type: "text", text }] } });
96
+ },
97
+ });
98
+ steps.push({
99
+ afterMs: 60,
100
+ run: () => {
101
+ scriptedStreaming = false;
102
+ emitScriptedEvent({ type: "agent_end" });
103
+ },
104
+ });
105
+ runScriptedSteps(steps);
106
+ }
107
+
108
+ function handleVoiceScriptPrompt(command, base) {
109
+ if (!voiceScriptsEnabled) return false;
110
+ const message = String(command.message || "");
111
+ const marker = ["voice test say", "voice test question", "voice test tool", "voice test slow"].find((item) => message.includes(item));
112
+ if (!marker) return false;
113
+ dynamicMessages.push({ role: "user", content: message, timestamp: Date.now() });
114
+ respond({ ...base, data: { output: "voice scripted prompt accepted" } });
115
+ if (marker === "voice test say") {
116
+ runVoiceScriptFlow({ text: "Okay, this is the spoken answer." });
117
+ } else if (marker === "voice test question") {
118
+ runVoiceScriptFlow({ text: "Should we proceed with the next step?" });
119
+ } else if (marker === "voice test tool") {
120
+ runVoiceScriptFlow({ text: "The tool has finished.", toolPhase: true });
121
+ } else {
122
+ runVoiceScriptFlow({
123
+ text: "This is a long streaming answer that keeps going.",
124
+ chunks: ["This is a long ", "streaming answer ", "that keeps ", "going", "."],
125
+ chunkSpacingMs: 500,
126
+ });
127
+ }
128
+ return true;
129
+ }
130
+
23
131
  function conversationStatusText() {
24
132
  return conversationEnabled ? "Voice: listening" : "";
25
133
  }
@@ -34,6 +142,38 @@ function emitConversationStatus() {
34
142
  });
35
143
  }
36
144
 
145
+ function handleWebuiHelperPrompt(command, base) {
146
+ const message = String(command.message || "").trim();
147
+ const match = message.match(/^\/webui-helper\s+(\{[\s\S]*\})$/);
148
+ if (!match) return false;
149
+ let request = {};
150
+ try {
151
+ request = JSON.parse(match[1]);
152
+ } catch {
153
+ request = {};
154
+ }
155
+ const requestId = String(request.requestId || "");
156
+ const respondHelper = (payload) => emitEvent({
157
+ type: "extension_ui_request",
158
+ id: randomUUID(),
159
+ method: "notify",
160
+ message: `__PI_WEBUI_HELPER_RESPONSE__:${JSON.stringify(payload)}`,
161
+ notifyType: payload.ok === false ? "error" : "info",
162
+ });
163
+ respond({ ...base, data: { output: "webui-helper handled" } });
164
+ switch (request.action) {
165
+ case "tools-state":
166
+ respondHelper({ requestId, ok: true, data: { tools: [], activeTools: [] } });
167
+ return true;
168
+ case "skills-state":
169
+ respondHelper({ requestId, ok: true, data: { skills: [] } });
170
+ return true;
171
+ default:
172
+ respondHelper({ requestId, ok: false, error: `Unknown webui-helper action: ${String(request.action || "")}` });
173
+ return true;
174
+ }
175
+ }
176
+
37
177
  function handleTalkPrompt(command, base) {
38
178
  const message = String(command.message || "").trim();
39
179
  const match = message.match(/^\/(talk|voice|conversation)(?:\s+(\S+))?/i);
@@ -69,6 +209,7 @@ rl.on("line", (line) => {
69
209
  }
70
210
  const { id, type } = command || {};
71
211
  if (!id || !type) return;
212
+ logJsonLine({ direction: "command", type, ...(command.message !== undefined ? { message: String(command.message) } : {}) });
72
213
  const base = { type: "response", id, command: type, success: true };
73
214
 
74
215
  switch (type) {
@@ -78,7 +219,7 @@ rl.on("line", (line) => {
78
219
  data: {
79
220
  model: { provider: "fake", id: "fake-model" },
80
221
  thinkingLevel,
81
- isStreaming: false,
222
+ isStreaming: scriptedStreaming,
82
223
  isCompacting: false,
83
224
  steeringMode: "one-at-a-time",
84
225
  followUpMode: "one-at-a-time",
@@ -99,6 +240,7 @@ rl.on("line", (line) => {
99
240
  { role: "user", content: "fake prompt", timestamp: 1000 },
100
241
  { role: "assistant", content: [{ type: "text", text: "fake answer" }], timestamp: 2000 },
101
242
  { role: "user", content: "fake follow-up", timestamp: 3000 },
243
+ ...dynamicMessages,
102
244
  ],
103
245
  },
104
246
  });
@@ -116,9 +258,14 @@ rl.on("line", (line) => {
116
258
  });
117
259
  return;
118
260
  case "prompt":
261
+ if (handleWebuiHelperPrompt(command, base)) return;
119
262
  if (handleTalkPrompt(command, base)) return;
263
+ if (handleVoiceScriptPrompt(command, base)) return;
120
264
  respond({ ...base, data: { output: "fake prompt accepted" } });
121
265
  return;
266
+ case "steer":
267
+ respond({ ...base, data: { output: "fake steer accepted" } });
268
+ return;
122
269
  case "set_thinking_level":
123
270
  thinkingLevel = String(command.level || "off");
124
271
  respond({ ...base, data: { level: thinkingLevel } });
@@ -145,3 +292,9 @@ rl.on("line", (line) => {
145
292
  respond({ ...base, data: {} });
146
293
  }
147
294
  });
295
+
296
+ rl.on("close", () => {
297
+ // The parent (pi-webui or a test driver) closed stdin or died; never let
298
+ // pending scripted voice events outlive it and write into a broken pipe.
299
+ cancelScriptedSteps();
300
+ });