@firstpick/pi-package-webui 0.5.9 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -1
- package/bin/pi-webui.mjs +333 -10
- package/lib/git-worktrees.mjs +5 -1
- package/package.json +2 -2
- package/public/app.js +568 -5
- package/public/index.html +10 -2
- package/public/service-worker.js +2 -1
- package/public/styles.css +158 -4
- package/public/voice-conversation.mjs +466 -0
- package/tests/fixtures/compacting-pi.mjs +0 -0
- package/tests/fixtures/fake-pi.mjs +154 -1
- package/tests/http-endpoints-harness.test.mjs +76 -3
- package/tests/mobile-static.test.mjs +54 -5
- package/tests/voice-conversation.test.mjs +361 -0
|
@@ -0,0 +1,466 @@
|
|
|
1
|
+
// Browser voice loop for Natural Conversation Mode (Phase 3).
|
|
2
|
+
// The controller is dependency-injected: speech recognition, speech synthesis,
|
|
3
|
+
// timers, and delivery callbacks all arrive through options so Node tests can
|
|
4
|
+
// drive the full listen -> prompt -> speak -> listen loop without a browser.
|
|
5
|
+
//
|
|
6
|
+
// Turn-taking contract (see docs/webui/natural-conversation/NATURAL_CONVERSATION_MODE_PLAN.md):
|
|
7
|
+
// - Final transcripts while Pi is idle are sent as normal prompts.
|
|
8
|
+
// - Transcripts while the assistant streams final text are interruptions.
|
|
9
|
+
// - Transcripts during a tool call/execution are queued until the tool phase
|
|
10
|
+
// ends; they are never injected into an active tool phase.
|
|
11
|
+
// - While TTS is speaking, the microphone is paused unless barge-in is enabled.
|
|
12
|
+
// - After the assistant asks a question, silence past the timeout emits a
|
|
13
|
+
// single structured silence event instead of guessing user intent.
|
|
14
|
+
// - The browser voice loop can be manually paused/resumed without disabling the
|
|
15
|
+
// package-owned conversation-mode safety constraints.
|
|
16
|
+
|
|
17
|
+
export const VOICE_UI_STATES = [
|
|
18
|
+
"off",
|
|
19
|
+
"listening",
|
|
20
|
+
"transcribing",
|
|
21
|
+
"answering",
|
|
22
|
+
"speaking",
|
|
23
|
+
"interrupting",
|
|
24
|
+
"silence",
|
|
25
|
+
"paused",
|
|
26
|
+
"error",
|
|
27
|
+
];
|
|
28
|
+
|
|
29
|
+
export const DEFAULT_SILENCE_TIMEOUT_MS = 8000;
|
|
30
|
+
|
|
31
|
+
export function voiceSilenceEventMessage(silentSeconds = Math.round(DEFAULT_SILENCE_TIMEOUT_MS / 1000)) {
|
|
32
|
+
const seconds = Number.isFinite(silentSeconds) && silentSeconds > 0 ? Math.round(silentSeconds) : 8;
|
|
33
|
+
return `[Conversation mode: the user stayed silent for ${seconds}s after your question. Treat the silence as possible confusion, discomfort, missing context, or an unneeded question; reframe, explain why you asked, or continue without pressuring the user. Do not invent intent from the silence.]`;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export function createVoiceConversationController(options = {}) {
|
|
37
|
+
const {
|
|
38
|
+
recognitionFactory = null,
|
|
39
|
+
synthesis = null,
|
|
40
|
+
utteranceFactory = null,
|
|
41
|
+
sendTranscript = async () => {},
|
|
42
|
+
sendSilenceEvent = async () => {},
|
|
43
|
+
onPartialTranscript = () => {},
|
|
44
|
+
onStateChange = () => {},
|
|
45
|
+
onEvent = () => {},
|
|
46
|
+
bargeInEnabled = false,
|
|
47
|
+
silenceTimeoutMs = DEFAULT_SILENCE_TIMEOUT_MS,
|
|
48
|
+
setTimeoutFn = (handler, delayMs) => setTimeout(handler, delayMs),
|
|
49
|
+
clearTimeoutFn = (id) => clearTimeout(id),
|
|
50
|
+
} = options;
|
|
51
|
+
|
|
52
|
+
const state = {
|
|
53
|
+
active: false,
|
|
54
|
+
uiState: "off",
|
|
55
|
+
sttAvailable: null,
|
|
56
|
+
ttsAvailable: !!(synthesis && typeof utteranceFactory === "function"),
|
|
57
|
+
assistantStreaming: false,
|
|
58
|
+
toolRunning: false,
|
|
59
|
+
speaking: false,
|
|
60
|
+
userPaused: false,
|
|
61
|
+
partialTranscript: "",
|
|
62
|
+
queuedInterrupt: "",
|
|
63
|
+
lastError: "",
|
|
64
|
+
};
|
|
65
|
+
|
|
66
|
+
let recognition = null;
|
|
67
|
+
let recognitionRunning = false;
|
|
68
|
+
let recognitionPausedForSpeech = false;
|
|
69
|
+
let silenceTimer = null;
|
|
70
|
+
let currentUtterance = null;
|
|
71
|
+
|
|
72
|
+
function snapshot() {
|
|
73
|
+
return { ...state };
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
function setUiState(uiState, detail = {}) {
|
|
77
|
+
state.uiState = uiState;
|
|
78
|
+
try {
|
|
79
|
+
onStateChange(uiState, { ...snapshot(), ...detail });
|
|
80
|
+
} catch {
|
|
81
|
+
// State listeners must never break the voice loop.
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function emitEvent(message, level = "info") {
|
|
86
|
+
try {
|
|
87
|
+
onEvent(message, level);
|
|
88
|
+
} catch {
|
|
89
|
+
// Log listeners must never break the voice loop.
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function setPartialTranscript(text = "") {
|
|
94
|
+
const partialTranscript = String(text || "").trim();
|
|
95
|
+
if (state.partialTranscript === partialTranscript) return;
|
|
96
|
+
state.partialTranscript = partialTranscript;
|
|
97
|
+
try {
|
|
98
|
+
onPartialTranscript(partialTranscript, snapshot());
|
|
99
|
+
} catch {
|
|
100
|
+
// Partial transcript listeners must never break the voice loop.
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
function clearPartialTranscript() {
|
|
105
|
+
setPartialTranscript("");
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
function clearSilenceTimer() {
|
|
109
|
+
if (silenceTimer === null) return;
|
|
110
|
+
clearTimeoutFn(silenceTimer);
|
|
111
|
+
silenceTimer = null;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
function startSilenceTimer() {
|
|
115
|
+
clearSilenceTimer();
|
|
116
|
+
if (!state.active || !silenceTimeoutMs || silenceTimeoutMs <= 0) return;
|
|
117
|
+
silenceTimer = setTimeoutFn(() => {
|
|
118
|
+
silenceTimer = null;
|
|
119
|
+
if (!state.active || state.speaking || state.assistantStreaming) return;
|
|
120
|
+
if (state.uiState !== "listening" && state.uiState !== "transcribing") return;
|
|
121
|
+
setUiState("silence");
|
|
122
|
+
const silentSeconds = Math.round(silenceTimeoutMs / 1000);
|
|
123
|
+
Promise.resolve(sendSilenceEvent({ silentSeconds }))
|
|
124
|
+
.then(() => {
|
|
125
|
+
if (state.active && state.uiState === "silence") setUiState("answering", { reason: "silence" });
|
|
126
|
+
})
|
|
127
|
+
.catch((error) => {
|
|
128
|
+
emitEvent(`Failed to send voice silence event: ${error?.message || error}`, "warn");
|
|
129
|
+
if (state.active && state.uiState === "silence") setUiState("listening");
|
|
130
|
+
});
|
|
131
|
+
}, silenceTimeoutMs);
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
function ensureRecognition() {
|
|
135
|
+
if (recognition) return recognition;
|
|
136
|
+
if (typeof recognitionFactory !== "function") {
|
|
137
|
+
state.sttAvailable = false;
|
|
138
|
+
return null;
|
|
139
|
+
}
|
|
140
|
+
let created = null;
|
|
141
|
+
try {
|
|
142
|
+
created = recognitionFactory();
|
|
143
|
+
} catch {
|
|
144
|
+
created = null;
|
|
145
|
+
}
|
|
146
|
+
if (!created) {
|
|
147
|
+
state.sttAvailable = false;
|
|
148
|
+
return null;
|
|
149
|
+
}
|
|
150
|
+
recognition = created;
|
|
151
|
+
try {
|
|
152
|
+
recognition.continuous = true;
|
|
153
|
+
recognition.interimResults = true;
|
|
154
|
+
} catch {
|
|
155
|
+
// Some engines expose read-only configuration; keep their defaults.
|
|
156
|
+
}
|
|
157
|
+
recognition.onresult = handleRecognitionResult;
|
|
158
|
+
recognition.onerror = handleRecognitionError;
|
|
159
|
+
recognition.onend = handleRecognitionEnd;
|
|
160
|
+
return recognition;
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
function startRecognition() {
|
|
164
|
+
const instance = ensureRecognition();
|
|
165
|
+
if (!instance) return false;
|
|
166
|
+
recognitionPausedForSpeech = false;
|
|
167
|
+
if (recognitionRunning) return true;
|
|
168
|
+
try {
|
|
169
|
+
instance.start();
|
|
170
|
+
recognitionRunning = true;
|
|
171
|
+
} catch {
|
|
172
|
+
// start() throws InvalidStateError when already started; treat as running.
|
|
173
|
+
recognitionRunning = true;
|
|
174
|
+
}
|
|
175
|
+
state.sttAvailable = true;
|
|
176
|
+
return true;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
function stopRecognition({ forSpeech = false } = {}) {
|
|
180
|
+
recognitionPausedForSpeech = forSpeech;
|
|
181
|
+
if (!recognition || !recognitionRunning) return;
|
|
182
|
+
recognitionRunning = false;
|
|
183
|
+
try {
|
|
184
|
+
if (typeof recognition.abort === "function") recognition.abort();
|
|
185
|
+
else recognition.stop();
|
|
186
|
+
} catch {
|
|
187
|
+
// Recognition may already be stopped by the engine.
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
function handleRecognitionEnd() {
|
|
192
|
+
recognitionRunning = false;
|
|
193
|
+
if (!state.active || state.userPaused || recognitionPausedForSpeech || state.uiState === "error") return;
|
|
194
|
+
// Browsers stop recognition spontaneously after silence; keep listening.
|
|
195
|
+
try {
|
|
196
|
+
recognition.start();
|
|
197
|
+
recognitionRunning = true;
|
|
198
|
+
} catch {
|
|
199
|
+
// Engine refused an immediate restart; the next turn end restarts it.
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
function handleRecognitionError(event) {
|
|
204
|
+
const code = String(event?.error || "");
|
|
205
|
+
if (code === "not-allowed" || code === "service-not-allowed") {
|
|
206
|
+
state.lastError = code;
|
|
207
|
+
// Enter the error state before stopping so onend does not auto-restart.
|
|
208
|
+
setUiState("error", { reason: code });
|
|
209
|
+
stopRecognition();
|
|
210
|
+
emitEvent("Microphone access was denied, so Natural Conversation Mode cannot listen. Allow microphone access for this site, then end and restart the conversation.", "error");
|
|
211
|
+
return;
|
|
212
|
+
}
|
|
213
|
+
if (code === "no-speech" || code === "aborted") return;
|
|
214
|
+
emitEvent(`Speech recognition error: ${code || "unknown"}`, "warn");
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
function handleRecognitionResult(event) {
|
|
218
|
+
if (!state.active) return;
|
|
219
|
+
const results = event?.results || [];
|
|
220
|
+
let finalText = "";
|
|
221
|
+
let interimText = "";
|
|
222
|
+
for (let index = event?.resultIndex || 0; index < results.length; index += 1) {
|
|
223
|
+
const result = results[index];
|
|
224
|
+
if (!result) continue;
|
|
225
|
+
const alternative = result[0];
|
|
226
|
+
const text = alternative && typeof alternative.transcript === "string" ? alternative.transcript : "";
|
|
227
|
+
if (result.isFinal) finalText += text;
|
|
228
|
+
else if (text.trim()) interimText += text;
|
|
229
|
+
}
|
|
230
|
+
if (finalText.trim()) {
|
|
231
|
+
clearPartialTranscript();
|
|
232
|
+
handleFinalTranscript(finalText.trim());
|
|
233
|
+
return;
|
|
234
|
+
}
|
|
235
|
+
if (interimText.trim()) {
|
|
236
|
+
setPartialTranscript(interimText.trim());
|
|
237
|
+
if (state.uiState === "listening") setUiState("transcribing");
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
function deliverTranscript(text, reason) {
|
|
242
|
+
Promise.resolve(sendTranscript(text, { reason }))
|
|
243
|
+
.then(() => {
|
|
244
|
+
if (state.active && state.uiState !== "error" && state.uiState !== "off") setUiState("answering", { reason });
|
|
245
|
+
})
|
|
246
|
+
.catch((error) => {
|
|
247
|
+
emitEvent(`Failed to send voice transcript: ${error?.message || error}`, "error");
|
|
248
|
+
if (state.active && state.uiState !== "error") setUiState("listening");
|
|
249
|
+
});
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
function handleFinalTranscript(text) {
|
|
253
|
+
const transcript = String(text || "").trim();
|
|
254
|
+
if (!transcript || !state.active || state.userPaused) return;
|
|
255
|
+
clearPartialTranscript();
|
|
256
|
+
clearSilenceTimer();
|
|
257
|
+
if (state.speaking) {
|
|
258
|
+
// Barge-in: only reachable when bargeInEnabled keeps the mic open during TTS.
|
|
259
|
+
stopSpeaking();
|
|
260
|
+
setUiState("interrupting", { transcript });
|
|
261
|
+
deliverTranscript(transcript, "interrupt");
|
|
262
|
+
return;
|
|
263
|
+
}
|
|
264
|
+
if (state.assistantStreaming && state.toolRunning) {
|
|
265
|
+
state.queuedInterrupt = state.queuedInterrupt ? `${state.queuedInterrupt} ${transcript}` : transcript;
|
|
266
|
+
setUiState("interrupting", { queued: true, transcript });
|
|
267
|
+
emitEvent("Voice interruption queued; it will be delivered after the current tool phase ends.", "info");
|
|
268
|
+
return;
|
|
269
|
+
}
|
|
270
|
+
if (state.assistantStreaming) {
|
|
271
|
+
setUiState("interrupting", { transcript });
|
|
272
|
+
deliverTranscript(transcript, "interrupt");
|
|
273
|
+
return;
|
|
274
|
+
}
|
|
275
|
+
setUiState("answering", { transcript });
|
|
276
|
+
deliverTranscript(transcript, "prompt");
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
function flushQueuedInterrupt() {
|
|
280
|
+
const text = state.queuedInterrupt;
|
|
281
|
+
state.queuedInterrupt = "";
|
|
282
|
+
if (!text || !state.active) return;
|
|
283
|
+
setUiState("interrupting", { queued: true });
|
|
284
|
+
deliverTranscript(text, state.assistantStreaming ? "interrupt" : "prompt");
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
function setAssistantActivity(patch = {}) {
|
|
288
|
+
if (!state.active) return snapshot();
|
|
289
|
+
const wasToolRunning = state.toolRunning;
|
|
290
|
+
const wasStreaming = state.assistantStreaming;
|
|
291
|
+
if (typeof patch.streaming === "boolean") state.assistantStreaming = patch.streaming;
|
|
292
|
+
if (typeof patch.toolRunning === "boolean") state.toolRunning = patch.toolRunning;
|
|
293
|
+
if (state.assistantStreaming && !wasStreaming) {
|
|
294
|
+
clearSilenceTimer();
|
|
295
|
+
if (!state.userPaused && !state.speaking && state.uiState !== "interrupting") setUiState("answering");
|
|
296
|
+
}
|
|
297
|
+
if (wasToolRunning && !state.toolRunning && state.queuedInterrupt) flushQueuedInterrupt();
|
|
298
|
+
else if (wasStreaming && !state.assistantStreaming && state.queuedInterrupt) flushQueuedInterrupt();
|
|
299
|
+
return snapshot();
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
function stopSpeaking() {
|
|
303
|
+
const utterance = currentUtterance;
|
|
304
|
+
currentUtterance = null;
|
|
305
|
+
state.speaking = false;
|
|
306
|
+
recognitionPausedForSpeech = false;
|
|
307
|
+
if (utterance) {
|
|
308
|
+
utterance.onend = null;
|
|
309
|
+
utterance.onerror = null;
|
|
310
|
+
}
|
|
311
|
+
try {
|
|
312
|
+
synthesis?.cancel?.();
|
|
313
|
+
} catch {
|
|
314
|
+
// Synthesis may already be idle.
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
function resumeListeningAfterTurn({ askedQuestion = false } = {}) {
|
|
319
|
+
if (!state.active) return;
|
|
320
|
+
if (state.userPaused) {
|
|
321
|
+
setUiState("paused", { reason: "manual" });
|
|
322
|
+
return;
|
|
323
|
+
}
|
|
324
|
+
const listening = startRecognition();
|
|
325
|
+
if (listening) setUiState("listening");
|
|
326
|
+
else if (state.uiState !== "error") setUiState("paused", { reason: "stt-unavailable" });
|
|
327
|
+
if (askedQuestion && listening) startSilenceTimer();
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
function speak(text, { askedQuestion = false } = {}) {
|
|
331
|
+
const spoken = String(text || "").trim();
|
|
332
|
+
if (state.userPaused) return;
|
|
333
|
+
if (!spoken || !state.ttsAvailable) {
|
|
334
|
+
resumeListeningAfterTurn({ askedQuestion: askedQuestion && !!spoken });
|
|
335
|
+
return;
|
|
336
|
+
}
|
|
337
|
+
stopSpeaking();
|
|
338
|
+
if (!bargeInEnabled) stopRecognition({ forSpeech: true });
|
|
339
|
+
let utterance = null;
|
|
340
|
+
try {
|
|
341
|
+
utterance = utteranceFactory(spoken);
|
|
342
|
+
} catch {
|
|
343
|
+
utterance = null;
|
|
344
|
+
}
|
|
345
|
+
if (!utterance) {
|
|
346
|
+
resumeListeningAfterTurn({ askedQuestion });
|
|
347
|
+
return;
|
|
348
|
+
}
|
|
349
|
+
state.speaking = true;
|
|
350
|
+
setUiState("speaking");
|
|
351
|
+
currentUtterance = utterance;
|
|
352
|
+
const finish = () => {
|
|
353
|
+
if (currentUtterance !== utterance) return;
|
|
354
|
+
currentUtterance = null;
|
|
355
|
+
state.speaking = false;
|
|
356
|
+
recognitionPausedForSpeech = false;
|
|
357
|
+
if (!state.active) return;
|
|
358
|
+
resumeListeningAfterTurn({ askedQuestion });
|
|
359
|
+
};
|
|
360
|
+
utterance.onend = finish;
|
|
361
|
+
utterance.onerror = finish;
|
|
362
|
+
try {
|
|
363
|
+
synthesis.speak(utterance);
|
|
364
|
+
} catch (error) {
|
|
365
|
+
emitEvent(`Speech synthesis failed: ${error?.message || error}`, "warn");
|
|
366
|
+
finish();
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
function handleAssistantTurnEnd(finalText, { askedQuestion = false } = {}) {
|
|
371
|
+
if (!state.active) return snapshot();
|
|
372
|
+
state.assistantStreaming = false;
|
|
373
|
+
state.toolRunning = false;
|
|
374
|
+
if (state.userPaused) return snapshot();
|
|
375
|
+
if (state.queuedInterrupt) {
|
|
376
|
+
flushQueuedInterrupt();
|
|
377
|
+
return snapshot();
|
|
378
|
+
}
|
|
379
|
+
speak(finalText, { askedQuestion });
|
|
380
|
+
return snapshot();
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
function handleUserActivity() {
|
|
384
|
+
clearSilenceTimer();
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
function start() {
|
|
388
|
+
if (state.active) return snapshot();
|
|
389
|
+
state.active = true;
|
|
390
|
+
state.userPaused = false;
|
|
391
|
+
state.queuedInterrupt = "";
|
|
392
|
+
state.assistantStreaming = false;
|
|
393
|
+
state.toolRunning = false;
|
|
394
|
+
state.lastError = "";
|
|
395
|
+
const listening = startRecognition();
|
|
396
|
+
if (listening) {
|
|
397
|
+
setUiState("listening");
|
|
398
|
+
} else if (state.ttsAvailable) {
|
|
399
|
+
setUiState("paused", { reason: "stt-unavailable" });
|
|
400
|
+
emitEvent("Browser speech recognition is unavailable here; Pi answers will still be spoken, but the microphone stays off. Type prompts normally or configure another STT provider in a later phase.", "warn");
|
|
401
|
+
} else {
|
|
402
|
+
setUiState("error", { reason: "speech-unsupported" });
|
|
403
|
+
emitEvent("This browser exposes neither speech recognition nor speech synthesis, so the Natural Conversation voice loop cannot run.", "error");
|
|
404
|
+
}
|
|
405
|
+
return snapshot();
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
function pause({ reason = "manual" } = {}) {
|
|
409
|
+
if (!state.active) return snapshot();
|
|
410
|
+
state.userPaused = true;
|
|
411
|
+
clearSilenceTimer();
|
|
412
|
+
clearPartialTranscript();
|
|
413
|
+
stopSpeaking();
|
|
414
|
+
stopRecognition({ forSpeech: true });
|
|
415
|
+
setUiState("paused", { reason });
|
|
416
|
+
return snapshot();
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
function resume() {
|
|
420
|
+
if (!state.active) return snapshot();
|
|
421
|
+
state.userPaused = false;
|
|
422
|
+
recognitionPausedForSpeech = false;
|
|
423
|
+
if (state.assistantStreaming) {
|
|
424
|
+
if (!state.toolRunning) startRecognition();
|
|
425
|
+
setUiState("answering", { reason: "manual-resume" });
|
|
426
|
+
return snapshot();
|
|
427
|
+
}
|
|
428
|
+
resumeListeningAfterTurn();
|
|
429
|
+
return snapshot();
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
function togglePaused() {
|
|
433
|
+
return state.userPaused ? resume() : pause();
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
function stop() {
|
|
437
|
+
if (!state.active) {
|
|
438
|
+
state.uiState = "off";
|
|
439
|
+
return snapshot();
|
|
440
|
+
}
|
|
441
|
+
state.active = false;
|
|
442
|
+
state.userPaused = false;
|
|
443
|
+
clearSilenceTimer();
|
|
444
|
+
clearPartialTranscript();
|
|
445
|
+
stopSpeaking();
|
|
446
|
+
stopRecognition();
|
|
447
|
+
state.assistantStreaming = false;
|
|
448
|
+
state.toolRunning = false;
|
|
449
|
+
state.queuedInterrupt = "";
|
|
450
|
+
setUiState("off");
|
|
451
|
+
return snapshot();
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
return {
|
|
455
|
+
start,
|
|
456
|
+
stop,
|
|
457
|
+
pause,
|
|
458
|
+
resume,
|
|
459
|
+
togglePaused,
|
|
460
|
+
getState: snapshot,
|
|
461
|
+
isActive: () => state.active,
|
|
462
|
+
setAssistantActivity,
|
|
463
|
+
handleAssistantTurnEnd,
|
|
464
|
+
handleUserActivity,
|
|
465
|
+
};
|
|
466
|
+
}
|
|
File without changes
|
|
@@ -1,8 +1,23 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
// Minimal JSONL RPC stub standing in for the pi coding agent so HTTP endpoint
|
|
3
3
|
// tests can boot the real pi-webui server without a model provider.
|
|
4
|
+
//
|
|
5
|
+
// Optional env-gated features (default off; without them the voice scripting
|
|
6
|
+
// and logging additions are inert and command responses are unchanged, except
|
|
7
|
+
// that "steer" now gets an explicit success payload instead of the generic
|
|
8
|
+
// default-case response — no test asserts on either shape):
|
|
9
|
+
// - FAKE_PI_LOG_FILE=<path>: append one JSON line per received RPC command
|
|
10
|
+
// ({direction:"command", type, message}) plus one per scripted event emitted
|
|
11
|
+
// ({direction:"event", type}), so browser-driver tests can assert ordering.
|
|
12
|
+
// - FAKE_PI_VOICE_SCRIPTS=1: prompts containing "voice test say|question|tool|slow"
|
|
13
|
+
// respond success and then asynchronously emit a scripted agent event flow
|
|
14
|
+
// (agent_start/message_*/tool_execution_*/agent_end) over stdout; the scripted
|
|
15
|
+
// assistant turns are appended to a dynamic transcript returned by
|
|
16
|
+
// get_messages AFTER the three baseline messages, and get_state reports
|
|
17
|
+
// isStreaming=true while a scripted flow runs.
|
|
4
18
|
import { createInterface } from "node:readline";
|
|
5
19
|
import { randomUUID } from "node:crypto";
|
|
20
|
+
import { appendFileSync } from "node:fs";
|
|
6
21
|
|
|
7
22
|
const sessionIndex = process.argv.indexOf("--session");
|
|
8
23
|
const sessionFile = sessionIndex !== -1 ? process.argv[sessionIndex + 1] : undefined;
|
|
@@ -12,6 +27,20 @@ let peakBash = 0;
|
|
|
12
27
|
let thinkingLevel = "off";
|
|
13
28
|
let conversationEnabled = false;
|
|
14
29
|
|
|
30
|
+
const voiceScriptsEnabled = process.env.FAKE_PI_VOICE_SCRIPTS === "1";
|
|
31
|
+
const commandLogFile = process.env.FAKE_PI_LOG_FILE || "";
|
|
32
|
+
const dynamicMessages = [];
|
|
33
|
+
let scriptedStreaming = false;
|
|
34
|
+
|
|
35
|
+
function logJsonLine(entry) {
|
|
36
|
+
if (!commandLogFile) return;
|
|
37
|
+
try {
|
|
38
|
+
appendFileSync(commandLogFile, `${JSON.stringify({ at: Date.now(), ...entry })}\n`);
|
|
39
|
+
} catch {
|
|
40
|
+
// Logging must never break the fixture.
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
|
|
15
44
|
function respond(payload) {
|
|
16
45
|
process.stdout.write(`${JSON.stringify(payload)}\n`);
|
|
17
46
|
}
|
|
@@ -20,6 +49,85 @@ function emitEvent(payload) {
|
|
|
20
49
|
process.stdout.write(`${JSON.stringify(payload)}\n`);
|
|
21
50
|
}
|
|
22
51
|
|
|
52
|
+
function emitScriptedEvent(payload) {
|
|
53
|
+
logJsonLine({ direction: "event", type: payload.type, ...(payload.toolName ? { toolName: payload.toolName } : {}) });
|
|
54
|
+
emitEvent(payload);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
const scriptedTimers = new Set();
|
|
58
|
+
|
|
59
|
+
function runScriptedSteps(steps) {
|
|
60
|
+
let at = 0;
|
|
61
|
+
for (const step of steps) {
|
|
62
|
+
at += step.afterMs ?? 0;
|
|
63
|
+
const timer = setTimeout(() => {
|
|
64
|
+
scriptedTimers.delete(timer);
|
|
65
|
+
step.run();
|
|
66
|
+
}, at);
|
|
67
|
+
scriptedTimers.add(timer);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function cancelScriptedSteps() {
|
|
72
|
+
for (const timer of scriptedTimers) clearTimeout(timer);
|
|
73
|
+
scriptedTimers.clear();
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
// Emits a full scripted assistant turn: agent_start, optional tool phase,
|
|
77
|
+
// streamed assistant text, message_end, agent_end. The final assistant text is
|
|
78
|
+
// appended to the dynamic transcript so get_messages returns it afterwards.
|
|
79
|
+
function runVoiceScriptFlow({ text, chunks, chunkSpacingMs = 60, toolPhase = false, toolDurationMs = 1500 }) {
|
|
80
|
+
scriptedStreaming = true;
|
|
81
|
+
const steps = [{ afterMs: 30, run: () => emitScriptedEvent({ type: "agent_start" }) }];
|
|
82
|
+
if (toolPhase) {
|
|
83
|
+
const toolCallId = `voice-tool-${randomUUID()}`;
|
|
84
|
+
steps.push({ afterMs: 50, run: () => emitScriptedEvent({ type: "tool_execution_start", toolCallId, toolName: "read", args: { path: "README.md" } }) });
|
|
85
|
+
steps.push({ afterMs: toolDurationMs, run: () => emitScriptedEvent({ type: "tool_execution_end", toolCallId, toolName: "read", isError: false, result: { content: [{ type: "text", text: "fake tool output" }] } }) });
|
|
86
|
+
}
|
|
87
|
+
steps.push({ afterMs: toolPhase ? 120 : 50, run: () => emitScriptedEvent({ type: "message_start", message: { role: "assistant" } }) });
|
|
88
|
+
for (const delta of chunks || [text.slice(0, Math.ceil(text.length / 2)), text.slice(Math.ceil(text.length / 2))]) {
|
|
89
|
+
steps.push({ afterMs: chunkSpacingMs, run: () => emitScriptedEvent({ type: "message_update", assistantMessageEvent: { type: "text_delta", delta } }) });
|
|
90
|
+
}
|
|
91
|
+
steps.push({
|
|
92
|
+
afterMs: chunkSpacingMs,
|
|
93
|
+
run: () => {
|
|
94
|
+
dynamicMessages.push({ role: "assistant", content: [{ type: "text", text }], timestamp: Date.now() });
|
|
95
|
+
emitScriptedEvent({ type: "message_end", message: { role: "assistant", content: [{ type: "text", text }] } });
|
|
96
|
+
},
|
|
97
|
+
});
|
|
98
|
+
steps.push({
|
|
99
|
+
afterMs: 60,
|
|
100
|
+
run: () => {
|
|
101
|
+
scriptedStreaming = false;
|
|
102
|
+
emitScriptedEvent({ type: "agent_end" });
|
|
103
|
+
},
|
|
104
|
+
});
|
|
105
|
+
runScriptedSteps(steps);
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
function handleVoiceScriptPrompt(command, base) {
|
|
109
|
+
if (!voiceScriptsEnabled) return false;
|
|
110
|
+
const message = String(command.message || "");
|
|
111
|
+
const marker = ["voice test say", "voice test question", "voice test tool", "voice test slow"].find((item) => message.includes(item));
|
|
112
|
+
if (!marker) return false;
|
|
113
|
+
dynamicMessages.push({ role: "user", content: message, timestamp: Date.now() });
|
|
114
|
+
respond({ ...base, data: { output: "voice scripted prompt accepted" } });
|
|
115
|
+
if (marker === "voice test say") {
|
|
116
|
+
runVoiceScriptFlow({ text: "Okay, this is the spoken answer." });
|
|
117
|
+
} else if (marker === "voice test question") {
|
|
118
|
+
runVoiceScriptFlow({ text: "Should we proceed with the next step?" });
|
|
119
|
+
} else if (marker === "voice test tool") {
|
|
120
|
+
runVoiceScriptFlow({ text: "The tool has finished.", toolPhase: true });
|
|
121
|
+
} else {
|
|
122
|
+
runVoiceScriptFlow({
|
|
123
|
+
text: "This is a long streaming answer that keeps going.",
|
|
124
|
+
chunks: ["This is a long ", "streaming answer ", "that keeps ", "going", "."],
|
|
125
|
+
chunkSpacingMs: 500,
|
|
126
|
+
});
|
|
127
|
+
}
|
|
128
|
+
return true;
|
|
129
|
+
}
|
|
130
|
+
|
|
23
131
|
function conversationStatusText() {
|
|
24
132
|
return conversationEnabled ? "Voice: listening" : "";
|
|
25
133
|
}
|
|
@@ -34,6 +142,38 @@ function emitConversationStatus() {
|
|
|
34
142
|
});
|
|
35
143
|
}
|
|
36
144
|
|
|
145
|
+
function handleWebuiHelperPrompt(command, base) {
|
|
146
|
+
const message = String(command.message || "").trim();
|
|
147
|
+
const match = message.match(/^\/webui-helper\s+(\{[\s\S]*\})$/);
|
|
148
|
+
if (!match) return false;
|
|
149
|
+
let request = {};
|
|
150
|
+
try {
|
|
151
|
+
request = JSON.parse(match[1]);
|
|
152
|
+
} catch {
|
|
153
|
+
request = {};
|
|
154
|
+
}
|
|
155
|
+
const requestId = String(request.requestId || "");
|
|
156
|
+
const respondHelper = (payload) => emitEvent({
|
|
157
|
+
type: "extension_ui_request",
|
|
158
|
+
id: randomUUID(),
|
|
159
|
+
method: "notify",
|
|
160
|
+
message: `__PI_WEBUI_HELPER_RESPONSE__:${JSON.stringify(payload)}`,
|
|
161
|
+
notifyType: payload.ok === false ? "error" : "info",
|
|
162
|
+
});
|
|
163
|
+
respond({ ...base, data: { output: "webui-helper handled" } });
|
|
164
|
+
switch (request.action) {
|
|
165
|
+
case "tools-state":
|
|
166
|
+
respondHelper({ requestId, ok: true, data: { tools: [], activeTools: [] } });
|
|
167
|
+
return true;
|
|
168
|
+
case "skills-state":
|
|
169
|
+
respondHelper({ requestId, ok: true, data: { skills: [] } });
|
|
170
|
+
return true;
|
|
171
|
+
default:
|
|
172
|
+
respondHelper({ requestId, ok: false, error: `Unknown webui-helper action: ${String(request.action || "")}` });
|
|
173
|
+
return true;
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
|
|
37
177
|
function handleTalkPrompt(command, base) {
|
|
38
178
|
const message = String(command.message || "").trim();
|
|
39
179
|
const match = message.match(/^\/(talk|voice|conversation)(?:\s+(\S+))?/i);
|
|
@@ -69,6 +209,7 @@ rl.on("line", (line) => {
|
|
|
69
209
|
}
|
|
70
210
|
const { id, type } = command || {};
|
|
71
211
|
if (!id || !type) return;
|
|
212
|
+
logJsonLine({ direction: "command", type, ...(command.message !== undefined ? { message: String(command.message) } : {}) });
|
|
72
213
|
const base = { type: "response", id, command: type, success: true };
|
|
73
214
|
|
|
74
215
|
switch (type) {
|
|
@@ -78,7 +219,7 @@ rl.on("line", (line) => {
|
|
|
78
219
|
data: {
|
|
79
220
|
model: { provider: "fake", id: "fake-model" },
|
|
80
221
|
thinkingLevel,
|
|
81
|
-
isStreaming:
|
|
222
|
+
isStreaming: scriptedStreaming,
|
|
82
223
|
isCompacting: false,
|
|
83
224
|
steeringMode: "one-at-a-time",
|
|
84
225
|
followUpMode: "one-at-a-time",
|
|
@@ -99,6 +240,7 @@ rl.on("line", (line) => {
|
|
|
99
240
|
{ role: "user", content: "fake prompt", timestamp: 1000 },
|
|
100
241
|
{ role: "assistant", content: [{ type: "text", text: "fake answer" }], timestamp: 2000 },
|
|
101
242
|
{ role: "user", content: "fake follow-up", timestamp: 3000 },
|
|
243
|
+
...dynamicMessages,
|
|
102
244
|
],
|
|
103
245
|
},
|
|
104
246
|
});
|
|
@@ -116,9 +258,14 @@ rl.on("line", (line) => {
|
|
|
116
258
|
});
|
|
117
259
|
return;
|
|
118
260
|
case "prompt":
|
|
261
|
+
if (handleWebuiHelperPrompt(command, base)) return;
|
|
119
262
|
if (handleTalkPrompt(command, base)) return;
|
|
263
|
+
if (handleVoiceScriptPrompt(command, base)) return;
|
|
120
264
|
respond({ ...base, data: { output: "fake prompt accepted" } });
|
|
121
265
|
return;
|
|
266
|
+
case "steer":
|
|
267
|
+
respond({ ...base, data: { output: "fake steer accepted" } });
|
|
268
|
+
return;
|
|
122
269
|
case "set_thinking_level":
|
|
123
270
|
thinkingLevel = String(command.level || "off");
|
|
124
271
|
respond({ ...base, data: { level: thinkingLevel } });
|
|
@@ -145,3 +292,9 @@ rl.on("line", (line) => {
|
|
|
145
292
|
respond({ ...base, data: {} });
|
|
146
293
|
}
|
|
147
294
|
});
|
|
295
|
+
|
|
296
|
+
rl.on("close", () => {
|
|
297
|
+
// The parent (pi-webui or a test driver) closed stdin or died; never let
|
|
298
|
+
// pending scripted voice events outlive it and write into a broken pipe.
|
|
299
|
+
cancelScriptedSteps();
|
|
300
|
+
});
|