@bojackduy/opencode-voice 0.11.2 → 0.11.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +77 -19
- package/index.js +8 -1
- package/lib/chatterbox-server.js +9 -1
- package/lib/conversation-status.js +269 -0
- package/lib/conversation.js +139 -1
- package/lib/streaming-stt.js +101 -0
- package/lib/streaming-transcript.js +59 -6
- package/lib/stt-log.js +39 -0
- package/lib/stt.js +474 -27
- package/lib/tts-player.js +356 -0
- package/lib/tts-status.js +266 -0
- package/lib/tts.js +856 -173
- package/lib/v2-compat.js +41 -0
- package/lib/v2-preview.js +242 -22
- package/lib/vieneu-server.js +287 -0
- package/package.json +12 -1
package/lib/conversation.js
CHANGED
|
@@ -136,6 +136,39 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
|
|
|
136
136
|
api.ui.toast({ message, variant, duration: 3000 });
|
|
137
137
|
}
|
|
138
138
|
|
|
139
|
+
// v2 persistent status line: phase-aware buttons. v1 has no surface, so
|
|
140
|
+
// these are no-ops there (api.conversationStatus is undefined).
|
|
141
|
+
const convStatus = api.conversationStatus || null;
|
|
142
|
+
const setConvPhase = api.setConvPhase || (() => {});
|
|
143
|
+
const setConvActions = api.setConvActions || (() => {});
|
|
144
|
+
function convBegin() {
|
|
145
|
+
try {
|
|
146
|
+
const p = convStatus?.begin?.();
|
|
147
|
+
if (p && typeof p.then === "function") p.catch(() => {});
|
|
148
|
+
} catch {}
|
|
149
|
+
}
|
|
150
|
+
function convUpdate() {
|
|
151
|
+
try {
|
|
152
|
+
convStatus?.update?.();
|
|
153
|
+
} catch {}
|
|
154
|
+
}
|
|
155
|
+
function convClear() {
|
|
156
|
+
try {
|
|
157
|
+
convStatus?.clear?.();
|
|
158
|
+
} catch {}
|
|
159
|
+
}
|
|
160
|
+
function convSetPhase(p) {
|
|
161
|
+
try {
|
|
162
|
+
setConvPhase(p);
|
|
163
|
+
convUpdate();
|
|
164
|
+
} catch {}
|
|
165
|
+
}
|
|
166
|
+
function convSetActions(actions) {
|
|
167
|
+
try {
|
|
168
|
+
setConvActions(actions);
|
|
169
|
+
} catch {}
|
|
170
|
+
}
|
|
171
|
+
|
|
139
172
|
function currentSessionID() {
|
|
140
173
|
const route = api?.route?.current;
|
|
141
174
|
return route?.name === "session" ? route?.params?.sessionID : null;
|
|
@@ -156,13 +189,34 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
|
|
|
156
189
|
turn += 1;
|
|
157
190
|
phase = "recording";
|
|
158
191
|
logger?.log("VOICE", `Conversation turn ${turn} listening`, "debug");
|
|
192
|
+
// Hands-free: auto-finalize after N seconds of silence so the user
|
|
193
|
+
// never has to press a key to submit.
|
|
194
|
+
stt.setSilenceAutoFinalize?.(1.5, () => {
|
|
195
|
+
if (active && phase === "recording") {
|
|
196
|
+
logger?.log("VOICE", "Conversation auto-finalize on silence", "debug");
|
|
197
|
+
finishTurnFlow();
|
|
198
|
+
}
|
|
199
|
+
});
|
|
200
|
+
convSetActions({
|
|
201
|
+
cancel: () => {
|
|
202
|
+
if (phase === "recording") {
|
|
203
|
+
stt.cancel();
|
|
204
|
+
phase = "paused";
|
|
205
|
+
convSetPhase("paused");
|
|
206
|
+
}
|
|
207
|
+
},
|
|
208
|
+
send: () => {
|
|
209
|
+
if (phase === "recording") finishTurnFlow();
|
|
210
|
+
},
|
|
211
|
+
});
|
|
212
|
+
convSetPhase("recording");
|
|
159
213
|
if (!stt.start()) {
|
|
160
214
|
phase = "idle";
|
|
161
215
|
stop("busy");
|
|
162
216
|
toast("STT busy, conversation off", "warning");
|
|
163
217
|
return;
|
|
164
218
|
}
|
|
165
|
-
toast(`Listening - press ${keyLabel()}
|
|
219
|
+
toast(`Listening - speak, then pause ${1.5}s to send (or press ${keyLabel()})`);
|
|
166
220
|
}
|
|
167
221
|
|
|
168
222
|
// The resting state: mic closed, one press to talk. Entered by start() and
|
|
@@ -170,6 +224,13 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
|
|
|
170
224
|
|
|
171
225
|
function enterPaused() {
|
|
172
226
|
phase = "paused";
|
|
227
|
+
convSetActions({
|
|
228
|
+
talk: () => {
|
|
229
|
+
if (phase === "paused") beginTurn();
|
|
230
|
+
},
|
|
231
|
+
end: () => stop("user"),
|
|
232
|
+
});
|
|
233
|
+
convSetPhase("paused");
|
|
173
234
|
toast(`Press ${keyLabel()} to talk`);
|
|
174
235
|
}
|
|
175
236
|
|
|
@@ -179,6 +240,7 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
|
|
|
179
240
|
|
|
180
241
|
function bargeIn() {
|
|
181
242
|
logger?.log("VOICE", "Conversation barge-in", "debug");
|
|
243
|
+
stt.setSilenceAutoFinalize?.(null, null);
|
|
182
244
|
discardStreamAudio();
|
|
183
245
|
clearProcessingToast();
|
|
184
246
|
// Settle the pending reply wait as "stopped" so the abandoned answer is
|
|
@@ -214,6 +276,7 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
|
|
|
214
276
|
active = false;
|
|
215
277
|
generation += 1;
|
|
216
278
|
phase = "idle";
|
|
279
|
+
stt.setSilenceAutoFinalize?.(null, null);
|
|
217
280
|
if (waitCancel) {
|
|
218
281
|
waitCancel("stopped");
|
|
219
282
|
waitCancel = null;
|
|
@@ -225,6 +288,7 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
|
|
|
225
288
|
clearProcessingToast();
|
|
226
289
|
tts.stop();
|
|
227
290
|
tts.setConversationActive(false);
|
|
291
|
+
convClear();
|
|
228
292
|
logger?.log("VOICE", `Conversation stopped reason=${reason}`, "debug");
|
|
229
293
|
}
|
|
230
294
|
|
|
@@ -338,8 +402,15 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
|
|
|
338
402
|
|
|
339
403
|
async function finishTurnFlow() {
|
|
340
404
|
if (!active || phase !== "recording") return;
|
|
405
|
+
stt.setSilenceAutoFinalize?.(null, null);
|
|
341
406
|
const myGen = generation;
|
|
342
407
|
phase = "processing";
|
|
408
|
+
convSetActions({
|
|
409
|
+
cancel: () => {
|
|
410
|
+
// Processing is too short to cancel safely; report busy.
|
|
411
|
+
},
|
|
412
|
+
});
|
|
413
|
+
convSetPhase("processing");
|
|
343
414
|
showProcessingToast("Transcribing...");
|
|
344
415
|
|
|
345
416
|
const res = await stt.transcribeTurn();
|
|
@@ -375,6 +446,11 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
|
|
|
375
446
|
let earlyWait = null;
|
|
376
447
|
if (api.client?.tui?.v2) {
|
|
377
448
|
phase = "waiting";
|
|
449
|
+
convSetActions({
|
|
450
|
+
cancel: () => {},
|
|
451
|
+
barge: () => bargeIn(),
|
|
452
|
+
});
|
|
453
|
+
convSetPhase("waiting");
|
|
378
454
|
startReplyStream(currentSessionID, turnStart);
|
|
379
455
|
earlyWait = waitForReply(currentSessionID);
|
|
380
456
|
}
|
|
@@ -386,6 +462,11 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
|
|
|
386
462
|
}
|
|
387
463
|
|
|
388
464
|
phase = "waiting";
|
|
465
|
+
convSetActions({
|
|
466
|
+
cancel: () => {},
|
|
467
|
+
barge: () => bargeIn(),
|
|
468
|
+
});
|
|
469
|
+
convSetPhase("waiting");
|
|
389
470
|
const sessionID = currentSessionID();
|
|
390
471
|
showProcessingToast(`Working - press ${keyLabel()} to talk`);
|
|
391
472
|
if (!earlyWait) startReplyStream(sessionID, turnStart);
|
|
@@ -423,6 +504,24 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
|
|
|
423
504
|
async function answerGateThenSpeak(myGen, sessionID, turnStart, outcome) {
|
|
424
505
|
discardStreamAudio();
|
|
425
506
|
phase = "speaking";
|
|
507
|
+
convSetActions({
|
|
508
|
+
stop: () => {
|
|
509
|
+
if (phase === "speaking") {
|
|
510
|
+
tts.stop();
|
|
511
|
+
phase = "paused";
|
|
512
|
+
convSetPhase("paused");
|
|
513
|
+
}
|
|
514
|
+
},
|
|
515
|
+
barge: () => bargeIn(),
|
|
516
|
+
pauseTts: () => {
|
|
517
|
+
if (phase === "speaking") {
|
|
518
|
+
tts.stop();
|
|
519
|
+
phase = "paused";
|
|
520
|
+
convSetPhase("paused");
|
|
521
|
+
}
|
|
522
|
+
},
|
|
523
|
+
});
|
|
524
|
+
convSetPhase("speaking");
|
|
426
525
|
showProcessingToast("Speaking...");
|
|
427
526
|
await tts.speak(
|
|
428
527
|
outcome === "permission"
|
|
@@ -435,6 +534,11 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
|
|
|
435
534
|
// Still "waiting": the user answers on screen, and a press here is an
|
|
436
535
|
// ordinary barge-in (drop the pending reply, open the mic).
|
|
437
536
|
phase = "waiting";
|
|
537
|
+
convSetActions({
|
|
538
|
+
cancel: () => {},
|
|
539
|
+
barge: () => bargeIn(),
|
|
540
|
+
});
|
|
541
|
+
convSetPhase("waiting");
|
|
438
542
|
showProcessingToast(`Answer on screen - press ${keyLabel()} to talk`);
|
|
439
543
|
startReplyStream(sessionID, turnStart);
|
|
440
544
|
const resumed = await waitForReply(sessionID, { gates: false });
|
|
@@ -457,6 +561,24 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
|
|
|
457
561
|
|
|
458
562
|
async function speakReply(myGen) {
|
|
459
563
|
phase = "speaking";
|
|
564
|
+
convSetActions({
|
|
565
|
+
stop: () => {
|
|
566
|
+
if (phase === "speaking") {
|
|
567
|
+
tts.stop();
|
|
568
|
+
phase = "paused";
|
|
569
|
+
convSetPhase("paused");
|
|
570
|
+
}
|
|
571
|
+
},
|
|
572
|
+
barge: () => bargeIn(),
|
|
573
|
+
pauseTts: () => {
|
|
574
|
+
if (phase === "speaking") {
|
|
575
|
+
tts.stop();
|
|
576
|
+
phase = "paused";
|
|
577
|
+
convSetPhase("paused");
|
|
578
|
+
}
|
|
579
|
+
},
|
|
580
|
+
});
|
|
581
|
+
convSetPhase("speaking");
|
|
460
582
|
showProcessingToast(`Speaking - press ${keyLabel()} to interrupt`);
|
|
461
583
|
if (streamSpokenChars > 0) {
|
|
462
584
|
flushStream(true);
|
|
@@ -466,6 +588,10 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
|
|
|
466
588
|
await delay(restartDelayMs);
|
|
467
589
|
if (!active || myGen !== generation || phase !== "speaking") return;
|
|
468
590
|
enterPaused();
|
|
591
|
+
// Hands-free: auto-listen after the reply instead of waiting for a keypress.
|
|
592
|
+
await delay(500);
|
|
593
|
+
if (!active || myGen !== generation || phase !== "paused") return;
|
|
594
|
+
beginTurn();
|
|
469
595
|
return;
|
|
470
596
|
}
|
|
471
597
|
const spoken = await tts.speakAssistantTurn();
|
|
@@ -479,6 +605,10 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
|
|
|
479
605
|
await delay(restartDelayMs);
|
|
480
606
|
if (!active || myGen !== generation || phase !== "speaking") return;
|
|
481
607
|
enterPaused();
|
|
608
|
+
// Hands-free: auto-listen after the reply instead of waiting for a keypress.
|
|
609
|
+
await delay(500);
|
|
610
|
+
if (!active || myGen !== generation || phase !== "paused") return;
|
|
611
|
+
beginTurn();
|
|
482
612
|
}
|
|
483
613
|
|
|
484
614
|
// Pause instead of auto-recording so an empty or failed turn never feels
|
|
@@ -560,6 +690,14 @@ export function registerConversation(api, opts, logger, { stt, tts, isLiveNotesA
|
|
|
560
690
|
consecErrors = 0;
|
|
561
691
|
phase = "paused";
|
|
562
692
|
tts.setConversationActive(true);
|
|
693
|
+
convBegin();
|
|
694
|
+
convSetActions({
|
|
695
|
+
talk: () => {
|
|
696
|
+
if (phase === "paused") beginTurn();
|
|
697
|
+
},
|
|
698
|
+
end: () => stop("user"),
|
|
699
|
+
});
|
|
700
|
+
convSetPhase("paused");
|
|
563
701
|
// Recording toasts must name the key the user actually has bound.
|
|
564
702
|
stt.setStopHint(kb("voice.conversation"));
|
|
565
703
|
logger?.log("VOICE", "Conversation started", "debug");
|
package/lib/streaming-stt.js
CHANGED
|
@@ -288,6 +288,30 @@ export function createSoxRollingCapture({
|
|
|
288
288
|
return spoolChain;
|
|
289
289
|
}
|
|
290
290
|
|
|
291
|
+
// Pause: SIGSTOP freezes the mic so paused spans never enter the ring
|
|
292
|
+
// (and the first post-resume snapshot re-reads pre-pause audio the
|
|
293
|
+
// overlap tracker already absorbed). Best-effort: missing/dead proc
|
|
294
|
+
// reports false, never throws.
|
|
295
|
+
function pauseCapture() {
|
|
296
|
+
if (!proc) return false;
|
|
297
|
+
try {
|
|
298
|
+
proc.kill("SIGSTOP");
|
|
299
|
+
return true;
|
|
300
|
+
} catch {
|
|
301
|
+
return false;
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
function resumeCapture() {
|
|
306
|
+
if (!proc) return false;
|
|
307
|
+
try {
|
|
308
|
+
proc.kill("SIGCONT");
|
|
309
|
+
return true;
|
|
310
|
+
} catch {
|
|
311
|
+
return false;
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
|
|
291
315
|
// Honest coverage of retained audio. The spool holds a contiguous prefix
|
|
292
316
|
// from t=0; the RAM ring holds the trailing window. With no overflow the
|
|
293
317
|
// session is fully retained; with overflow the middle is explicitly listed
|
|
@@ -436,6 +460,16 @@ export function createSoxRollingCapture({
|
|
|
436
460
|
});
|
|
437
461
|
}
|
|
438
462
|
|
|
463
|
+
// Cheap tail read for live level meters: copies only the requested bytes
|
|
464
|
+
// (no WAV wrap, no snapshot sequence bookkeeping). Returns empty when the
|
|
465
|
+
// capture has no audio yet.
|
|
466
|
+
function peekTail(nBytes = 2048) {
|
|
467
|
+
const want = Math.max(0, Math.min(Math.floor(Number(nBytes) || 0), capacityBytes));
|
|
468
|
+
if (want === 0) return null;
|
|
469
|
+
const pcm = ring.snapshotLast(want);
|
|
470
|
+
return pcm.length > 0 ? pcm : null;
|
|
471
|
+
}
|
|
472
|
+
|
|
439
473
|
function snapshot() {
|
|
440
474
|
const pcm = ring.snapshotLast(Math.ceil((windowMs / 1000) * STREAMING_DEFAULTS.bytesPerSecond));
|
|
441
475
|
if (pcm.length === 0) return null;
|
|
@@ -477,6 +511,9 @@ export function createSoxRollingCapture({
|
|
|
477
511
|
// coverage descriptor that says exactly what is retained. Bounded: SIGINT,
|
|
478
512
|
// then SIGKILL after soxExitTimeoutMs. Consumes the spool file (cleanup).
|
|
479
513
|
async function stopFinal({ soxExitTimeoutMs = STREAMING_DEFAULTS.soxExitTimeoutMs } = {}) {
|
|
514
|
+
// A SIGSTOPped sox ignores SIGINT: always continue first so teardown
|
|
515
|
+
// can never hang on a paused mic.
|
|
516
|
+
resumeCapture();
|
|
480
517
|
if (proc) {
|
|
481
518
|
try {
|
|
482
519
|
proc.kill("SIGINT");
|
|
@@ -519,6 +556,9 @@ export function createSoxRollingCapture({
|
|
|
519
556
|
}
|
|
520
557
|
|
|
521
558
|
async function cancel() {
|
|
559
|
+
// SIGKILL terminates even a stopped process, but continuing first lets
|
|
560
|
+
// stdio flush and the close handler release the handle cleanly.
|
|
561
|
+
resumeCapture();
|
|
522
562
|
if (proc) {
|
|
523
563
|
const owned = proc;
|
|
524
564
|
proc = null;
|
|
@@ -551,6 +591,9 @@ export function createSoxRollingCapture({
|
|
|
551
591
|
return {
|
|
552
592
|
start,
|
|
553
593
|
snapshot,
|
|
594
|
+
peekTail,
|
|
595
|
+
pause: pauseCapture,
|
|
596
|
+
resume: resumeCapture,
|
|
554
597
|
stopFinal,
|
|
555
598
|
cancel,
|
|
556
599
|
dispose,
|
|
@@ -679,6 +722,11 @@ export function createStreamingController({
|
|
|
679
722
|
let coalesced = false;
|
|
680
723
|
let finalizing = false;
|
|
681
724
|
let generation = 0;
|
|
725
|
+
// Dictation pause: ticks are skipped (no inference cost) and the mic is
|
|
726
|
+
// SIGSTOPped so paused spans never enter the ring. The tracker needs no
|
|
727
|
+
// changes: the first post-resume snapshot re-reads pre-pause audio it
|
|
728
|
+
// already absorbed, so overlap-join dedupes it.
|
|
729
|
+
let paused = false;
|
|
682
730
|
let lastError = null;
|
|
683
731
|
let fallbackAudio = null;
|
|
684
732
|
let fallbackCoverage = null;
|
|
@@ -825,6 +873,12 @@ export function createStreamingController({
|
|
|
825
873
|
|
|
826
874
|
async function tick() {
|
|
827
875
|
if (status !== "streaming" || finalizing) return;
|
|
876
|
+
if (paused) {
|
|
877
|
+
// Mic frozen, nothing new to transcribe: keep the cadence timer alive
|
|
878
|
+
// for an instant resume, with zero inference cost while paused.
|
|
879
|
+
schedule(cadenceMs);
|
|
880
|
+
return;
|
|
881
|
+
}
|
|
828
882
|
if (inFlight) {
|
|
829
883
|
// Exactly one in-flight request: coalesce to the newest audio instead
|
|
830
884
|
// of queueing a stale snapshot behind the slow one.
|
|
@@ -960,6 +1014,7 @@ export function createStreamingController({
|
|
|
960
1014
|
function start() {
|
|
961
1015
|
if (status === "streaming" || status === "starting" || status === "stopping") return false;
|
|
962
1016
|
generation += 1; // invalidate any ancient late responses
|
|
1017
|
+
paused = false; // a parked-then-reused lease never resumes paused
|
|
963
1018
|
abortActiveRequest(); // defensive: no lingering signal from a dead session
|
|
964
1019
|
stopPromise = null;
|
|
965
1020
|
tracker = trackerFactory();
|
|
@@ -1060,6 +1115,7 @@ export function createStreamingController({
|
|
|
1060
1115
|
async function stop() {
|
|
1061
1116
|
if (status === "idle" || status === "cancelled") return currentTranscript();
|
|
1062
1117
|
if (stopPromise) return stopPromise;
|
|
1118
|
+
paused = false;
|
|
1063
1119
|
const stoppingFrom = status;
|
|
1064
1120
|
const myGen = generation;
|
|
1065
1121
|
setStatus("stopping");
|
|
@@ -1202,12 +1258,44 @@ export function createStreamingController({
|
|
|
1202
1258
|
return await stopFromActive(myGen, "streaming", fin);
|
|
1203
1259
|
}
|
|
1204
1260
|
|
|
1261
|
+
// Pause: freeze the mic (SIGSTOP) and skip recognition ticks so paused
|
|
1262
|
+
// spans cost no inference and never enter the transcript. Only valid
|
|
1263
|
+
// while streaming; teardown paths (stop/cancel/dispose) clear it, and the
|
|
1264
|
+
// capture resumes the mic first so teardown can never hang on SIGINT.
|
|
1265
|
+
// An in-flight transcription is left to settle - it is pre-pause speech
|
|
1266
|
+
// the overlap tracker already handles.
|
|
1267
|
+
function pause() {
|
|
1268
|
+
if (status !== "streaming" || paused) return false;
|
|
1269
|
+
paused = true;
|
|
1270
|
+
try {
|
|
1271
|
+
capture?.pause?.();
|
|
1272
|
+
} catch {}
|
|
1273
|
+
try {
|
|
1274
|
+
onStateChange?.("paused");
|
|
1275
|
+
} catch {}
|
|
1276
|
+
return true;
|
|
1277
|
+
}
|
|
1278
|
+
|
|
1279
|
+
function resume() {
|
|
1280
|
+
if (status !== "streaming" || !paused) return false;
|
|
1281
|
+
try {
|
|
1282
|
+
capture?.resume?.();
|
|
1283
|
+
} catch {}
|
|
1284
|
+
paused = false;
|
|
1285
|
+
try {
|
|
1286
|
+
onStateChange?.("resumed");
|
|
1287
|
+
} catch {}
|
|
1288
|
+
schedule(0);
|
|
1289
|
+
return true;
|
|
1290
|
+
}
|
|
1291
|
+
|
|
1205
1292
|
// Cancel: prompt, bounded, invalidates late responses via generation.
|
|
1206
1293
|
// Aborts the physical in-flight request (no overlapping inference on
|
|
1207
1294
|
// restart), stops capture, and wins over any concurrent stop/error path
|
|
1208
1295
|
// via the generation bump. Never waits for inference.
|
|
1209
1296
|
async function cancel() {
|
|
1210
1297
|
if (status === "idle") return false;
|
|
1298
|
+
paused = false;
|
|
1211
1299
|
generation += 1;
|
|
1212
1300
|
abortActiveRequest();
|
|
1213
1301
|
setStatus("cancelled");
|
|
@@ -1225,6 +1313,7 @@ export function createStreamingController({
|
|
|
1225
1313
|
|
|
1226
1314
|
async function dispose() {
|
|
1227
1315
|
generation += 1;
|
|
1316
|
+
paused = false;
|
|
1228
1317
|
abortActiveRequest();
|
|
1229
1318
|
stopPromise = null;
|
|
1230
1319
|
startupPromise = null;
|
|
@@ -1298,6 +1387,7 @@ export function createStreamingController({
|
|
|
1298
1387
|
stableText: t.stableText,
|
|
1299
1388
|
tentativeText: t.tentativeText,
|
|
1300
1389
|
inFlight,
|
|
1390
|
+
paused,
|
|
1301
1391
|
lastError,
|
|
1302
1392
|
hasFallbackAudio: fallbackAudio !== null,
|
|
1303
1393
|
fallbackCoverage,
|
|
@@ -1309,6 +1399,8 @@ export function createStreamingController({
|
|
|
1309
1399
|
return {
|
|
1310
1400
|
start,
|
|
1311
1401
|
stop,
|
|
1402
|
+
pause,
|
|
1403
|
+
resume,
|
|
1312
1404
|
cancel,
|
|
1313
1405
|
dispose,
|
|
1314
1406
|
setModel,
|
|
@@ -1316,6 +1408,15 @@ export function createStreamingController({
|
|
|
1316
1408
|
getState,
|
|
1317
1409
|
getServerPort,
|
|
1318
1410
|
getTranscript: currentTranscript,
|
|
1411
|
+
// Live level meter source: newest PCM tail (16-bit mono). Null when the
|
|
1412
|
+
// capture is gone or has no audio yet; never throws.
|
|
1413
|
+
peekPcm: (nBytes) => {
|
|
1414
|
+
try {
|
|
1415
|
+
return capture?.peekTail?.(nBytes) || null;
|
|
1416
|
+
} catch {
|
|
1417
|
+
return null;
|
|
1418
|
+
}
|
|
1419
|
+
},
|
|
1319
1420
|
getFallbackAudio: () => fallbackAudio,
|
|
1320
1421
|
getFallbackCoverage: () => fallbackCoverage,
|
|
1321
1422
|
};
|
|
@@ -41,6 +41,9 @@
|
|
|
41
41
|
// prefix it still shares, so no word is ever emitted twice.
|
|
42
42
|
//
|
|
43
43
|
// Update rule summary:
|
|
44
|
+
// 0. Final stop-tail decode covering the utterance from t=0 supersedes the
|
|
45
|
+
// rolling assembly outright (tentative = hypothesis); guarded by length
|
|
46
|
+
// so a collapsed decode can never wipe a good transcript.
|
|
44
47
|
// 1. Empty hypothesis: keep stable, clear the tail (pause/silence).
|
|
45
48
|
// 2. c = common word-prefix(assembled, hypothesis), k = overlap(assembled,
|
|
46
49
|
// hypothesis). If c > k: revision path (see above).
|
|
@@ -51,16 +54,29 @@
|
|
|
51
54
|
// 4. The first hypothesis only fills tentative (nothing is confirmed by a
|
|
52
55
|
// single observation).
|
|
53
56
|
//
|
|
54
|
-
// Comparison is word level, punctuation/case-insensitive; original
|
|
57
|
+
// Comparison is word level, punctuation/case/diacritic-insensitive; original
|
|
58
|
+
// words are preserved for display. Pure punctuation tokens must match exactly
|
|
59
|
+
// to count.
|
|
55
60
|
// preserved for display. Pure punctuation tokens must match exactly to count.
|
|
56
61
|
//
|
|
57
62
|
// Memory: O(transcript words). The tracker holds stable words, the tentative
|
|
58
63
|
// tail, and the last window hints - no history list, no per-tick
|
|
59
64
|
// accumulation. Long dictations grow only the transcript itself (the output).
|
|
65
|
+
import { sttLog } from "./stt-log.js";
|
|
60
66
|
|
|
61
67
|
export function normalizeStabilityWord(word) {
|
|
68
|
+
// Diacritic-insensitive: whisper re-decodes shared Vietnamese audio with
|
|
69
|
+
// shifting tone marks between windows, so exact matching breaks anchors on
|
|
70
|
+
// every tick (nothing stabilizes) or appends whole hypotheses as
|
|
71
|
+
// duplicates. Strip combining marks after NFD; Vietnamese đ/Đ do not
|
|
72
|
+
// decompose, so map them explicitly (after lowercase, only đ occurs).
|
|
73
|
+
// Genuine repetitions still work: overlap needs multi-word contiguous
|
|
74
|
+
// runs, not single words.
|
|
62
75
|
return String(word || "")
|
|
63
76
|
.toLowerCase()
|
|
77
|
+
.normalize("NFD")
|
|
78
|
+
.replace(/[\u0300-\u036f]/g, "")
|
|
79
|
+
.replace(/đ/g, "d")
|
|
64
80
|
.replace(/[^\p{L}\p{N}]/gu, "");
|
|
65
81
|
}
|
|
66
82
|
|
|
@@ -132,11 +148,13 @@ export function stripStablePrefix(stableWords, hypothesisWords) {
|
|
|
132
148
|
* streaming controller - this stays a pure text function for testability.
|
|
133
149
|
*
|
|
134
150
|
* `update(hypothesisText, windowHints)` accepts optional snapshot metadata
|
|
135
|
-
* ({ absoluteStartMs, absoluteEndMs, seq }) describing which audio
|
|
136
|
-
* hypothesis was decoded from. The hints are retained for
|
|
137
|
-
* accounting (see getState().lastWindow) and future alignment
|
|
138
|
-
* assembly itself is overlap-based because snapshot-relative
|
|
139
|
-
* cannot anchor a cross-window commit pointer.
|
|
151
|
+
* ({ absoluteStartMs, absoluteEndMs, seq, final }) describing which audio
|
|
152
|
+
* window the hypothesis was decoded from. The hints are retained for
|
|
153
|
+
* coverage accounting (see getState().lastWindow) and future alignment
|
|
154
|
+
* work; text assembly itself is overlap-based because snapshot-relative
|
|
155
|
+
* segment times cannot anchor a cross-window commit pointer. One exception:
|
|
156
|
+
* a final stop-tail decode covering the utterance from t=0 supersedes the
|
|
157
|
+
* rolling assembly outright (see update) instead of joining it.
|
|
140
158
|
*/
|
|
141
159
|
export function createStabilityTracker() {
|
|
142
160
|
let stableWords = [];
|
|
@@ -158,6 +176,41 @@ export function createStabilityTracker() {
|
|
|
158
176
|
return snapshot();
|
|
159
177
|
}
|
|
160
178
|
|
|
179
|
+
// Full-utterance stop-tail decode supersedes the rolling assembly.
|
|
180
|
+
// update() is also fed the final transcription (hints.final) decoded
|
|
181
|
+
// from the stop-tail WAV; when that window covers the utterance from t=0
|
|
182
|
+
// (spool-backed, or a ring tail starting at 0 for short dictations), a
|
|
183
|
+
// single full-context decode beats stitched windows. Joining it would
|
|
184
|
+
// append nearly the whole transcript again whenever the re-decode
|
|
185
|
+
// rewords the head and breaks every anchor - so replace instead.
|
|
186
|
+
// Guarded by length so a collapsed model output can never wipe a good
|
|
187
|
+
// transcript; tail-only and suspiciously short finals still go through
|
|
188
|
+
// the normal join below.
|
|
189
|
+
if (
|
|
190
|
+
windowHints &&
|
|
191
|
+
windowHints.final === true &&
|
|
192
|
+
(windowHints.absoluteStartMs ?? Infinity) <= 1000
|
|
193
|
+
) {
|
|
194
|
+
const total = stableWords.length + tentativeWords.length;
|
|
195
|
+
if (total === 0 || hypoWords.length * 2 >= total) {
|
|
196
|
+
try {
|
|
197
|
+
sttLog(
|
|
198
|
+
"transcript.final-replace",
|
|
199
|
+
`hypoWords=${hypoWords.length} assembledWords=${total} absoluteStartMs=${windowHints.absoluteStartMs}`,
|
|
200
|
+
);
|
|
201
|
+
} catch {}
|
|
202
|
+
stableWords = [];
|
|
203
|
+
tentativeWords = hypoWords.slice();
|
|
204
|
+
return snapshot();
|
|
205
|
+
}
|
|
206
|
+
try {
|
|
207
|
+
sttLog(
|
|
208
|
+
"transcript.final-join",
|
|
209
|
+
`hypoWords=${hypoWords.length} assembledWords=${total} absoluteStartMs=${windowHints.absoluteStartMs}`,
|
|
210
|
+
);
|
|
211
|
+
} catch {}
|
|
212
|
+
}
|
|
213
|
+
|
|
161
214
|
const assembled = stableWords.concat(tentativeWords);
|
|
162
215
|
|
|
163
216
|
// First observation of the session: everything is tentative.
|
package/lib/stt-log.js
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
// Local file log for STT/streaming diagnosis (user-requested).
|
|
2
|
+
//
|
|
3
|
+
// Why this exists: v2 toasts are one-shot and the slot preview's fate is
|
|
4
|
+
// invisible from the UI. This log records which branch ran (batch vs
|
|
5
|
+
// streaming), what the preview did, and toast emissions, so the next
|
|
6
|
+
// recording answers "what happened" without guessing.
|
|
7
|
+
//
|
|
8
|
+
// Privacy: local /tmp only, never shipped. Free text (partials, messages)
|
|
9
|
+
// is truncated to 120 chars; options/endpoints/keys are never logged.
|
|
10
|
+
// Disable with OPENCODE_VOICE_DEBUG_LOG=0. Rotates at ~512KB.
|
|
11
|
+
import fs from "node:fs";
|
|
12
|
+
|
|
13
|
+
const LOG_PATH = "/tmp/opencode-voice-stt.log";
|
|
14
|
+
const MAX_BYTES = 512 * 1024;
|
|
15
|
+
const MAX_TEXT = 120;
|
|
16
|
+
|
|
17
|
+
function enabled() {
|
|
18
|
+
return process.env.OPENCODE_VOICE_DEBUG_LOG !== "0";
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export function trunc(value) {
|
|
22
|
+
const s = String(value ?? "");
|
|
23
|
+
return s.length > MAX_TEXT ? `${s.slice(0, MAX_TEXT)}…` : s;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export function sttLog(event, details = "") {
|
|
27
|
+
if (!enabled()) return;
|
|
28
|
+
try {
|
|
29
|
+
let header = "";
|
|
30
|
+
try {
|
|
31
|
+
if (fs.statSync(LOG_PATH).size > MAX_BYTES) {
|
|
32
|
+
header = `--- rotated ${new Date().toISOString()} ---\n`;
|
|
33
|
+
fs.writeFileSync(LOG_PATH, header);
|
|
34
|
+
}
|
|
35
|
+
} catch {}
|
|
36
|
+
const line = `${new Date().toISOString()} pid=${process.pid} ${event}${details ? ` ${trunc(details)}` : ""}\n`;
|
|
37
|
+
fs.appendFileSync(LOG_PATH, header + line);
|
|
38
|
+
} catch {}
|
|
39
|
+
}
|