@bojackduy/opencode-voice 0.11.2 → 0.11.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +77 -19
- package/index.js +8 -1
- package/lib/chatterbox-server.js +9 -1
- package/lib/conversation-status.js +269 -0
- package/lib/conversation.js +139 -1
- package/lib/streaming-stt.js +101 -0
- package/lib/streaming-transcript.js +59 -6
- package/lib/stt-log.js +39 -0
- package/lib/stt.js +474 -27
- package/lib/tts-player.js +356 -0
- package/lib/tts-status.js +266 -0
- package/lib/tts.js +856 -173
- package/lib/v2-compat.js +41 -0
- package/lib/v2-preview.js +242 -22
- package/lib/vieneu-server.js +287 -0
- package/package.json +12 -1
package/lib/stt.js
CHANGED
|
@@ -5,6 +5,7 @@ import path from "node:path";
|
|
|
5
5
|
import os from "node:os";
|
|
6
6
|
import { spawn, execSync } from "node:child_process";
|
|
7
7
|
import { getActiveSessionTitle, getRecentConversationContext } from "./session.js";
|
|
8
|
+
import { sttLog } from "./stt-log.js";
|
|
8
9
|
|
|
9
10
|
let sttApiEndpoint = null;
|
|
10
11
|
let sttApiModel = null;
|
|
@@ -337,10 +338,24 @@ export const STREAMING_MODE_DEFAULTS = {
|
|
|
337
338
|
cadenceMs: 1000,
|
|
338
339
|
};
|
|
339
340
|
|
|
340
|
-
export function resolveSttMode(opts) {
|
|
341
|
+
export function resolveSttMode(opts, kv = null) {
|
|
342
|
+
// kv wins when set: the v2 host has been observed delivering only a
|
|
343
|
+
// subset of file-plugin options, so a persisted choice must override.
|
|
344
|
+
try {
|
|
345
|
+
const saved = kv?.get("stt.mode");
|
|
346
|
+
if (saved === "streaming" || saved === "batch") return saved;
|
|
347
|
+
} catch {}
|
|
341
348
|
return opts?.sttMode === "streaming" ? "streaming" : "batch";
|
|
342
349
|
}
|
|
343
350
|
|
|
351
|
+
export function resolveSttAutoSubmit(opts, kv = null) {
|
|
352
|
+
try {
|
|
353
|
+
const saved = kv?.get("stt.autoSubmit");
|
|
354
|
+
if (saved === true || saved === false) return saved;
|
|
355
|
+
} catch {}
|
|
356
|
+
return opts?.sttAutoSubmit === true;
|
|
357
|
+
}
|
|
358
|
+
|
|
344
359
|
export function resolveStreamTimings(opts) {
|
|
345
360
|
const windowMs =
|
|
346
361
|
Number(opts?.sttStreamWindowMs) > 0
|
|
@@ -427,6 +442,14 @@ const WAVE_LEN = 16;
|
|
|
427
442
|
const WAVE_INTERVAL = 35;
|
|
428
443
|
const WAVE_DURATION = 70;
|
|
429
444
|
let useWave = true;
|
|
445
|
+
// v2's toast API has no update/replace: every call stacks as a new
|
|
446
|
+
// notification, so the v1 polling pattern becomes a toast storm. v2 shows
|
|
447
|
+
// live state in the slot preview instead and uses one-shot toasts.
|
|
448
|
+
let toastPolling = true;
|
|
449
|
+
|
|
450
|
+
export function __setToastPolling(v) {
|
|
451
|
+
toastPolling = !!v;
|
|
452
|
+
}
|
|
430
453
|
|
|
431
454
|
export function __setRecordingToastFn(fn) {
|
|
432
455
|
recordingToastFn = fn;
|
|
@@ -478,6 +501,7 @@ export function showProcessingToast(message) {
|
|
|
478
501
|
variant: "info",
|
|
479
502
|
duration: PROCESSING_TOAST_DURATION,
|
|
480
503
|
});
|
|
504
|
+
if (!toastPolling) return;
|
|
481
505
|
processingToastTimer = setInterval(() => {
|
|
482
506
|
if (processingToastFn) {
|
|
483
507
|
processingToastFn({
|
|
@@ -525,6 +549,7 @@ let streamingToastFn = null;
|
|
|
525
549
|
let streamingToastBase = "";
|
|
526
550
|
let streamingToastStartMs = null;
|
|
527
551
|
let streamingLastPartialMs = null;
|
|
552
|
+
let lastPartialText = null;
|
|
528
553
|
let streamingSlowAfterMs = 2000;
|
|
529
554
|
const STREAMING_TOAST_DURATION = 3000;
|
|
530
555
|
const STREAMING_TOAST_INTERVAL = 2500;
|
|
@@ -542,6 +567,7 @@ export function __clearStreamingToastState() {
|
|
|
542
567
|
streamingToastBase = "";
|
|
543
568
|
streamingToastStartMs = null;
|
|
544
569
|
streamingLastPartialMs = null;
|
|
570
|
+
lastPartialText = null;
|
|
545
571
|
}
|
|
546
572
|
|
|
547
573
|
export function isStreamingToastActive() {
|
|
@@ -577,9 +603,12 @@ export function showStreamingToast(message, { slowAfterMs } = {}) {
|
|
|
577
603
|
streamingToastBase = message;
|
|
578
604
|
streamingToastStartMs = Date.now();
|
|
579
605
|
streamingLastPartialMs = null;
|
|
606
|
+
lastPartialText = null;
|
|
580
607
|
if (Number(slowAfterMs) > 0) streamingSlowAfterMs = Math.floor(Number(slowAfterMs));
|
|
581
608
|
if (!streamingToastFn) return;
|
|
609
|
+
sttLog("toast.streaming", `base=${streamingToastBase} polling=${toastPolling}`);
|
|
582
610
|
emitStreamingToast();
|
|
611
|
+
if (!toastPolling) return;
|
|
583
612
|
streamingToastTimer = setInterval(() => {
|
|
584
613
|
emitStreamingToast();
|
|
585
614
|
}, STREAMING_TOAST_INTERVAL);
|
|
@@ -594,7 +623,13 @@ export function updateStreamingToast(message) {
|
|
|
594
623
|
emitStreamingToast();
|
|
595
624
|
}
|
|
596
625
|
|
|
597
|
-
export function noteStreamingPartial() {
|
|
626
|
+
export function noteStreamingPartial(partial) {
|
|
627
|
+
// Don't reset the silence timer if the partial is identical to the
|
|
628
|
+
// previous one - whisper hallucinating on silence (repeating the same
|
|
629
|
+
// phrase) must not keep the auto-end timer alive.
|
|
630
|
+
const text = partial ? `${partial.stableText || ""} ${partial.tentativeText || ""}`.trim() : "";
|
|
631
|
+
if (text && text === lastPartialText) return;
|
|
632
|
+
lastPartialText = text;
|
|
598
633
|
streamingLastPartialMs = Date.now();
|
|
599
634
|
}
|
|
600
635
|
|
|
@@ -749,6 +784,63 @@ export function getWaveString() {
|
|
|
749
784
|
}
|
|
750
785
|
}
|
|
751
786
|
|
|
787
|
+
// ---- Streaming waveform (live transcript + wave) ----
|
|
788
|
+
// The batch meter samples a WAV file on disk; streaming keeps PCM in memory,
|
|
789
|
+
// so the controller exposes its newest tail instead. Same thresholds and
|
|
790
|
+
// noise-floor gating as the batch meter, with slightly slower decay because
|
|
791
|
+
// this samples at ~80 ms rather than ~35 ms.
|
|
792
|
+
|
|
793
|
+
const STREAM_WAVE_BYTES = 2560; // ~80 ms of 16 kHz mono 16-bit PCM
|
|
794
|
+
let streamWavePrev = [];
|
|
795
|
+
let streamNoiseFloor = null;
|
|
796
|
+
|
|
797
|
+
export function __resetStreamWave() {
|
|
798
|
+
streamWavePrev = [];
|
|
799
|
+
streamNoiseFloor = null;
|
|
800
|
+
}
|
|
801
|
+
|
|
802
|
+
export function waveLevelsFromPcm(pcm) {
|
|
803
|
+
if (!pcm || pcm.length < 2) return Array(WAVE_LEN).fill(0);
|
|
804
|
+
const chunkBytes = Math.max(2, Math.floor(pcm.length / WAVE_LEN));
|
|
805
|
+
const raw = [];
|
|
806
|
+
for (let column = 0; column < WAVE_LEN; column++) {
|
|
807
|
+
const offset = column * chunkBytes;
|
|
808
|
+
if (offset >= pcm.length) {
|
|
809
|
+
raw.push(0);
|
|
810
|
+
continue;
|
|
811
|
+
}
|
|
812
|
+
const chunk = pcm.subarray(offset, Math.min(offset + chunkBytes, pcm.length));
|
|
813
|
+
const samples = Math.floor(chunk.length / 2);
|
|
814
|
+
if (samples === 0) {
|
|
815
|
+
raw.push(0);
|
|
816
|
+
continue;
|
|
817
|
+
}
|
|
818
|
+
let sum = 0;
|
|
819
|
+
for (let i = 0; i + 1 < chunk.length; i += 2) {
|
|
820
|
+
const sample = chunk.readInt16LE(i) / 32768;
|
|
821
|
+
sum += sample * sample;
|
|
822
|
+
}
|
|
823
|
+
raw.push(Math.sqrt(sum / samples));
|
|
824
|
+
}
|
|
825
|
+
const floor = Math.min(...raw);
|
|
826
|
+
if (streamNoiseFloor === null) streamNoiseFloor = floor;
|
|
827
|
+
else streamNoiseFloor = Math.min(streamNoiseFloor, floor) * 0.995 + floor * 0.005;
|
|
828
|
+
const levels = raw.map((rms) => {
|
|
829
|
+
const effective = Math.max(0, rms - streamNoiseFloor * 1.15);
|
|
830
|
+
if (effective < 0.005) return 0;
|
|
831
|
+
return rmsToLevel(effective * 1.6);
|
|
832
|
+
});
|
|
833
|
+
if (streamWavePrev.length !== WAVE_LEN) streamWavePrev = Array(WAVE_LEN).fill(0);
|
|
834
|
+
const smoothed = levels.map((level, index) => {
|
|
835
|
+
const prev = streamWavePrev[index] || 0;
|
|
836
|
+
if (level > prev) return level;
|
|
837
|
+
const decayed = Math.floor(prev * 0.35);
|
|
838
|
+
return level > decayed ? level : decayed;
|
|
839
|
+
});
|
|
840
|
+
streamWavePrev = smoothed;
|
|
841
|
+
return smoothed;
|
|
842
|
+
}
|
|
843
|
+
|
|
752
844
|
let currentRecordKeybind = "<leader>[";
|
|
753
845
|
export function __setRecordKeybind(kb) {
|
|
754
846
|
if (kb) currentRecordKeybind = kb.replace(/^<leader>/, "leader+");
|
|
@@ -767,10 +859,14 @@ export function __setStopHint(hint) {
|
|
|
767
859
|
savedRecordKeybind = null;
|
|
768
860
|
}
|
|
769
861
|
}
|
|
770
|
-
|
|
862
|
+
function recordingModeSuffix() {
|
|
771
863
|
// When the conversation loop overrides the stop hint, the toast says so
|
|
772
864
|
// explicitly - otherwise users wonder why it names a key they did not press.
|
|
773
|
-
|
|
865
|
+
return savedRecordKeybind !== null ? " (conversation)" : "";
|
|
866
|
+
}
|
|
867
|
+
|
|
868
|
+
export function buildRecordingMessage() {
|
|
869
|
+
const mode = recordingModeSuffix();
|
|
774
870
|
if (!useWave || recordingStartMs === null)
|
|
775
871
|
return `${RECORDING_TOAST_BASE}${mode} · ${currentRecordKeybind} to stop`;
|
|
776
872
|
const elapsed = formatElapsed(Date.now() - recordingStartMs);
|
|
@@ -781,10 +877,19 @@ export function buildRecordingMessage() {
|
|
|
781
877
|
export function showRecordingToast() {
|
|
782
878
|
clearRecordingToast();
|
|
783
879
|
if (!recordingToastFn) return;
|
|
880
|
+
sttLog("toast.recording", `polling=${toastPolling} wave=${useWave}`);
|
|
784
881
|
recordingStartMs = Date.now();
|
|
785
882
|
prevWaveLevels = Array(WAVE_LEN).fill(0);
|
|
786
883
|
noiseFloorRms = null;
|
|
787
884
|
calibrationEndMs = Date.now() + 120;
|
|
885
|
+
if (!toastPolling) {
|
|
886
|
+
recordingToastFn({
|
|
887
|
+
message: `${RECORDING_TOAST_BASE}${recordingModeSuffix()} · ${currentRecordKeybind} to stop`,
|
|
888
|
+
variant: "info",
|
|
889
|
+
duration: RECORDING_TOAST_DURATION,
|
|
890
|
+
});
|
|
891
|
+
return;
|
|
892
|
+
}
|
|
788
893
|
const interval = useWave ? WAVE_INTERVAL : RECORDING_TOAST_INTERVAL;
|
|
789
894
|
const duration = useWave ? WAVE_DURATION : RECORDING_TOAST_DURATION;
|
|
790
895
|
recordingToastFn({
|
|
@@ -826,7 +931,15 @@ function forceKillSox(logger) {
|
|
|
826
931
|
} catch {}
|
|
827
932
|
}
|
|
828
933
|
|
|
829
|
-
function startRecording(
|
|
934
|
+
function startRecording(
|
|
935
|
+
kv,
|
|
936
|
+
backend,
|
|
937
|
+
toast,
|
|
938
|
+
logger,
|
|
939
|
+
trimSilence = true,
|
|
940
|
+
silenceStopSec = null,
|
|
941
|
+
onStopped = null,
|
|
942
|
+
) {
|
|
830
943
|
if (soxProc) {
|
|
831
944
|
logger?.log("STT", "Start recording skipped: sox already running", "debug");
|
|
832
945
|
return;
|
|
@@ -844,11 +957,16 @@ function startRecording(kv, backend, toast, logger, trimSilence = true) {
|
|
|
844
957
|
const spawnT0 = Date.now();
|
|
845
958
|
logger?.log(
|
|
846
959
|
"STT",
|
|
847
|
-
`Starting recording backend=${backend} mic=${mic || "system default"}`,
|
|
960
|
+
`Starting recording backend=${backend} mic=${mic || "system default"} silenceStop=${silenceStopSec ?? "off"}`,
|
|
848
961
|
"debug",
|
|
849
962
|
);
|
|
850
963
|
|
|
964
|
+
// Trim leading silence, then (optionally) auto-stop after N seconds of
|
|
965
|
+
// silence so hands-free modes (conversation) can finalize without a keypress.
|
|
851
966
|
const silenceArgs = trimSilence ? ["silence", "1", "0.05", "1%"] : [];
|
|
967
|
+
if (silenceStopSec && silenceStopSec > 0) {
|
|
968
|
+
silenceArgs.push("1", String(silenceStopSec), "1%");
|
|
969
|
+
}
|
|
852
970
|
soxProc = spawn(
|
|
853
971
|
"sox",
|
|
854
972
|
[...inputArgs, "-r", "16000", "-c", "1", "-b", "16", wavFile, ...silenceArgs],
|
|
@@ -895,7 +1013,15 @@ function startRecording(kv, backend, toast, logger, trimSilence = true) {
|
|
|
895
1013
|
"error",
|
|
896
1014
|
);
|
|
897
1015
|
} else {
|
|
898
|
-
|
|
1016
|
+
// Normal exit (including silence auto-stop): fire the onStopped
|
|
1017
|
+
// callback if registered, otherwise show the generic toast.
|
|
1018
|
+
if (typeof onStopped === "function") {
|
|
1019
|
+
try {
|
|
1020
|
+
onStopped();
|
|
1021
|
+
} catch {}
|
|
1022
|
+
} else {
|
|
1023
|
+
toast("Recording stopped unexpectedly", "warning");
|
|
1024
|
+
}
|
|
899
1025
|
}
|
|
900
1026
|
}
|
|
901
1027
|
});
|
|
@@ -1605,6 +1731,23 @@ export function registerSTT(api, kv, complete, prompts, opts, logger, deps = {})
|
|
|
1605
1731
|
const client = api.client;
|
|
1606
1732
|
const renderer = opts?.focusMode === "primary" ? null : api.renderer;
|
|
1607
1733
|
const preview = api.voicePreview;
|
|
1734
|
+
// Seed engine from opts if the host delivered it.
|
|
1735
|
+
try {
|
|
1736
|
+
if (opts?.ttsEngine && kv.get("tts.engine") == null) {
|
|
1737
|
+
kv.set("tts.engine", opts.ttsEngine);
|
|
1738
|
+
}
|
|
1739
|
+
} catch {}
|
|
1740
|
+
__setToastPolling(!preview);
|
|
1741
|
+
const bootMode = resolveSttMode(opts, kv);
|
|
1742
|
+
let modeSource = "default-batch";
|
|
1743
|
+
try {
|
|
1744
|
+
if (kv?.get("stt.mode") === bootMode) modeSource = "kv";
|
|
1745
|
+
else if (opts?.sttMode === bootMode) modeSource = "options";
|
|
1746
|
+
} catch {}
|
|
1747
|
+
sttLog(
|
|
1748
|
+
"register",
|
|
1749
|
+
`sttMode=${bootMode} source=${modeSource} preview=${preview ? "yes" : "no"} toastPolling=${!preview} optKeys=${Object.keys(opts || {}).join(",")}`,
|
|
1750
|
+
);
|
|
1608
1751
|
let streamGeneration = 0;
|
|
1609
1752
|
let streamDisposed = false;
|
|
1610
1753
|
let finalizingController = null;
|
|
@@ -1683,9 +1826,23 @@ export function registerSTT(api, kv, complete, prompts, opts, logger, deps = {})
|
|
|
1683
1826
|
|
|
1684
1827
|
// Start a recording for the voice-conversation loop. Returns false when busy.
|
|
1685
1828
|
|
|
1829
|
+
// Conversation-mode silence auto-finalize: when set, the recording uses
|
|
1830
|
+
// sox silence detection to auto-stop after this many seconds of silence,
|
|
1831
|
+
// and the onStopped callback fires. Default null = manual stop only.
|
|
1832
|
+
let conversationSilenceStopSec = null;
|
|
1833
|
+
let conversationOnStopped = null;
|
|
1834
|
+
|
|
1686
1835
|
function startLoopRecording() {
|
|
1687
1836
|
if (recording || processing || streamingDictationActive) return false;
|
|
1688
|
-
startRecording(
|
|
1837
|
+
startRecording(
|
|
1838
|
+
kv,
|
|
1839
|
+
backend,
|
|
1840
|
+
toast,
|
|
1841
|
+
logger,
|
|
1842
|
+
opts?.trimSilence,
|
|
1843
|
+
conversationSilenceStopSec,
|
|
1844
|
+
conversationOnStopped,
|
|
1845
|
+
);
|
|
1689
1846
|
if (recording) {
|
|
1690
1847
|
showRecordingToast();
|
|
1691
1848
|
// Fetch title + recent turns while the user speaks, so the context is
|
|
@@ -1696,7 +1853,30 @@ export function registerSTT(api, kv, complete, prompts, opts, logger, deps = {})
|
|
|
1696
1853
|
}
|
|
1697
1854
|
|
|
1698
1855
|
// ---- Streaming dictation session (stage 2, local-only, no LLM) ----
|
|
1699
|
-
|
|
1856
|
+
// Mode/auto-submit resolve live from kv first: options seed kv on first
|
|
1857
|
+
// boot, then the /stt-mode and /stt-autosubmit commands own the runtime
|
|
1858
|
+
// value. This survives hosts that deliver partial plugin options.
|
|
1859
|
+
const seedKvFromOpts = () => {
|
|
1860
|
+
try {
|
|
1861
|
+
if (
|
|
1862
|
+
(opts?.sttMode === "streaming" || opts?.sttMode === "batch") &&
|
|
1863
|
+
kv.get("stt.mode") == null
|
|
1864
|
+
) {
|
|
1865
|
+
kv.set("stt.mode", opts.sttMode);
|
|
1866
|
+
}
|
|
1867
|
+
} catch {}
|
|
1868
|
+
try {
|
|
1869
|
+
if (opts?.sttAutoSubmit !== undefined && kv.get("stt.autoSubmit") == null) {
|
|
1870
|
+
kv.set("stt.autoSubmit", opts.sttAutoSubmit === true);
|
|
1871
|
+
}
|
|
1872
|
+
} catch {}
|
|
1873
|
+
};
|
|
1874
|
+
seedKvFromOpts();
|
|
1875
|
+
const currentSttMode = () => resolveSttMode(opts, kv);
|
|
1876
|
+
const sttAutoSubmitEnabled = () => resolveSttAutoSubmit(opts, kv);
|
|
1877
|
+
// Boot-time snapshot for the mode log line below; behavior branches call
|
|
1878
|
+
// currentSttMode() live so /stt-mode takes effect without a restart.
|
|
1879
|
+
const sttMode = currentSttMode();
|
|
1700
1880
|
const streamTimings = resolveStreamTimings(opts);
|
|
1701
1881
|
logger?.log(
|
|
1702
1882
|
"STT",
|
|
@@ -1707,6 +1887,67 @@ export function registerSTT(api, kv, complete, prompts, opts, logger, deps = {})
|
|
|
1707
1887
|
let streamEditor = null;
|
|
1708
1888
|
let streamTarget = null;
|
|
1709
1889
|
let streamStarting = false;
|
|
1890
|
+
// Auto-end-of-speech: when enabled, a timer checks if no new partial has
|
|
1891
|
+
// arrived for AUTO_END_SILENCE_MS and triggers finalize automatically.
|
|
1892
|
+
let streamAutoEndTimer = null;
|
|
1893
|
+
const AUTO_END_SILENCE_MS = 3000;
|
|
1894
|
+
|
|
1895
|
+
function enableStreamAutoEnd() {
|
|
1896
|
+
disableStreamAutoEnd();
|
|
1897
|
+
streamAutoEndTimer = setInterval(() => {
|
|
1898
|
+
if (!streamingDictationActive || !streamController) {
|
|
1899
|
+
disableStreamAutoEnd();
|
|
1900
|
+
return;
|
|
1901
|
+
}
|
|
1902
|
+
// Only act if auto-end is actually enabled.
|
|
1903
|
+
if (!preview?.getAutoEndOfSpeech?.()) return;
|
|
1904
|
+
// Audio-level silence check: if the mic is picking up nothing,
|
|
1905
|
+
// the user has stopped speaking regardless of what whisper returns.
|
|
1906
|
+
let audioSilent = false;
|
|
1907
|
+
let rms = 0;
|
|
1908
|
+
try {
|
|
1909
|
+
const pcm = streamController.peekPcm?.(STREAM_WAVE_BYTES);
|
|
1910
|
+
if (pcm && pcm.length >= 2) {
|
|
1911
|
+
let sum = 0;
|
|
1912
|
+
const samples = Math.floor(pcm.length / 2);
|
|
1913
|
+
for (let i = 0; i + 1 < pcm.length; i += 2) {
|
|
1914
|
+
const s = pcm.readInt16LE(i) / 32768;
|
|
1915
|
+
sum += s * s;
|
|
1916
|
+
}
|
|
1917
|
+
rms = Math.sqrt(sum / samples);
|
|
1918
|
+
if (rms < 0.01) audioSilent = true;
|
|
1919
|
+
}
|
|
1920
|
+
} catch {}
|
|
1921
|
+
const last = streamingLastPartialMs;
|
|
1922
|
+
const elapsed = last ? Date.now() - last : 0;
|
|
1923
|
+
if (audioSilent && last && elapsed > AUTO_END_SILENCE_MS) {
|
|
1924
|
+
sttLog(
|
|
1925
|
+
"stream.autoEnd",
|
|
1926
|
+
`triggered by silence (audio+partial) rms=${rms.toFixed(4)} elapsed=${elapsed}ms`,
|
|
1927
|
+
);
|
|
1928
|
+
disableStreamAutoEnd();
|
|
1929
|
+
void finalizeStreamingDictation({ submit: preview?.getAutoSubmit?.() ?? false });
|
|
1930
|
+
return;
|
|
1931
|
+
}
|
|
1932
|
+
// Fallback: partial-based silence (no audio data available).
|
|
1933
|
+
if (last && elapsed > AUTO_END_SILENCE_MS * 2) {
|
|
1934
|
+
sttLog(
|
|
1935
|
+
"stream.autoEnd",
|
|
1936
|
+
`triggered by silence (partial only) rms=${rms.toFixed(4)} elapsed=${elapsed}ms`,
|
|
1937
|
+
);
|
|
1938
|
+
disableStreamAutoEnd();
|
|
1939
|
+
void finalizeStreamingDictation({ submit: preview?.getAutoSubmit?.() ?? false });
|
|
1940
|
+
}
|
|
1941
|
+
}, 500);
|
|
1942
|
+
if (typeof streamAutoEndTimer.unref === "function") streamAutoEndTimer.unref();
|
|
1943
|
+
}
|
|
1944
|
+
|
|
1945
|
+
function disableStreamAutoEnd() {
|
|
1946
|
+
if (streamAutoEndTimer) {
|
|
1947
|
+
clearInterval(streamAutoEndTimer);
|
|
1948
|
+
streamAutoEndTimer = null;
|
|
1949
|
+
}
|
|
1950
|
+
}
|
|
1710
1951
|
// Warm server lease retained across dictations: the idle controller (mic
|
|
1711
1952
|
// fully stopped, server still loaded) parks here between sessions. At most
|
|
1712
1953
|
// one warm lease ever exists; the next start reuses it when the model key
|
|
@@ -1797,6 +2038,55 @@ export function registerSTT(api, kv, complete, prompts, opts, logger, deps = {})
|
|
|
1797
2038
|
});
|
|
1798
2039
|
if (preview) await preview.begin();
|
|
1799
2040
|
else editor.begin(target);
|
|
2041
|
+
sttLog("stream.begin", `preview=${preview ? "yes" : "no"}`);
|
|
2042
|
+
// Clickable pause lives on the preview line (v2 slot). The toggle
|
|
2043
|
+
// action funnels through the same pause/resume path as the command.
|
|
2044
|
+
try {
|
|
2045
|
+
preview?.setToggleAction?.(() => toggleStreamingPause());
|
|
2046
|
+
} catch {}
|
|
2047
|
+
// Restore persisted toggle states from kv so they survive restarts.
|
|
2048
|
+
try {
|
|
2049
|
+
const savedAutoSubmit = kv.get("stt.autoSubmit");
|
|
2050
|
+
if (savedAutoSubmit === true || savedAutoSubmit === false) {
|
|
2051
|
+
preview?.setAutoSubmit?.(savedAutoSubmit);
|
|
2052
|
+
}
|
|
2053
|
+
} catch {}
|
|
2054
|
+
try {
|
|
2055
|
+
const savedAutoEnd = kv.get("stt.autoEndOfSpeech");
|
|
2056
|
+
if (savedAutoEnd === true || savedAutoEnd === false) {
|
|
2057
|
+
preview?.setAutoEndOfSpeech?.(savedAutoEnd);
|
|
2058
|
+
}
|
|
2059
|
+
} catch {}
|
|
2060
|
+
try {
|
|
2061
|
+
preview?.setAutoSubmitToggleAction?.(() => {
|
|
2062
|
+
const next = !preview.getAutoSubmit?.();
|
|
2063
|
+
preview.setAutoSubmit?.(next);
|
|
2064
|
+
// Persist in kv so it survives restarts.
|
|
2065
|
+
try {
|
|
2066
|
+
kv.set("stt.autoSubmit", next);
|
|
2067
|
+
} catch {}
|
|
2068
|
+
sttLog("stream.autoSubmit", `toggled to ${next}`);
|
|
2069
|
+
});
|
|
2070
|
+
} catch {}
|
|
2071
|
+
try {
|
|
2072
|
+
preview?.setAutoEndToggleAction?.(() => {
|
|
2073
|
+
const next = !preview.getAutoEndOfSpeech?.();
|
|
2074
|
+
preview.setAutoEndOfSpeech?.(next);
|
|
2075
|
+
// Persist so it survives restarts.
|
|
2076
|
+
try {
|
|
2077
|
+
kv.set("stt.autoEndOfSpeech", next);
|
|
2078
|
+
} catch {}
|
|
2079
|
+
sttLog("stream.autoEnd", `toggled to ${next}`);
|
|
2080
|
+
// If recording is active, enable/disable silence auto-finalize.
|
|
2081
|
+
if (streamingDictationActive && streamController) {
|
|
2082
|
+
if (next) {
|
|
2083
|
+
enableStreamAutoEnd();
|
|
2084
|
+
} else {
|
|
2085
|
+
disableStreamAutoEnd();
|
|
2086
|
+
}
|
|
2087
|
+
}
|
|
2088
|
+
});
|
|
2089
|
+
} catch {}
|
|
1800
2090
|
if (streamDisposed || myGen !== streamGeneration) {
|
|
1801
2091
|
preview?.clear();
|
|
1802
2092
|
return false;
|
|
@@ -1805,13 +2095,18 @@ export function registerSTT(api, kv, complete, prompts, opts, logger, deps = {})
|
|
|
1805
2095
|
onPartial: (p) => {
|
|
1806
2096
|
if (streamDisposed || myGen !== streamGeneration) return;
|
|
1807
2097
|
try {
|
|
1808
|
-
noteStreamingPartial();
|
|
2098
|
+
noteStreamingPartial(p);
|
|
1809
2099
|
} catch {}
|
|
1810
2100
|
try {
|
|
1811
2101
|
ed.applyPartial(p);
|
|
1812
2102
|
} catch {}
|
|
2103
|
+
sttLog(
|
|
2104
|
+
"stream.partial",
|
|
2105
|
+
`stable=${(p?.stableText || "").length} tentative=${(p?.tentativeText || "").length}`,
|
|
2106
|
+
);
|
|
1813
2107
|
},
|
|
1814
2108
|
onStateChange: (s) => {
|
|
2109
|
+
sttLog("stream.state", s);
|
|
1815
2110
|
try {
|
|
1816
2111
|
if (s === "streaming") {
|
|
1817
2112
|
updateStreamingToast("● Streaming dictation — listening");
|
|
@@ -1865,6 +2160,24 @@ export function registerSTT(api, kv, complete, prompts, opts, logger, deps = {})
|
|
|
1865
2160
|
return false;
|
|
1866
2161
|
}
|
|
1867
2162
|
__setStreamingActive(true);
|
|
2163
|
+
sttLog("stream.started", `reusedWarm=${reusedWarm}`);
|
|
2164
|
+
// Live waveform rides the v2 preview slot next to the live transcript.
|
|
2165
|
+
// v1 has no persistent preview surface; its batch wave is unchanged.
|
|
2166
|
+
if (preview) {
|
|
2167
|
+
__resetStreamWave();
|
|
2168
|
+
let pcmLogCount = 0;
|
|
2169
|
+
preview.startWave?.(() => {
|
|
2170
|
+
const pcm = controller.peekPcm?.(STREAM_WAVE_BYTES);
|
|
2171
|
+
if (pcmLogCount < 3 && pcm) {
|
|
2172
|
+
pcmLogCount += 1;
|
|
2173
|
+
sttLog("stream.pcm", `bytes=${pcm.length}`);
|
|
2174
|
+
}
|
|
2175
|
+
return pcm ? waveLevelsFromPcm(pcm) : null;
|
|
2176
|
+
});
|
|
2177
|
+
// Auto-end: always run the timer; it checks the preview's toggle
|
|
2178
|
+
// state internally so toggling mid-recording works without restart.
|
|
2179
|
+
enableStreamAutoEnd();
|
|
2180
|
+
}
|
|
1868
2181
|
// Sticky, never silent: covers the cold model load ("Loading…") until
|
|
1869
2182
|
// the controller reports streaming, then live capture with elapsed
|
|
1870
2183
|
// time. Partial preview toasts continue alongside via the editor.
|
|
@@ -1875,6 +2188,7 @@ export function registerSTT(api, kv, complete, prompts, opts, logger, deps = {})
|
|
|
1875
2188
|
return true;
|
|
1876
2189
|
} catch (err) {
|
|
1877
2190
|
preview?.clear();
|
|
2191
|
+
sttLog("stream.start-failed", err?.message || String(err));
|
|
1878
2192
|
logger?.log("STT", `Streaming start failed: ${err?.message || err}`, "error");
|
|
1879
2193
|
try {
|
|
1880
2194
|
clearStreamingToast();
|
|
@@ -1901,6 +2215,60 @@ export function registerSTT(api, kv, complete, prompts, opts, logger, deps = {})
|
|
|
1901
2215
|
}
|
|
1902
2216
|
}
|
|
1903
2217
|
|
|
2218
|
+
// Pause/resume for live dictation: freezes the mic (SIGSTOP) and skips
|
|
2219
|
+
// recognition ticks so paused spans cost nothing and never enter the
|
|
2220
|
+
// transcript. The record key resumes a paused session instead of
|
|
2221
|
+
// finalizing it; explicit stop stays on /stt-stop.
|
|
2222
|
+
function toggleStreamingPause() {
|
|
2223
|
+
if (!streamingDictationActive || !streamController) return false;
|
|
2224
|
+
let pausedNow = false;
|
|
2225
|
+
try {
|
|
2226
|
+
pausedNow = !!streamController.getState?.().paused;
|
|
2227
|
+
} catch {
|
|
2228
|
+
return false;
|
|
2229
|
+
}
|
|
2230
|
+
if (pausedNow) {
|
|
2231
|
+
let ok = false;
|
|
2232
|
+
try {
|
|
2233
|
+
ok = streamController.resume() === true;
|
|
2234
|
+
} catch {
|
|
2235
|
+
ok = false;
|
|
2236
|
+
}
|
|
2237
|
+
if (ok) {
|
|
2238
|
+
preview?.setPaused?.(false);
|
|
2239
|
+
updateStreamingToast("● Streaming dictation — listening");
|
|
2240
|
+
sttLog("stream.resume", "ok");
|
|
2241
|
+
}
|
|
2242
|
+
return ok;
|
|
2243
|
+
}
|
|
2244
|
+
let ok = false;
|
|
2245
|
+
try {
|
|
2246
|
+
ok = streamController.pause?.() === true;
|
|
2247
|
+
} catch {
|
|
2248
|
+
ok = false;
|
|
2249
|
+
}
|
|
2250
|
+
if (ok) {
|
|
2251
|
+
preview?.setPaused?.(true);
|
|
2252
|
+
updateStreamingToast("⏸ Dictation paused — press record or click ▶ to resume");
|
|
2253
|
+
sttLog("stream.pause", "ok");
|
|
2254
|
+
}
|
|
2255
|
+
return ok;
|
|
2256
|
+
}
|
|
2257
|
+
|
|
2258
|
+
// Pause/resume entry point shared by the /stt-pause command. Returns false
|
|
2259
|
+
// with a hint toast when there is nothing pausable.
|
|
2260
|
+
async function pauseStreamingDictation() {
|
|
2261
|
+
if (!streamingDictationActive || !streamController) {
|
|
2262
|
+
toast("No streaming dictation in progress", "warning");
|
|
2263
|
+
return false;
|
|
2264
|
+
}
|
|
2265
|
+
if (!toggleStreamingPause()) {
|
|
2266
|
+
toast("Pause unavailable right now - use /stt-stop to cancel", "warning");
|
|
2267
|
+
return false;
|
|
2268
|
+
}
|
|
2269
|
+
return true;
|
|
2270
|
+
}
|
|
2271
|
+
|
|
1904
2272
|
// Finalize: stop the controller (single tail transcription, no LLM),
|
|
1905
2273
|
// commit the editor range, then insert EXACTLY once via the focused-input
|
|
1906
2274
|
// helper. Never falls through to primary-chat appendPrompt. When `submit`
|
|
@@ -1911,6 +2279,8 @@ export function registerSTT(api, kv, complete, prompts, opts, logger, deps = {})
|
|
|
1911
2279
|
// is at fault, on model/language change, or on plugin unload.
|
|
1912
2280
|
async function finalizeStreamingDictation({ submit = false } = {}) {
|
|
1913
2281
|
if (!streamingDictationActive || !streamController) return null;
|
|
2282
|
+
sttLog("stream.finalize", `submit=${submit}`);
|
|
2283
|
+
disableStreamAutoEnd();
|
|
1914
2284
|
const controller = streamController;
|
|
1915
2285
|
const editor = streamEditor;
|
|
1916
2286
|
const target = streamTarget;
|
|
@@ -1988,18 +2358,9 @@ export function registerSTT(api, kv, complete, prompts, opts, logger, deps = {})
|
|
|
1988
2358
|
retainWarmStreamController(controller);
|
|
1989
2359
|
return { text: "" };
|
|
1990
2360
|
}
|
|
1991
|
-
if (preview) {
|
|
1992
|
-
// v2 has a visible preview but no composer append. Finalize submits
|
|
1993
|
-
// once through the same home/create/cancellation fence as batch STT.
|
|
1994
|
-
processing = true;
|
|
1995
|
-
const result = await submitTurnText(client, null, toast, text, logger);
|
|
1996
|
-
if (!streamDisposed && myGen === streamGeneration) retainWarmStreamController(controller);
|
|
1997
|
-
else await controller.dispose?.();
|
|
1998
|
-
return { ...result, submitted: !result.error };
|
|
1999
|
-
}
|
|
2000
2361
|
if (fin?.inserted) {
|
|
2001
2362
|
logger?.log("STT", `Streaming finalized via owned range chars=${text.length}`, "debug");
|
|
2002
|
-
if (submit ||
|
|
2363
|
+
if (submit || autoSubmitNow()) {
|
|
2003
2364
|
if (target && typeof target.submit === "function") {
|
|
2004
2365
|
try {
|
|
2005
2366
|
target.submit();
|
|
@@ -2013,6 +2374,7 @@ export function registerSTT(api, kv, complete, prompts, opts, logger, deps = {})
|
|
|
2013
2374
|
} else {
|
|
2014
2375
|
toast("Streaming transcription added", "success");
|
|
2015
2376
|
}
|
|
2377
|
+
if (preview) processing = false;
|
|
2016
2378
|
retainWarmStreamController(controller);
|
|
2017
2379
|
return { text, inserted: true };
|
|
2018
2380
|
}
|
|
@@ -2033,11 +2395,12 @@ export function registerSTT(api, kv, complete, prompts, opts, logger, deps = {})
|
|
|
2033
2395
|
if (!inserted) {
|
|
2034
2396
|
logger?.log("STT", "Streaming finalize: no editable field, transcript retained", "warn");
|
|
2035
2397
|
toast("No editable field - dictated text kept (not inserted elsewhere)", "warning");
|
|
2398
|
+
if (preview) processing = false;
|
|
2036
2399
|
retainWarmStreamController(controller);
|
|
2037
2400
|
return { text, inserted: false };
|
|
2038
2401
|
}
|
|
2039
2402
|
logger?.log("STT", `Streaming finalized chars=${text.length}`, "debug");
|
|
2040
|
-
if (submit ||
|
|
2403
|
+
if (submit || autoSubmitNow()) {
|
|
2041
2404
|
if (target && typeof target.submit === "function") {
|
|
2042
2405
|
try {
|
|
2043
2406
|
target.submit();
|
|
@@ -2051,6 +2414,7 @@ export function registerSTT(api, kv, complete, prompts, opts, logger, deps = {})
|
|
|
2051
2414
|
} else {
|
|
2052
2415
|
toast("Streaming transcription added", "success");
|
|
2053
2416
|
}
|
|
2417
|
+
if (preview) processing = false;
|
|
2054
2418
|
retainWarmStreamController(controller);
|
|
2055
2419
|
return { text, inserted: true };
|
|
2056
2420
|
}
|
|
@@ -2061,6 +2425,8 @@ export function registerSTT(api, kv, complete, prompts, opts, logger, deps = {})
|
|
|
2061
2425
|
// next dictation (user cancel is per-dictation, never a server fault).
|
|
2062
2426
|
async function cancelStreamingDictation() {
|
|
2063
2427
|
streamGeneration += 1;
|
|
2428
|
+
sttLog("stream.cancel");
|
|
2429
|
+
disableStreamAutoEnd();
|
|
2064
2430
|
preview?.clear();
|
|
2065
2431
|
if (preview && processing) cancelRecording(logger);
|
|
2066
2432
|
if (finalizingController) {
|
|
@@ -2136,6 +2502,10 @@ export function registerSTT(api, kv, complete, prompts, opts, logger, deps = {})
|
|
|
2136
2502
|
isProcessing: () => processing,
|
|
2137
2503
|
isStreaming,
|
|
2138
2504
|
start: startLoopRecording,
|
|
2505
|
+
setSilenceAutoFinalize: (sec, onStopped) => {
|
|
2506
|
+
conversationSilenceStopSec = sec && sec > 0 ? sec : null;
|
|
2507
|
+
conversationOnStopped = typeof onStopped === "function" ? onStopped : null;
|
|
2508
|
+
},
|
|
2139
2509
|
setStopHint: (hint) => __setStopHint(hint),
|
|
2140
2510
|
cancel: () => cancelRecording(logger),
|
|
2141
2511
|
discard: () => discardTurn(),
|
|
@@ -2153,7 +2523,9 @@ export function registerSTT(api, kv, complete, prompts, opts, logger, deps = {})
|
|
|
2153
2523
|
|
|
2154
2524
|
// One-shot auto-submit: speak, press again, and the text submits itself
|
|
2155
2525
|
// without landing in the box first. Off by default (append-then-edit).
|
|
2156
|
-
|
|
2526
|
+
// Live reader: /stt-autosubmit flips the kv value mid-session.
|
|
2527
|
+
const autoSubmit = sttAutoSubmitEnabled();
|
|
2528
|
+
const autoSubmitNow = () => sttAutoSubmitEnabled();
|
|
2157
2529
|
|
|
2158
2530
|
const commands = [
|
|
2159
2531
|
{
|
|
@@ -2171,10 +2543,18 @@ export function registerSTT(api, kv, complete, prompts, opts, logger, deps = {})
|
|
|
2171
2543
|
...(kb("stt.record") ? { keybind: kb("stt.record") } : {}),
|
|
2172
2544
|
slash: { name: "stt-record" },
|
|
2173
2545
|
onSelect() {
|
|
2174
|
-
// Streaming mode: toggle streaming record/finalize
|
|
2175
|
-
//
|
|
2176
|
-
|
|
2546
|
+
// Streaming mode: toggle streaming record/finalize, or resume a
|
|
2547
|
+
// paused session (cheaper than re-speaking; explicit stop stays on
|
|
2548
|
+
// /stt-stop). Batch default below is behavior-identical to before.
|
|
2549
|
+
sttLog("toggle", `sttMode=${currentSttMode()} streamingActive=${streamingDictationActive}`);
|
|
2550
|
+
if (currentSttMode() === "streaming") {
|
|
2177
2551
|
if (streamingDictationActive) {
|
|
2552
|
+
try {
|
|
2553
|
+
if (streamController?.getState?.().paused) {
|
|
2554
|
+
toggleStreamingPause();
|
|
2555
|
+
return;
|
|
2556
|
+
}
|
|
2557
|
+
} catch {}
|
|
2178
2558
|
void finalizeStreamingDictation({ submit: false });
|
|
2179
2559
|
return;
|
|
2180
2560
|
}
|
|
@@ -2240,7 +2620,7 @@ export function registerSTT(api, kv, complete, prompts, opts, logger, deps = {})
|
|
|
2240
2620
|
onSelect() {
|
|
2241
2621
|
// Streaming mode: finalize then submit ONLY via the captured
|
|
2242
2622
|
// target's own submit(). Never falls through to primary-chat submit.
|
|
2243
|
-
if (
|
|
2623
|
+
if (currentSttMode() === "streaming") {
|
|
2244
2624
|
if (!streamingDictationActive) {
|
|
2245
2625
|
toast("No streaming dictation in progress", "warning");
|
|
2246
2626
|
return;
|
|
@@ -2292,7 +2672,7 @@ export function registerSTT(api, kv, complete, prompts, opts, logger, deps = {})
|
|
|
2292
2672
|
onSelect() {
|
|
2293
2673
|
// Streaming mode: cancel and remove ONLY plugin-owned dictation
|
|
2294
2674
|
// text, never user-typed text.
|
|
2295
|
-
if (
|
|
2675
|
+
if (currentSttMode() === "streaming" || streamingDictationActive) {
|
|
2296
2676
|
if (streamingDictationActive) {
|
|
2297
2677
|
void cancelStreamingDictation();
|
|
2298
2678
|
return;
|
|
@@ -2322,6 +2702,73 @@ export function registerSTT(api, kv, complete, prompts, opts, logger, deps = {})
|
|
|
2322
2702
|
}
|
|
2323
2703
|
},
|
|
2324
2704
|
},
|
|
2705
|
+
{
|
|
2706
|
+
title: "STT: pause / resume streaming dictation",
|
|
2707
|
+
value: "stt.pause",
|
|
2708
|
+
category: "opencode-voice",
|
|
2709
|
+
description: "Freeze the mic mid-dictation (skips noise); run again to resume",
|
|
2710
|
+
slash: { name: "stt-pause" },
|
|
2711
|
+
onSelect() {
|
|
2712
|
+
// Batch mode has no pausable session (one-shot record); the mic
|
|
2713
|
+
// toggle is start/stop there.
|
|
2714
|
+
if (currentSttMode() !== "streaming" && !streamingDictationActive) {
|
|
2715
|
+
toast("Pause needs streaming mode - use /stt-mode to switch", "warning");
|
|
2716
|
+
return;
|
|
2717
|
+
}
|
|
2718
|
+
void pauseStreamingDictation();
|
|
2719
|
+
},
|
|
2720
|
+
},
|
|
2721
|
+
{
|
|
2722
|
+
title: "STT: select dictation mode",
|
|
2723
|
+
value: "stt.mode",
|
|
2724
|
+
category: "opencode-voice",
|
|
2725
|
+
description: "Choose one-shot batch or live streaming dictation (saved, survives restarts)",
|
|
2726
|
+
slash: { name: "stt-mode" },
|
|
2727
|
+
onSelect() {
|
|
2728
|
+
const current = currentSttMode();
|
|
2729
|
+
api.ui.dialog.replace(() =>
|
|
2730
|
+
api.ui.DialogSelect({
|
|
2731
|
+
title: "Select dictation mode",
|
|
2732
|
+
current,
|
|
2733
|
+
options: [
|
|
2734
|
+
{
|
|
2735
|
+
title: "Streaming (live transcript + wave, local whisper, no LLM)",
|
|
2736
|
+
value: "streaming",
|
|
2737
|
+
onSelect() {
|
|
2738
|
+
kv.set("stt.mode", "streaming");
|
|
2739
|
+
sttLog("mode.set", "streaming via /stt-mode");
|
|
2740
|
+
toast("Dictation mode: streaming");
|
|
2741
|
+
api.ui.dialog.clear();
|
|
2742
|
+
},
|
|
2743
|
+
},
|
|
2744
|
+
{
|
|
2745
|
+
title: "Batch (record, then transcribe)",
|
|
2746
|
+
value: "batch",
|
|
2747
|
+
onSelect() {
|
|
2748
|
+
kv.set("stt.mode", "batch");
|
|
2749
|
+
sttLog("mode.set", "batch via /stt-mode");
|
|
2750
|
+
toast("Dictation mode: batch");
|
|
2751
|
+
api.ui.dialog.clear();
|
|
2752
|
+
},
|
|
2753
|
+
},
|
|
2754
|
+
],
|
|
2755
|
+
}),
|
|
2756
|
+
);
|
|
2757
|
+
},
|
|
2758
|
+
},
|
|
2759
|
+
{
|
|
2760
|
+
title: "STT: toggle auto-submit",
|
|
2761
|
+
value: "stt.autosubmit",
|
|
2762
|
+
category: "opencode-voice",
|
|
2763
|
+
description: "Toggle submitting the transcript automatically after dictation (saved)",
|
|
2764
|
+
slash: { name: "stt-autosubmit" },
|
|
2765
|
+
onSelect() {
|
|
2766
|
+
const next = !sttAutoSubmitEnabled();
|
|
2767
|
+
kv.set("stt.autoSubmit", next);
|
|
2768
|
+
sttLog("mode.set", `autoSubmit=${next} via /stt-autosubmit`);
|
|
2769
|
+
toast(`STT auto-submit: ${next ? "on" : "off"}`);
|
|
2770
|
+
},
|
|
2771
|
+
},
|
|
2325
2772
|
{
|
|
2326
2773
|
title: sttApiEndpoint ? "STT: select model (API)" : "STT: select model",
|
|
2327
2774
|
value: "stt.model",
|