dexterm-cli 0.0.0-stage → 0.1.1

package/dist/packages/voice/src/controller.js

@@ -0,0 +1,395 @@
+// VoiceController (task-1282): the push-to-talk state machine that ties speech-in, the safe router,
+// and speech-out together, measures latency, and stays entirely UI-agnostic so it runs headless in
+// tests (fake engines) and in the app (real macOS engines). It emits a decision the app performs; it
+// NEVER performs a trade itself. The mic is open only in the 'listening' mode — the single place
+// listen() is called — so the on-screen indicator and the real mic state can never disagree.
+import { route, isSafe } from "./router.js";
+import { spokenSummary } from "./summary.js";
+export class VoiceController {
+    stt;
+    tts;
+    opts;
+    now;
+    enabled = false;
+    mode = 'off';
+    muted;
+    voice;
+    rate;
+    voices = [];
+    lastTranscript = null;
+    lastSay = null;
+    message = null;
+    session = null;
+    speaking = null;
+    /** Key-release timestamp awaiting its first-spoken-word, and the path it will be attributed to. */
+    pendingLatency = null;
+    maxListenMs;
+    setTimer;
+    clearTimer;
+    /** The live mic's auto-stop handle; non-null only while listening. */
+    listenTimer = null;
+    /**
+     * True from the talk key until stt.listen() has resolved (task-1339, re-verify N3). A second talk key
+     * in that window is ignored instead of opening a second, orphaned mic session; a cancel in that
+     * window closes the session the moment it arrives.
+     */
+    starting = false;
+    startCancelled = false;
+    samples = [];
+    constructor(opts) {
+        this.opts = opts;
+        this.stt = opts.stt;
+        this.tts = opts.tts;
+        this.muted = opts.muted ?? false;
+        this.voice = opts.voice ?? null;
+        this.rate = opts.rate ?? 190;
+        this.now = opts.now ?? (() => Date.now());
+        this.maxListenMs = opts.maxListenMs ?? 15_000;
+        this.setTimer = opts.setTimer ?? ((fn, ms) => { const h = setTimeout(fn, ms); h.unref?.(); return h; });
+        this.clearTimer = opts.clearTimer ?? ((h) => clearTimeout(h));
+    }
+    status() {
+        return {
+            enabled: this.enabled,
+            mode: this.mode,
+            starting: this.starting,
+            muted: this.muted,
+            voice: this.voice,
+            sttLabel: this.stt?.label ?? null,
+            ttsLabel: this.tts?.label ?? null,
+            lastTranscript: this.lastTranscript,
+            lastSay: this.lastSay,
+            message: this.message,
+        };
+    }
+    notify() { try {
+        this.opts.onState?.(this.status());
+    }
+    catch { /* UI handler must not break us */ } }
+    setMode(m) { this.mode = m; this.notify(); }
+    get isEnabled() { return this.enabled; }
+    get isListening() { return this.mode === 'listening'; }
+    /** The mic is live OR about to be (listen() in flight): the states a key must cancel. */
+    get micEngaged() { return this.mode === 'listening' || this.starting; }
+    /** Load the voice list lazily (used by cycleVoice and the status line). */
+    async loadVoices() {
+        if (!this.tts)
+            return;
+        try {
+            this.voices = await this.tts.listVoices();
+            if (!this.voice && this.voices.length)
+                this.voice = this.voices[0];
+        }
+        catch {
+            this.voices = [];
+        }
+    }
+    /** Turn voice mode on/off. Enabling with no STT engine still works for spoken replies to typed Qs. */
+    enable() {
+        if (this.enabled)
+            return;
+        this.enabled = true;
+        this.message = this.stt ? null : 'voice-in unavailable on this machine — type questions; replies are still spoken';
+        this.setMode('idle');
+        void this.loadVoices();
+    }
+    disable() {
+        if (!this.enabled)
+            return;
+        this.cancelListen();
+        this.stopSpeaking();
+        this.enabled = false;
+        this.pendingLatency = null;
+        this.setMode('off');
+    }
+    toggle() { this.enabled ? this.disable() : this.enable(); }
+    /**
+     * The push-to-talk key. Terminals deliver no key-up, so this toggles: first press opens the mic
+     * (mode 'listening' — the only place the mic is live), second press closes it, transcribes, and
+     * routes. Starting to talk also interrupts any reply in progress. Returns when the decision (if any)
+     * has been emitted, so tests can await it.
+     */
+    async toggleTalk() {
+        if (!this.enabled)
+            this.enable();
+        if (this.mode === 'listening')
+            return this.stopTalkAndRoute();
+        if (this.mode === 'transcribing' || this.mode === 'thinking')
+            return; // busy; ignore re-press
+        if (this.starting)
+            return; // a second press while the mic is opening: never a second session (N3)
+        return this.startTalk();
+    }
+    async startTalk() {
+        if (!this.stt) {
+            this.message = 'voice-in unavailable — type your question instead';
+            this.notify();
+            return;
+        }
+        this.stopSpeaking(); // starting to talk cuts off any reply
+        this.starting = true;
+        this.startCancelled = false;
+        this.notify();
+        let session = null;
+        try {
+            session = await this.stt.listen();
+        }
+        catch (e) {
+            this.starting = false;
+            this.session = null;
+            this.message = `mic/STT could not start: ${describe(e)}`;
+            this.setMode('idle');
+            return;
+        }
+        this.starting = false;
+        // The user moved on (a key, esc, disable) while the mic was opening: close it at once (N3/N8).
+        if (this.startCancelled || !this.enabled) {
+            try {
+                session.cancel();
+            }
+            catch { /* ignore */ }
+            this.session = null;
+            this.setMode(this.enabled ? 'idle' : 'off');
+            return;
+        }
+        this.session = session;
+        this.setMode('listening');
+        this.armListenTimeout(); // the mic can never stay live indefinitely (task-1282 rework D2)
+    }
+    /** Start the hard cap that auto-stops a live mic. Cleared whenever we leave 'listening'. */
+    armListenTimeout() {
+        this.clearListenTimeout();
+        if (this.maxListenMs <= 0)
+            return;
+        this.listenTimer = this.setTimer(() => {
+            this.listenTimer = null;
+            if (this.mode === 'listening') {
+                this.cancelListen();
+                this.message = 'mic timed out — press v to talk again';
+                this.notify();
+            }
+        }, this.maxListenMs);
+    }
+    clearListenTimeout() {
+        if (this.listenTimer !== null) {
+            try {
+                this.clearTimer(this.listenTimer);
+            }
+            catch { /* ignore */ }
+            this.listenTimer = null;
+        }
+    }
+    async stopTalkAndRoute() {
+        this.clearListenTimeout(); // leaving 'listening': the mic is no longer live
+        const session = this.session;
+        this.session = null;
+        this.setMode('transcribing');
+        // "Key release": the clock for key-release → first-spoken-word starts now.
+        const releaseAt = this.now();
+        let result;
+        try {
+            result = session ? await session.stop() : { text: '', confidence: 0 };
+        }
+        catch (e) {
+            // Always release the capture on a transcription error so the mic can never be left open.
+            try {
+                session?.cancel();
+            }
+            catch { /* ignore */ }
+            this.message = `transcription failed: ${describe(e)}`;
+            this.setMode('idle');
+            return;
+        }
+        this.lastTranscript = result.text || null;
+        // An empty capture used to route to a silent 'ignore': the user saw the badge vanish and nothing else.
+        if (!result.text.trim()) {
+            this.message = "didn't hear anything — press v, wait for the red LISTENING badge, speak, then press v";
+            this.setMode('idle');
+            return;
+        }
+        const ss = this.opts.getScreenState?.() ?? {};
+        const prov = {
+            pttActive: true, // this transcript came from a real PTT session, by construction
+            speakerCount: result.speakerCount ?? 1,
+            echoFlagged: result.echoFlagged ?? false,
+            sttConfidence: result.confidence,
+            lowConfTokens: result.lowConfTokens ?? [],
+        };
+        const decision = route(result.text, prov, ss);
+        // Hard safety gate: never act on a decision that is not in the safe set (defence in depth).
+        if (!isSafe(decision)) {
+            this.message = 'blocked an unsafe voice decision';
+            this.setMode('idle');
+            return;
+        }
+        this.dispatch(decision, releaseAt);
+    }
+    /** Emit the decision for the app, speak any immediate confirmation, and arm latency timing. */
+    dispatch(d, releaseAt) {
+        if (d.kind === 'analyst') {
+            // The spoken output is the analyst's answer; the app calls speakAnswer() when it lands. Time the
+            // key-release → first word of that answer.
+            this.pendingLatency = { at: releaseAt, path: 'analyst', transcript: this.lastTranscript ?? '' };
+            this.setMode('thinking');
+            try {
+                this.opts.onDecision?.(d);
+            }
+            catch { /* app handler */ }
+            return;
+        }
+        // Instant paths (navigate/open/filter/watchlist/open-ticket/popup/clarify) speak their own one-liner —
+        // unless the app answered itself (returned false): then its reply stands and `say` is NOT read over
+        // it, so a clarify such as "I see 2 tokens called X" is what the user actually hears (N1).
+        this.pendingLatency = { at: releaseAt, path: 'instant', transcript: this.lastTranscript ?? '' };
+        let appAnswered = false;
+        try {
+            appAnswered = this.opts.onDecision?.(d) === false;
+        }
+        catch { /* app handler */ }
+        if (appAnswered) {
+            if (this.mode !== 'speaking')
+                this.setMode('idle');
+            return;
+        }
+        const line = d.kind === 'clarify' ? d.question : ('say' in d ? d.say : null);
+        if (line)
+            this.speak(line);
+        else {
+            this.setMode('idle');
+        }
+    }
+    /**
+     * Speak the analyst's answer (first ~2 sentences). Called by the app when a turn settles. Honours
+     * mute, is interruptible, and records the pending analyst-path latency on the first audio sample.
+     */
+    speakAnswer(src) {
+        const line = spokenSummary(src);
+        if (!line) {
+            this.setMode('idle');
+            return;
+        }
+        this.speak(line);
+    }
+    /** Low-level speak: stops any current reply, then reads `text`. Muted → skip audio but update state. */
+    speak(text) {
+        this.lastSay = text;
+        if (!this.tts || this.muted) {
+            // Still resolve the latency sample (the "spoken word" is just suppressed) so the table is complete.
+            this.recordFirstAudio();
+            this.setMode('idle');
+            this.notify();
+            return;
+        }
+        this.stopSpeaking();
+        this.setMode('speaking');
+        const handle = this.tts.speak(text, {
+            voice: this.voice ?? undefined,
+            rate: this.rate,
+            onFirstAudio: () => this.recordFirstAudio(),
+        });
+        this.speaking = handle;
+        handle.done.then(() => {
+            if (this.speaking === handle) {
+                this.speaking = null;
+                if (this.mode === 'speaking')
+                    this.setMode('idle');
+            }
+        });
+    }
+    recordFirstAudio() {
+        const p = this.pendingLatency;
+        if (!p)
+            return;
+        this.pendingLatency = null;
+        this.samples.push({ path: p.path, ms: this.now() - p.at, transcript: p.transcript });
+    }
+    /** Interrupt a spoken reply (a key, or the user starting to talk). */
+    stopSpeaking() {
+        if (this.speaking) {
+            try {
+                this.speaking.stop();
+            }
+            catch { /* ignore */ }
+            this.speaking = null;
+        }
+        if (this.mode === 'speaking')
+            this.setMode('idle');
+    }
+    /** Abandon a live capture without transcribing. Always closes the mic and the auto-stop timer. */
+    cancelListen() {
+        this.clearListenTimeout();
+        if (this.starting)
+            this.startCancelled = true; // the session is closed the moment listen() resolves
+        if (this.session) {
+            try {
+                this.session.cancel();
+            }
+            catch { /* ignore */ }
+            this.session = null;
+        }
+        if (this.mode === 'listening')
+            this.setMode('idle');
+    }
+    /**
+     * Interrupt whatever voice is doing and return to idle — a mic capture, a spoken reply, OR a pending
+     * analyst turn ('thinking') / in-flight 'transcribing'. Resetting 'thinking' is the task-1282 rework
+     * D3 fix: an analyst error used to leave the mode stuck in 'thinking', where toggleTalk ignores every
+     * press, so voice was dead until restart. Now any key (esc) or an error cleanly returns to idle.
+     */
+    interrupt() {
+        this.stopSpeaking();
+        this.cancelListen();
+        this.pendingLatency = null;
+        if (this.enabled && this.mode !== 'idle' && this.mode !== 'off')
+            this.setMode('idle');
+    }
+    /**
+     * Abort the current turn because the analyst run failed/was cancelled, and surface a clear error both
+     * on screen (the HUD message) and aloud. Always leaves voice in idle so the next `v` works
+     * (task-1282 rework D3).
+     */
+    abortTurn(message = "Sorry — I couldn't get an answer. Try again.") {
+        this.interrupt();
+        if (!this.enabled) {
+            this.setMode('off');
+            return;
+        }
+        this.message = message;
+        this.mode = 'idle'; // speak()'s done handler returns to idle from 'speaking'
+        this.speak(message);
+    }
+    setMuted(m) { this.muted = m; if (m)
+        this.stopSpeaking(); this.notify(); }
+    toggleMute() { this.setMuted(!this.muted); }
+    get isMuted() { return this.muted; }
+    /** Cycle to the next installed voice (macOS voices). No-op when there are none. */
+    cycleVoice() {
+        if (!this.voices.length) {
+            this.notify();
+            return;
+        }
+        const i = this.voice ? this.voices.indexOf(this.voice) : -1;
+        this.voice = this.voices[(i + 1) % this.voices.length];
+        this.notify();
+    }
+    currentVoice() { return this.voice; }
+    availableVoices() { return this.voices.slice(); }
+    /** A compact latency report for the handoff / `--latency` demo. */
+    report() {
+        const paths = ['instant', 'analyst'];
+        return paths.map((path) => {
+            const xs = this.samples.filter((s) => s.path === path).map((s) => s.ms).sort((a, b) => a - b);
+            return { path, n: xs.length, p50: pct(xs, 0.5), min: xs[0] ?? null, max: xs[xs.length - 1] ?? null };
+        });
+    }
+}
+function pct(sorted, p) {
+    if (!sorted.length)
+        return null;
+    const i = Math.min(sorted.length - 1, Math.floor(p * sorted.length));
+    return sorted[i];
+}
+function describe(e) {
+    const m = e?.message;
+    return typeof m === 'string' ? m : String(e);
+}

package/dist/packages/voice/src/doctor.js

@@ -0,0 +1,63 @@
+// Voice-mode doctor checks (task-1282 scope item 4). Returns the same check shape the CLI doctor uses
+// so `dexterm doctor` can render them inline. Everything the checks depend on (platform, which engines
+// resolved, whether swiftc / a whisper model / a mic tool are present) is injected, so the checks are
+// deterministic in tests and never spawn or hang. The guiding rule: if a component is missing, say
+// exactly what to install, and make clear voice DEGRADES TO TEXT rather than breaking the app.
+/** Produce the voice checks. Never throws. */
+export function voiceDoctor(deps) {
+    const checks = [];
+    if (deps.platform !== 'darwin') {
+        checks.push({
+            name: 'Voice mode', status: 'warn',
+            detail: `voice mode is macOS-only for now (speech uses the built-in \`say\` and Apple Speech); on ${deps.platform} it stays off and dexterm runs as text`,
+            fix: 'Use dexterm on macOS for voice, or just use the keyboard — every voice action has a key.',
+        });
+        return checks;
+    }
+    // Speech OUT — macOS say.
+    if (deps.sayAvailable) {
+        checks.push({ name: 'Voice out (say)', status: 'pass', detail: `macOS \`say\` ready · ${deps.voiceCount} voice${deps.voiceCount === 1 ? '' : 's'} (press the voice key, then cycle the voice)` });
+    }
+    else {
+        checks.push({
+            name: 'Voice out (say)', status: 'warn',
+            detail: 'macOS `say` not found; spoken replies are off, the analyst still answers in text',
+            fix: '`say` ships with macOS — if it is missing, reinstall the Command Line Tools (`xcode-select --install`).',
+        });
+    }
+    // Speech IN — Apple Speech (preferred) or whisper (fallback).
+    if (deps.resolvedStt === 'apple' || deps.swiftcPresent) {
+        checks.push({ name: 'Voice in (Apple Speech)', status: 'pass', detail: 'on-device Apple Speech via swiftc (no key, streaming) — the first run asks for Microphone + Speech permission at the Mac' });
+    }
+    else if (deps.resolvedStt === 'whisper' || (deps.whisperBinPresent && deps.whisperModelPresent && deps.micToolPresent)) {
+        checks.push({ name: 'Voice in (whisper)', status: 'pass', detail: 'whisper.cpp base.en + a mic tool (local, no key)' });
+    }
+    else {
+        // Nothing for voice-in: steer to the fastest free path and make clear text still works.
+        const bits = [];
+        if (!deps.swiftcPresent)
+            bits.push('no swiftc (Apple Speech)');
+        if (!deps.whisperBinPresent)
+            bits.push('no whisper-cli');
+        else if (!deps.whisperModelPresent)
+            bits.push('no base.en model');
+        if (!deps.micToolPresent)
+            bits.push('no mic tool (sox/ffmpeg)');
+        checks.push({
+            name: 'Voice in', status: 'warn',
+            detail: `speech-to-text is off (${bits.join(', ')}); you can still type questions — voice degrades to text`,
+            fix: 'Fastest free path: install Xcode command line tools for Apple Speech (`xcode-select --install`). '
+                + 'Or whisper: `brew install whisper-cpp sox`, then download base.en to '
+                + '`~/.dexterm/models/ggml-base.en.bin` (from the whisper.cpp model repo). Never use large-v3-turbo on a CPU-only Mac — it is unusably slow.',
+        });
+    }
+    // Mic-capture tool note (only relevant to the whisper path; Apple Speech captures its own audio).
+    if (!deps.micToolPresent && deps.resolvedStt !== 'apple') {
+        checks.push({
+            name: 'Mic capture', status: 'warn',
+            detail: 'no sox/ffmpeg found; the whisper voice-in path needs one (Apple Speech does not)',
+            fix: '`brew install sox` (smallest) or `brew install ffmpeg`.',
+        });
+    }
+    return checks;
+}

package/dist/packages/voice/src/engines.js

@@ -0,0 +1,44 @@
+// Engine selection (task-1282). Picks the STT and TTS engines for this machine, honouring explicit
+// env overrides and falling back gracefully. The app degrades to text (never crashes) when nothing is
+// available; selectStt() returns null in that case and the controller says so.
+import { AppleStt } from "./stt/apple.js";
+import { WhisperStt } from "./stt/whisper.js";
+import { FakeStt } from "./stt/fake.js";
+import { SayTts } from "./tts/say.js";
+import { FakeTts } from "./tts/fake.js";
+/**
+ * Resolve the STT engine. `auto` prefers Apple Speech (on-device, streaming), then whisper base.en,
+ * then none. An explicit choice is returned even if it later proves unavailable, so the doctor and the
+ * status line can report exactly why voice-in is off.
+ */
+export async function selectStt(choice = process.env.DEXTERM_VOICE_STT || 'auto') {
+    if (choice === 'none')
+        return null;
+    if (choice === 'fake')
+        return new FakeStt();
+    if (choice === 'apple')
+        return new AppleStt();
+    if (choice === 'whisper')
+        return new WhisperStt();
+    // auto: probe in preference order.
+    const apple = new AppleStt();
+    if (await apple.available())
+        return apple;
+    const whisper = new WhisperStt();
+    if (await whisper.available())
+        return whisper;
+    return null;
+}
+/** Resolve the TTS engine. `auto` uses macOS `say` when present, else none (voice replies off). */
+export async function selectTts(choice = process.env.DEXTERM_VOICE_TTS || 'auto', defaultVoice) {
+    if (choice === 'none')
+        return null;
+    if (choice === 'fake')
+        return new FakeTts();
+    if (choice === 'say')
+        return new SayTts(defaultVoice);
+    const say = new SayTts(defaultVoice);
+    if (await say.available())
+        return say;
+    return null;
+}

package/dist/packages/voice/src/index.js

@@ -0,0 +1,16 @@
+// @dexterm/voice — JARVIS voice mode (task-1282). Public surface.
+//
+// Free and portable by requirement: macOS `say` for speech-out, on-device Apple Speech (compiled with
+// swiftc) or local whisper.cpp for speech-in — no paid APIs, no shipped keys. Voice can only ask the
+// analyst or trigger its existing safe actions; it can NEVER confirm, sign, or execute a trade.
+export * from "./types.js";
+export { route, isSafe } from "./router.js";
+export { spokenSummary, splitSentences } from "./summary.js";
+export { VoiceController } from "./controller.js";
+export { selectStt, selectTts } from "./engines.js";
+export { voiceDoctor } from "./doctor.js";
+export { SayTts } from "./tts/say.js";
+export { FakeTts } from "./tts/fake.js";
+export { WhisperStt, WHISPER_MODEL, WHISPER_MODEL_DIR } from "./stt/whisper.js";
+export { AppleStt, ensureHelper } from "./stt/apple.js";
+export { FakeStt } from "./stt/fake.js";