@staff0rd/assist 0.79.0 → 0.81.0

package/README.md

@@ -43,6 +43,11 @@ After installation, the `assist` command will be available globally.
 - `/verify` - Run all verification commands in parallel
 - `/transcript-format` - Format meeting transcripts from VTT files
 - `/transcript-summarise` - Summarise transcripts missing summaries
+- `/voice-setup` - Download required voice models (VAD, STT)
+- `/voice-start` - Start the voice interaction daemon
+- `/voice-stop` - Stop the voice interaction daemon
+- `/voice-status` - Check voice daemon status
+- `/voice-logs` - Show recent voice daemon logs
 
 ## CLI Commands
 
@@ -96,4 +101,11 @@ After installation, the `assist` command will be available globally.
 - `assist transcript configure` - Configure transcript directories
 - `assist transcript format` - Convert VTT files to formatted markdown transcripts
 - `assist transcript summarise` - List transcripts that do not have summaries
+- `assist voice setup` - Download required voice models (VAD, STT)
+- `assist voice start` - Start the voice daemon (always-on, listens for wake word)
+- `assist voice start --foreground` - Start in foreground for debugging
+- `assist voice stop` - Stop the voice daemon
+- `assist voice status` - Check voice daemon status and recent events
+- `assist voice devices` - List available audio input devices
+- `assist voice logs [-n <count>]` - Show recent voice daemon log entries
 

package/claude/commands/voice-logs.md

@@ -0,0 +1,5 @@
+---
+description: Show recent voice daemon logs
+---
+
+Run `assist voice logs 2>&1` to show recent voice daemon logs. Show the result to the user.

package/claude/commands/voice-setup.md

@@ -0,0 +1,5 @@
+---
+description: Download required voice models (VAD, STT)
+---
+
+Run `assist voice setup 2>&1` to download the required voice models. Show the result to the user.

package/claude/commands/voice-start.md

@@ -0,0 +1,5 @@
+---
+description: Start the voice interaction daemon
+---
+
+Run `assist voice start 2>&1` to start the voice daemon. If it fails, show the error to the user.

package/claude/commands/voice-status.md

@@ -0,0 +1,5 @@
+---
+description: Check voice daemon status
+---
+
+Run `assist voice status 2>&1` to check the voice daemon status. Show the result to the user.

package/claude/commands/voice-stop.md

@@ -0,0 +1,5 @@
+---
+description: Stop the voice interaction daemon
+---
+
+Run `assist voice stop 2>&1` to stop the voice daemon. Show the result to the user.

package/claude/settings.json

@@ -27,6 +27,7 @@
 			"Bash(assist transcript summarise:*)",
 			"Bash(assist complexity:*)",
 			"Bash(assist transcript format:*)",
+			"Bash(assist voice:*)",
 			"Bash(date:*)",
 			"Bash(git add:*)",
 			"Bash(git status:*)",
@@ -48,6 +49,11 @@
 			"SlashCommand(/review-comments)",
 			"SlashCommand(/transcript-format)",
 			"SlashCommand(/transcript-summarise)",
+			"SlashCommand(/voice-setup)",
+			"SlashCommand(/voice-start)",
+			"SlashCommand(/voice-stop)",
+			"SlashCommand(/voice-status)",
+			"SlashCommand(/voice-logs)",
 			"SlashCommand(/journal)",
 			"SlashCommand(/standup)",
 			"Skill(next-backlog-item)",
@@ -60,6 +66,11 @@
 			"Skill(transcript-summarise)",
 			"Skill(journal)",
 			"Skill(standup)",
+			"Skill(voice-setup)",
+			"Skill(voice-start)",
+			"Skill(voice-stop)",
+			"Skill(voice-status)",
+			"Skill(voice-logs)",
 			"WebFetch(domain:staffordwilliams.com)"
 		],
 		"deny": ["Bash(git commit:*)", "Bash(npm run:*)", "Bash(npx assist:*)"]

package/dist/commands/voice/python/audio_capture.py

@@ -0,0 +1,49 @@
+"""Microphone capture via sounddevice (16kHz PCM)."""
+
+import os
+import queue
+import numpy as np
+import sounddevice as sd
+
+from logger import log
+
+SAMPLE_RATE = 16000
+BLOCK_SIZE = 512  # Silero VAD requires exactly 512 samples at 16kHz
+
+
+class AudioCapture:
+    def __init__(self):
+        self._queue: queue.Queue[np.ndarray] = queue.Queue()
+        self._stream: sd.InputStream | None = None
+        device_name = os.environ.get("VOICE_MIC")
+        self._device = device_name if device_name else None
+
+    def _callback(self, indata: np.ndarray, frames: int, time_info, status) -> None:
+        if status:
+            log("audio_status", str(status), level="warn")
+        self._queue.put(indata[:, 0].copy())
+
+    def start(self) -> None:
+        log("audio_start", f"device={self._device}, rate={SAMPLE_RATE}, block={BLOCK_SIZE}")
+        self._stream = sd.InputStream(
+            samplerate=SAMPLE_RATE,
+            channels=1,
+            dtype="float32",
+            blocksize=BLOCK_SIZE,
+            device=self._device,
+            callback=self._callback,
+        )
+        self._stream.start()
+
+    def read(self, timeout: float = 1.0) -> np.ndarray | None:
+        try:
+            return self._queue.get(timeout=timeout)
+        except queue.Empty:
+            return None
+
+    def stop(self) -> None:
+        if self._stream:
+            self._stream.stop()
+            self._stream.close()
+            self._stream = None
+            log("audio_stop")

package/dist/commands/voice/python/dispatch.py

@@ -0,0 +1,14 @@
+"""Dispatch — keyboard-based input into the active terminal."""
+
+from logger import log
+
+import keyboard
+
+
+def dispatch(command: str) -> str:
+    """Type the command and press Enter."""
+    log("dispatch_start", command)
+    keyboard.type_text(command)
+    keyboard.press_enter()
+    log("dispatch_done", command)
+    return command

package/dist/commands/voice/python/keyboard.py

@@ -0,0 +1,73 @@
+"""Simulate keyboard input on Windows via SendInput."""
+
+import ctypes
+import ctypes.wintypes as w
+
+user32 = ctypes.windll.user32
+
+INPUT_KEYBOARD = 1
+KEYEVENTF_UNICODE = 0x0004
+KEYEVENTF_KEYUP = 0x0002
+KEYEVENTF_SCANCODE = 0x0008
+VK_RETURN = 0x0D
+VK_BACK = 0x08
+SCAN_RETURN = 0x1C
+SCAN_BACK = 0x0E
+
+
+class KEYBDINPUT(ctypes.Structure):
+    _fields_ = [
+        ("wVk", w.WORD),
+        ("wScan", w.WORD),
+        ("dwFlags", w.DWORD),
+        ("time", w.DWORD),
+        ("dwExtraInfo", ctypes.POINTER(ctypes.c_ulong)),
+    ]
+
+
+class MOUSEINPUT(ctypes.Structure):
+    _fields_ = [
+        ("dx", ctypes.c_long),
+        ("dy", ctypes.c_long),
+        ("mouseData", w.DWORD),
+        ("dwFlags", w.DWORD),
+        ("time", w.DWORD),
+        ("dwExtraInfo", ctypes.POINTER(ctypes.c_ulong)),
+    ]
+
+
+class INPUT(ctypes.Structure):
+    class _INPUT(ctypes.Union):
+        _fields_ = [("mi", MOUSEINPUT), ("ki", KEYBDINPUT)]
+
+    _anonymous_ = ("_input",)
+    _fields_ = [("type", w.DWORD), ("_input", _INPUT)]
+
+
+def _send_key(vk: int = 0, scan: int = 0, flags: int = 0) -> None:
+    inp = INPUT(type=INPUT_KEYBOARD)
+    inp.ki.wVk = vk
+    inp.ki.wScan = scan
+    inp.ki.dwFlags = flags
+    user32.SendInput(1, ctypes.byref(inp), ctypes.sizeof(inp))
+
+
+def type_text(text: str) -> None:
+    """Type a string by sending Unicode keystrokes."""
+    for ch in text:
+        code = ord(ch)
+        _send_key(scan=code, flags=KEYEVENTF_UNICODE)
+        _send_key(scan=code, flags=KEYEVENTF_UNICODE | KEYEVENTF_KEYUP)
+
+
+def backspace(n: int = 1) -> None:
+    """Press backspace n times."""
+    for _ in range(n):
+        _send_key(vk=VK_BACK, scan=SCAN_BACK)
+        _send_key(vk=VK_BACK, scan=SCAN_BACK, flags=KEYEVENTF_KEYUP)
+
+
+def press_enter() -> None:
+    """Press the Enter key."""
+    _send_key(vk=VK_RETURN, scan=SCAN_RETURN)
+    _send_key(vk=VK_RETURN, scan=SCAN_RETURN, flags=KEYEVENTF_KEYUP)

package/dist/commands/voice/python/list_devices.py

@@ -0,0 +1,20 @@
+"""List available audio input devices."""
+
+import sounddevice as sd
+
+
+def main() -> None:
+    devices = sd.query_devices()
+    print("Audio input devices:\n")
+    for i, dev in enumerate(devices):
+        if dev["max_input_channels"] > 0:
+            default = " (default)" if i == sd.default.device[0] else ""
+            print(f"  [{i}] {dev['name']}{default}")
+            ch = dev["max_input_channels"]
+            rate = dev["default_samplerate"]
+            print(f"      channels={ch}, rate={rate}")
+            print()
+
+
+if __name__ == "__main__":
+    main()

package/dist/commands/voice/python/logger.py

@@ -0,0 +1,38 @@
+"""JSON Lines structured logging to voice.log."""
+
+import json
+import os
+import sys
+from datetime import datetime, timezone
+
+
+LOG_FILE = os.environ.get(
+    "VOICE_LOG_FILE", os.path.expanduser("~/.assist/voice/voice.log")
+)
+
+DEBUG = os.environ.get("VOICE_DEBUG", "") == "1"
+
+
+def _write(entry: dict) -> None:
+    entry["timestamp"] = datetime.now(timezone.utc).isoformat()
+    line = json.dumps(entry)
+    try:
+        with open(LOG_FILE, "a", encoding="utf-8") as f:
+            f.write(line + "\n")
+    except OSError:
+        pass
+    if DEBUG:
+        ts = entry["timestamp"][11:19]
+        level = entry.get("level", "info").upper()
+        event = entry.get("event", "")
+        msg = entry.get("message", "")
+        print(f"{ts} {level:5s} [{event}] {msg}", file=sys.stderr, flush=True)
+
+
+def log(event: str, message: str = "", *, level: str = "info", **data) -> None:
+    entry: dict = {"event": event, "level": level}
+    if message:
+        entry["message"] = message
+    if data:
+        entry["data"] = data
+    _write(entry)

package/dist/commands/voice/python/pyproject.toml

@@ -0,0 +1,34 @@
+[project]
+name = "assist-voice"
+version = "0.1.0"
+requires-python = ">=3.10"
+dependencies = [
+    "onnxruntime>=1.17",
+    "sounddevice>=0.4",
+    "numpy>=1.24",
+    "nemo_toolkit[asr]>=1.22",
+    "silero-vad>=5.1",
+]
+
+[project.optional-dependencies]
+dev = [
+    "ruff>=0.8",
+]
+
+[tool.setuptools]
+py-modules = [
+    "audio_capture",
+    "dispatch",
+    "logger",
+    "smart_turn",
+    "stt",
+    "vad",
+    "voice_daemon",
+    "wake_word",
+    "setup_models",
+    "list_devices",
+]
+
+[build-system]
+requires = ["setuptools>=68"]
+build-backend = "setuptools.build_meta"

package/dist/commands/voice/python/setup_models.py

@@ -0,0 +1,91 @@
+"""Download and prepare all required voice models."""
+
+import os
+import sys
+
+from logger import log
+
+
+def get_models_dir() -> str:
+    return os.environ.get(
+        "VOICE_MODELS_DIR",
+        os.path.expanduser("~/.assist/voice/models"),
+    )
+
+
+def setup_silero_vad(models_dir: str) -> None:
+    target = os.path.join(models_dir, "silero_vad.onnx")
+    if os.path.exists(target):
+        print("  silero_vad.onnx already exists")
+        return
+
+    print("  Downloading Silero VAD ONNX model...")
+    import urllib.request
+
+    url = "https://github.com/snakers4/silero-vad/raw/master/src/silero_vad/data/silero_vad.onnx"
+    urllib.request.urlretrieve(url, target)
+    log("setup_vad", f"Downloaded to {target}")
+    print("  silero_vad.onnx downloaded")
+
+
+def setup_smart_turn(models_dir: str) -> None:
+    target = os.path.join(models_dir, "smart-turn-v3.2-cpu.onnx")
+    if os.path.exists(target):
+        print("  smart-turn-v3.2-cpu.onnx already exists")
+        return
+
+    print("  Downloading Smart Turn ONNX model from HuggingFace...")
+    from huggingface_hub import hf_hub_download
+
+    path = hf_hub_download(
+        repo_id="pipecat-ai/smart-turn-v3",
+        filename="smart-turn-v3.2-cpu.onnx",
+        local_dir=models_dir,
+    )
+    log("setup_smart_turn", f"Downloaded to {path}")
+    print("  smart-turn-v3.2-cpu.onnx downloaded")
+
+
+def setup_stt(models_dir: str) -> None:
+    model_name = os.environ.get("VOICE_MODEL_STT", "nvidia/parakeet-ctc-1.1b")
+    print(f"  Downloading STT model: {model_name}...")
+    print("  (this may take a while on first run)")
+
+    import nemo.collections.asr as nemo_asr
+
+    nemo_asr.models.EncDecCTCModelBPE.from_pretrained(model_name)
+    log("setup_stt", f"Model ready: {model_name}")
+    print(f"  STT model ready: {model_name}")
+
+
+def main() -> None:
+    models_dir = get_models_dir()
+    os.makedirs(models_dir, exist_ok=True)
+    print(f"Models directory: {models_dir}\n")
+
+    print("[1/3] Silero VAD")
+    try:
+        setup_silero_vad(models_dir)
+    except Exception as e:
+        log("setup_vad_error", str(e), level="error")
+        print(f"  ERROR: {e}", file=sys.stderr)
+
+    print("\n[2/3] Smart Turn (pipecat-ai)")
+    try:
+        setup_smart_turn(models_dir)
+    except Exception as e:
+        log("setup_smart_turn_error", str(e), level="error")
+        print(f"  ERROR: {e}", file=sys.stderr)
+
+    print("\n[3/3] Parakeet STT (NeMo)")
+    try:
+        setup_stt(models_dir)
+    except Exception as e:
+        log("setup_stt_error", str(e), level="error")
+        print(f"  ERROR: {e}", file=sys.stderr)
+
+    print("\nSetup complete.")
+
+
+if __name__ == "__main__":
+    main()

package/dist/commands/voice/python/smart_turn.py

@@ -0,0 +1,63 @@
+"""Smart Turn end-of-utterance detection (ONNX) via pipecat-ai/smart-turn."""
+
+import os
+
+import numpy as np
+import onnxruntime as ort
+from transformers import WhisperFeatureExtractor
+
+from logger import log
+
+END_THRESHOLD = 0.5
+CHUNK_SECONDS = 8
+SAMPLE_RATE = 16000
+
+
+def _truncate_or_pad(audio: np.ndarray) -> np.ndarray:
+    max_samples = CHUNK_SECONDS * SAMPLE_RATE
+    if len(audio) > max_samples:
+        return audio[-max_samples:]
+    if len(audio) < max_samples:
+        padding = max_samples - len(audio)
+        return np.pad(audio, (padding, 0), mode="constant", constant_values=0)
+    return audio
+
+
+class SmartTurn:
+    def __init__(self):
+        model_path = os.environ.get("VOICE_MODEL_SMART_TURN")
+        if not model_path:
+            models_dir = os.environ.get(
+                "VOICE_MODELS_DIR",
+                os.path.expanduser("~/.assist/voice/models"),
+            )
+            model_path = os.path.join(models_dir, "smart-turn-v3.2-cpu.onnx")
+
+        log("smart_turn_init", f"model={model_path}")
+        so = ort.SessionOptions()
+        so.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL
+        so.inter_op_num_threads = 1
+        so.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
+        self._session = ort.InferenceSession(
+            model_path, sess_options=so, providers=["CPUExecutionProvider"]
+        )
+        self._feature_extractor = WhisperFeatureExtractor(chunk_length=CHUNK_SECONDS)
+        self.threshold = END_THRESHOLD
+
+    def is_end_of_turn(self, audio: np.ndarray) -> bool:
+        """Check if the accumulated audio indicates end of utterance."""
+        audio = _truncate_or_pad(audio)
+        inputs = self._feature_extractor(
+            audio,
+            sampling_rate=SAMPLE_RATE,
+            return_tensors="np",
+            padding="max_length",
+            max_length=CHUNK_SECONDS * SAMPLE_RATE,
+            truncation=True,
+            do_normalize=True,
+        )
+        features = inputs.input_features.squeeze(0).astype(np.float32)
+        features = np.expand_dims(features, axis=0)
+        outputs = self._session.run(None, {"input_features": features})
+        prob = float(outputs[0][0].item())
+        return prob > self.threshold

package/dist/commands/voice/python/stt.py

@@ -0,0 +1,51 @@
+"""Parakeet NeMo STT wrapper (GPU)."""
+
+import os
+import numpy as np
+
+from logger import log
+
+DEFAULT_MODEL = "nvidia/parakeet-ctc-1.1b"
+
+
+class ParakeetSTT:
+    def __init__(self):
+        model_name = os.environ.get("VOICE_MODEL_STT", DEFAULT_MODEL)
+        log("stt_init", f"model={model_name}")
+
+        import nemo.collections.asr as nemo_asr
+
+        self._model = nemo_asr.models.EncDecCTCModelBPE.from_pretrained(model_name)
+        self._model.eval()
+        log("stt_ready")
+
+    def transcribe(self, audio: np.ndarray, sample_rate: int = 16000) -> str:
+        """Transcribe audio buffer to text via direct forward pass."""
+        import torch
+
+        audio_tensor = torch.tensor(audio, dtype=torch.float32).unsqueeze(0)
+        audio_len = torch.tensor([audio.shape[0]], dtype=torch.long)
+
+        with torch.no_grad():
+            logits, logits_len, _ = self._model.forward(
+                input_signal=audio_tensor, input_signal_length=audio_len
+            )
+            # Greedy CTC decode
+            preds = torch.argmax(logits, dim=-1)
+            text = self._model.decoding.ctc_decoder_predictions_tensor(
+                preds, decoder_lengths=logits_len
+            )
+
+        # Result may be nested: tuple of lists of Hypothesis objects
+        if isinstance(text, tuple):
+            text = text[0]
+        if isinstance(text, list):
+            text = text[0]
+        # NeMo returns Hypothesis namedtuples with a .text field
+        if hasattr(text, "text"):
+            text = text.text
+        if not isinstance(text, str):
+            text = str(text)
+
+        log("stt_result", text)
+        return text