@geravant/sinain 1.0.19 → 1.2.0

package/sinain-core/src/audio/pipeline.ts

@@ -0,0 +1,335 @@
+import { EventEmitter } from "node:events";
+import type { ChildProcess } from "node:child_process";
+import type { AudioPipelineConfig, AudioChunk, AudioSourceTag } from "../types.js";
+import type { CaptureSpawner } from "./capture-spawner.js";
+import type { Profiler } from "../profiler.js";
+import { log, warn, error, debug } from "../log.js";
+
+const TAG = "audio";
+
+/**
+ * Creates a 44-byte WAV header for raw PCM data.
+ * Format: PCM (1), 16-bit, mono/stereo, given sample rate.
+ */
+function createWavHeader(
+  dataLength: number,
+  sampleRate: number,
+  channels: number,
+  bitsPerSample: number = 16
+): Buffer {
+  const header = Buffer.alloc(44);
+  const byteRate = sampleRate * channels * (bitsPerSample / 8);
+  const blockAlign = channels * (bitsPerSample / 8);
+
+  header.write("RIFF", 0);
+  header.writeUInt32LE(36 + dataLength, 4);
+  header.write("WAVE", 8);
+
+  header.write("fmt ", 12);
+  header.writeUInt32LE(16, 16);
+  header.writeUInt16LE(1, 20);
+  header.writeUInt16LE(channels, 22);
+  header.writeUInt32LE(sampleRate, 24);
+  header.writeUInt32LE(byteRate, 28);
+  header.writeUInt16LE(blockAlign, 32);
+  header.writeUInt16LE(bitsPerSample, 34);
+
+  header.write("data", 36);
+  header.writeUInt32LE(dataLength, 40);
+
+  return header;
+}
+
+/** Calculate RMS energy of 16-bit PCM samples (0.0 to 1.0). */
+function calculateRmsEnergy(pcmData: Buffer): number {
+  if (pcmData.length < 2) return 0;
+  const sampleCount = Math.floor(pcmData.length / 2);
+  let sumSquares = 0;
+  for (let i = 0; i < sampleCount; i++) {
+    const sample = pcmData.readInt16LE(i * 2);
+    const normalized = sample / 32768;
+    sumSquares += normalized * normalized;
+  }
+  return Math.sqrt(sumSquares / sampleCount);
+}
+
+/**
+ * Audio capture pipeline.
+ * Spawns sox or ffmpeg to capture audio from a macOS device,
+ * accumulates raw PCM data, and emits WAV chunks at regular intervals.
+ *
+ * Events: 'chunk' (AudioChunk), 'started', 'stopped', 'muted', 'unmuted', 'error' (Error)
+ */
+/**
+ * Pre-allocated buffer size for audio accumulation.
+ * 320KB is sufficient for 5s of 16-bit mono audio at 16kHz (160KB)
+ * with 2x headroom for stereo or higher sample rates.
+ */
+const PREALLOC_BUFFER_SIZE = 320 * 1024;
+
+export class AudioPipeline extends EventEmitter {
+  private config: AudioPipelineConfig;
+  private audioSourceTag: AudioSourceTag;
+  private captureSpawner: CaptureSpawner;
+  private process: ChildProcess | null = null;
+  // Pre-allocated buffer to reduce GC pressure (vs Buffer.concat per chunk)
+  private preallocBuffer: Buffer = Buffer.allocUnsafe(PREALLOC_BUFFER_SIZE);
+  private bufferWriteOffset: number = 0;
+  private chunkTimer: ReturnType<typeof setInterval> | null = null;
+  private running: boolean = false;
+  private silentChunks: number = 0;
+  private speechChunks: number = 0;
+  private errorCount: number = 0;
+  private muted: boolean = false;
+  private profiler: Profiler | null = null;
+
+  setProfiler(p: Profiler): void { this.profiler = p; }
+
+  constructor(config: AudioPipelineConfig, audioSourceTag: AudioSourceTag = "system", captureSpawner?: CaptureSpawner) {
+    super();
+    this.config = config;
+    this.audioSourceTag = audioSourceTag;
+    // If no spawner provided, lazily import the platform default
+    this.captureSpawner = captureSpawner!;
+  }
+
+  start(): void {
+    if (this.running) {
+      warn(TAG, "already running, ignoring start()");
+      return;
+    }
+
+    log(TAG, `starting capture: device=${this.config.device} cmd=${this.config.captureCommand} rate=${this.config.sampleRate}`);
+
+    try {
+      this.spawnCaptureProcess();
+    } catch (err) {
+      error(TAG, "failed to spawn capture process:", err);
+      this.emit("error", err);
+      return;
+    }
+
+    this.muted = false;
+    this.chunkTimer = setInterval(() => {
+      this.emitChunk();
+    }, this.config.chunkDurationMs);
+
+    this.running = true;
+    this.emit("started");
+    log(TAG, "capture started");
+  }
+
+  stop(): void {
+    if (!this.running) return;
+
+    log(TAG, "stopping capture...");
+    this.running = false;
+    this.muted = false;
+
+    if (this.chunkTimer) {
+      clearInterval(this.chunkTimer);
+      this.chunkTimer = null;
+    }
+
+    if (this.process) {
+      this.process.removeAllListeners();
+      this.process.kill("SIGTERM");
+      const proc = this.process;
+      setTimeout(() => {
+        try { proc.kill("SIGKILL"); } catch { /* already dead */ }
+      }, 2000);
+      this.process = null;
+    }
+
+    if (this.bufferWriteOffset > 0) {
+      this.emitChunk();
+    }
+
+    this.bufferWriteOffset = 0;
+    this.emit("stopped");
+    log(TAG, "capture stopped");
+  }
+
+  isRunning(): boolean {
+    return this.running;
+  }
+
+  mute(): void {
+    if (!this.running || this.muted) return;
+    this.muted = true;
+    if (this.chunkTimer) {
+      clearInterval(this.chunkTimer);
+      this.chunkTimer = null;
+    }
+    this.bufferWriteOffset = 0;
+    log(TAG, `muted (${this.config.captureCommand} process still running)`);
+    this.emit("muted");
+  }
+
+  unmute(): void {
+    if (!this.running || !this.muted) return;
+    this.muted = false;
+    this.chunkTimer = setInterval(() => {
+      this.emitChunk();
+    }, this.config.chunkDurationMs);
+    log(TAG, "unmuted");
+    this.emit("unmuted");
+  }
+
+  isMuted(): boolean {
+    return this.muted;
+  }
+
+  getDevice(): string {
+    return this.config.device;
+  }
+
+  getCaptureCommand(): "sox" | "ffmpeg" | "screencapturekit" {
+    return this.config.captureCommand;
+  }
+
+  switchDevice(device: string): void {
+    const wasRunning = this.running;
+    if (wasRunning) this.stop();
+    this.config = { ...this.config, device };
+    log(TAG, `device switched to: ${device}`);
+    if (wasRunning) this.start();
+  }
+
+  // ── Capture process spawn (platform-dispatched via CaptureSpawner) ──
+
+  private spawnCaptureProcess(): void {
+    this.process = this.captureSpawner.spawn(this.config, this.audioSourceTag);
+    const name = process.platform === "win32" ? "win-audio-capture" : "sck-capture";
+    this.wireProcessEvents(name);
+  }
+
+  // ── Process event wiring ──
+
+  private wireProcessEvents(name: string): void {
+    const proc = this.process;
+    if (!proc) return;
+
+    let headerSkipped = name !== "sox";
+    let headerBuf = Buffer.alloc(0);
+
+    proc.stdout?.on("data", (data: Buffer) => {
+      if (!this.running) return;
+
+      if (!headerSkipped) {
+        headerBuf = Buffer.concat([headerBuf, data]);
+        if (headerBuf.length >= 44) {
+          const remaining = headerBuf.subarray(44);
+          headerSkipped = true;
+          headerBuf = Buffer.alloc(0);
+          if (remaining.length > 0) {
+            this.writeToBuffer(remaining);
+          }
+        }
+        return;
+      }
+
+      this.writeToBuffer(data);
+    });
+
+    proc.stderr?.on("data", (data: Buffer) => {
+      const msg = data.toString().trim();
+      if (msg && !/^In:.*Out:/.test(msg)) {
+        log(TAG, `${name} stderr: ${msg.slice(0, 200)}`);
+      }
+    });
+
+    proc.on("error", (err) => {
+      error(TAG, `${name} process error:`, err.message);
+      this.errorCount++;
+      this.profiler?.gauge("audio.errors", this.errorCount);
+      this.emit("error", new Error(`${name} process error: ${err.message}`));
+      if (this.running) this.stop();
+    });
+
+    proc.on("exit", (code, signal) => {
+      log(TAG, `${name} exited: code=${code} signal=${signal}`);
+      if (this.running && code !== 0) {
+        this.errorCount++;
+        this.profiler?.gauge("audio.errors", this.errorCount);
+        warn(TAG, `${name} exited unexpectedly, stopping pipeline`);
+        this.stop();
+      }
+    });
+  }
+
+  // ── Buffer management ──
+
+  /**
+   * Write data to pre-allocated buffer.
+   * Falls back to growing buffer if needed (rare case for very long audio).
+   */
+  private writeToBuffer(data: Buffer): void {
+    if (this.muted) return;
+    // Check if we need to grow the buffer (rare case)
+    if (this.bufferWriteOffset + data.length > this.preallocBuffer.length) {
+      // Grow to 2x current size
+      const newSize = Math.max(this.preallocBuffer.length * 2, this.bufferWriteOffset + data.length);
+      const newBuffer = Buffer.allocUnsafe(newSize);
+      this.preallocBuffer.copy(newBuffer, 0, 0, this.bufferWriteOffset);
+      this.preallocBuffer = newBuffer;
+    }
+
+    data.copy(this.preallocBuffer, this.bufferWriteOffset);
+    this.bufferWriteOffset += data.length;
+    this.profiler?.gauge("audio.accumulatorKb", Math.round(this.bufferWriteOffset / 1024));
+  }
+
+  // ── Chunk emission ──
+
+  private emitChunk(): void {
+    if (this.bufferWriteOffset === 0) return;
+
+    // Extract PCM data from pre-allocated buffer (no concat allocation)
+    const pcmData = this.preallocBuffer.subarray(0, this.bufferWriteOffset);
+    this.bufferWriteOffset = 0;
+
+    const alignedLength = pcmData.length - (pcmData.length % 2);
+    // Copy aligned portion to new buffer since we'll reuse preallocBuffer
+    const alignedPcm = Buffer.from(pcmData.subarray(0, alignedLength));
+
+    if (alignedPcm.length === 0) return;
+
+    const energy = calculateRmsEnergy(alignedPcm);
+    this.profiler?.gauge("audio.lastChunkKb", Math.round(alignedPcm.length / 1024));
+
+    if (this.config.vadEnabled && energy < this.config.vadThreshold) {
+      this.silentChunks++;
+      this.profiler?.gauge("audio.silentChunks", this.silentChunks);
+      debug(TAG, `VAD: silent (energy=${energy.toFixed(4)} < ${this.config.vadThreshold}), ${this.silentChunks} silent chunk(s)`);
+      return;
+    }
+
+    if (this.silentChunks > 0) {
+      debug(TAG, `VAD: speech detected after ${this.silentChunks} silent chunk(s) (energy=${energy.toFixed(4)})`);
+      this.silentChunks = 0;
+    }
+
+    this.speechChunks++;
+    this.profiler?.gauge("audio.speechChunks", this.speechChunks);
+
+    const wavHeader = createWavHeader(alignedPcm.length, this.config.sampleRate, this.config.channels, 16);
+    const wavBuffer = Buffer.concat([wavHeader, alignedPcm]);
+
+    const bytesPerSample = 2 * this.config.channels;
+    const sampleCount = alignedPcm.length / bytesPerSample;
+    const durationMs = Math.round((sampleCount / this.config.sampleRate) * 1000);
+
+    const chunk: AudioChunk = {
+      buffer: wavBuffer,
+      source: this.config.device,
+      ts: Date.now(),
+      durationMs,
+      energy,
+      audioSource: this.audioSourceTag,
+    };
+
+    debug(TAG, `chunk: ${durationMs}ms, ${wavBuffer.length} bytes, energy=${energy.toFixed(4)}`);
+    this.emit("chunk", chunk);
+  }
+}

package/sinain-core/src/audio/transcription-local.ts

@@ -0,0 +1,141 @@
+import { spawn } from "node:child_process";
+import { writeFile, unlink, rmdir, mkdtemp } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import type { AudioChunk, TranscriptResult } from "../types.js";
+import { log, warn, error, debug } from "../log.js";
+
+const TAG = "transcribe-local";
+
+export interface LocalTranscriptionConfig {
+  /** Path to whisper-cpp binary (default: "whisper-cli") */
+  bin: string;
+  /** Path to GGUF model file */
+  modelPath: string;
+  /** Language code, e.g. "en", "ru" (default: "en") */
+  language: string;
+  /** Timeout per chunk in ms (default: 15000) */
+  timeoutMs: number;
+}
+
+/**
+ * Local transcription via whisper.cpp CLI.
+ *
+ * Writes WAV chunk to a temp file, runs whisper-cli, parses stdout.
+ * Fully isolated — does not touch the OpenRouter path.
+ */
+export class LocalTranscriptionBackend {
+  private config: LocalTranscriptionConfig;
+  private destroyed = false;
+
+  constructor(config: LocalTranscriptionConfig) {
+    this.config = config;
+    log(TAG, `initialized: bin=${config.bin} model=${config.modelPath} lang=${config.language}`);
+  }
+
+  async transcribe(chunk: AudioChunk): Promise<TranscriptResult | null> {
+    if (this.destroyed) return null;
+
+    const tmpDir = await mkdtemp(join(tmpdir(), "sinain-whisper-"));
+    const wavPath = join(tmpDir, "chunk.wav");
+
+    try {
+      await writeFile(wavPath, chunk.buffer);
+
+      const startTs = Date.now();
+      const text = await this.runWhisper(wavPath);
+      const elapsed = Date.now() - startTs;
+
+      if (!text) {
+        debug(TAG, `empty result (${elapsed}ms)`);
+        return null;
+      }
+
+      log(TAG, `transcript (${elapsed}ms): "${text.slice(0, 100)}${text.length > 100 ? "..." : ""}"`);
+
+      return {
+        text,
+        source: "whisper",
+        refined: false,
+        confidence: 0.85,
+        ts: Date.now(),
+        audioSource: chunk.audioSource,
+      };
+    } catch (err) {
+      error(TAG, "local transcription failed:", err instanceof Error ? err.message : err);
+      throw err;
+    } finally {
+      // Cleanup temp files
+      await unlink(wavPath).catch(() => {});
+      await rmdir(tmpDir).catch(() => {});
+    }
+  }
+
+  private runWhisper(wavPath: string): Promise<string> {
+    return new Promise((resolve, reject) => {
+      // whisper-cli expects ISO 639-1 codes ("en"), not BCP-47 ("en-US")
+      const lang = this.config.language.split("-")[0].toLowerCase();
+      const args = [
+        "-m", this.config.modelPath,
+        "-f", wavPath,
+        "--no-timestamps",
+        "-l", lang,
+        "-np",
+      ];
+
+      debug(TAG, `exec: ${this.config.bin} ${args.join(" ")}`);
+
+      const proc = spawn(this.config.bin, args, {
+        stdio: ["ignore", "pipe", "pipe"],
+      });
+
+      let stdout = "";
+      let stderr = "";
+
+      proc.stdout.on("data", (d: Buffer) => { stdout += d.toString(); });
+      proc.stderr.on("data", (d: Buffer) => { stderr += d.toString(); });
+
+      const timer = setTimeout(() => {
+        proc.kill("SIGKILL");
+        reject(new Error(`whisper-cpp timed out after ${this.config.timeoutMs}ms`));
+      }, this.config.timeoutMs);
+
+      proc.on("error", (err) => {
+        clearTimeout(timer);
+        reject(new Error(`whisper-cpp spawn error: ${err.message}`));
+      });
+
+      proc.on("close", (code) => {
+        clearTimeout(timer);
+
+        // whisper-cli may print errors to stderr but still exit 0
+        if (stderr.includes("unknown language") || stderr.includes("error:")) {
+          const msg = stderr.trim().slice(0, 300);
+          reject(new Error(`whisper-cpp stderr: ${msg}`));
+          return;
+        }
+
+        if (code !== 0) {
+          const msg = stderr.trim().slice(0, 300) || `exit code ${code}`;
+          reject(new Error(`whisper-cpp failed: ${msg}`));
+          return;
+        }
+
+        // whisper-cpp outputs lines like "  [text]" — strip whitespace and join
+        const text = stdout
+          .split("\n")
+          .map(l => l.trim())
+          .filter(l => l.length > 0 && !l.startsWith("["))
+          .join(" ")
+          .trim();
+
+        resolve(text);
+      });
+    });
+  }
+
+  destroy(): void {
+    this.destroyed = true;
+    log(TAG, "destroyed");
+  }
+}