loukai-app 0.3.0

package/src/main/creator/python/whisper_runner.py

@@ -0,0 +1,172 @@
+#!/usr/bin/env python3
+"""
+Whisper Runner - Lyrics transcription for Loukai Creator
+
+Usage: python whisper_runner.py '{"input": "path/to/audio.wav", "model": "large-v3-turbo", "initial_prompt": "Song title. vocabulary hints"}'
+
+Outputs transcription with word timestamps as JSON to stdout.
+Progress updates are sent to stderr in format: PROGRESS:percent:message
+"""
+
+import json
+import sys
+import os
+
+def progress(percent, message):
+    """Send progress update to stderr"""
+    print(f"PROGRESS:{percent}:{message}", file=sys.stderr, flush=True)
+
+def main():
+    if len(sys.argv) < 2:
+        print(json.dumps({"error": "Missing arguments"}))
+        sys.exit(1)
+
+    try:
+        args = json.loads(sys.argv[1])
+    except json.JSONDecodeError as e:
+        print(json.dumps({"error": f"Invalid JSON arguments: {e}"}))
+        sys.exit(1)
+
+    input_path = args.get("input")
+    model_name = args.get("model", "large-v3-turbo")
+    initial_prompt = args.get("initial_prompt")
+    language = args.get("language", "en")
+
+    if not input_path:
+        print(json.dumps({"error": "Missing input path"}))
+        sys.exit(1)
+
+    try:
+        import torch
+        import whisper
+        import numpy as np
+
+        # Detect device
+        if torch.cuda.is_available():
+            device = "cuda"
+            device_name = torch.cuda.get_device_name(0)
+        elif hasattr(torch.backends, 'mps') and torch.backends.mps.is_available():
+            device = "mps"
+            device_name = "Apple Silicon GPU"
+        else:
+            device = "cpu"
+            device_name = "CPU"
+
+        progress(0, f"Loading Whisper {model_name} on {device_name}")
+
+        # Load model
+        model = whisper.load_model(model_name, device=device)
+
+        progress(15, "Loading audio for transcription")
+
+        # Load audio to get duration
+        audio = whisper.load_audio(input_path)
+        duration = len(audio) / whisper.audio.SAMPLE_RATE
+
+        progress(20, f"Transcribing {duration:.1f}s of audio...")
+
+        # Build transcription parameters
+        # Note: We don't use word_timestamps because:
+        # 1. MPS doesn't support the DTW algorithm (requires float64)
+        # 2. Singing has different timing than speech (stretching/compression)
+        # 3. LLM corrections break word alignment anyway
+        # We use line-level timing and estimate word positions
+        transcribe_params = {
+            "word_timestamps": False,
+            "language": language,
+            "task": "transcribe",
+            "verbose": True,  # Show transcription progress and lyrics in console
+            "condition_on_previous_text": False,  # Reduces repetition in singing
+            "no_speech_threshold": 0.3,  # More permissive for singing
+        }
+
+        if initial_prompt:
+            transcribe_params["initial_prompt"] = initial_prompt
+            progress(22, f"Using vocabulary hints ({len(initial_prompt.split())} words)")
+
+        # Transcribe - this is the long operation
+        # Whisper processes in 30-second chunks internally
+        num_chunks = max(1, int(duration / 30))
+        if num_chunks > 1:
+            progress(25, f"Processing ~{num_chunks} segments...")
+
+        # Redirect stdout to stderr during transcription so verbose output doesn't interfere with JSON
+        old_stdout = sys.stdout
+        sys.stdout = sys.stderr
+
+        result = model.transcribe(audio, **transcribe_params)
+
+        # Restore stdout
+        sys.stdout = old_stdout
+
+        progress(85, "Extracting line timestamps")
+
+        # Extract line-level timestamps and estimate word positions
+        words = []
+        lines = []
+
+        for segment in result.get("segments", []):
+            segment_text = segment["text"].strip()
+            segment_start = segment["start"]
+            segment_end = segment["end"]
+            segment_duration = segment_end - segment_start
+
+            # Estimate word timings by evenly distributing across segment
+            text_words = segment_text.split()
+            segment_words = []
+
+            if text_words:
+                word_duration = segment_duration / len(text_words)
+                for i, word_text in enumerate(text_words):
+                    word_start = segment_start + (i * word_duration)
+                    word_end = word_start + word_duration
+                    word_data = {
+                        "word": word_text,
+                        "start": round(word_start, 3),
+                        "end": round(word_end, 3)
+                    }
+                    segment_words.append(word_data)
+                    words.append({
+                        **word_data,
+                        "probability": 0.9  # Good confidence in text, estimated timing
+                    })
+
+            if segment_words:
+                lines.append({
+                    "text": segment_text,
+                    "start": round(segment_start, 3),
+                    "end": round(segment_end, 3),
+                    "words": segment_words
+                })
+
+        progress(95, f"Organized into {len(lines)} lines")
+
+        # Calculate some stats for the UI
+        avg_confidence = sum(w["probability"] for w in words) / len(words) if words else 0
+
+        progress(100, f"✓ Transcribed {len(words)} words, {len(lines)} lines")
+
+        # Output result
+        output = {
+            "success": True,
+            "text": result.get("text", "").strip(),
+            "language": result.get("language", language),
+            "lines": lines,
+            "words": words,
+            "model": model_name,
+            "device": device,
+            "duration": duration,
+            "avgConfidence": round(avg_confidence, 3)
+        }
+        print(json.dumps(output))
+
+    except Exception as e:
+        import traceback
+        print(json.dumps({
+            "error": str(e),
+            "traceback": traceback.format_exc()
+        }))
+        sys.exit(1)
+
+if __name__ == "__main__":
+    main()

package/src/main/creator/pythonRunner.js

@@ -0,0 +1,268 @@
+/**
+ * Python Runner - Spawns Python ML scripts from Node.js
+ *
+ * Handles:
+ * - Demucs stem separation
+ * - Whisper transcription
+ * - CREPE pitch detection
+ *
+ * Progress is parsed from stderr and emitted via callbacks.
+ * Results are parsed from stdout JSON.
+ */
+
+import { spawn } from 'child_process';
+import { join, dirname } from 'path';
+import { fileURLToPath } from 'url';
+import { getPythonPath, getPythonEnv } from './systemChecker.js';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+
+// Path to Python scripts - handle unpacked asar for packaged apps
+let PYTHON_SCRIPTS_DIR = join(__dirname, 'python');
+
+// When running from asar, scripts are in app.asar.unpacked instead
+if (PYTHON_SCRIPTS_DIR.includes('app.asar')) {
+  PYTHON_SCRIPTS_DIR = PYTHON_SCRIPTS_DIR.replace('app.asar', 'app.asar.unpacked');
+}
+
+/**
+ * Run a Python script with JSON arguments
+ *
+ * @param {string} scriptName - Name of script (e.g., 'demucs_runner.py')
+ * @param {Object} args - Arguments to pass as JSON
+ * @param {Function} onProgress - Progress callback (step, message, progress%)
+ * @param {Function} onConsoleOutput - Raw console output callback (line)
+ * @param {Function} onProcess - Callback to receive the spawned process (for cancellation)
+ * @returns {Promise<Object>} Parsed JSON result from script
+ */
+export function runPythonScript(
+  scriptName,
+  args,
+  onProgress = null,
+  onConsoleOutput = null,
+  onProcess = null
+) {
+  return new Promise((resolve, reject) => {
+    const pythonPath = getPythonPath();
+    const scriptPath = join(PYTHON_SCRIPTS_DIR, scriptName);
+    const argsJson = JSON.stringify(args);
+
+    const proc = spawn(pythonPath, [scriptPath, argsJson], {
+      env: getPythonEnv(),
+      timeout: 3600000, // 1 hour timeout for long operations
+    });
+
+    // Allow caller to track the process for cancellation
+    if (onProcess) {
+      onProcess(proc);
+    }
+
+    let stdout = '';
+    let stderr = '';
+
+    proc.stdout.on('data', (data) => {
+      stdout += data.toString();
+    });
+
+    // Track last known progress message for tqdm updates
+    let lastProgressMessage = '';
+    let lastProgressPercent = 0;
+
+    proc.stderr.on('data', (data) => {
+      const text = data.toString();
+      stderr += text;
+
+      const lines = text.split('\n');
+
+      for (const line of lines) {
+        // Emit raw console output
+        if (onConsoleOutput && line.trim()) {
+          onConsoleOutput(line);
+        }
+
+        // Parse our PROGRESS: updates
+        if (line.startsWith('PROGRESS:')) {
+          const parts = line.split(':');
+          if (parts.length >= 3) {
+            lastProgressPercent = parseInt(parts[1], 10);
+            lastProgressMessage = parts.slice(2).join(':');
+            if (onProgress) {
+              onProgress(lastProgressPercent, lastProgressMessage);
+            }
+          }
+          continue;
+        }
+
+        // Parse tqdm progress bars: " 14%|█▍        | 4/28 [00:10<01:02,  2.60s/it]"
+        // or simpler: "  4/28" pattern
+        const tqdmMatch = line.match(/(\d+)%\|.*?\|\s*(\d+)\/(\d+)/);
+        if (tqdmMatch && onProgress) {
+          const current = parseInt(tqdmMatch[2], 10);
+          const total = parseInt(tqdmMatch[3], 10);
+          if (total > 0) {
+            // Map tqdm progress (0-100%) to our 10-80% range for separation
+            const tqdmPercent = (current / total) * 100;
+            const mappedPercent = Math.floor(10 + tqdmPercent * 0.7);
+            onProgress(mappedPercent, `${lastProgressMessage} (${current}/${total})`);
+          }
+          continue;
+        }
+
+        // Also try simpler pattern for tqdm without percentage
+        const simpleMatch = line.match(/\s+(\d+)\/(\d+)\s+\[/);
+        if (simpleMatch && onProgress) {
+          const current = parseInt(simpleMatch[1], 10);
+          const total = parseInt(simpleMatch[2], 10);
+          if (total > 0) {
+            const tqdmPercent = (current / total) * 100;
+            const mappedPercent = Math.floor(10 + tqdmPercent * 0.7);
+            onProgress(mappedPercent, `${lastProgressMessage} (${current}/${total})`);
+          }
+        }
+      }
+    });
+
+    proc.on('close', (code) => {
+      try {
+        const result = JSON.parse(stdout.trim());
+
+        if (result.error) {
+          // Include full traceback if available
+          const errorMsg = result.traceback
+            ? `${result.error}\n\nTraceback:\n${result.traceback}`
+            : result.error;
+          console.error('❌ Python script error:', errorMsg);
+          reject(new Error(errorMsg));
+        } else {
+          resolve(result);
+        }
+      } catch (e) {
+        if (code !== 0) {
+          const errorMsg = `Python script failed (code ${code}):\n${stderr}`;
+          console.error('❌ Python script failed:', errorMsg);
+          reject(new Error(errorMsg));
+        } else {
+          const errorMsg = `Failed to parse Python output: ${e.message}\nStderr: ${stderr}\nStdout: ${stdout.slice(-500)}`;
+          console.error('❌ Failed to parse Python output:', errorMsg);
+          reject(new Error(errorMsg));
+        }
+      }
+    });
+
+    proc.on('error', (err) => {
+      reject(new Error(`Failed to spawn Python: ${err.message}`));
+    });
+  });
+}
+
+/**
+ * Run Demucs stem separation
+ *
+ * @param {string} inputPath - Path to input audio file
+ * @param {string} outputDir - Directory to save stems
+ * @param {Object} options - Options
+ * @param {string} options.model - Demucs model (default 'htdemucs_ft')
+ * @param {number} options.numStems - Number of stems (2 or 4)
+ * @param {Function} onProgress - Progress callback
+ * @param {Function} onConsoleOutput - Raw console output callback
+ * @param {Function} onProcess - Process callback for cancellation
+ * @returns {Promise<Object>} Result with stem file paths
+ */
+export function runDemucs(
+  inputPath,
+  outputDir,
+  options = {},
+  onProgress = null,
+  onConsoleOutput = null,
+  onProcess = null
+) {
+  return runPythonScript(
+    'demucs_runner.py',
+    {
+      input: inputPath,
+      output_dir: outputDir,
+      model: options.model || 'htdemucs_ft',
+      num_stems: options.numStems || 4,
+    },
+    onProgress,
+    onConsoleOutput,
+    onProcess
+  );
+}
+
+/**
+ * Run Whisper transcription
+ *
+ * @param {string} inputPath - Path to audio file (preferably vocals)
+ * @param {Object} options - Options
+ * @param {string} options.model - Whisper model (default 'large-v3-turbo')
+ * @param {string} options.initialPrompt - Initial prompt with vocabulary hints
+ * @param {string} options.language - Language code (default 'en')
+ * @param {Function} onProgress - Progress callback
+ * @param {Function} onConsoleOutput - Raw console output callback
+ * @param {Function} onProcess - Process callback for cancellation
+ * @returns {Promise<Object>} Transcription result with word timestamps
+ */
+export function runWhisper(
+  inputPath,
+  options = {},
+  onProgress = null,
+  onConsoleOutput = null,
+  onProcess = null
+) {
+  return runPythonScript(
+    'whisper_runner.py',
+    {
+      input: inputPath,
+      model: options.model || 'large-v3-turbo',
+      initial_prompt: options.initialPrompt || null,
+      language: options.language || 'en',
+    },
+    onProgress,
+    onConsoleOutput,
+    onProcess
+  );
+}
+
+/**
+ * Run CREPE pitch detection
+ *
+ * @param {string} inputPath - Path to vocal audio file
+ * @param {string} outputPath - Path to save pitch data JSON (optional)
+ * @param {Object} options - Options
+ * @param {string} options.model - CREPE model capacity (default 'full')
+ * @param {number} options.hopLength - Hop length in samples (default 512)
+ * @param {Function} onProgress - Progress callback
+ * @param {Function} onConsoleOutput - Raw console output callback
+ * @param {Function} onProcess - Process callback for cancellation
+ * @returns {Promise<Object>} Pitch detection result
+ */
+export function runCrepe(
+  inputPath,
+  outputPath = null,
+  options = {},
+  onProgress = null,
+  onConsoleOutput = null,
+  onProcess = null
+) {
+  return runPythonScript(
+    'crepe_runner.py',
+    {
+      input: inputPath,
+      output: outputPath,
+      model: options.model || 'tiny',
+      hop_length: options.hopLength || 512,
+    },
+    onProgress,
+    onConsoleOutput,
+    onProcess
+  );
+}
+
+export default {
+  runPythonScript,
+  runDemucs,
+  runWhisper,
+  runCrepe,
+};