omnius 1.0.639 → 1.0.641

package/dist/scripts/audio-speaker-embedding-worker.py

@@ -6,9 +6,11 @@ WAV paths.  It never records audio, installs dependencies, downloads models,
 or enables CUDA.  The parent runtime checksums the pinned ONNX artifact before
 this worker can start.
 
-The fbank path is deliberately the official WeSpeaker ONNX preprocessing:
-torchaudio.compliance.kaldi.fbank with 80 bins, 25 ms / 10 ms Hamming frames,
-dither=0 and utterance CMN (without CVN).
+The fbank path is a pinned, CPU-only NumPy implementation of the Kaldi
+configuration used by WeSpeaker's ONNX example: 80 bins, 25 ms / 10 ms Hamming
+frames, dither=0 and utterance CMN (without CVN). It intentionally imports
+neither Torch nor Torchaudio, so an upstream Torchaudio binary can never bind
+against or replace JetPack's vendor Torch ABI.
 """
 from __future__ import annotations
 
@@ -22,8 +24,8 @@ import traceback
 import wave
 
 
-# Set before importing Torch or ONNX Runtime so a globally configured Jetson
-# CUDA device cannot become an accidental provider for speaker embeddings.
+# Set before importing ONNX Runtime so a globally configured Jetson CUDA device
+# cannot become an accidental provider for speaker embeddings.
 os.environ["CUDA_VISIBLE_DEVICES"] = ""
 
 MODEL_NAME = "wespeaker-voxceleb-campplus"
@@ -34,6 +36,12 @@ MAX_DURATION_SECONDS = 30.0
 MIN_RMS = 0.003
 MIN_PEAK = 0.01
 MAX_CLIPPED_FRACTION = 0.01
+FBANK_IMPLEMENTATION = "kaldi-fbank-numpy-v1"
+# A CPU FFT may differ in harmless last bits across x86/aarch64 and NumPy
+# builds. Validate the public Kaldi configuration with numeric invariants,
+# not a byte hash of implementation-dependent FFT output.
+FBANK_VALIDATION = "kaldi-fbank-invariants-v2"
+CALIBRATION_BAND_MEANS = (11.1405, 17.5231, 12.8845, 9.2557, 7.7213, 6.7842, 6.6405)
 
 
 def emit(payload: dict) -> None:
@@ -98,18 +106,114 @@ def quality_gate(samples, metrics):
     return "usable"
 
 
+def kaldi_fbank80_numpy(waveform, np):
+    """Kaldi-compatible fbank for the fixed WeSpeaker CAM++ ONNX contract.
+
+    This mirrors WeSpeaker's published Kaldi fbank configuration: mono 16k
+    input, snip_edges=True,
+    remove_dc_offset=True, preemphasis=0.97, round_to_power_of_two=True,
+    80 triangular 20Hz..Nyquist mel bins, Hamming window and log power. The
+    caller passes PCM-scaled float32 samples (not normalized [-1, 1] values).
+    """
+    dtype = np.float32
+    waveform = np.ascontiguousarray(waveform, dtype=dtype)
+    frame_size = 400
+    frame_shift = 160
+    fft_size = 512
+    if waveform.ndim != 1 or waveform.size < frame_size:
+        raise RuntimeError("Kaldi fbank requires at least one 25 ms mono frame")
+    frame_count = 1 + (waveform.size - frame_size) // frame_shift
+    frames = np.lib.stride_tricks.as_strided(
+        waveform,
+        shape=(frame_count, frame_size),
+        strides=(frame_shift * waveform.strides[0], waveform.strides[0]),
+        writeable=False,
+    ).copy()
+    # Published Kaldi fbank frame/window defaults.
+    frames -= np.mean(frames, axis=1, dtype=dtype, keepdims=True)
+    frames[:, 1:] -= dtype(0.97) * frames[:, :-1]
+    frames[:, 0] -= dtype(0.97) * frames[:, 0]
+    sample_index = np.arange(frame_size, dtype=dtype)
+    hamming = dtype(0.54) - dtype(0.46) * np.cos(
+        dtype(2.0 * np.pi) * sample_index / dtype(frame_size - 1)
+    )
+    frames *= hamming
+    padded = np.pad(frames, ((0, 0), (0, fft_size - frame_size)), mode="constant")
+    # NumPy's rfft may calculate internally at float64 precision. Cast its
+    # power spectrum back to float32 before the Kaldi-style matrix multiply.
+    spectrum = np.asarray(np.abs(np.fft.rfft(padded, axis=1)) ** 2, dtype=dtype)
+
+    nyquist = dtype(8000.0)
+    low_hz = dtype(20.0)
+    mel = lambda hz: dtype(1127.0) * np.log(dtype(1.0) + hz / dtype(700.0))
+    mel_low = mel(low_hz)
+    mel_high = mel(nyquist)
+    mel_step = (mel_high - mel_low) / dtype(81.0)
+    bin_index = np.arange(80, dtype=dtype)[:, None]
+    left = mel_low + bin_index * mel_step
+    center = mel_low + (bin_index + dtype(1.0)) * mel_step
+    right = mel_low + (bin_index + dtype(2.0)) * mel_step
+    fft_bin_hz = dtype(16000.0 / fft_size) * np.arange(fft_size // 2, dtype=dtype)[None, :]
+    fft_bin_mel = mel(fft_bin_hz)
+    up = (fft_bin_mel - left) / (center - left)
+    down = (right - fft_bin_mel) / (right - center)
+    banks = np.maximum(dtype(0.0), np.minimum(up, down))
+    banks = np.pad(banks, ((0, 0), (0, 1)), mode="constant")
+    energies = np.matmul(spectrum, np.asarray(banks.T, dtype=dtype))
+    return np.log(np.maximum(energies, np.finfo(dtype).eps)).astype(dtype, copy=False)
+
+
+def validate_kaldi_fbank_numpy(np):
+    """Validate the no-Torch preprocessing path before readiness is true.
+
+    The calibration intentionally asserts shape, finite/CMN invariants, the
+    expected 220/440 Hz Kaldi mel-band profile and energy range with tolerances
+    that are stable across supported CPU FFT implementations. It must reject a
+    changed window, frame layout, PCM scale, mel bank, pre-emphasis or CMN, but
+    must not reject Jetson due to last-bit numerical differences.
+    """
+    t = np.arange(SAMPLE_RATE * 2, dtype=np.float32) / np.float32(SAMPLE_RATE)
+    samples = (
+        np.float32(0.075) * np.sin(np.float32(2.0 * np.pi * 220.0) * t)
+        + np.float32(0.025) * np.sin(np.float32(2.0 * np.pi * 440.0) * t)
+    ).astype(np.float32)
+    raw_features = kaldi_fbank80_numpy(samples * np.float32(1 << 15), np)
+    features = (
+        raw_features - np.mean(raw_features, axis=0, dtype=np.float32, keepdims=True)
+    ).astype(np.float32)
+    if features.shape != (198, 80) or not np.all(np.isfinite(features)):
+        raise RuntimeError("NumPy Kaldi fbank calibration produced an invalid feature matrix")
+    cmn_abs_mean_max = float(np.max(np.abs(np.mean(features, axis=0))))
+    if cmn_abs_mean_max > 1e-3:
+        raise RuntimeError("NumPy Kaldi fbank calibration failed utterance CMN")
+    raw_min = float(np.min(raw_features))
+    raw_max = float(np.max(raw_features))
+    cmn_rms = float(np.sqrt(np.mean(np.square(features))))
+    band_means = raw_features.mean(axis=0)[[0, 5, 10, 20, 40, 60, 79]]
+    if not (4.5 < raw_min < 6.0 and 19.0 < raw_max < 21.5 and 0.55 < cmn_rms < 0.75):
+        raise RuntimeError(
+            "NumPy Kaldi fbank calibration energy/range invariant failed"
+        )
+    if not np.allclose(band_means, np.asarray(CALIBRATION_BAND_MEANS, dtype=np.float32), rtol=0.0, atol=0.25):
+        raise RuntimeError("NumPy Kaldi fbank calibration mel-band invariant failed")
+    return {
+        "validation": FBANK_VALIDATION,
+        "frames": int(features.shape[0]),
+        "bins": int(features.shape[1]),
+        "cmn_abs_mean_max": round(cmn_abs_mean_max, 8),
+        "cmn_rms": round(cmn_rms, 6),
+        "raw_feature_range": [round(raw_min, 6), round(raw_max, 6)],
+    }
+
+
 class WeSpeakerCamPlus:
     def __init__(self, model_path: str):
         started = time.perf_counter()
         import numpy as np
         import onnxruntime as ort
-        import torch
-        from torchaudio.compliance import kaldi
 
         self.np = np
-        self.torch = torch
-        self.kaldi = kaldi
-        torch.set_num_threads(1)
+        self.preprocessing_validation = validate_kaldi_fbank_numpy(np)
         options = ort.SessionOptions()
         options.inter_op_num_threads = 1
         options.intra_op_num_threads = 1
@@ -134,27 +238,14 @@ class WeSpeakerCamPlus:
         self.model_load_ms = (time.perf_counter() - started) * 1000
 
     def features(self, samples):
-        # Mirrors wespeaker/bin/infer_onnx.py exactly. Its torchaudio.load
-        # values are normalized PCM16 floats and multiplied by 2**15 before
-        # Kaldi fbank; samples are decoded PCM16 values in that same range.
-        waveform = self.torch.from_numpy(
-            self.np.ascontiguousarray(samples * (1 << 15), dtype=self.np.float32)
-        ).unsqueeze(0)
-        mat = self.kaldi.fbank(
-            waveform,
-            num_mel_bins=80,
-            frame_length=25,
-            frame_shift=10,
-            dither=0.0,
-            sample_frequency=SAMPLE_RATE,
-            window_type="hamming",
-            use_energy=False,
-        )
+        # WeSpeaker's ONNX inference multiplies normalized PCM values by 2**15
+        # before Kaldi fbank. Decode follows the same convention above.
+        mat = kaldi_fbank80_numpy(samples * self.np.float32(1 << 15), self.np)
         if mat.ndim != 2 or mat.shape[0] < 1 or mat.shape[1] != 80:
-            raise RuntimeError(f"Official WeSpeaker fbank produced an invalid shape: {tuple(mat.shape)}")
+            raise RuntimeError(f"NumPy Kaldi fbank produced an invalid shape: {tuple(mat.shape)}")
         # CMN without CVN, over the full input utterance, per official script.
-        mat = mat - self.torch.mean(mat, dim=0)
-        return mat.unsqueeze(0).detach().cpu().numpy().astype(self.np.float32, copy=False)
+        mat = mat - self.np.mean(mat, axis=0, dtype=self.np.float32, keepdims=True)
+        return mat[None, :, :].astype(self.np.float32, copy=False)
 
     def embed(self, samples):
         started = time.perf_counter()
@@ -186,9 +277,22 @@ class WeSpeakerCamPlus:
 
 def main() -> int:
     parser = argparse.ArgumentParser()
-    parser.add_argument("--model", required=True)
-    parser.add_argument("--model-digest", required=True)
+    parser.add_argument("--model")
+    parser.add_argument("--model-digest")
+    parser.add_argument("--preprocessing-probe", action="store_true")
     args = parser.parse_args()
+    if args.preprocessing_probe:
+        import numpy as np
+        emit(
+            {
+                "type": "preprocessing_probe",
+                "implementation": FBANK_IMPLEMENTATION,
+                "validation": validate_kaldi_fbank_numpy(np),
+            }
+        )
+        return 0
+    if not args.model or not args.model_digest:
+        parser.error("--model and --model-digest are required unless --preprocessing-probe is used")
     worker = WeSpeakerCamPlus(args.model)
     worker.warm()
     emit(
@@ -202,6 +306,8 @@ def main() -> int:
             "dimension": EMBEDDING_DIMENSION,
             "model_load_ms": round(worker.model_load_ms, 3),
             "warmed": True,
+            "preprocessing": FBANK_IMPLEMENTATION,
+            "preprocessing_validation": worker.preprocessing_validation,
         }
     )
     for line in sys.stdin:

package/dist/scripts/live-whisper.py

@@ -463,6 +463,38 @@ def amplify(samples):
         return samples
     return np.clip(samples * gain, -1.0, 1.0)
 
+
+def pcm16_signal_metrics(samples):
+    """Authoritative stats for the raw PCM16 intake before any ASR call."""
+    count = int(len(samples))
+    if count <= 0:
+        return {
+            "sampleRateHz": SAMPLE_RATE, "channels": CHANNELS, "bitsPerSample": 16,
+            "sampleCount": 0, "durationMs": 0.0, "peakPcm16": 0,
+            "rmsPcm16": 0.0, "activeSampleCount": 0, "activeSampleRatio": 0.0,
+        }
+    # Input originates in int16 stdin. Rounding only restores that exact integer
+    # scale for reporting; it never modifies the samples supplied to Whisper.
+    pcm = np.rint(samples * 32768.0).astype(np.int32, copy=False)
+    magnitude = np.abs(pcm)
+    peak = int(np.max(magnitude))
+    active = int(np.count_nonzero(magnitude >= 8))
+    return {
+        "sampleRateHz": SAMPLE_RATE, "channels": CHANNELS, "bitsPerSample": 16,
+        "sampleCount": count, "durationMs": count * 1000.0 / SAMPLE_RATE,
+        "peakPcm16": peak, "rmsPcm16": float(np.sqrt(np.mean(pcm.astype(np.float64) ** 2))),
+        "activeSampleCount": active, "activeSampleRatio": active / count,
+    }
+
+
+def digitally_silent_pcm16(samples):
+    metrics = pcm16_signal_metrics(samples)
+    return (
+        metrics["peakPcm16"] <= 4
+        and metrics["rmsPcm16"] <= 1.0
+        and metrics["activeSampleCount"] == 0
+    ), metrics
+
 # ---------------------------------------------------------------------------
 # Main transcription loop — utterance FSM
 # ---------------------------------------------------------------------------
@@ -546,6 +578,11 @@ def main():
     })
 
     def run_transcribe(m, samples):
+        # Every live PCM ASR call goes through this exact gate. It is stricter
+        # than VAD only for digital silence, so it cannot reject quiet speech.
+        silent, _ = digitally_silent_pcm16(samples)
+        if silent:
+            return {"text": "", "segments": []}
         return m.transcribe(
             samples,
             fp16=fp16,
@@ -644,6 +681,10 @@ def main():
 
     def transcribe_final(samples, speech_block_count=0, rms_values=None, peak=0.0):
         nonlocal last_final_text
+        silent, signal = digitally_silent_pcm16(samples)
+        if silent:
+            emit({"type": "no_speech", "kind": "no_speech", "reason": "digital_silence", "text": "", "signal": signal})
+            return
         if not utterance_quality_ok(samples, speech_block_count, rms_values or [], peak):
             return
         samples = amplify(samples)