@gobing-ai/knowledge-kit 0.0.12 → 0.0.14

package/plugins/generations/omni-voice-gen/src/omni_voice_gen/audio.py

@@ -0,0 +1,190 @@
+"""WAV parse/concat with gaps + loudness-dip detector — Python port of voice-gen concat.ts
+and the detectLoudnessDip half of qc.ts (spec layout: dip detection lives here, not qc.py).
+
+D8.1 delta (bug fix, not a port): gaps_ms[i] is silence inserted BEFORE segment i. The
+voice-gen `gapsMs[0] === 0 ? gapsMs[i] : gapsMs[i - 1]` special case is not replicated.
+"""
+
+from __future__ import annotations
+
+import struct
+from dataclasses import dataclass
+
+# Dip thresholds — identical to voice-gen qc.ts (D7).
+DIP_RATIO = 0.4
+DIP_SILENCE_FLOOR = 0.008
+DIP_ABS_CEILING = 0.06
+DIP_NEIGHBOR_FLOOR = 0.02
+
+
+@dataclass
+class ParsedWav:
+    audio_format: int
+    num_channels: int
+    sample_rate: int
+    byte_rate: int
+    block_align: int
+    bits_per_sample: int
+    data_bytes: bytes
+
+
+def parse_wav(buffer: bytes) -> ParsedWav:
+    """Parse a RIFF WAVE buffer into its format fields and raw audio data."""
+    if len(buffer) < 44:
+        raise ValueError("Invalid WAV buffer: buffer too short")
+
+    riff_tag = buffer[0:4].decode("latin-1")
+    wave_tag = buffer[8:12].decode("latin-1")
+    if riff_tag != "RIFF" or wave_tag != "WAVE":
+        raise ValueError(f"Invalid WAV buffer: expected RIFF/WAVE header, got {riff_tag}/{wave_tag}")
+
+    offset = 12
+    format_info: dict[str, int] | None = None
+    data_bytes: bytes | None = None
+
+    while offset + 8 <= len(buffer):
+        chunk_id = buffer[offset : offset + 4].decode("latin-1")
+        (chunk_size,) = struct.unpack_from("<I", buffer, offset + 4)
+        chunk_data_offset = offset + 8
+
+        if chunk_id == "fmt ":
+            if chunk_size < 16:
+                raise ValueError("Invalid WAV fmt chunk size")
+            audio_format, num_channels, sample_rate, byte_rate, block_align, bits_per_sample = struct.unpack_from(
+                "<HHIIHH", buffer, chunk_data_offset
+            )
+            format_info = {
+                "audio_format": audio_format,
+                "num_channels": num_channels,
+                "sample_rate": sample_rate,
+                "byte_rate": byte_rate,
+                "block_align": block_align,
+                "bits_per_sample": bits_per_sample,
+            }
+        elif chunk_id == "data":
+            end = min(chunk_data_offset + chunk_size, len(buffer))
+            data_bytes = buffer[chunk_data_offset:end]
+
+        # Advance to next chunk (aligned to 2 bytes)
+        offset = chunk_data_offset + chunk_size + (1 if chunk_size % 2 == 1 else 0)
+
+    if format_info is None:
+        raise ValueError('Invalid WAV: missing "fmt " chunk')
+    if data_bytes is None:
+        raise ValueError('Invalid WAV: missing "data" chunk')
+
+    return ParsedWav(**format_info, data_bytes=data_bytes)
+
+
+def concat_wavs(wav_buffers: list[bytes], gaps_ms: list[int] | None = None) -> bytes:
+    """Concatenate WAV buffers in order; gaps_ms[i] is silence inserted before segment i (D8.1)."""
+    if not wav_buffers:
+        return b""
+
+    gaps = gaps_ms or []
+    parsed_wavs = [parse_wav(buf) for buf in wav_buffers]
+    first = parsed_wavs[0]
+
+    # Verify format compatibility
+    for i in range(1, len(parsed_wavs)):
+        current = parsed_wavs[i]
+        if current.sample_rate != first.sample_rate:
+            raise ValueError(
+                f"WAV sample rate mismatch: expected {first.sample_rate} Hz, got {current.sample_rate} Hz"
+            )
+        if (
+            current.audio_format != first.audio_format
+            or current.num_channels != first.num_channels
+            or current.bits_per_sample != first.bits_per_sample
+        ):
+            raise ValueError(
+                f"WAV format mismatch between segment 0 and segment {i}: format {first.audio_format} vs "
+                f"{current.audio_format}, channels {first.num_channels} vs {current.num_channels}, bits "
+                f"{first.bits_per_sample} vs {current.bits_per_sample}"
+            )
+
+    # Compute total combined data length
+    total_data_length = 0
+    pieces: list[bytes] = []
+
+    for i, wav in enumerate(parsed_wavs):
+        if i > 0:
+            gap = gaps[i] if i < len(gaps) else 0
+            if gap > 0:
+                num_gap_bytes = int(first.sample_rate * (gap / 1000)) * first.block_align
+                if num_gap_bytes > 0:
+                    pieces.append(bytes(num_gap_bytes))
+                    total_data_length += num_gap_bytes
+        pieces.append(wav.data_bytes)
+        total_data_length += len(wav.data_bytes)
+
+    # Build the consolidated WAV buffer
+    header = struct.pack(
+        "<4sI4s4sIHHIIHH4sI",
+        b"RIFF",
+        36 + total_data_length,
+        b"WAVE",
+        b"fmt ",
+        16,
+        first.audio_format,
+        first.num_channels,
+        first.sample_rate,
+        first.byte_rate,
+        first.block_align,
+        first.bits_per_sample,
+        b"data",
+        total_data_length,
+    )
+    return header + b"".join(pieces)
+
+
+def detect_loudness_dip(wav: bytes) -> tuple[bool, float | None]:
+    """Detect a 'collapsed-volume' window: center RMS dips vs both neighbors.
+
+    Returns (dip, dip_at_sec). Any parse failure -> (False, None): a dip check must
+    never crash the run (voice-gen parity).
+    """
+    try:
+        parsed = parse_wav(wav)
+        if parsed.bits_per_sample != 16 or parsed.num_channels < 1 or len(parsed.data_bytes) < 2:
+            return False, None
+
+        data = parsed.data_bytes
+        total_frames = len(data) // parsed.block_align
+        window_frames = parsed.sample_rate  # 1s windows
+        if total_frames < window_frames * 5:
+            return False, None
+        n_wins = total_frames // window_frames
+        if n_wins < 7:
+            return False, None
+
+        bytes_per_frame = parsed.block_align
+        bytes_per_window = window_frames * bytes_per_frame
+
+        def window_rms(win_idx: int) -> float:
+            start = win_idx * bytes_per_window
+            sum_sq = 0.0
+            count = 0
+            for off in range(start, start + bytes_per_window, bytes_per_frame):
+                (sample,) = struct.unpack_from("<h", data, off)
+                value = sample / 32768.0
+                sum_sq += value * value
+                count += 1
+            return (sum_sq / count) ** 0.5 if count else 0.0
+
+        rms = [window_rms(i) for i in range(n_wins)]
+        for j in range(1, n_wins - 1):
+            prev_rms = rms[j - 1]
+            center = rms[j]
+            next_rms = rms[j + 1]
+            nbr = max(prev_rms, next_rms)
+            if (
+                center > DIP_SILENCE_FLOOR
+                and center < DIP_ABS_CEILING
+                and nbr > DIP_NEIGHBOR_FLOOR
+                and center < nbr * DIP_RATIO
+            ):
+                return True, round(j + 0.5, 2)
+        return False, None
+    except Exception:  # noqa: BLE001 — a dip check must never crash the run (voice-gen parity)
+        return False, None

package/plugins/generations/omni-voice-gen/src/omni_voice_gen/backend.py

@@ -0,0 +1,150 @@
+"""Backend seam (D6): one in-process OmniVoice load renders every segment.
+
+This is the only torch/omnivoice-importing module, behind the Backend Protocol so every
+other module and all tests run without a GPU/model. torch/omnivoice imports stay lazy inside
+methods; FakeBackend replays canned data and never touches the model stack. WAV codec is
+stdlib `wave` only (float32 [-1,1] -> int16 via np.clip; 24kHz mono both directions).
+"""
+
+from __future__ import annotations
+
+import io
+import os
+import wave
+from typing import Any, Protocol, TypedDict
+
+import numpy as np
+
+from .profiles import ProfileEntry
+
+DEFAULT_MODEL = "k2-fsa/OmniVoice"
+SAMPLE_RATE = 24000  # OmniVoice output rate; ASR transcribe is fixed at 24000 (task R4)
+
+
+class SegmentSpec(TypedDict):
+    text: str
+    language: str
+    profile: ProfileEntry
+    instruct: str | None
+    seed: int | None
+    speed: float | None
+
+
+class Backend(Protocol):
+    # Concrete `raise` bodies (not `...`): coverage.py excludes ellipsis lines from
+    # measurement, which would leave these permanently "uncovered" for the D2 gate.
+    def load(self) -> None:
+        raise NotImplementedError
+
+    def generate(self, spec: SegmentSpec) -> tuple[bytes, float]:  # wav bytes, duration sec
+        raise NotImplementedError
+
+    def transcribe(self, wav: bytes, language: str) -> str:  # lazy whisper; may raise
+        raise NotImplementedError
+
+
+def _float_to_wav(audio: np.ndarray, sample_rate: int) -> bytes:
+    pcm = (np.clip(audio, -1.0, 1.0) * 32767).astype(np.int16)
+    buf = io.BytesIO()
+    with wave.open(buf, "wb") as wf:
+        wf.setnchannels(1)
+        wf.setsampwidth(2)
+        wf.setframerate(sample_rate)
+        wf.writeframes(pcm.tobytes())
+    return buf.getvalue()
+
+
+def _wav_to_float(wav: bytes) -> np.ndarray:
+    with wave.open(io.BytesIO(wav), "rb") as wf:
+        frames = wf.readframes(wf.getnframes())
+    return np.frombuffer(frames, dtype=np.int16).astype(np.float32) / 32768.0
+
+
+class FakeBackend:
+    """Canned test backend: replays wavs/durations/transcriptions by call order."""
+
+    def __init__(self, wavs: list[bytes], durations: list[float], transcriptions: dict[int, str]) -> None:
+        self._wavs = wavs
+        self._durations = durations
+        self._transcriptions = transcriptions
+        self.calls: list[SegmentSpec] = []
+        self.loaded = False
+        self._transcribe_count = 0
+
+    def load(self) -> None:
+        self.loaded = True
+
+    def generate(self, spec: SegmentSpec) -> tuple[bytes, float]:
+        self.calls.append(spec)
+        idx = len(self.calls) - 1
+        return self._wavs[idx], self._durations[idx]
+
+    def transcribe(self, wav: bytes, language: str) -> str:
+        text = self._transcriptions.get(self._transcribe_count, "")
+        self._transcribe_count += 1
+        return text
+
+
+def _default_device() -> str:
+    import torch
+
+    if torch.cuda.is_available():
+        return "cuda"
+    if torch.backends.mps.is_available():
+        return "mps"
+    return "cpu"
+
+
+class OmnivoiceBackend:
+    """Real backend: single `OmniVoice.from_pretrained` per instance; ASR lazy (D6/D7)."""
+
+    def __init__(self, model: str | None = None, device: str | None = None, speed: float | None = None) -> None:
+        self._model_id = model or os.environ.get("OMNIVOICE_MODEL", DEFAULT_MODEL)
+        self._device = device or os.environ.get("OMNIVOICE_DEVICE")
+        self._speed = speed if speed is not None else float(os.environ.get("OMNIVOICE_SPEED", "1.0"))
+        self._asr_model = os.environ.get("OMNIVOICE_ASR_MODEL")  # None -> omnivoice default whisper
+        self._model: Any = None
+        self._asr_loaded = False
+
+    def load(self) -> None:
+        if self._model is not None:
+            return
+        from omnivoice import OmniVoice
+
+        self._model = OmniVoice.from_pretrained(self._model_id, device_map=self._device or _default_device())
+
+    def generate(self, spec: SegmentSpec) -> tuple[bytes, float]:
+        if self._model is None:
+            raise RuntimeError("OmnivoiceBackend.load() must be called before generate()")
+        profile = spec["profile"]
+        kwargs: dict[str, Any] = {
+            "text": spec["text"],
+            "language": spec["language"],  # straight through — OmniVoice validates (task R5)
+            "speed": spec["speed"] if spec["speed"] is not None else self._speed,
+            "normalize_text": False,
+        }
+        if profile.kind == "clone":
+            kwargs["ref_audio"] = profile.ref_audio
+            kwargs["ref_text"] = profile.ref_text
+        elif profile.kind == "prompt":
+            from omnivoice import VoiceClonePrompt
+
+            assert profile.prompt is not None  # guaranteed by registry validation
+            kwargs["voice_clone_prompt"] = VoiceClonePrompt.load(profile.prompt)
+        else:  # instruct — spec.instruct wins over the profile instruct
+            kwargs["instruct"] = spec["instruct"] if spec["instruct"] is not None else profile.instruct
+        if spec["seed"] is not None:
+            import torch
+
+            torch.manual_seed(spec["seed"])
+        audios = self._model.generate(**kwargs)
+        audio = audios[0]
+        return _float_to_wav(audio, SAMPLE_RATE), len(audio) / SAMPLE_RATE
+
+    def transcribe(self, wav: bytes, language: str) -> str:
+        if self._model is None:
+            raise RuntimeError("OmnivoiceBackend.load() must be called before transcribe()")
+        if not self._asr_loaded:
+            self._model.load_asr_model(model_name=self._asr_model)
+            self._asr_loaded = True
+        return str(self._model.transcribe((_wav_to_float(wav), SAMPLE_RATE)))

package/plugins/generations/omni-voice-gen/src/omni_voice_gen/contract.py

@@ -0,0 +1,76 @@
+"""Pydantic contract mirroring @gobing-ai/kk-core Doc/Content (packages/kk-core/src/schemas.ts).
+
+Doc requires only `id` + `body`; Content requires `body` with `format` defaulting to
+"markdown". Unknown fields are ignored (zod-strip parity). Every violation surfaces as
+ValueError so __main__ maps it onto the single fail-loud exit path.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from pydantic import BaseModel, ValidationError
+
+
+class Reference(BaseModel):
+    """Optional citation entry inside Content.references."""
+
+    url: str | None = None
+    title: str | None = None
+    cite: str | None = None
+
+
+class Doc(BaseModel):
+    """One ingested document; `body` is the primary text/markdown payload."""
+
+    id: str
+    body: str
+    title: str | None = None
+    sourceUri: str | None = None
+    mediaType: str | None = None
+    metadata: dict[str, Any] | None = None
+
+
+class Content(BaseModel):
+    """Generator output validated by the kk executor's ContentSchema."""
+
+    body: str
+    title: str | None = None
+    format: str = "markdown"
+    outline: str | None = None
+    references: list[Reference] | None = None
+    metadata: dict[str, Any] | None = None
+
+
+NOTICE_CONTENT = Content(
+    title="Notice",
+    body="# Notice\nNo documents provided for generation.",
+    format="markdown",
+    references=[],
+)
+
+
+def parse_doc_list(raw: str) -> list[Doc]:
+    """Parse a Doc[] JSON string; raise ValueError carrying the reason on any violation."""
+    try:
+        value = json.loads(raw)
+    except json.JSONDecodeError as exc:
+        raise ValueError(f"input is not valid JSON: {exc.msg} at line {exc.lineno} column {exc.colno}") from exc
+    if not isinstance(value, list):
+        # ValueError, not TypeError: every parse failure must funnel into the single
+        # `Invalid DocList input:` exit path (task R4 contract).
+        raise ValueError(  # noqa: TRY004
+            f"input must be a JSON array of Doc objects, got {type(value).__name__}"
+        )
+    try:
+        return [Doc.model_validate(item) for item in value]
+    except ValidationError as exc:
+        first = exc.errors()[0]
+        loc = ".".join(str(part) for part in first.get("loc", ())) or "root"
+        raise ValueError(f"Doc validation failed at [{loc}]: {first['msg']}") from exc
+
+
+def content_to_json(content: Content) -> str:
+    """2-space-indent JSON with nulls dropped — voice-gen JSON.stringify output parity."""
+    return json.dumps(content.model_dump(exclude_none=True), ensure_ascii=False, indent=2)

package/plugins/generations/omni-voice-gen/src/omni_voice_gen/mp3.py

@@ -0,0 +1,60 @@
+"""VOICE_GEN_MP3 truthiness + ffmpeg transcode — Python port of voice-gen mp3.ts.
+
+Same flags (`-y -hide_banner -loglevel error -i <wav> -codec:a libmp3lame -qscale:a 2`),
+same fail-loud contract: missing input, missing ffmpeg, non-zero exit, or no output file
+all raise with the voice-gen message text.
+"""
+
+from __future__ import annotations
+
+import os
+import subprocess
+from collections.abc import Mapping
+
+MP3_TRUTHY = frozenset({"1", "true", "yes", "on"})
+
+
+def is_mp3_requested(env: Mapping[str, str] | None = None) -> bool:
+    """True when the operator asked for a sibling MP3 (VOICE_GEN_MP3=true|1|yes|on)."""
+    raw = (os.environ if env is None else env).get("VOICE_GEN_MP3")
+    if raw is None or raw == "":
+        return False
+    return raw.strip().lower() in MP3_TRUTHY
+
+
+def transcode_wav_to_mp3(wav_path: str, mp3_path: str, ffmpeg_bin: str = "ffmpeg") -> None:
+    """WAV → MP3 via ffmpeg + libmp3lame (VBR -qscale:a 2). Fail-loud on any error."""
+    if not os.path.exists(wav_path):
+        raise RuntimeError(f"ffmpeg mp3 transcode failed: WAV input not found: {wav_path}")
+
+    try:
+        proc = subprocess.run(
+            [
+                ffmpeg_bin,
+                "-y",
+                "-hide_banner",
+                "-loglevel",
+                "error",
+                "-i",
+                wav_path,
+                "-codec:a",
+                "libmp3lame",
+                "-qscale:a",
+                "2",
+                mp3_path,
+            ],
+            capture_output=True,
+            text=True,
+            check=False,
+        )
+    except FileNotFoundError as err:
+        raise RuntimeError(
+            f"ffmpeg is required for MP3 output (VOICE_GEN_MP3=true) but was not found on PATH ({err})"
+        ) from err
+
+    if proc.returncode != 0:
+        detail = proc.stderr.strip() or proc.stdout.strip()
+        raise RuntimeError(f"ffmpeg mp3 transcode failed (exit {proc.returncode}): {detail or 'no stderr'}")
+
+    if not os.path.exists(mp3_path):
+        raise RuntimeError(f"ffmpeg mp3 transcode produced no file: {mp3_path}")