@qvac/tts-ggml 0.10.0 → 0.11.0

package/examples/moss-speech.js

@@ -0,0 +1,136 @@
+'use strict'
+
+/**
+ * MOSS-Speech speech-to-speech for @qvac/tts-ggml.
+ *
+ * MOSS-Speech answers a spoken question with speech, with no text step in
+ * between: a 9B language model with separate text and audio branches reads
+ * the question's speech tokens and writes the reply's, and the codec speaks
+ * them in a 24 kHz voice (the codec's default voice, or a reference WAV).
+ *
+ * Usage:
+ *   bare examples/moss-speech.js question.wav [reply-voice.wav]
+ *
+ * Examples:
+ *   bare examples/moss-speech.js test/reference-audio/jfk.wav
+ *   QVAC_TTS_MOSS_SPEECH_GPU=1 bare examples/moss-speech.js question.wav my-voice.wav
+ *   QVAC_TTS_MOSS_SPEECH_TEXT=1 bare examples/moss-speech.js question.wav
+ *
+ * Expects the MOSS-Speech language model (moss-speech-q8_0.gguf or
+ * moss-speech-bf16.gguf) and its codec (moss-speech-codec-f16.gguf) under:
+ *   models/
+ * Produce them with engines/tts/scripts/convert-moss-speech-to-gguf.py and
+ * convert-moss-speech-codec-to-gguf.py in qvac-fabric-speech.cpp until they
+ * are published to the model registry. Desktop only; use a GPU.
+ */
+
+const path = require('bare-path')
+const proc = require('bare-process')
+const TTSGgml = require('../')
+const { createWav, readWavAsFloat32 } = require('./wav-helper')
+const { setLogger, releaseLogger } = require('../addonLogging')
+
+const SPEECH_SAMPLE_RATE = 24000
+
+const argv = global.Bare ? global.Bare.argv : process.argv
+const questionArg = argv[2]
+const voiceArg = argv[3]
+const textReply = proc.env.QVAC_TTS_MOSS_SPEECH_TEXT === '1'
+
+function fail(message) {
+  console.error(message)
+  if (global.Bare) global.Bare.exit(1)
+  else process.exit(1)
+}
+
+if (!questionArg) {
+  fail('Usage: moss-speech.js <question.wav> [reply-voice.wav]')
+}
+
+const pkgRoot = path.join(__dirname, '..')
+const modelDir = path.join(pkgRoot, 'models')
+
+function buildModel() {
+  return new TTSGgml({
+    engine: TTSGgml.ENGINE_MOSS_SPEECH,
+    files: { modelDir },
+    config: { useGPU: proc.env.QVAC_TTS_MOSS_SPEECH_GPU === '1' },
+    logger: console,
+    opts: { stats: true }
+  })
+}
+
+function replyVoiceFields() {
+  if (!voiceArg) return {}
+  const voice = readWavAsFloat32(path.resolve(voiceArg))
+  return { replyVoice: voice.samples, replyVoiceSampleRate: voice.sampleRate }
+}
+
+function reportStats(stats) {
+  if (!stats) return
+  console.log(
+    `Reply stats: totalTime=${stats.totalTime.toFixed(2)}s, ` +
+      `audioDuration=${stats.audioDurationMs}ms, promptTokens=${stats.promptTokens}, ` +
+      `replyTokens=${stats.replyTokens}, truncated=${stats.truncated}, ` +
+      `backendDevice=${stats.backendDevice}, backendId=${stats.backendId}`
+  )
+}
+
+async function collectReply(response) {
+  const reply = { pcm: [], text: '' }
+  await response
+    .onUpdate((data) => {
+      if (data && data.outputArray) reply.pcm = reply.pcm.concat(Array.from(data.outputArray))
+      if (data && typeof data.text === 'string') reply.text += data.text
+    })
+    .await()
+  return reply
+}
+
+async function main() {
+  setLogger((priority, message) => {
+    if (priority > 1) return
+    const names = { 0: 'ERROR', 1: 'WARNING', 2: 'INFO', 3: 'DEBUG', 4: 'OFF' }
+    const name = names[priority] || 'UNKNOWN'
+    console.log(`[${new Date().toISOString()}] [C++ log] [${name}]: ${message}`)
+  })
+
+  const outputFile = path.join(__dirname, 'moss-speech-reply.wav')
+  const question = readWavAsFloat32(path.resolve(questionArg))
+  const model = buildModel()
+
+  try {
+    console.log('Loading MOSS-Speech model...')
+    await model.load()
+    console.log('Model loaded.')
+
+    console.log(`Answering ${questionArg} (${(question.samples.length / question.sampleRate).toFixed(1)} s)`)
+    const response = await model.run({
+      type: 'text',
+      input: '',
+      audio: question.samples,
+      sampleRate: question.sampleRate,
+      textReply,
+      ...replyVoiceFields()
+    })
+    const reply = await collectReply(response)
+
+    console.log('Reply finished!')
+    if (reply.text) console.log(`Reply text: ${reply.text}`)
+    reportStats(response.stats)
+
+    if (reply.pcm.length > 0) {
+      createWav(reply.pcm, SPEECH_SAMPLE_RATE, outputFile)
+      console.log(`Finished writing to ${outputFile}`)
+    }
+  } finally {
+    await model.unload()
+    releaseLogger()
+  }
+}
+
+main().catch((err) => {
+  console.error(err)
+  if (global.Bare) global.Bare.exit(1)
+  else process.exit(1)
+})

package/examples/pcm-chunk-player.js

@@ -28,18 +28,41 @@
 const fs = require('bare-fs')
 const os = require('bare-os')
 const path = require('bare-path')
-const { spawn, spawnSync } = require('bare-subprocess')
 const { createWav } = require('./wav-helper')
 
+const MISSING_SUBPROCESS_WARNING =
+  '[pcm-chunk-player] bare-subprocess is not installed, so audio will not play. ' +
+  'Install it for live playback: `npm install bare-subprocess`.'
+
+const subprocess = loadSubprocess()
+
 let _seq = 0
 let _hasFfplay
 let _hasPlay
 let _hasAplay
+let _subprocessWarned = false
+
+function loadSubprocess () {
+  try {
+    return require('bare-subprocess')
+  } catch {
+    return null
+  }
+}
+
+function hasSubprocess () {
+  if (subprocess !== null) return true
+  if (!_subprocessWarned) {
+    console.warn(MISSING_SUBPROCESS_WARNING)
+    _subprocessWarned = true
+  }
+  return false
+}
 
 function syncOk (cmd, args) {
   try {
-    const r = spawnSync(cmd, args, { stdio: ['ignore', 'ignore', 'ignore'] })
-    return (r.status | 0) === 0
+    const r = subprocess.spawnSync(cmd, args, { stdio: ['ignore', 'ignore', 'ignore'] })
+    return !r.error && r.status === 0
   } catch {
     return false
   }
@@ -64,6 +87,7 @@ function detectAplay () {
 }
 
 function canPlayPcmChunks () {
+  if (!hasSubprocess()) return false
   if (detectFfplay()) return true
   if (detectPlay()) return true
   if (detectAplay()) return true
@@ -87,7 +111,7 @@ function unlinkQuiet (p) {
 function spawnAsync (cmd, args, opts) {
   return new Promise((resolve, reject) => {
     try {
-      const child = spawn(cmd, args, opts)
+      const child = subprocess.spawn(cmd, args, opts)
       child.on('exit', (code) => resolve(code))
       child.on('error', reject)
     } catch (err) {
@@ -106,6 +130,7 @@ function spawnAsync (cmd, args, opts) {
  * `sampleRate` defaults to 24000 (Chatterbox native rate).
  */
 function createStreamingPlayer ({ sampleRate = 24000, channels = 1 } = {}) {
+  if (!hasSubprocess()) return null
   // sox `play` is preferred on darwin: on some macOS builds ffplay's
   // SDL output is silent for raw-piped audio; sox uses CoreAudio
   // directly and works reliably.  (qvac-tts.cpp's README documents
@@ -164,7 +189,7 @@ function tryAplay (sampleRate, channels) {
 }
 
 function spawnStreamingPlayer (cmd, args) {
-  const child = spawn(cmd, args, { stdio: ['pipe', 'ignore', 'pipe'] })
+  const child = subprocess.spawn(cmd, args, { stdio: ['pipe', 'ignore', 'pipe'] })
   let exited = false
   let exitResolve
   const exitPromise = new Promise((resolve) => { exitResolve = resolve })
@@ -255,7 +280,7 @@ function createAfplayFallback (sampleRate) {
 
 function playInt16ChunkSync (samples, sampleRate) {
   const arr = samples instanceof Int16Array ? samples : Int16Array.from(samples)
-  if (arr.length === 0) return
+  if (arr.length === 0 || !hasSubprocess()) return
 
   const id = `${Date.now()}-${++_seq}`
   const tmpDir = os.tmpdir()
@@ -264,14 +289,14 @@ function playInt16ChunkSync (samples, sampleRate) {
   if (plat === 'darwin') {
     const tmpWav = path.join(tmpDir, `qvac-tts-stream-${id}.wav`)
     createWav(Array.from(arr), sampleRate, tmpWav)
-    spawnSync('afplay', [tmpWav], { stdio: 'ignore' })
+    subprocess.spawnSync('afplay', [tmpWav], { stdio: 'ignore' })
     unlinkQuiet(tmpWav)
     return
   }
   if (detectFfplay()) {
     const tmpWav = path.join(tmpDir, `qvac-tts-stream-${id}.wav`)
     createWav(Array.from(arr), sampleRate, tmpWav)
-    spawnSync(
+    subprocess.spawnSync(
       'ffplay',
       ['-nodisp', '-autoexit', '-loglevel', 'error', '-i', tmpWav],
       { stdio: 'ignore' }
@@ -282,7 +307,7 @@ function playInt16ChunkSync (samples, sampleRate) {
   if (detectAplay()) {
     const rawPath = path.join(tmpDir, `qvac-tts-stream-${id}.raw`)
     fs.writeFileSync(rawPath, toInt16Buffer(arr))
-    spawnSync(
+    subprocess.spawnSync(
       'aplay',
       ['-q', '-t', 'raw', '-f', 'S16_LE', '-r', String(sampleRate), '-c', '1', rawPath],
       { stdio: 'ignore' }
@@ -293,7 +318,7 @@ function playInt16ChunkSync (samples, sampleRate) {
 
 async function playInt16Chunk (samples, sampleRate) {
   const arr = samples instanceof Int16Array ? samples : Int16Array.from(samples)
-  if (arr.length === 0) return
+  if (arr.length === 0 || !hasSubprocess()) return
 
   const id = `${Date.now()}-${++_seq}`
   const tmpDir = os.tmpdir()

package/index.d.ts

@@ -1,7 +1,7 @@
 import { type QvacResponse } from "@qvac/infer-base";
 import * as errorModule from "./lib/error";
 import { resolveBackendsDir as resolveBackendsDirImpl } from "./lib/backends";
-import { assessFit as assessFitImpl, type Audio8FitRequest, type MossFitRequest, type ChatterboxFitRequest, type CosyvoiceFitRequest, type ParlerFitRequest, type SupertonicFitRequest, type TtsFitEngine, type TtsFitRequest, type TtsFitResult, type TtsFitStatus } from "./lib/fit";
+import { assessFit as assessFitImpl, type Audio8FitRequest, type MossFitRequest, type MossSoundEffectFitRequest, type ChatterboxFitRequest, type CosyvoiceFitRequest, type ParlerFitRequest, type SupertonicFitRequest, type TtsFitEngine, type TtsFitRequest, type TtsFitResult, type TtsFitStatus } from "./lib/fit";
 import { type SentenceDelimiterPreset } from "./lib/textStreamAccumulator";
 declare const ENGINE_CHATTERBOX = "chatterbox";
 declare const ENGINE_SUPERTONIC = "supertonic";
@@ -9,6 +9,8 @@ declare const ENGINE_COSYVOICE3 = "cosyvoice3";
 declare const ENGINE_PARLER = "parler";
 declare const ENGINE_AUDIO8 = "audio8";
 declare const ENGINE_MOSS = "moss";
+declare const ENGINE_MOSS_SFX = "moss-sfx";
+declare const ENGINE_MOSS_SPEECH = "moss-speech";
 declare const ENGINE_POCKET = "pocket";
 declare const COSYVOICE_DIALECTS: {
     readonly cantonese: "广东话";
@@ -57,7 +59,8 @@ declare const EMOTIONS: readonly ["command", "anger", "narration", "conversation
 declare const PACES: readonly ["slow", "moderate", "fast"];
 type Emotion = (typeof EMOTIONS)[number];
 type Pace = (typeof PACES)[number];
-type EngineType = typeof ENGINE_CHATTERBOX | typeof ENGINE_SUPERTONIC | typeof ENGINE_COSYVOICE3 | typeof ENGINE_PARLER | typeof ENGINE_AUDIO8 | typeof ENGINE_MOSS | typeof ENGINE_POCKET;
+declare const MOSS_SPEECH_ROLES: readonly ["system", "user", "assistant"];
+type EngineType = typeof ENGINE_CHATTERBOX | typeof ENGINE_SUPERTONIC | typeof ENGINE_COSYVOICE3 | typeof ENGINE_PARLER | typeof ENGINE_AUDIO8 | typeof ENGINE_MOSS | typeof ENGINE_MOSS_SFX | typeof ENGINE_MOSS_SPEECH | typeof ENGINE_POCKET;
 /**
  * Model file paths for the GGML TTS backend. Engine is auto-detected
  * from these fields (Chatterbox vs Supertonic) unless overridden via
@@ -120,6 +123,25 @@ interface TTSGgmlFiles {
      */
     mossCodecEncoder?: string;
     mossCodecEncoderPath?: string;
+    /**
+     * MOSS-SoundEffect GGUF (`moss-sfx-*.gguf`): text encoder, DiT and VAE in
+     * one file. Routes to the `moss-sfx` engine. Overrides `modelDir`.
+     */
+    mossSoundEffect?: string;
+    mossSoundEffectPath?: string;
+    /**
+     * MOSS-Speech language model GGUF (`moss-speech-*.gguf`, e.g.
+     * `moss-speech-q8_0.gguf`). Routes to the `moss-speech` engine together
+     * with `mossSpeechCodec`. Overrides `modelDir`.
+     */
+    mossSpeechModel?: string;
+    mossSpeechModelPath?: string;
+    /**
+     * MOSS-Speech codec GGUF (`moss-speech-codec-*.gguf`): speech tokenizer,
+     * reply decoder and the default reply voice. Overrides `modelDir`.
+     */
+    mossSpeechCodec?: string;
+    mossSpeechCodecPath?: string;
     /**
      * CosyVoice3 model directory holding the sub-model GGUFs
      * (`cosyvoice3-{llm,flow,hift}-*.gguf`) plus `voice.gguf`, `vocab.json` and
@@ -310,6 +332,66 @@ interface Audio8VoiceFields {
     /** Audio8: what `referenceAudio` says. Required when cloning. */
     referenceText?: string;
 }
+/**
+ * MOSS-SoundEffect per-call generation controls (moss-sfx only). Each one is
+ * optional; unset `steps` / `guidance` / `shift` use the defaults stored in the
+ * model file (100 steps, guidance 4, shift 5).
+ */
+interface MossSoundEffectFields {
+    /** Length of the sound effect, in (0, 30] seconds, rounded to 0.1 s. Defaults to 10. */
+    seconds?: number;
+    /** What the sound effect should avoid (classifier-free guidance negative). */
+    negativePrompt?: string;
+    /** Diffusion steps, 1..1000. More steps cost proportionally more time. */
+    steps?: number;
+    /** Classifier-free guidance scale, 1..50; 1 skips the negative branch. */
+    guidance?: number;
+    /** Flow-matching schedule shift, (0, 100]. */
+    shift?: number;
+}
+/** Mono PCM accepted by the MOSS-Speech engine: 16-bit integers or floats in [-1, 1]. */
+type MossSpeechPcm = Int16Array | Float32Array;
+/** One conversation turn for the MOSS-Speech engine: text or audio, never both. */
+interface MossSpeechMessage {
+    role: (typeof MOSS_SPEECH_ROLES)[number];
+    text?: string;
+    audio?: MossSpeechPcm;
+    /** Sample rate of `audio`, 8000..192000 Hz. */
+    sampleRate?: number;
+}
+/**
+ * MOSS-Speech per-call fields (moss-speech only). The user turn is `audio`
+ * (with `sampleRate`) or the run `input` text; `messages` carries the earlier
+ * conversation.
+ */
+interface MossSpeechFields {
+    /** The user's spoken turn, mono PCM. */
+    audio?: MossSpeechPcm;
+    /** Sample rate of `audio`, 8000..192000 Hz. */
+    sampleRate?: number;
+    /** Earlier turns, oldest first; the new user turn is appended after them. */
+    messages?: MossSpeechMessage[];
+    /** A system turn placed first; replaces the model's default system prompt. */
+    systemPrompt?: string;
+    /** Reply voice prompt (up to 60 s of mono PCM); the codec's default voice otherwise. */
+    replyVoice?: MossSpeechPcm;
+    /** Sample rate of `replyVoice`, 8000..192000 Hz. */
+    replyVoiceSampleRate?: number;
+    /** Reply with text only (no audio); the text comes back as `text`. */
+    textReply?: boolean;
+    /** Cut the spoken reply after this many seconds (0 = no cut). */
+    maxReplySeconds?: number;
+    /** Explicit bound on generated rows, 1..4096; omitted uses remaining model context. */
+    maxNewTokens?: number;
+    /** Greedy decoding instead of sampling. */
+    greedy?: boolean;
+    /** Sampling temperature, (0, 10] (default 0.7). */
+    temperature?: number;
+    /** Nucleus sampling, (0, 1] (default 0.95). */
+    topP?: number;
+    /** Top-k sampling, >= 0 (default 20). */
+    topK?: number;
+}
 /**
  * CosyVoice3 per-call instruction. Same values as the constructor's
  * `instruct`; a per-call conditioning control replaces the configured one for
@@ -333,9 +415,10 @@ interface TTSGgmlOptions extends ParlerDescriptionFields, Audio8VoiceFields, TTS
     /** Chatterbox: directory of baked voice-conditioning tensors. */
     voiceDir?: string;
     /**
-     * RNG seed for Chatterbox CFM/SineGen, Supertonic latent generation, or
-     * Pocket's portable sampling RNG. Pocket accepts integers from 0 to
-     * 4294967295 (inclusive).
+     * RNG seed for Chatterbox CFM/SineGen, Supertonic latent generation,
+     * the MOSS-SoundEffect diffusion noise (0 when unset, so an unseeded prompt
+     * repeats its clip), or Pocket's portable sampling RNG. Pocket accepts
+     * integers from 0 to 4294967295 (inclusive).
      */
     seed?: number;
     /**
@@ -590,6 +673,8 @@ interface TTSOutputChunk {
     isLast?: boolean;
     /** Signed 16-bit mono PCM audio payload. */
     outputArray: Int16Array;
+    /** MOSS-Speech: the reply's text channel (the whole answer with `textReply`). */
+    text?: string;
     /**
      * Output sample rate. The native engine rate (24000 for Chatterbox,
      * CosyVoice3 and MOSS; 44100 for Supertonic, Parler, and Audio8), or 48000
@@ -718,7 +803,7 @@ interface RunStreamingOptions extends ParlerDescriptionFields, Audio8VoiceFields
 }
 /** Input accepted by `runStreaming`. */
 type TextStreamInput = string | string[] | Iterable<string> | AsyncIterable<string>;
-interface TTSRunInput extends ParlerDescriptionFields, Audio8VoiceFields, TTSConditioningFields, CosyvoiceJobFields {
+interface TTSRunInput extends ParlerDescriptionFields, Audio8VoiceFields, TTSConditioningFields, CosyvoiceJobFields, MossSoundEffectFields, MossSpeechFields {
     type?: string;
     input: string;
     streamOutput?: boolean;
@@ -750,6 +835,8 @@ declare class TTSGgml {
     static readonly ENGINE_PARLER = "parler";
     static readonly ENGINE_AUDIO8 = "audio8";
     static readonly ENGINE_MOSS = "moss";
+    static readonly ENGINE_MOSS_SFX = "moss-sfx";
+    static readonly ENGINE_MOSS_SPEECH = "moss-speech";
     static readonly ENGINE_POCKET = "pocket";
     opts: object;
     exclusiveRun: boolean;
@@ -829,6 +916,9 @@ declare class TTSGgml {
     private _mossBackbonePath?;
     private _mossCodecDecoderPath?;
     private _mossCodecEncoderPath?;
+    private _mossSoundEffectPath?;
+    private _mossSpeechModelPath?;
+    private _mossSpeechCodecPath?;
     private _dialogueReferences?;
     private _durationTokens?;
     private _referenceText?;
@@ -865,6 +955,10 @@ declare class TTSGgml {
      */
     private _assertEngineScopedOptions;
     private _assertAudio8OptionConsistency;
+    private _assertMossSoundEffectOptionConsistency;
+    private _assertMossSpeechOptionConsistency;
+    private _assertMossSpeechOutputRate;
+    private _assertMossSoundEffectOutputRate;
     private _assertMossOptionConsistency;
     private _assertNoMossOnlyOptions;
     private _assertMossDialogueReferences;
@@ -917,6 +1011,18 @@ declare class TTSGgml {
      * streaming options. Returns undefined when none are present.
      */
     private _resolveAudio8JobFields;
+    private _assertNoSpeechFields;
+    private _assertNoOtherEngineFields;
+    /**
+     * Validate the MOSS-Speech call and fold the new user turn (spoken `audio`
+     * or text `input`) and `systemPrompt` into the native `messages` list.
+     */
+    private _resolveSpeechJobFields;
+    /**
+     * Extract + validate the per-call MOSS-SoundEffect controls from a run
+     * input. Returns undefined when none are present.
+     */
+    private _resolveSoundEffectJobFields;
     /**
      * The per-call fields of whichever engine is loaded, if any are set. Parler
      * takes the full description/template surface, Audio8 its voice override,
@@ -969,6 +1075,8 @@ declare class TTSGgml {
     private _buildParlerParams;
     private _buildAudio8Params;
     private _buildMossParams;
+    private _buildMossSoundEffectParams;
+    private _buildMossSpeechParams;
     /** The knobs every engine in SAMPLING_ENGINES reads. */
     private _assignSamplingParams;
     /**
@@ -1072,6 +1180,7 @@ type NamespaceParlerFit = ParlerFitRequest;
 type NamespaceChatterboxFit = ChatterboxFitRequest;
 type NamespaceAudio8Fit = Audio8FitRequest;
 type NamespaceMossFit = MossFitRequest;
+type NamespaceMossSoundEffectFit = MossSoundEffectFitRequest;
 type NamespaceCosyvoiceFit = CosyvoiceFitRequest;
 declare namespace TTSGgml {
     export import QvacErrorAddonTTSGgml = errorModule.QvacErrorAddonTTSGgml;
@@ -1100,6 +1209,7 @@ declare namespace TTSGgml {
     type ChatterboxFitRequest = NamespaceChatterboxFit;
     type Audio8FitRequest = NamespaceAudio8Fit;
     type MossFitRequest = NamespaceMossFit;
+    type MossSoundEffectFitRequest = NamespaceMossSoundEffectFit;
     type CosyvoiceFitRequest = NamespaceCosyvoiceFit;
     const resolveBackendsDir: typeof resolveBackendsDirImpl;
     const assessFit: typeof assessFitImpl;