@qvac/inference 0.19.1 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +43 -11
- package/dist/api/audio-gen.d.ts +42 -1
- package/dist/api/audio-gen.js +137 -51
- package/dist/api/index.d.ts +2 -1
- package/dist/api/index.js +2 -1
- package/dist/api/text-to-speech.d.ts +31 -4
- package/dist/api/text-to-speech.js +117 -21
- package/dist/api/vector-index.d.ts +142 -0
- package/dist/api/vector-index.js +168 -0
- package/dist/api/world.d.ts +2 -2
- package/dist/errors/index.d.ts +12 -0
- package/dist/errors/index.js +21 -0
- package/dist/handlers/vector-index.d.ts +2 -0
- package/dist/handlers/vector-index.js +40 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/models/registry/models.d.ts +351 -12
- package/dist/models/registry/models.js +2271 -1957
- package/dist/models/registry/resource-profiles.js +135 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +2 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +65 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +9 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +24 -7
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +46 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +218 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +1 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +56 -225
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +8 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +28 -0
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +248 -0
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +17 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +5 -2
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +1 -2
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.js +1 -2
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +6 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.d.ts +1 -2
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +0 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +3 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +9 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +96 -89
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.d.ts +0 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.js +0 -31
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +40 -24
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +187 -123
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +11 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +18 -16
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +8 -4
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +1 -2
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +1 -2
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -38
- package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +9 -0
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +42 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +1 -1
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +1 -2
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +1 -1
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +1 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +1 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +8 -2
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +9 -4
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +51 -19
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +29 -0
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +66 -0
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +10 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +3 -5
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +143 -28
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +5 -2
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +182 -66
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +267 -22
- package/dist/plugins/builtin/tts-ggml/plugin.js +94 -35
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +1 -1
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +1 -2
- package/dist/plugins/index.d.ts +1 -1
- package/dist/plugins/index.js +1 -1
- package/dist/plugins/ops/load-model.js +22 -2
- package/dist/plugins/ops/translate.js +58 -12
- package/dist/plugins/ops/translatepsy-utils.d.ts +17 -0
- package/dist/plugins/ops/translatepsy-utils.js +32 -0
- package/dist/plugins/registry.d.ts +15 -0
- package/dist/plugins/registry.js +133 -15
- package/dist/plugins/turbovec-backend.d.ts +17 -0
- package/dist/plugins/turbovec-backend.js +99 -0
- package/dist/registry.js +4 -0
- package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +46 -0
- package/dist/resources/model-fit/native-probe/advisory-fit.js +214 -0
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +26 -0
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +171 -0
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +42 -0
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +192 -0
- package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +70 -0
- package/dist/resources/model-fit/native-probe/run-isolated-fit.js +372 -0
- package/dist/runtime/env.d.ts +2 -0
- package/dist/runtime/env.js +9 -2
- package/dist/runtime/lifecycle.js +2 -0
- package/dist/runtime/model-registry.d.ts +3 -1
- package/dist/runtime/model-registry.js +2 -1
- package/dist/runtime/request-context.js +17 -0
- package/dist/runtime/vector-index-backend.d.ts +25 -0
- package/dist/runtime/vector-index-backend.js +1 -0
- package/dist/runtime/vector-index-registry.d.ts +76 -0
- package/dist/runtime/vector-index-registry.js +161 -0
- package/dist/schemas/assess-model-fit.d.ts +44 -0
- package/dist/schemas/assess-model-fit.js +40 -0
- package/dist/schemas/audio-gen.d.ts +479 -4
- package/dist/schemas/audio-gen.js +376 -17
- package/dist/schemas/batch-completion-stream.d.ts +7 -0
- package/dist/schemas/batch-completion-stream.js +5 -3
- package/dist/schemas/common.d.ts +749 -15
- package/dist/schemas/common.js +8 -1
- package/dist/schemas/completion-event.d.ts +11 -1
- package/dist/schemas/completion-event.js +3 -0
- package/dist/schemas/completion-stream.d.ts +20 -0
- package/dist/schemas/completion-stream.js +41 -4
- package/dist/schemas/config.d.ts +48 -30
- package/dist/schemas/errors.d.ts +4 -0
- package/dist/schemas/errors.js +22 -0
- package/dist/schemas/index.d.ts +1 -0
- package/dist/schemas/index.js +1 -0
- package/dist/schemas/llamacpp-config.d.ts +4 -8
- package/dist/schemas/llamacpp-config.js +4 -4
- package/dist/schemas/load-model.d.ts +2503 -139
- package/dist/schemas/model-config-schemas.d.ts +228 -5
- package/dist/schemas/plugin.d.ts +10 -0
- package/dist/schemas/sdcpp-config.d.ts +221 -4
- package/dist/schemas/sdcpp-config.js +128 -22
- package/dist/schemas/text-to-speech.d.ts +780 -19
- package/dist/schemas/text-to-speech.js +298 -44
- package/dist/schemas/transcription-config.js +7 -4
- package/dist/schemas/vector-index.d.ts +194 -0
- package/dist/schemas/vector-index.js +185 -0
- package/dist/utils/addon-responses.d.ts +29 -3
- package/dist/utils/aggregate-events.d.ts +3 -1
- package/dist/utils/aggregate-events.js +7 -1
- package/dist/utils/tts-stats.d.ts +15 -11
- package/dist/utils/tts-stats.js +33 -11
- package/package.json +28 -25
|
@@ -63,6 +63,12 @@ export const TTS_SUPERTONIC_LANGUAGES = [
|
|
|
63
63
|
'uk', // Ukrainian
|
|
64
64
|
'vi' // Vietnamese
|
|
65
65
|
];
|
|
66
|
+
// The engines @qvac/tts-ggml exposes as `TTSGgml.ENGINE_*`, in the order the
|
|
67
|
+
// addon lists them. Exported so callers can discriminate a TTS config without
|
|
68
|
+
// re-declaring the union (the CLI used to).
|
|
69
|
+
export const TTS_ENGINES = ['chatterbox', 'supertonic', 'parler', 'cosyvoice3', 'audio8'];
|
|
70
|
+
// Mirror of the addon's `SentenceDelimiterPreset` (./text-stream-accumulator).
|
|
71
|
+
export const TTS_SENTENCE_DELIMITER_PRESETS = ['latin', 'cjk', 'multilingual'];
|
|
66
72
|
export const TTS_PARLER_EMOTIONS = [
|
|
67
73
|
'command',
|
|
68
74
|
'anger',
|
|
@@ -136,9 +142,9 @@ const ttsSupertonicLanguageSchema = z.enum(TTS_SUPERTONIC_LANGUAGES);
|
|
|
136
142
|
const ttsParlerEmotionSchema = z.enum(TTS_PARLER_EMOTIONS);
|
|
137
143
|
const ttsCosyvoice3EmotionSchema = z.enum(TTS_COSYVOICE3_EMOTIONS);
|
|
138
144
|
const ttsPaceSchema = z.enum(TTS_PACES);
|
|
145
|
+
// Every integer knob is int32: the addon narrows them with `| 0` before the
|
|
146
|
+
// native call, so a wider value would wrap silently instead of failing.
|
|
139
147
|
const ttsIntegerSchema = z.number().int();
|
|
140
|
-
const ttsNonNegativeIntegerSchema = ttsIntegerSchema.nonnegative();
|
|
141
|
-
const ttsPositiveIntegerSchema = ttsIntegerSchema.positive();
|
|
142
148
|
const ttsInt32Schema = ttsIntegerSchema.min(-2147483648).max(2147483647);
|
|
143
149
|
const ttsNonNegativeInt32Schema = ttsInt32Schema.nonnegative();
|
|
144
150
|
const ttsPositiveInt32Schema = ttsInt32Schema.positive();
|
|
@@ -146,17 +152,48 @@ const ttsPositiveInt32Schema = ttsInt32Schema.positive();
|
|
|
146
152
|
const TTS_USE_GPU_DESC = 'Route inference through a GPU backend (Metal / CUDA / Vulkan / OpenCL) when available. Default false.';
|
|
147
153
|
const TTS_SEED_DESC = 'RNG seed for the engine’s stochastic stages (e.g. Chatterbox CFM/SineGen, Supertonic latent generation).';
|
|
148
154
|
const TTS_THREADS_DESC = 'CPU thread count; overrides the hardware default.';
|
|
149
|
-
const TTS_NGPU_LAYERS_DESC = 'Model layers to offload to the GPU backend (99 = all).
|
|
155
|
+
const TTS_NGPU_LAYERS_DESC = 'Model layers to offload to the GPU backend (99 = all, 0 = CPU). Takes effect on its own and wins over `useGPU`; when both are set they must agree.';
|
|
150
156
|
const TTS_STREAM_CHUNK_TOKENS_DESC = 'Speech tokens per native streaming chunk; 0 disables native chunk streaming.';
|
|
151
157
|
const TTS_STREAM_FIRST_CHUNK_TOKENS_DESC = 'Smaller first streaming chunk for lower first-audio latency.';
|
|
152
158
|
const TTS_TEMPERATURE_DESC = 'Sampling temperature; unset defers to the engine default (Parler 1.0, Audio8 0.7).';
|
|
153
159
|
const TTS_TOP_K_DESC = 'Top-k sampling cutoff; unset defers to the engine default (50).';
|
|
154
160
|
const TTS_TOP_P_DESC = 'Top-p (nucleus) sampling cutoff (0 < p ≤ 1); unset defers to the engine default.';
|
|
155
161
|
const TTS_MAX_FRAMES_DESC = 'Generation-length cap in decoder frames; 0 = engine default (Parler ≈86 frames/s, Audio8 ≈21.5).';
|
|
162
|
+
// Same wording as the transcription and llama.cpp configs — these are the
|
|
163
|
+
// generic ggml backend-loading knobs, not TTS-specific ones.
|
|
164
|
+
const TTS_BACKENDS_DIR_DESC = 'Root directory for dynamically-loaded ggml backend `.so` files. Defaults to `prebuilds/`.';
|
|
165
|
+
const TTS_OPENCL_CACHE_DIR_DESC = "Persistent directory for ggml-opencl's compiled-program cache (Android only).";
|
|
166
|
+
// The ggml backend knobs every engine honours. `openclCacheDir` is added only
|
|
167
|
+
// on the three engines whose native config reads it (Parler and Audio8 build
|
|
168
|
+
// their params through the addon's _assignBackendParams, which does not).
|
|
169
|
+
const ttsBackendDirFieldsShape = {
|
|
170
|
+
backendsDir: z.string().min(1).optional().describe(TTS_BACKENDS_DIR_DESC)
|
|
171
|
+
};
|
|
172
|
+
const ttsOpenclCacheFieldShape = {
|
|
173
|
+
openclCacheDir: z.string().min(1).optional().describe(TTS_OPENCL_CACHE_DIR_DESC)
|
|
174
|
+
};
|
|
175
|
+
// Mirrors the addon's assertGpuIntentConsistent: when both are stated they
|
|
176
|
+
// must agree, because `nGpuLayers` wins at the native layer and would silently
|
|
177
|
+
// contradict an explicit `useGPU`. Rejecting it here names both fields instead
|
|
178
|
+
// of surfacing a raw addon throw at load.
|
|
179
|
+
function refineGpuIntent(config, ctx) {
|
|
180
|
+
if (typeof config.useGPU !== 'boolean' || config.nGpuLayers === undefined)
|
|
181
|
+
return;
|
|
182
|
+
if (config.useGPU === (config.nGpuLayers !== 0))
|
|
183
|
+
return;
|
|
184
|
+
ctx.addIssue({
|
|
185
|
+
code: 'custom',
|
|
186
|
+
path: ['nGpuLayers'],
|
|
187
|
+
message: `useGPU=${config.useGPU} conflicts with nGpuLayers=${config.nGpuLayers}. ` +
|
|
188
|
+
'Drop one, or make them agree (useGPU true with a non-zero nGpuLayers, ' +
|
|
189
|
+
'or useGPU false with nGpuLayers 0).'
|
|
190
|
+
});
|
|
191
|
+
}
|
|
156
192
|
// Desired output sample rate in Hz. Matches the @qvac/tts-ggml addon's
|
|
157
193
|
// accepted range; omit to keep the engine's native rate (or 48 kHz when the
|
|
158
|
-
// LavaSR enhancer is active).
|
|
159
|
-
//
|
|
194
|
+
// LavaSR enhancer is active). Every engine resamples, so this is exposed on
|
|
195
|
+
// all five configs; the engines whose native chunk streaming cannot resample
|
|
196
|
+
// seam-free (Parler, CosyVoice3) constrain it in their own refinements.
|
|
160
197
|
const ttsOutputSampleRateSchema = ttsIntegerSchema
|
|
161
198
|
.min(8000)
|
|
162
199
|
.max(192000)
|
|
@@ -226,8 +263,12 @@ function refineParlerDescriptionFields(config, ctx) {
|
|
|
226
263
|
});
|
|
227
264
|
}
|
|
228
265
|
}
|
|
229
|
-
|
|
230
|
-
|
|
266
|
+
// Parler's native chunk streaming cannot resample across chunk seams, so the
|
|
267
|
+
// rate is pinned to 44.1 kHz — unless the LavaSR enhancer is loaded, whose
|
|
268
|
+
// overlap-reprocess window resamples seam-free (ParlerModel.cpp waives the
|
|
269
|
+
// check on enhancerGgufPath). The load schema applies the waiver; the runtime
|
|
270
|
+
// schema has no enhancer field and always pins.
|
|
271
|
+
function refineParlerNativeStreamingRate(config, ctx) {
|
|
231
272
|
const nativeStreamingEnabled = (config.streamChunkTokens ?? 0) > 0;
|
|
232
273
|
if (nativeStreamingEnabled &&
|
|
233
274
|
config.outputSampleRate !== undefined &&
|
|
@@ -235,11 +276,18 @@ function refineParlerRuntimeConfig(config, ctx) {
|
|
|
235
276
|
ctx.addIssue({
|
|
236
277
|
code: 'custom',
|
|
237
278
|
path: ['outputSampleRate'],
|
|
238
|
-
message: 'Parler native streaming emits at 44100 Hz; omit outputSampleRate, use 44100,
|
|
279
|
+
message: 'Parler native streaming emits at 44100 Hz; omit outputSampleRate, use 44100, ' +
|
|
280
|
+
'enable the LavaSR enhancer (which resamples seam-free), or disable native streaming.'
|
|
239
281
|
});
|
|
240
282
|
}
|
|
241
283
|
}
|
|
242
|
-
|
|
284
|
+
function refineParlerRuntimeConfig(config, ctx) {
|
|
285
|
+
refineParlerDescriptionFields(config, ctx);
|
|
286
|
+
refineGpuIntent(config, ctx);
|
|
287
|
+
refineParlerNativeStreamingRate(config, ctx);
|
|
288
|
+
}
|
|
289
|
+
export const ttsChatterboxRuntimeConfigSchema = z
|
|
290
|
+
.object({
|
|
243
291
|
ttsEngine: z
|
|
244
292
|
.literal('chatterbox')
|
|
245
293
|
.describe('TTS engine: Chatterbox (multilingual, voice cloning).'),
|
|
@@ -249,12 +297,29 @@ export const ttsChatterboxRuntimeConfigSchema = z.object({
|
|
|
249
297
|
.optional()
|
|
250
298
|
.describe('Ignored by Chatterbox; use `referenceAudioSrc` for voice cloning.'),
|
|
251
299
|
useGPU: z.boolean().optional().describe(TTS_USE_GPU_DESC),
|
|
300
|
+
outputSampleRate: ttsOutputSampleRateSchema.optional(),
|
|
301
|
+
// Chatterbox has no pace channel — it time-stretches the rendered waveform
|
|
302
|
+
// instead — so the multiplier is the only speech-rate control it offers,
|
|
303
|
+
// and the native WSOLA stretcher rejects anything outside [0.25, 4.0].
|
|
304
|
+
ttsSpeed: z
|
|
305
|
+
.number()
|
|
306
|
+
.min(0.25)
|
|
307
|
+
.max(4)
|
|
308
|
+
.optional()
|
|
309
|
+
.describe('Speech-rate multiplier (1.0 = unchanged, <1 slower, >1 faster), applied as a pitch-preserving WSOLA time-stretch. Range 0.25–4.0.'),
|
|
310
|
+
nCtx: ttsNonNegativeInt32Schema
|
|
311
|
+
.optional()
|
|
312
|
+
.describe('Cap on the T3 context length in tokens (prompt + generated speech, ~25 tokens ≈ 1 s of audio). The KV cache is allocated up front at this length, so it directly bounds memory; 0 uses the GGUF’s full context.'),
|
|
313
|
+
kvCacheType: z
|
|
314
|
+
.enum(['f32', 'f16', 'q8_0'])
|
|
315
|
+
.optional()
|
|
316
|
+
.describe('T3 KV-cache storage dtype. `f16` is the safe cross-backend default; `q8_0` is smaller and faster where the backend implements its ops.'),
|
|
252
317
|
// Chatterbox-only native streaming controls.
|
|
253
|
-
streamChunkTokens:
|
|
254
|
-
streamFirstChunkTokens:
|
|
318
|
+
streamChunkTokens: ttsNonNegativeInt32Schema.optional().describe(TTS_STREAM_CHUNK_TOKENS_DESC),
|
|
319
|
+
streamFirstChunkTokens: ttsNonNegativeInt32Schema
|
|
255
320
|
.optional()
|
|
256
321
|
.describe(TTS_STREAM_FIRST_CHUNK_TOKENS_DESC),
|
|
257
|
-
cfmSteps:
|
|
322
|
+
cfmSteps: ttsNonNegativeInt32Schema
|
|
258
323
|
.optional()
|
|
259
324
|
.describe('Chatterbox CFM Euler step count. Default 2.'),
|
|
260
325
|
cfgRate: z
|
|
@@ -262,32 +327,67 @@ export const ttsChatterboxRuntimeConfigSchema = z.object({
|
|
|
262
327
|
.nonnegative()
|
|
263
328
|
.optional()
|
|
264
329
|
.describe('Chatterbox S3Gen classifier-free-guidance rate; `0` skips the unconditioned pass, a positive value overrides the model’s baked rate. Omit to keep the baked rate.'),
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
330
|
+
// Int32 like the other engines: the addon narrows with `| 0`, so a wider
|
|
331
|
+
// value would wrap silently rather than fail.
|
|
332
|
+
threads: ttsPositiveInt32Schema.optional().describe(TTS_THREADS_DESC),
|
|
333
|
+
nGpuLayers: ttsInt32Schema.optional().describe(TTS_NGPU_LAYERS_DESC),
|
|
334
|
+
seed: ttsInt32Schema.optional().describe(TTS_SEED_DESC),
|
|
335
|
+
...ttsBackendDirFieldsShape,
|
|
336
|
+
...ttsOpenclCacheFieldShape
|
|
337
|
+
})
|
|
338
|
+
.superRefine(refineGpuIntent);
|
|
339
|
+
// Supertonic drives its duration predictor from either an exact multiplier
|
|
340
|
+
// (`ttsSpeed`) or the canonical pace vocabulary, never both — the engine
|
|
341
|
+
// rejects the pair rather than picking a winner.
|
|
342
|
+
function refineSupertonicRateControls(config, ctx) {
|
|
343
|
+
refineGpuIntent(config, ctx);
|
|
344
|
+
if (config.ttsSpeed !== undefined && config.pace !== undefined) {
|
|
345
|
+
ctx.addIssue({
|
|
346
|
+
code: 'custom',
|
|
347
|
+
path: ['pace'],
|
|
348
|
+
message: 'Supertonic accepts either ttsSpeed (an exact multiplier) or pace (slow/moderate/fast), not both.'
|
|
349
|
+
});
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
export const ttsSupertonicRuntimeConfigSchema = z
|
|
353
|
+
.object({
|
|
270
354
|
ttsEngine: z.literal('supertonic').describe('TTS engine: Supertonic.'),
|
|
271
355
|
language: ttsSupertonicLanguageSchema.describe('Language code. Default `en`.'),
|
|
272
356
|
voice: z.string().optional().describe('Supertonic baked voice id, e.g. `F1` or `M1`.'),
|
|
273
357
|
ttsSpeed: z
|
|
274
358
|
.number()
|
|
359
|
+
.nonnegative()
|
|
275
360
|
.optional()
|
|
276
|
-
.describe('Speech-rate / duration multiplier (1.0 = unchanged, <1 slower, >1 faster). Supertonic scales its native duration predictor.'),
|
|
277
|
-
|
|
278
|
-
|
|
361
|
+
.describe('Speech-rate / duration multiplier (1.0 = unchanged, <1 slower, >1 faster). Supertonic scales its native duration predictor. Mutually exclusive with `pace`.'),
|
|
362
|
+
// Supertonic maps the pace step onto a multiplier relative to the GGUF's
|
|
363
|
+
// own default_speed. It conditions the engine at construction, so unlike
|
|
364
|
+
// Parler and CosyVoice3 it cannot take a per-request pace.
|
|
365
|
+
pace: ttsPaceSchema
|
|
366
|
+
.optional()
|
|
367
|
+
.describe("Speaking rate: `'slow'`, `'moderate'`, or `'fast'`, applied when the engine is built. Mutually exclusive with `ttsSpeed`; cannot be changed per request."),
|
|
368
|
+
// SupertonicModel::validateConfig rejects negatives ('steps must be >= 0')
|
|
369
|
+
// and the addon narrows with `| 0`, so bound it like its sibling ttsSpeed.
|
|
370
|
+
ttsNumInferenceSteps: ttsNonNegativeInt32Schema
|
|
279
371
|
.optional()
|
|
280
372
|
.describe('Supertonic vector-estimator CFM steps; 0 uses the GGUF default.'),
|
|
281
373
|
useGPU: z.boolean().optional().describe(TTS_USE_GPU_DESC),
|
|
282
374
|
outputSampleRate: ttsOutputSampleRateSchema.optional(),
|
|
375
|
+
threads: ttsPositiveInt32Schema.optional().describe(TTS_THREADS_DESC),
|
|
376
|
+
nGpuLayers: ttsInt32Schema.optional().describe(TTS_NGPU_LAYERS_DESC),
|
|
377
|
+
seed: ttsInt32Schema.optional().describe(TTS_SEED_DESC),
|
|
283
378
|
vulkanCacheDir: z
|
|
284
379
|
.string()
|
|
285
380
|
.min(1)
|
|
286
381
|
.optional()
|
|
287
|
-
.describe('Supertonic + `useGPU` only: directory where the Vulkan backend persists its compiled pipeline cache.')
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
382
|
+
.describe('Supertonic + `useGPU` only: directory where the Vulkan backend persists its compiled pipeline cache.'),
|
|
383
|
+
...ttsBackendDirFieldsShape,
|
|
384
|
+
...ttsOpenclCacheFieldShape
|
|
385
|
+
})
|
|
386
|
+
.superRefine(refineSupertonicRateControls);
|
|
387
|
+
// A shape rather than a schema so the load schema below can be built from it
|
|
388
|
+
// and carry its own refinement set (the enhancer waiver needs the load-only
|
|
389
|
+
// lavasrEnhancerModelSrc field, which the runtime schema never sees).
|
|
390
|
+
const ttsParlerRuntimeConfigShape = {
|
|
291
391
|
ttsEngine: z.literal('parler').describe('TTS engine: Parler.'),
|
|
292
392
|
...ttsParlerDescriptionFieldsShape,
|
|
293
393
|
useGPU: z.boolean().optional().describe(TTS_USE_GPU_DESC),
|
|
@@ -313,8 +413,11 @@ export const ttsParlerRuntimeConfigSchema = z
|
|
|
313
413
|
normalizeNumbers: z
|
|
314
414
|
.boolean()
|
|
315
415
|
.optional()
|
|
316
|
-
.describe('Parler prompt digit expansion (engine default: enabled).')
|
|
317
|
-
|
|
416
|
+
.describe('Parler prompt digit expansion (engine default: enabled).'),
|
|
417
|
+
...ttsBackendDirFieldsShape
|
|
418
|
+
};
|
|
419
|
+
export const ttsParlerRuntimeConfigSchema = z
|
|
420
|
+
.object(ttsParlerRuntimeConfigShape)
|
|
318
421
|
.superRefine(refineParlerRuntimeConfig);
|
|
319
422
|
// CosyVoice3 structured instruct: a raw string passes through to the engine as
|
|
320
423
|
// the exact instruction text (advanced escape hatch); the object form renders
|
|
@@ -377,6 +480,7 @@ function refineCosyvoice3NativeStreamingRate(config, ctx) {
|
|
|
377
480
|
function refineCosyvoice3RuntimeConfig(config, ctx) {
|
|
378
481
|
refineCosyvoice3ConditioningControls(config, ctx);
|
|
379
482
|
refineCosyvoice3NativeStreamingRate(config, ctx);
|
|
483
|
+
refineGpuIntent(config, ctx);
|
|
380
484
|
}
|
|
381
485
|
const ttsCosyvoice3RuntimeConfigShape = {
|
|
382
486
|
ttsEngine: z.literal('cosyvoice3').describe('TTS engine: CosyVoice3.'),
|
|
@@ -389,6 +493,14 @@ const ttsCosyvoice3RuntimeConfigShape = {
|
|
|
389
493
|
instruct: ttsCosyvoice3InstructSchema
|
|
390
494
|
.optional()
|
|
391
495
|
.describe('Natural-language control: a structured object (one of dialect / volume / style) or a raw instruction string. One conditioning control per synthesis.'),
|
|
496
|
+
// Trimmed for the same reason as the instruct string: a whitespace-only
|
|
497
|
+
// transcript would reach the engine as a non-empty prompt.
|
|
498
|
+
promptText: z
|
|
499
|
+
.string()
|
|
500
|
+
.trim()
|
|
501
|
+
.min(1)
|
|
502
|
+
.optional()
|
|
503
|
+
.describe('Verbatim transcript of `referenceAudioSrc`. Setting it selects zero-shot cloning (best fidelity in the reference’s own language); omitting it selects cross-lingual cloning (timbre only). Without a reference it overrides the baked voice’s transcript metadata.'),
|
|
392
504
|
useGPU: z.boolean().optional().describe(TTS_USE_GPU_DESC),
|
|
393
505
|
outputSampleRate: ttsOutputSampleRateSchema.optional(),
|
|
394
506
|
streamChunkTokens: ttsNonNegativeInt32Schema.optional().describe(TTS_STREAM_CHUNK_TOKENS_DESC),
|
|
@@ -397,7 +509,9 @@ const ttsCosyvoice3RuntimeConfigShape = {
|
|
|
397
509
|
.describe(TTS_STREAM_FIRST_CHUNK_TOKENS_DESC),
|
|
398
510
|
threads: ttsPositiveInt32Schema.optional().describe(TTS_THREADS_DESC),
|
|
399
511
|
nGpuLayers: ttsInt32Schema.optional().describe(TTS_NGPU_LAYERS_DESC),
|
|
400
|
-
seed: ttsInt32Schema.optional().describe(TTS_SEED_DESC)
|
|
512
|
+
seed: ttsInt32Schema.optional().describe(TTS_SEED_DESC),
|
|
513
|
+
...ttsBackendDirFieldsShape,
|
|
514
|
+
...ttsOpenclCacheFieldShape
|
|
401
515
|
};
|
|
402
516
|
export const ttsCosyvoice3RuntimeConfigSchema = z
|
|
403
517
|
.object(ttsCosyvoice3RuntimeConfigShape)
|
|
@@ -424,9 +538,12 @@ const ttsAudio8RuntimeConfigShape = {
|
|
|
424
538
|
outputSampleRate: ttsOutputSampleRateSchema.optional(),
|
|
425
539
|
threads: ttsPositiveInt32Schema.optional().describe(TTS_THREADS_DESC),
|
|
426
540
|
nGpuLayers: ttsInt32Schema.optional().describe(TTS_NGPU_LAYERS_DESC),
|
|
427
|
-
seed: ttsInt32Schema.optional().describe(TTS_SEED_DESC)
|
|
541
|
+
seed: ttsInt32Schema.optional().describe(TTS_SEED_DESC),
|
|
542
|
+
...ttsBackendDirFieldsShape
|
|
428
543
|
};
|
|
429
|
-
export const ttsAudio8RuntimeConfigSchema = z
|
|
544
|
+
export const ttsAudio8RuntimeConfigSchema = z
|
|
545
|
+
.object(ttsAudio8RuntimeConfigShape)
|
|
546
|
+
.superRefine(refineGpuIntent);
|
|
430
547
|
export const ttsRuntimeConfigSchema = z.discriminatedUnion('ttsEngine', [
|
|
431
548
|
ttsChatterboxRuntimeConfigSchema,
|
|
432
549
|
ttsSupertonicRuntimeConfigSchema,
|
|
@@ -446,7 +563,20 @@ const ttsLavasrLoadFieldsShape = {
|
|
|
446
563
|
.optional()
|
|
447
564
|
.describe('LavaSR denoiser model source; runs before the enhancer, rate-preserving (batch synthesis only).')
|
|
448
565
|
};
|
|
449
|
-
|
|
566
|
+
// The denoiser needs the whole utterance, so every engine with native chunk
|
|
567
|
+
// streaming rejects the pair at load. Shared so Chatterbox, Parler and
|
|
568
|
+
// CosyVoice3 fail the same way with the same message.
|
|
569
|
+
function refineLavasrDenoiserStreaming(config, ctx) {
|
|
570
|
+
if (config.lavasrDenoiserModelSrc !== undefined && (config.streamChunkTokens ?? 0) > 0) {
|
|
571
|
+
ctx.addIssue({
|
|
572
|
+
code: 'custom',
|
|
573
|
+
path: ['lavasrDenoiserModelSrc'],
|
|
574
|
+
message: 'The LavaSR denoiser is batch-only; disable native streaming (streamChunkTokens) to use it.'
|
|
575
|
+
});
|
|
576
|
+
}
|
|
577
|
+
}
|
|
578
|
+
export const ttsChatterboxLoadConfigSchema = ttsChatterboxRuntimeConfigSchema
|
|
579
|
+
.extend({
|
|
450
580
|
// Optional at schema time so legacy ONNX configs (no s3genModelSrc) reach
|
|
451
581
|
// the plugin's resolveConfig and raise LegacyTtsModelDeprecatedError.
|
|
452
582
|
s3genModelSrc: modelSrcInputSchema
|
|
@@ -462,37 +592,102 @@ export const ttsChatterboxLoadConfigSchema = ttsChatterboxRuntimeConfigSchema.ex
|
|
|
462
592
|
.optional()
|
|
463
593
|
.describe('Chatterbox MTL only: Cangjie TSV source for Chinese romanisation (required for language `zh`).'),
|
|
464
594
|
...ttsLavasrLoadFieldsShape
|
|
465
|
-
})
|
|
595
|
+
})
|
|
596
|
+
.superRefine(refineLavasrDenoiserStreaming);
|
|
597
|
+
// Supertonic has no native chunk streaming, so the denoiser is always reachable.
|
|
466
598
|
export const ttsSupertonicLoadConfigSchema = ttsSupertonicRuntimeConfigSchema.extend({
|
|
467
599
|
...ttsLavasrLoadFieldsShape
|
|
468
600
|
});
|
|
469
|
-
|
|
601
|
+
function refineParlerLoadConfig(config, ctx) {
|
|
602
|
+
refineParlerDescriptionFields(config, ctx);
|
|
603
|
+
refineGpuIntent(config, ctx);
|
|
604
|
+
// Same waiver CosyVoice3 gets: with the enhancer active the addon resamples
|
|
605
|
+
// seam-free, so the 44.1 kHz native-streaming pin only applies without it.
|
|
606
|
+
if (config.lavasrEnhancerModelSrc === undefined) {
|
|
607
|
+
refineParlerNativeStreamingRate(config, ctx);
|
|
608
|
+
}
|
|
609
|
+
refineLavasrDenoiserStreaming(config, ctx);
|
|
610
|
+
}
|
|
611
|
+
export const ttsParlerLoadConfigSchema = z
|
|
612
|
+
.object({
|
|
613
|
+
...ttsParlerRuntimeConfigShape,
|
|
614
|
+
...ttsLavasrLoadFieldsShape
|
|
615
|
+
})
|
|
616
|
+
.superRefine(refineParlerLoadConfig);
|
|
617
|
+
// CosyVoice3 zero-shot / cross-lingual voice cloning. The speech tokenizer and
|
|
618
|
+
// the CAM++ speaker encoder are NOT in the LLM GGUF's companion set — they are
|
|
619
|
+
// a ~300 MB opt-in, so they are requested explicitly here and forwarded to the
|
|
620
|
+
// addon as `files.cosyvoiceS3tokModel` / `files.cosyvoiceCampplusModel`.
|
|
621
|
+
const ttsCosyvoice3CloningFieldsShape = {
|
|
622
|
+
referenceAudioSrc: modelSrcInputSchema
|
|
623
|
+
.optional()
|
|
624
|
+
.describe('CosyVoice3 voice-cloning reference recording source (wav; 0.5–30 s, 5–15 s of clean speech recommended). Replaces the baked voice.'),
|
|
625
|
+
cosyvoice3S3tokModelSrc: modelSrcInputSchema
|
|
626
|
+
.optional()
|
|
627
|
+
.describe('CosyVoice3 speech-tokenizer (speech_tokenizer_v3) model source; required with `referenceAudioSrc`.'),
|
|
628
|
+
cosyvoice3CampplusModelSrc: modelSrcInputSchema
|
|
629
|
+
.optional()
|
|
630
|
+
.describe('CosyVoice3 CAM++ speaker-encoder model source; required with `referenceAudioSrc`.')
|
|
631
|
+
};
|
|
632
|
+
// The addon fails the native load — never silently falls back to the baked
|
|
633
|
+
// voice — when a reference recording arrives without the two cloning GGUFs, so
|
|
634
|
+
// catch the incomplete set here where the message can name the missing source.
|
|
635
|
+
// `promptText` is deliberately not required: its absence selects cross-lingual.
|
|
636
|
+
function refineCosyvoice3Cloning(config, ctx) {
|
|
637
|
+
if (config.referenceAudioSrc !== undefined) {
|
|
638
|
+
if (config.cosyvoice3S3tokModelSrc === undefined) {
|
|
639
|
+
ctx.addIssue({
|
|
640
|
+
code: 'custom',
|
|
641
|
+
path: ['cosyvoice3S3tokModelSrc'],
|
|
642
|
+
message: 'CosyVoice3 voice cloning requires the speech-tokenizer GGUF (cosyvoice3S3tokModelSrc).'
|
|
643
|
+
});
|
|
644
|
+
}
|
|
645
|
+
if (config.cosyvoice3CampplusModelSrc === undefined) {
|
|
646
|
+
ctx.addIssue({
|
|
647
|
+
code: 'custom',
|
|
648
|
+
path: ['cosyvoice3CampplusModelSrc'],
|
|
649
|
+
message: 'CosyVoice3 voice cloning requires the CAM++ speaker-encoder GGUF (cosyvoice3CampplusModelSrc).'
|
|
650
|
+
});
|
|
651
|
+
}
|
|
652
|
+
return;
|
|
653
|
+
}
|
|
654
|
+
for (const field of ['cosyvoice3S3tokModelSrc', 'cosyvoice3CampplusModelSrc']) {
|
|
655
|
+
if (config[field] !== undefined) {
|
|
656
|
+
ctx.addIssue({
|
|
657
|
+
code: 'custom',
|
|
658
|
+
path: ['referenceAudioSrc'],
|
|
659
|
+
message: `referenceAudioSrc is required when ${field} is set; the cloning models are only used to clone a voice.`
|
|
660
|
+
});
|
|
661
|
+
}
|
|
662
|
+
}
|
|
663
|
+
}
|
|
470
664
|
function refineCosyvoice3LoadConfig(config, ctx) {
|
|
471
665
|
refineCosyvoice3ConditioningControls(config, ctx);
|
|
666
|
+
refineGpuIntent(config, ctx);
|
|
472
667
|
// With the LavaSR enhancer active the addon resamples seam-free, so the
|
|
473
668
|
// 24 kHz native-streaming restriction only applies without it.
|
|
474
669
|
if (config.lavasrEnhancerModelSrc === undefined) {
|
|
475
670
|
refineCosyvoice3NativeStreamingRate(config, ctx);
|
|
476
671
|
}
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
code: 'custom',
|
|
480
|
-
path: ['lavasrDenoiserModelSrc'],
|
|
481
|
-
message: 'The LavaSR denoiser is batch-only; disable native streaming (streamChunkTokens) to use it.'
|
|
482
|
-
});
|
|
483
|
-
}
|
|
672
|
+
refineLavasrDenoiserStreaming(config, ctx);
|
|
673
|
+
refineCosyvoice3Cloning(config, ctx);
|
|
484
674
|
}
|
|
485
675
|
// CosyVoice3 loads from a model *directory* (LLM/flow/HiFT GGUFs plus
|
|
486
676
|
// voice.gguf, vocab.json and merges.txt). The primary `modelSrc` is the LLM
|
|
487
677
|
// GGUF, whose registry companion set co-locates the remaining files, so the
|
|
488
|
-
// load config
|
|
678
|
+
// load config adds the optional LavaSR post-processing sources and the
|
|
679
|
+
// voice-cloning add-ons, which are NOT part of that companion set.
|
|
489
680
|
export const ttsCosyvoice3LoadConfigSchema = z
|
|
490
681
|
.object({
|
|
491
682
|
...ttsCosyvoice3RuntimeConfigShape,
|
|
492
|
-
...ttsLavasrLoadFieldsShape
|
|
683
|
+
...ttsLavasrLoadFieldsShape,
|
|
684
|
+
...ttsCosyvoice3CloningFieldsShape
|
|
493
685
|
})
|
|
494
686
|
.superRefine(refineCosyvoice3LoadConfig);
|
|
687
|
+
// Built from the runtime *shape* rather than the runtime schema, so the
|
|
688
|
+
// refinements attached to that schema do not carry over — re-run them here.
|
|
495
689
|
function refineAudio8LoadConfig(config, ctx) {
|
|
690
|
+
refineGpuIntent(config, ctx);
|
|
496
691
|
if (config.referenceAudioSrc !== undefined && config.referenceText === undefined) {
|
|
497
692
|
ctx.addIssue({
|
|
498
693
|
code: 'custom',
|
|
@@ -596,8 +791,17 @@ export const ttsConfigSchema = z
|
|
|
596
791
|
ttsAudio8LoadConfigSchema.strict()
|
|
597
792
|
])
|
|
598
793
|
.superRefine(refineChatterboxTokenizerAssets);
|
|
794
|
+
// Cancel targeting. Both TTS handlers declare `cancel: { scope: 'model', hard:
|
|
795
|
+
// true }`, which the runtime only honours for requests registered in the
|
|
796
|
+
// request registry — so the id has to reach the engine on the request.
|
|
797
|
+
const ttsRequestIdSchema = z
|
|
798
|
+
.string()
|
|
799
|
+
.min(1)
|
|
800
|
+
.optional()
|
|
801
|
+
.describe('Client-generated id for targeting this run with `cancel({ requestId })`.');
|
|
599
802
|
const ttsClientParamsShape = {
|
|
600
803
|
modelId: z.string(),
|
|
804
|
+
requestId: ttsRequestIdSchema,
|
|
601
805
|
inputType: z.string().default('text'),
|
|
602
806
|
text: z.string().trim().min(1, 'text must not be empty or whitespace-only'),
|
|
603
807
|
stream: z.boolean().default(true),
|
|
@@ -619,28 +823,70 @@ export const ttsRequestSchema = z
|
|
|
619
823
|
type: z.literal('textToSpeech')
|
|
620
824
|
})
|
|
621
825
|
.superRefine(refineParlerDescriptionFields);
|
|
826
|
+
// Mirrors @qvac/tts-ggml's `RuntimeStats`, using the same field names as the
|
|
827
|
+
// transcription and audio-gen stats so a caller reads one vocabulary across
|
|
828
|
+
// the audio plugins. `audioDuration` keeps its existing name (the addon calls
|
|
829
|
+
// it `audioDurationMs`; both are milliseconds). `totalTime` is in SECONDS —
|
|
830
|
+
// `realTimeFactor` = `totalTime` * 1000 / `audioDuration`.
|
|
831
|
+
//
|
|
832
|
+
// The chunked paths (`stream: true`, `sentenceStream`, and the duplex session)
|
|
833
|
+
// run one native job per sentence and the addon's aggregate carries only the
|
|
834
|
+
// four summable fields (totalTime, audioDuration, totalSamples,
|
|
835
|
+
// generatedFrames); the five backend fields below are reported by batch
|
|
836
|
+
// synthesis (`stream: false`) only.
|
|
622
837
|
export const ttsStatsSchema = z.object({
|
|
623
838
|
audioDuration: z.number().optional(),
|
|
839
|
+
totalTime: z.number().optional(),
|
|
840
|
+
realTimeFactor: z.number().optional(),
|
|
841
|
+
tokensPerSecond: z.number().optional(),
|
|
624
842
|
totalSamples: z.number().optional(),
|
|
843
|
+
// Audio8 counts codec frames on a fixed 46 ms grid rather than tokens, and
|
|
844
|
+
// reports them as the unit behind its `tokensPerSecond`.
|
|
845
|
+
generatedFrames: z.number().optional(),
|
|
846
|
+
// Backend selection captured once at model load. `0` CPU / `1` GPU;
|
|
847
|
+
// `backendId` codes the family (0 CPU, 1 Metal, 2 CUDA, 3 Vulkan, 4 OpenCL,
|
|
848
|
+
// 99 other GPU). The enhancer fields report the LavaSR stage separately and
|
|
849
|
+
// are `-1` when no enhancer is loaded.
|
|
850
|
+
backendDevice: z.number().optional(),
|
|
851
|
+
backendId: z.number().optional(),
|
|
852
|
+
gpuUnsupported: z.number().optional(),
|
|
625
853
|
enhancerBackendDevice: z.number().optional(),
|
|
626
854
|
enhancerBackendId: z.number().optional()
|
|
627
855
|
});
|
|
856
|
+
// The rate of `buffer`. It is not a constant of the engine: `outputSampleRate`
|
|
857
|
+
// and the LavaSR enhancer both move it, so a caller writing a WAV or feeding
|
|
858
|
+
// an audio device has to read it off the frame rather than assume 24/44.1 kHz.
|
|
859
|
+
const ttsResponseSampleRateSchema = z
|
|
860
|
+
.number()
|
|
861
|
+
.int()
|
|
862
|
+
.positive()
|
|
863
|
+
.optional()
|
|
864
|
+
.describe('Sample rate in Hz of the signed 16-bit mono PCM in `buffer`.');
|
|
628
865
|
export const ttsResponseSchema = z.object({
|
|
629
866
|
type: z.literal('textToSpeech'),
|
|
630
867
|
buffer: z.array(z.number()),
|
|
631
868
|
done: z.boolean().default(false),
|
|
869
|
+
sampleRate: ttsResponseSampleRateSchema,
|
|
632
870
|
stats: ttsStatsSchema.optional(),
|
|
633
871
|
chunkIndex: z.number().int().nonnegative().optional(),
|
|
634
|
-
sentenceChunk: z.string().optional()
|
|
872
|
+
sentenceChunk: z.string().optional(),
|
|
873
|
+
isLast: z
|
|
874
|
+
.boolean()
|
|
875
|
+
.optional()
|
|
876
|
+
.describe('True on the final audio-bearing chunk of a pre-chunked synthesis. Absent when the chunk count is not known up front (streamed text in).'),
|
|
877
|
+
// On the terminal frame only: how the run ended. A run stopped by
|
|
878
|
+
// `cancel()` ends cleanly with `cancelled` rather than as an error.
|
|
879
|
+
stopReason: z.enum(['completed', 'cancelled']).optional()
|
|
635
880
|
});
|
|
636
881
|
// Internal: kept un-exported to present a single request-schema surface to
|
|
637
882
|
// consumers. The inferred `TextToSpeechStreamClientParams` type below uses
|
|
638
883
|
// this shape via `typeof`, no runtime export needed.
|
|
639
884
|
const textToSpeechStreamRequestBaseShape = {
|
|
640
885
|
modelId: z.string(),
|
|
886
|
+
requestId: ttsRequestIdSchema,
|
|
641
887
|
inputType: z.string().default('text'),
|
|
642
888
|
accumulateSentences: z.boolean().optional(),
|
|
643
|
-
sentenceDelimiterPreset: z.enum(
|
|
889
|
+
sentenceDelimiterPreset: z.enum(TTS_SENTENCE_DELIMITER_PRESETS).optional(),
|
|
644
890
|
maxBufferScalars: z.number().positive().optional(),
|
|
645
891
|
flushAfterMs: z.number().positive().optional(),
|
|
646
892
|
...ttsParlerDescriptionFieldsShape
|
|
@@ -655,7 +901,15 @@ export const textToSpeechStreamResponseSchema = z.object({
|
|
|
655
901
|
type: z.literal('textToSpeechStream'),
|
|
656
902
|
buffer: z.array(z.number()),
|
|
657
903
|
done: z.boolean().default(false),
|
|
904
|
+
sampleRate: ttsResponseSampleRateSchema,
|
|
658
905
|
stats: ttsStatsSchema.optional(),
|
|
659
906
|
chunkIndex: z.number().int().nonnegative().optional(),
|
|
660
|
-
sentenceChunk: z.string().optional()
|
|
907
|
+
sentenceChunk: z.string().optional(),
|
|
908
|
+
isLast: z
|
|
909
|
+
.boolean()
|
|
910
|
+
.optional()
|
|
911
|
+
.describe('True on the final audio-bearing chunk of a pre-chunked synthesis. Absent when the chunk count is not known up front (streamed text in).'),
|
|
912
|
+
// On the terminal frame only: how the run ended. A run stopped by
|
|
913
|
+
// `cancel()` ends cleanly with `cancelled` rather than as an error.
|
|
914
|
+
stopReason: z.enum(['completed', 'cancelled']).optional()
|
|
661
915
|
});
|
|
@@ -175,10 +175,12 @@ export const whisperConfigSchema = z.object({
|
|
|
175
175
|
.optional()
|
|
176
176
|
.describe('Voice-activity-detection (VAD) model source; enables VAD when set.')
|
|
177
177
|
});
|
|
178
|
+
const PARAKEET_LANGUAGE_PATTERN = /^(|auto|[a-zA-Z]{2,3}(-[a-zA-Z]{2,4})?)$/;
|
|
178
179
|
// === Parakeet (NVIDIA NeMo GGML) engine config ===
|
|
179
180
|
//
|
|
180
181
|
// Backed by the ggml-based qvac-parakeet.cpp engine. A single GGUF
|
|
181
|
-
// checkpoint covers every variant (TDT, CTC, EOU,
|
|
182
|
+
// checkpoint covers every variant (TDT, RNN-T, CTC, EOU, Nemotron,
|
|
183
|
+
// Sortformer); the
|
|
182
184
|
// addon auto-detects the model type from `parakeet.model.type` GGUF
|
|
183
185
|
// metadata, so callers no longer pass a `modelType` discriminator and
|
|
184
186
|
// only ever supply a single `modelSrc` at `loadModel` time.
|
|
@@ -213,7 +215,7 @@ export const parakeetRuntimeConfigSchema = z.object({
|
|
|
213
215
|
.int()
|
|
214
216
|
.positive()
|
|
215
217
|
.optional()
|
|
216
|
-
.describe('Streaming chunk cadence in ms.
|
|
218
|
+
.describe('Streaming chunk cadence in ms. Defaults to 320 for Nemotron and 2000 for existing models. Nemotron supports 80, 160, 320, 560, or 1120.'),
|
|
217
219
|
streamingHistoryMs: z
|
|
218
220
|
.number()
|
|
219
221
|
.int()
|
|
@@ -227,7 +229,7 @@ export const parakeetRuntimeConfigSchema = z.object({
|
|
|
227
229
|
streamingEnergyVad: z
|
|
228
230
|
.boolean()
|
|
229
231
|
.optional()
|
|
230
|
-
.describe('CTC/TDT-only energy-based voice-activity hint; affects speech segmentation but adds no new event types. For standalone VAD `speaking`/`probability` events, use the whisper engine. Default false.'),
|
|
232
|
+
.describe('CTC/TDT/Nemotron-only energy-based voice-activity hint; affects speech segmentation but adds no new event types. For standalone VAD `speaking`/`probability` events, use the whisper engine. Default false.'),
|
|
231
233
|
streamingLeftContextMs: z
|
|
232
234
|
.number()
|
|
233
235
|
.int()
|
|
@@ -242,8 +244,9 @@ export const parakeetRuntimeConfigSchema = z.object({
|
|
|
242
244
|
.describe('ASR encoder right-lookahead window in ms; omit to keep the model default (2000).'),
|
|
243
245
|
language: z
|
|
244
246
|
.string()
|
|
247
|
+
.regex(PARAKEET_LANGUAGE_PATTERN)
|
|
245
248
|
.optional()
|
|
246
|
-
.describe('
|
|
249
|
+
.describe('Indic CTC language id or Nemotron locale alias (e.g. `hi`, `ta`, `en-US`, `hi-IN`, or `auto`). Empty selects `auto` for Nemotron and keeps full-vocabulary CTC decoding.'),
|
|
247
250
|
// === AOSC (Audio-Online Speaker Cache; v2.1+ Sortformer only) =========
|
|
248
251
|
// Auto-enabled when the loaded GGUF carries
|
|
249
252
|
// `parakeet.model_variant == "sortformer-streaming-v2.1-aosc"`. Ignored
|