@qvac/inference 0.21.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +50 -25
- package/README.md +44 -1
- package/dist/api/assess-model-fit.d.ts +4 -4
- package/dist/api/assess-model-fit.js +4 -4
- package/dist/api/audio-gen.d.ts +4 -1
- package/dist/api/audio-gen.js +4 -1
- package/dist/api/batch-completion.js +1 -1
- package/dist/api/completion-stream.js +1 -1
- package/dist/api/decide.d.ts +26 -0
- package/dist/api/decide.js +26 -0
- package/dist/api/get-model-info.d.ts +1 -1
- package/dist/api/index.d.ts +1 -0
- package/dist/api/index.js +1 -0
- package/dist/api/text-to-speech.d.ts +1 -0
- package/dist/api/text-to-speech.js +15 -19
- package/dist/api/transcribe.js +5 -2
- package/dist/api/video-request.d.ts +5 -5
- package/dist/api/video.d.ts +5 -4
- package/dist/api/video.js +5 -4
- package/dist/errors/index.d.ts +3 -0
- package/dist/errors/index.js +5 -0
- package/dist/handlers/assess-model-fit.d.ts +5 -3
- package/dist/handlers/assess-model-fit.js +80 -28
- package/dist/handlers/get-loaded-model-info.js +1 -2
- package/dist/handlers/load-model/safe-fetch.js +5 -6
- package/dist/handlers/registry.d.ts +3 -1
- package/dist/handlers/registry.js +3 -3
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/logging/logger.js +1 -1
- package/dist/logging/utils.js +1 -1
- package/dist/models/registry/models.d.ts +563 -816
- package/dist/models/registry/models.js +2873 -3110
- package/dist/models/registry/resource-profiles.d.ts +1 -2
- package/dist/models/registry/resource-profiles.js +203 -2
- package/dist/plugins/builtin/asr-ggml/config.d.ts +7 -1
- package/dist/plugins/builtin/asr-ggml/config.js +6 -0
- package/dist/plugins/builtin/asr-ggml/logging.d.ts +1 -1
- package/dist/plugins/builtin/audiogen-ggml/config.d.ts +1 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +28 -10
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +27 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +23 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +6 -27
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +147 -60
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +1 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +40 -24
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +1 -1
- package/dist/plugins/builtin/ggml-classification/plugin.d.ts +4 -4
- package/dist/plugins/builtin/ggml-ocr/params.d.ts +18 -0
- package/dist/plugins/builtin/ggml-ocr/params.js +37 -0
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +18 -10
- package/dist/plugins/builtin/ggml-ocr/plugin.js +16 -38
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +21 -14
- package/dist/plugins/builtin/ggml-vla/plugin.js +1 -1
- package/dist/plugins/builtin/index.d.ts +2 -0
- package/dist/plugins/builtin/index.js +2 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +9 -7
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +109 -189
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +32 -90
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +60 -211
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +2 -49
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +2 -46
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +185 -87
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +4 -5
- package/dist/plugins/builtin/llamacpp-completion/transform.js +1 -1
- package/dist/plugins/builtin/llamacpp-decision/helpers.d.ts +6 -0
- package/dist/plugins/builtin/llamacpp-decision/helpers.js +97 -0
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +91 -0
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +129 -0
- package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +216 -0
- package/dist/plugins/builtin/llamacpp-decision/plugin.js +43 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +12 -5
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +1 -1
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +3 -0
- package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +143 -0
- package/dist/plugins/builtin/moss-transcribe/plugin.js +96 -0
- package/dist/plugins/builtin/nmtcpp-translation/load-files.d.ts +6 -0
- package/dist/plugins/builtin/nmtcpp-translation/load-files.js +70 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +25 -13
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +9 -73
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +52 -31
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +18 -3
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +10 -6
- package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +2 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/world.js +1 -1
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +59 -33
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +12 -9
- package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +22 -0
- package/dist/plugins/builtin/tts-ggml/ops/moss-options.js +79 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +4 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +17 -5
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +164 -29
- package/dist/plugins/builtin/tts-ggml/plugin.js +38 -1
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +58 -35
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +6 -2
- package/dist/plugins/ops/kv-cache-utils.d.ts +1 -1
- package/dist/plugins/ops/kv-cache-utils.js +10 -25
- package/dist/plugins/ops/load-model.js +21 -16
- package/dist/plugins/ops/transcribe.js +17 -1
- package/dist/registry.js +1 -0
- package/dist/resources/model-fit/assess.d.ts +15 -23
- package/dist/resources/model-fit/assess.js +207 -271
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.d.ts +3 -1
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.js +6 -1
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.js +2 -0
- package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts +5 -3
- package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.js +9 -1
- package/dist/resources/model-fit/floor.d.ts +10 -10
- package/dist/resources/model-fit/floor.js +13 -13
- package/dist/resources/model-fit/kv-cache.d.ts +23 -0
- package/dist/resources/model-fit/kv-cache.js +85 -0
- package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +11 -6
- package/dist/resources/model-fit/native-probe/advisory-fit.js +12 -27
- package/dist/resources/model-fit/native-probe/child-engine-fit.js +8 -0
- package/dist/resources/model-fit/native-probe/classify-fit.js +32 -1
- package/dist/resources/model-fit/native-probe/create-fit-request.d.ts +4 -1
- package/dist/resources/model-fit/native-probe/create-fit-request.js +24 -14
- package/dist/resources/model-fit/native-probe/engine-fit.d.ts +14 -0
- package/dist/resources/model-fit/native-probe/engine-fit.js +1 -1
- package/dist/resources/model-fit/native-probe/engines/asr.d.ts +4 -0
- package/dist/resources/model-fit/native-probe/engines/asr.js +23 -0
- package/dist/resources/model-fit/native-probe/engines/audiogen.js +29 -1
- package/dist/resources/model-fit/native-probe/engines/diffusion.js +16 -6
- package/dist/resources/model-fit/native-probe/engines/llama.d.ts +0 -1
- package/dist/resources/model-fit/native-probe/engines/llama.js +19 -123
- package/dist/resources/model-fit/native-probe/engines/nmt.d.ts +8 -0
- package/dist/resources/model-fit/native-probe/engines/nmt.js +11 -0
- package/dist/resources/model-fit/native-probe/engines/ocr.d.ts +9 -0
- package/dist/resources/model-fit/native-probe/engines/ocr.js +13 -0
- package/dist/resources/model-fit/native-probe/engines/tts.d.ts +2 -0
- package/dist/resources/model-fit/native-probe/engines/tts.js +49 -3
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +5 -3
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +38 -23
- package/dist/resources/model-fit/platform.d.ts +4 -4
- package/dist/resources/model-fit/platform.js +6 -8
- package/dist/resources/model-fit/types.d.ts +3 -117
- package/dist/rpc/inventory.js +5 -7
- package/dist/runtime/env.d.ts +0 -2
- package/dist/runtime/env.js +2 -9
- package/dist/runtime/model-config-utils.js +6 -1
- package/dist/runtime/request-context.d.ts +1 -1
- package/dist/runtime/request-context.js +8 -0
- package/dist/runtime/state.js +5 -0
- package/dist/schemas/assess-model-fit.d.ts +100 -43
- package/dist/schemas/assess-model-fit.js +66 -29
- package/dist/schemas/audio-gen.d.ts +186 -18
- package/dist/schemas/audio-gen.js +180 -48
- package/dist/schemas/batch-completion-stream.d.ts +202 -34
- package/dist/schemas/bci-config.d.ts +2 -0
- package/dist/schemas/bci.d.ts +23 -2
- package/dist/schemas/cancel.d.ts +5 -0
- package/dist/schemas/cancel.js +1 -0
- package/dist/schemas/common.d.ts +1083 -158
- package/dist/schemas/common.js +5 -2
- package/dist/schemas/completion-stream.d.ts +232 -37
- package/dist/schemas/completion-stream.js +64 -20
- package/dist/schemas/config.d.ts +477 -12
- package/dist/schemas/config.js +21 -1
- package/dist/schemas/decision.d.ts +602 -0
- package/dist/schemas/decision.js +205 -0
- package/dist/schemas/download-asset.d.ts +7 -3
- package/dist/schemas/engine-addon-map.d.ts +7 -0
- package/dist/schemas/engine-addon-map.js +23 -3
- package/dist/schemas/errors.d.ts +1 -0
- package/dist/schemas/errors.js +5 -0
- package/dist/schemas/index.d.ts +2 -0
- package/dist/schemas/index.js +2 -0
- package/dist/schemas/llamacpp-config.d.ts +42 -2
- package/dist/schemas/llamacpp-config.js +27 -0
- package/dist/schemas/load-model.d.ts +5247 -1309
- package/dist/schemas/load-model.js +88 -9
- package/dist/schemas/model-config-schemas.d.ts +233 -8
- package/dist/schemas/model-config-schemas.js +4 -1
- package/dist/schemas/model-fit-policy.d.ts +17 -0
- package/dist/schemas/model-fit-policy.js +17 -0
- package/dist/schemas/model-info.d.ts +20 -2
- package/dist/schemas/model-info.js +2 -1
- package/dist/schemas/model-resource-profile.d.ts +3 -4
- package/dist/schemas/model-resource-profile.js +1 -4
- package/dist/schemas/model-src-utils.d.ts +10 -2
- package/dist/schemas/model-types.d.ts +10 -1
- package/dist/schemas/model-types.js +3 -0
- package/dist/schemas/ocr.d.ts +74 -0
- package/dist/schemas/ocr.js +42 -0
- package/dist/schemas/plugin.d.ts +8 -2
- package/dist/schemas/plugin.js +6 -0
- package/dist/schemas/profiling.d.ts +4 -4
- package/dist/schemas/registry.d.ts +26 -0
- package/dist/schemas/registry.js +3 -0
- package/dist/schemas/reload-config.d.ts +10 -2
- package/dist/schemas/rpc-server.js +3 -3
- package/dist/schemas/sdcpp-config.d.ts +26 -0
- package/dist/schemas/sdcpp-config.js +8 -7
- package/dist/schemas/system-resources.d.ts +4 -4
- package/dist/schemas/text-to-speech.d.ts +402 -3
- package/dist/schemas/text-to-speech.js +129 -11
- package/dist/schemas/tools.d.ts +13 -4
- package/dist/schemas/tools.js +18 -6
- package/dist/schemas/transcription-config.d.ts +15 -0
- package/dist/schemas/transcription-config.js +38 -3
- package/dist/schemas/transcription.d.ts +46 -4
- package/dist/schemas/transcription.js +51 -4
- package/dist/schemas/translation-config.d.ts +30 -10
- package/dist/schemas/vla.d.ts +9 -2
- package/dist/schemas/vla.js +9 -5
- package/dist/surface.d.ts +1 -1
- package/dist/surface.js +1 -1
- package/dist/utils/addon-responses.d.ts +1 -0
- package/dist/utils/asr-events.d.ts +1 -0
- package/dist/utils/asr-events.js +2 -1
- package/dist/utils/cache/types.d.ts +7 -0
- package/dist/utils/mcp-adapter.js +12 -8
- package/dist/utils/tool-helpers.d.ts +1 -1
- package/dist/utils/tool-helpers.js +20 -52
- package/dist/utils/tools/parsers/dsml.js +8 -5
- package/dist/utils/tools/parsers/qwen35.js +7 -4
- package/dist/utils/tools/shared.d.ts +3 -0
- package/dist/utils/tools/shared.js +62 -0
- package/dist/utils/transcribe-metadata.d.ts +8 -1
- package/dist/utils/transcribe-metadata.js +13 -3
- package/dist/utils/tts-stats.d.ts +1 -0
- package/package.json +41 -32
- package/dist/resources/model-fit/calibration/darwin-arm64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/darwin-arm64.js +0 -36
- package/dist/resources/model-fit/calibration/darwin-x64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/darwin-x64.js +0 -38
- package/dist/resources/model-fit/calibration/fit.d.ts +0 -73
- package/dist/resources/model-fit/calibration/fit.js +0 -152
- package/dist/resources/model-fit/calibration/index.d.ts +0 -37
- package/dist/resources/model-fit/calibration/index.js +0 -75
- package/dist/resources/model-fit/calibration/linux-arm64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/linux-arm64.js +0 -38
- package/dist/resources/model-fit/calibration/linux-x64-vulkan.d.ts +0 -8
- package/dist/resources/model-fit/calibration/linux-x64-vulkan.js +0 -39
- package/dist/resources/model-fit/calibration/linux-x64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/linux-x64.js +0 -38
- package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.d.ts +0 -8
- package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.js +0 -39
- package/dist/resources/model-fit/calibration/win32-x64-vulkan.d.ts +0 -8
- package/dist/resources/model-fit/calibration/win32-x64-vulkan.js +0 -36
- package/dist/resources/model-fit/calibration/win32-x64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/win32-x64.js +0 -38
- package/dist/resources/model-fit/estimators/llm.d.ts +0 -46
- package/dist/resources/model-fit/estimators/llm.js +0 -222
- package/dist/resources/model-fit/estimators/whisper.d.ts +0 -10
- package/dist/resources/model-fit/estimators/whisper.js +0 -76
|
@@ -4,12 +4,20 @@ import { audioInputSchema } from './transcription.js';
|
|
|
4
4
|
import { inferenceBackendDiagnosticsSchema } from './system-resources.js';
|
|
5
5
|
import { encodeBase64 } from '../utils/encoding.js';
|
|
6
6
|
const base64Schema = z.string().min(1);
|
|
7
|
-
//
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
const
|
|
7
|
+
// Mirror requireInferenceSteps, requireAcestepShift, requireMinimaxCfgScale and
|
|
8
|
+
// optionalFloat32 in @qvac/audiogen-ggml, so a value the addon would refuse
|
|
9
|
+
// fails here instead.
|
|
10
|
+
const MAX_INFERENCE_STEPS = 1000;
|
|
11
|
+
const FLOAT32_MAX = 3.4028234663852886e38;
|
|
12
|
+
const FLOAT32_MIN_POSITIVE = 1.401298464324817e-45;
|
|
11
13
|
export const AUDIOGEN_ENGINES = ['acestep', 'minimax'];
|
|
12
14
|
export const audioGenEngineSchema = z.enum(AUDIOGEN_ENGINES);
|
|
15
|
+
/**
|
|
16
|
+
* Where MiniMax-Music3 runs: `gpu` requires a usable GPU and fails the load
|
|
17
|
+
* otherwise, `auto` (what `useGPU: true` means) falls back to the CPU, `cpu`
|
|
18
|
+
* never touches a GPU.
|
|
19
|
+
*/
|
|
20
|
+
const MINIMAX_DEVICES = ['cpu', 'gpu', 'auto'];
|
|
13
21
|
/** Sample rate the ACE-Step engine expects for reference and source audio. */
|
|
14
22
|
export const AUDIOGEN_INPUT_SAMPLE_RATE = 48000;
|
|
15
23
|
/** Channel count (interleaved stereo) the ACE-Step engine expects for input audio. */
|
|
@@ -82,21 +90,30 @@ const commonAudioGenRuntimeConfigShape = {
|
|
|
82
90
|
.string()
|
|
83
91
|
.min(1)
|
|
84
92
|
.optional()
|
|
85
|
-
.describe('Advanced:
|
|
93
|
+
.describe('Advanced: directory scanned as given for dlopen’d ggml backend modules. Unset scans the directory the native module was loaded from. Needed on arm64, where the CPU backend ships as per-microarch module `.so` files.')
|
|
86
94
|
};
|
|
95
|
+
/** Diffusion steps, 0..1000. `0` lets the engine pick its default. */
|
|
96
|
+
const inferenceStepsSchema = z.number().int().min(0).max(MAX_INFERENCE_STEPS);
|
|
97
|
+
/**
|
|
98
|
+
* ACE-Step DiT timestep shift. `0` lets the engine pick; a positive value too
|
|
99
|
+
* small for a float32 would round to `0` and silently select that default, so
|
|
100
|
+
* it is refused instead.
|
|
101
|
+
*/
|
|
102
|
+
const acestepShiftSchema = z
|
|
103
|
+
.number()
|
|
104
|
+
.min(0)
|
|
105
|
+
.max(FLOAT32_MAX)
|
|
106
|
+
.refine((value) => value === 0 || value >= FLOAT32_MIN_POSITIVE, {
|
|
107
|
+
message: 'shift must be 0 or a positive float32 value'
|
|
108
|
+
});
|
|
87
109
|
const acestepRuntimeConfigShape = {
|
|
88
110
|
...commonAudioGenRuntimeConfigShape,
|
|
89
|
-
inferenceSteps:
|
|
90
|
-
.number()
|
|
91
|
-
.int()
|
|
92
|
-
.nonnegative()
|
|
111
|
+
inferenceSteps: inferenceStepsSchema
|
|
93
112
|
.optional()
|
|
94
|
-
.describe('DiT sampling steps; `0` (default) lets ACE-Step auto-pick per DiT architecture (turbo 8 / sft 50).'),
|
|
95
|
-
shift:
|
|
96
|
-
.number()
|
|
97
|
-
.nonnegative()
|
|
113
|
+
.describe('DiT sampling steps, 0..1000; `0` (default) lets ACE-Step auto-pick per DiT architecture (turbo 8 / base and sft 50). A generation or edit can override it per run.'),
|
|
114
|
+
shift: acestepShiftSchema
|
|
98
115
|
.optional()
|
|
99
|
-
.describe('Flow-matching time-shift; `0` (default) lets ACE-Step auto-pick per DiT architecture (turbo 3.0 / sft 1.0).'),
|
|
116
|
+
.describe('Flow-matching time-shift; `0` (default) lets ACE-Step auto-pick per DiT architecture (turbo 3.0 / base and sft 1.0). A generation or edit can override it per run.'),
|
|
100
117
|
nGpuLayers: z
|
|
101
118
|
.number()
|
|
102
119
|
.int()
|
|
@@ -107,34 +124,46 @@ const acestepRuntimeConfigShape = {
|
|
|
107
124
|
const minimaxCfgScaleSchema = z
|
|
108
125
|
.number()
|
|
109
126
|
.min(0)
|
|
110
|
-
.max(
|
|
111
|
-
.refine((value) => value === 0 || value >=
|
|
127
|
+
.max(FLOAT32_MAX)
|
|
128
|
+
.refine((value) => value === 0 || value >= FLOAT32_MIN_POSITIVE, {
|
|
112
129
|
message: 'cfgScale must be 0 or a positive float32 value'
|
|
113
130
|
})
|
|
114
131
|
.describe('MiniMax flow classifier-free guidance scale; `0` uses the model default.');
|
|
115
132
|
const minimaxRuntimeConfigShape = {
|
|
116
133
|
...commonAudioGenRuntimeConfigShape,
|
|
117
|
-
|
|
118
|
-
.
|
|
119
|
-
.
|
|
120
|
-
.
|
|
121
|
-
|
|
134
|
+
device: z
|
|
135
|
+
.enum(MINIMAX_DEVICES)
|
|
136
|
+
.optional()
|
|
137
|
+
.describe("Where MiniMax runs, instead of `useGPU`: `'gpu'` requires a usable GPU and fails the load otherwise, `'auto'` (what `useGPU: true` means) falls back to the CPU, `'cpu'` never touches a GPU. Cannot be combined with `useGPU`."),
|
|
138
|
+
inferenceSteps: inferenceStepsSchema
|
|
122
139
|
.optional()
|
|
123
140
|
.describe('MiniMax flow sampling steps; `0` uses the engine default (20).'),
|
|
124
141
|
cfgScale: minimaxCfgScaleSchema.optional()
|
|
125
142
|
};
|
|
143
|
+
/** The addon takes `useGPU` or `device` for MiniMax, never both. */
|
|
144
|
+
function refineMinimaxDevice(value, ctx) {
|
|
145
|
+
if (value.device === undefined || value.useGPU === undefined)
|
|
146
|
+
return;
|
|
147
|
+
ctx.addIssue({
|
|
148
|
+
code: 'custom',
|
|
149
|
+
path: ['device'],
|
|
150
|
+
message: 'MiniMax accepts either useGPU or device, not both'
|
|
151
|
+
});
|
|
152
|
+
}
|
|
126
153
|
const acestepAudioGenRuntimeConfigSchema = z
|
|
127
154
|
.object({
|
|
128
155
|
engine: z.literal('acestep').optional().describe('Use the ACE-Step music-generation engine.'),
|
|
129
156
|
...acestepRuntimeConfigShape
|
|
130
157
|
})
|
|
131
158
|
.strict();
|
|
132
|
-
const
|
|
133
|
-
.object({
|
|
159
|
+
const minimaxEngineShape = {
|
|
134
160
|
engine: z.literal('minimax').describe('Use the MiniMax-Music3 generation engine.'),
|
|
135
161
|
...minimaxRuntimeConfigShape
|
|
136
|
-
}
|
|
137
|
-
|
|
162
|
+
};
|
|
163
|
+
const minimaxAudioGenRuntimeConfigSchema = z
|
|
164
|
+
.object(minimaxEngineShape)
|
|
165
|
+
.strict()
|
|
166
|
+
.superRefine(refineMinimaxDevice);
|
|
138
167
|
export const audioGenRuntimeConfigSchema = z.discriminatedUnion('engine', [
|
|
139
168
|
acestepAudioGenRuntimeConfigSchema,
|
|
140
169
|
minimaxAudioGenRuntimeConfigSchema
|
|
@@ -147,12 +176,16 @@ const acestepAudioGenConfigSchema = acestepAudioGenRuntimeConfigSchema
|
|
|
147
176
|
vaeModelSrc: modelSrcInputSchema.describe('VAE model source; decodes the latent into the output waveform.')
|
|
148
177
|
})
|
|
149
178
|
.strict();
|
|
150
|
-
|
|
151
|
-
|
|
179
|
+
// Built from the shape rather than extended from the runtime schema, so the
|
|
180
|
+
// device refinement is declared on the schema that carries it.
|
|
181
|
+
const minimaxAudioGenConfigSchema = z
|
|
182
|
+
.object({
|
|
183
|
+
...minimaxEngineShape,
|
|
152
184
|
lmModelSrc: modelSrcInputSchema.describe('MiniMax language-model source; generates semantic music tokens.'),
|
|
153
185
|
synthModelSrc: modelSrcInputSchema.describe('MiniMax synthesis-model source; converts semantic tokens into the output waveform.')
|
|
154
186
|
})
|
|
155
|
-
.strict()
|
|
187
|
+
.strict()
|
|
188
|
+
.superRefine(refineMinimaxDevice);
|
|
156
189
|
export const audioGenConfigSchema = z.discriminatedUnion('engine', [
|
|
157
190
|
acestepAudioGenConfigSchema,
|
|
158
191
|
minimaxAudioGenConfigSchema
|
|
@@ -207,13 +240,26 @@ export const audioGenClientAudioCodesSchema = z
|
|
|
207
240
|
z.array(z.number())
|
|
208
241
|
])
|
|
209
242
|
.pipe(audioCodesWireSchema);
|
|
243
|
+
/**
|
|
244
|
+
* The ACE-Step controls a generation and an edit session's run share. The
|
|
245
|
+
* addon validates them identically on both paths: `seed` a safe integer (which
|
|
246
|
+
* zod's `int()` already is), `bpm` a non-negative int32, the DCW scalers
|
|
247
|
+
* within the float32 range.
|
|
248
|
+
*/
|
|
249
|
+
const seedSchema = z.number().int();
|
|
250
|
+
const bpmSchema = z.number().int().min(0).max(INT32_MAX);
|
|
251
|
+
const dcwScalerSchema = z.number().nonnegative().max(FLOAT32_MAX);
|
|
210
252
|
const audioGenGenerationShape = {
|
|
211
253
|
modelId: z.string().min(1),
|
|
212
254
|
caption: z.string().trim().min(1, 'caption must not be empty or whitespace-only'),
|
|
213
255
|
lyrics: z.string().optional(),
|
|
214
|
-
seed:
|
|
256
|
+
seed: seedSchema
|
|
257
|
+
.optional()
|
|
258
|
+
.describe('RNG seed; omit for a random one. An ACE-Step run reports the seed it used in `stats.metadata.seed`, so an unseeded take can be replayed.'),
|
|
215
259
|
vocalLanguage: z.string().min(1).optional(),
|
|
216
|
-
bpm:
|
|
260
|
+
bpm: bpmSchema
|
|
261
|
+
.optional()
|
|
262
|
+
.describe('Beats per minute; `0` or unset lets the LM infer it. ACE-Step only.'),
|
|
217
263
|
keyscale: z.string().min(1).optional(),
|
|
218
264
|
timesignature: z.string().min(1).optional(),
|
|
219
265
|
augmentCaptionWithMetadata: z
|
|
@@ -260,13 +306,12 @@ const audioGenGenerationShape = {
|
|
|
260
306
|
.max(Number.MAX_SAFE_INTEGER)
|
|
261
307
|
.optional()
|
|
262
308
|
.describe('MiniMax semantic-frame cap. Cannot be combined with duration. MiniMax only; rejected by ACE-Step.'),
|
|
263
|
-
inferenceSteps:
|
|
264
|
-
.
|
|
265
|
-
.
|
|
266
|
-
|
|
267
|
-
.max(MINIMAX_MAX_INFERENCE_STEPS)
|
|
309
|
+
inferenceSteps: inferenceStepsSchema
|
|
310
|
+
.optional()
|
|
311
|
+
.describe('Diffusion steps for this generation, 0..1000; omit to keep the load-time `inferenceSteps`. 0 is the engine default: MiniMax 20 flow steps, ACE-Step 8 on turbo DiTs and 50 on base/sft.'),
|
|
312
|
+
shift: acestepShiftSchema
|
|
268
313
|
.optional()
|
|
269
|
-
.describe('
|
|
314
|
+
.describe('DiT timestep shift for this generation; omit to keep the load-time `shift`. 0 is the engine default: 3.0 on turbo DiTs, 1.0 on base/sft. ACE-Step only; rejected by MiniMax.'),
|
|
270
315
|
cfgScale: minimaxCfgScaleSchema
|
|
271
316
|
.optional()
|
|
272
317
|
.describe('MiniMax flow classifier-free guidance scale for this generation. MiniMax only; rejected by ACE-Step.'),
|
|
@@ -290,14 +335,10 @@ const audioGenGenerationShape = {
|
|
|
290
335
|
.boolean()
|
|
291
336
|
.optional()
|
|
292
337
|
.describe('Apply the official ACE-Step Haar DCW correction during DiT sampling (default: true).'),
|
|
293
|
-
dcwScaler:
|
|
294
|
-
.number()
|
|
295
|
-
.nonnegative()
|
|
338
|
+
dcwScaler: dcwScalerSchema
|
|
296
339
|
.optional()
|
|
297
340
|
.describe('DCW low-frequency correction strength (official default: 0.05).'),
|
|
298
|
-
dcwHighScaler:
|
|
299
|
-
.number()
|
|
300
|
-
.nonnegative()
|
|
341
|
+
dcwHighScaler: dcwScalerSchema
|
|
301
342
|
.optional()
|
|
302
343
|
.describe('DCW high-frequency correction strength (official default: 0.02).'),
|
|
303
344
|
taskType: audioGenTaskTypeSchema
|
|
@@ -506,11 +547,47 @@ const audioEditShape = {
|
|
|
506
547
|
.array(audioEditOperationSchema)
|
|
507
548
|
.min(1)
|
|
508
549
|
.describe('Ordered edit pipeline: operations run in array order and may repeat or mix. Flow-Edit is supported on turbo DiT variants only.'),
|
|
509
|
-
seed:
|
|
510
|
-
.
|
|
511
|
-
.
|
|
550
|
+
seed: seedSchema
|
|
551
|
+
.optional()
|
|
552
|
+
.describe('Seeds the first operation; each following operation uses seed + its index. Omit for a random one, reported in `stats.metadata.seed`.'),
|
|
553
|
+
vocalLanguage: z
|
|
554
|
+
.string()
|
|
555
|
+
.min(1)
|
|
556
|
+
.optional()
|
|
557
|
+
.describe("Vocal language hint for the edit prompts; the engine defaults to 'unknown'."),
|
|
558
|
+
bpm: bpmSchema
|
|
559
|
+
.optional()
|
|
560
|
+
.describe('Beats per minute for the edit prompts; `0` or unset leaves it unset.'),
|
|
561
|
+
keyscale: z
|
|
562
|
+
.string()
|
|
563
|
+
.min(1)
|
|
564
|
+
.optional()
|
|
565
|
+
.describe("Key and scale for the edit prompts, e.g. 'C minor'."),
|
|
566
|
+
timesignature: z
|
|
567
|
+
.string()
|
|
568
|
+
.min(1)
|
|
569
|
+
.optional()
|
|
570
|
+
.describe("Time signature for the edit prompts, e.g. '4/4'."),
|
|
571
|
+
augmentCaptionWithMetadata: z
|
|
572
|
+
.boolean()
|
|
512
573
|
.optional()
|
|
513
|
-
.describe(
|
|
574
|
+
.describe("Append BPM, time signature and key to each operation's conditioning caption (default: false)."),
|
|
575
|
+
dcwEnabled: z
|
|
576
|
+
.boolean()
|
|
577
|
+
.optional()
|
|
578
|
+
.describe('Repaint only: apply the Haar DCW sampler correction (default: true).'),
|
|
579
|
+
dcwScaler: dcwScalerSchema
|
|
580
|
+
.optional()
|
|
581
|
+
.describe('Repaint only: DCW low-frequency correction strength (default: 0.05).'),
|
|
582
|
+
dcwHighScaler: dcwScalerSchema
|
|
583
|
+
.optional()
|
|
584
|
+
.describe('Repaint only: DCW high-frequency correction strength (default: 0.02).'),
|
|
585
|
+
inferenceSteps: inferenceStepsSchema
|
|
586
|
+
.optional()
|
|
587
|
+
.describe('DiT steps for every operation, 0..1000; omit to keep the load-time `inferenceSteps`.'),
|
|
588
|
+
shift: acestepShiftSchema
|
|
589
|
+
.optional()
|
|
590
|
+
.describe('DiT timestep shift for every operation; omit to keep the load-time `shift`.')
|
|
514
591
|
};
|
|
515
592
|
/**
|
|
516
593
|
* Cross-field rules the addon enforces per operation, checked up front so a
|
|
@@ -540,7 +617,8 @@ function validateAudioEditOperations(value, ctx) {
|
|
|
540
617
|
export const audioEditClientParamsSchema = z
|
|
541
618
|
.object({
|
|
542
619
|
...audioEditShape,
|
|
543
|
-
sourceAudio: audioGenClientAudioInputSchema
|
|
620
|
+
sourceAudio: audioGenClientAudioInputSchema,
|
|
621
|
+
referenceAudio: audioGenClientAudioInputSchema.optional()
|
|
544
622
|
})
|
|
545
623
|
.strict()
|
|
546
624
|
.superRefine(validateAudioEditOperations);
|
|
@@ -548,6 +626,9 @@ export const audioEditStreamRequestSchema = z
|
|
|
548
626
|
.object({
|
|
549
627
|
...audioEditShape,
|
|
550
628
|
sourceAudio: audioGenAudioInputSchema.describe('Recording to edit: a file path decoded server-side, or raw interleaved stereo 48 kHz Float32 LE PCM in [-1, 1].'),
|
|
629
|
+
referenceAudio: audioGenAudioInputSchema
|
|
630
|
+
.optional()
|
|
631
|
+
.describe('Optional timbre reference for every operation: a file path decoded server-side, or raw interleaved stereo 48 kHz Float32 LE PCM.'),
|
|
551
632
|
type: z.literal('audioEditStream'),
|
|
552
633
|
requestId: z.string().min(1).optional()
|
|
553
634
|
})
|
|
@@ -575,7 +656,38 @@ export const audioGenUnderstandResultSchema = z.object({
|
|
|
575
656
|
audioCodes: z
|
|
576
657
|
.array(z.number().int().min(INT32_MIN).max(INT32_MAX))
|
|
577
658
|
.max(AUDIOGEN_MAX_AUDIO_CODES)
|
|
578
|
-
.describe("FSQ semantic codes recovered from the clip, reusable as a generation's `audioCodes` input.")
|
|
659
|
+
.describe("FSQ semantic codes recovered from the clip, reusable as a generation's `audioCodes` input."),
|
|
660
|
+
seed: z
|
|
661
|
+
.number()
|
|
662
|
+
.describe('The seed the LM decode used: the engine-drawn one when none was passed, so the description can be reproduced.')
|
|
663
|
+
});
|
|
664
|
+
/**
|
|
665
|
+
* What an ACE-Step run rendered, after the LM filled in what the caller left
|
|
666
|
+
* unset (Phase 1, Simple Mode, Query Rewriting). An edit reports its base seed
|
|
667
|
+
* and the prompt metadata it was given, with an empty caption and no code
|
|
668
|
+
* frames. MiniMax runs report none.
|
|
669
|
+
*/
|
|
670
|
+
export const audioGenGenerationMetadataSchema = z.object({
|
|
671
|
+
caption: z
|
|
672
|
+
.string()
|
|
673
|
+
.describe('The caption the run used: the LM-composed one under Simple Mode or Query Rewriting. With `augmentCaptionWithMetadata` it is always your own caption, before the metadata suffix. Empty for an edit, whose operations carry their own.'),
|
|
674
|
+
lyrics: z.string().describe('The lyrics the run used (LM-written under Simple Mode).'),
|
|
675
|
+
bpm: z.number().describe('Beats per minute; 0 when unresolved.'),
|
|
676
|
+
keyscale: z.string(),
|
|
677
|
+
beatsPerBar: z
|
|
678
|
+
.number()
|
|
679
|
+
.describe('Beats per bar, the time-signature numerator (4 for 4/4); 0 when unresolved.'),
|
|
680
|
+
vocalLanguage: z.string(),
|
|
681
|
+
seed: z
|
|
682
|
+
.number()
|
|
683
|
+
.describe('The seed the run used: the engine-drawn one when none was passed, so the take can be replayed.'),
|
|
684
|
+
codeFrames: z
|
|
685
|
+
.number()
|
|
686
|
+
.describe('5 Hz semantic code frames the DiT was conditioned on; 0 when the LM and detokenizer are skipped (cover-nofsq, lego, edits).'),
|
|
687
|
+
qualityReport: z
|
|
688
|
+
.string()
|
|
689
|
+
.optional()
|
|
690
|
+
.describe('Per-condition breakdown of `qualityScore`. Present only when the run set `computeQualityScore`.')
|
|
579
691
|
});
|
|
580
692
|
export const audioGenStatsSchema = z.object({
|
|
581
693
|
audioDurationMs: z.number().optional(),
|
|
@@ -583,6 +695,26 @@ export const audioGenStatsSchema = z.object({
|
|
|
583
695
|
realTimeFactor: z.number().optional(),
|
|
584
696
|
backendDevice: z.number().optional(),
|
|
585
697
|
backendId: z.number().optional(),
|
|
698
|
+
emittedFrames: z
|
|
699
|
+
.number()
|
|
700
|
+
.optional()
|
|
701
|
+
.describe('MiniMax only: semantic frames the autoregressive stage emitted (25 per second). Below the frame cap when the song ended at the model end-of-sequence.'),
|
|
702
|
+
arMs: z
|
|
703
|
+
.number()
|
|
704
|
+
.optional()
|
|
705
|
+
.describe('MiniMax only: engine time in the autoregressive semantic stage, in ms.'),
|
|
706
|
+
conditionMs: z
|
|
707
|
+
.number()
|
|
708
|
+
.optional()
|
|
709
|
+
.describe('MiniMax only: engine time building the flow conditioning, in ms.'),
|
|
710
|
+
flowMs: z
|
|
711
|
+
.number()
|
|
712
|
+
.optional()
|
|
713
|
+
.describe('MiniMax only: engine time in the flow-matching DiT, in ms.'),
|
|
714
|
+
vocoderMs: z.number().optional().describe('MiniMax only: engine time in the vocoder, in ms.'),
|
|
715
|
+
metadata: audioGenGenerationMetadataSchema
|
|
716
|
+
.optional()
|
|
717
|
+
.describe('What an ACE-Step generation or edit rendered: the caption, lyrics and musical metadata after the LM filled in what was left unset, and the seed the run used. Absent for MiniMax and for `audioUnderstand()`.'),
|
|
586
718
|
lyricsScore: z
|
|
587
719
|
.number()
|
|
588
720
|
.optional()
|