@qvac/inference 0.21.1 → 0.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (246) hide show
  1. package/NOTICE +30 -1
  2. package/README.md +44 -1
  3. package/dist/api/assess-model-fit.d.ts +4 -4
  4. package/dist/api/assess-model-fit.js +4 -4
  5. package/dist/api/audio-gen.d.ts +4 -1
  6. package/dist/api/audio-gen.js +4 -1
  7. package/dist/api/batch-completion.js +1 -1
  8. package/dist/api/completion-stream.js +1 -1
  9. package/dist/api/decide.d.ts +26 -0
  10. package/dist/api/decide.js +26 -0
  11. package/dist/api/get-model-info.d.ts +1 -1
  12. package/dist/api/index.d.ts +1 -0
  13. package/dist/api/index.js +1 -0
  14. package/dist/api/text-to-speech.d.ts +1 -0
  15. package/dist/api/text-to-speech.js +15 -19
  16. package/dist/api/transcribe.js +5 -2
  17. package/dist/api/video-request.d.ts +5 -5
  18. package/dist/api/video.d.ts +5 -4
  19. package/dist/api/video.js +5 -4
  20. package/dist/errors/index.d.ts +3 -0
  21. package/dist/errors/index.js +5 -0
  22. package/dist/handlers/assess-model-fit.d.ts +5 -3
  23. package/dist/handlers/assess-model-fit.js +80 -28
  24. package/dist/handlers/get-loaded-model-info.js +1 -2
  25. package/dist/handlers/load-model/safe-fetch.js +5 -6
  26. package/dist/handlers/registry.d.ts +3 -1
  27. package/dist/handlers/registry.js +3 -3
  28. package/dist/index.d.ts +1 -1
  29. package/dist/index.js +1 -1
  30. package/dist/logging/logger.js +1 -1
  31. package/dist/logging/utils.js +1 -1
  32. package/dist/models/registry/models.d.ts +563 -816
  33. package/dist/models/registry/models.js +2873 -3110
  34. package/dist/models/registry/resource-profiles.d.ts +1 -2
  35. package/dist/models/registry/resource-profiles.js +203 -2
  36. package/dist/plugins/builtin/asr-ggml/config.d.ts +7 -1
  37. package/dist/plugins/builtin/asr-ggml/config.js +6 -0
  38. package/dist/plugins/builtin/asr-ggml/logging.d.ts +1 -1
  39. package/dist/plugins/builtin/audiogen-ggml/config.d.ts +1 -0
  40. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +28 -10
  41. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +27 -0
  42. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +23 -0
  43. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +6 -27
  44. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +147 -60
  45. package/dist/plugins/builtin/audiogen-ggml/plugin.js +1 -0
  46. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +40 -24
  47. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +1 -1
  48. package/dist/plugins/builtin/ggml-classification/plugin.d.ts +4 -4
  49. package/dist/plugins/builtin/ggml-ocr/params.d.ts +18 -0
  50. package/dist/plugins/builtin/ggml-ocr/params.js +37 -0
  51. package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +18 -10
  52. package/dist/plugins/builtin/ggml-ocr/plugin.js +16 -38
  53. package/dist/plugins/builtin/ggml-vla/plugin.d.ts +21 -14
  54. package/dist/plugins/builtin/ggml-vla/plugin.js +1 -1
  55. package/dist/plugins/builtin/index.d.ts +2 -0
  56. package/dist/plugins/builtin/index.js +2 -0
  57. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +9 -7
  58. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +109 -189
  59. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +32 -90
  60. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +60 -211
  61. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +2 -49
  62. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +2 -46
  63. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +185 -87
  64. package/dist/plugins/builtin/llamacpp-completion/plugin.js +4 -5
  65. package/dist/plugins/builtin/llamacpp-completion/transform.js +1 -1
  66. package/dist/plugins/builtin/llamacpp-decision/helpers.d.ts +6 -0
  67. package/dist/plugins/builtin/llamacpp-decision/helpers.js +97 -0
  68. package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +91 -0
  69. package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +129 -0
  70. package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +216 -0
  71. package/dist/plugins/builtin/llamacpp-decision/plugin.js +43 -0
  72. package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +12 -5
  73. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +1 -1
  74. package/dist/plugins/builtin/llamacpp-embedding/transform.js +3 -0
  75. package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +143 -0
  76. package/dist/plugins/builtin/moss-transcribe/plugin.js +96 -0
  77. package/dist/plugins/builtin/nmtcpp-translation/load-files.d.ts +6 -0
  78. package/dist/plugins/builtin/nmtcpp-translation/load-files.js +70 -0
  79. package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +25 -13
  80. package/dist/plugins/builtin/nmtcpp-translation/plugin.js +9 -73
  81. package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +52 -31
  82. package/dist/plugins/builtin/parakeet-transcription/plugin.js +18 -3
  83. package/dist/plugins/builtin/sdcpp-generation/ops/video.js +10 -6
  84. package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +2 -2
  85. package/dist/plugins/builtin/sdcpp-generation/ops/world.js +1 -1
  86. package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +59 -33
  87. package/dist/plugins/builtin/sdcpp-generation/plugin.js +12 -9
  88. package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +22 -0
  89. package/dist/plugins/builtin/tts-ggml/ops/moss-options.js +79 -0
  90. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +4 -1
  91. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +17 -5
  92. package/dist/plugins/builtin/tts-ggml/plugin.d.ts +164 -29
  93. package/dist/plugins/builtin/tts-ggml/plugin.js +38 -1
  94. package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +58 -35
  95. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +6 -2
  96. package/dist/plugins/ops/kv-cache-utils.d.ts +1 -1
  97. package/dist/plugins/ops/kv-cache-utils.js +10 -25
  98. package/dist/plugins/ops/load-model.js +21 -16
  99. package/dist/plugins/ops/transcribe.js +17 -1
  100. package/dist/registry.js +1 -0
  101. package/dist/resources/model-fit/assess.d.ts +15 -23
  102. package/dist/resources/model-fit/assess.js +207 -271
  103. package/dist/resources/model-fit/fit-stub/fetch-fit-stub.d.ts +3 -1
  104. package/dist/resources/model-fit/fit-stub/fetch-fit-stub.js +6 -1
  105. package/dist/resources/model-fit/fit-stub/project-fit-from-load.js +2 -0
  106. package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts +5 -3
  107. package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.js +9 -1
  108. package/dist/resources/model-fit/floor.d.ts +10 -10
  109. package/dist/resources/model-fit/floor.js +13 -13
  110. package/dist/resources/model-fit/kv-cache.d.ts +23 -0
  111. package/dist/resources/model-fit/kv-cache.js +85 -0
  112. package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +11 -6
  113. package/dist/resources/model-fit/native-probe/advisory-fit.js +12 -27
  114. package/dist/resources/model-fit/native-probe/child-engine-fit.js +8 -0
  115. package/dist/resources/model-fit/native-probe/classify-fit.js +32 -1
  116. package/dist/resources/model-fit/native-probe/create-fit-request.d.ts +4 -1
  117. package/dist/resources/model-fit/native-probe/create-fit-request.js +24 -14
  118. package/dist/resources/model-fit/native-probe/engine-fit.d.ts +14 -0
  119. package/dist/resources/model-fit/native-probe/engine-fit.js +1 -1
  120. package/dist/resources/model-fit/native-probe/engines/asr.d.ts +4 -0
  121. package/dist/resources/model-fit/native-probe/engines/asr.js +23 -0
  122. package/dist/resources/model-fit/native-probe/engines/audiogen.js +29 -1
  123. package/dist/resources/model-fit/native-probe/engines/diffusion.js +16 -6
  124. package/dist/resources/model-fit/native-probe/engines/llama.d.ts +0 -1
  125. package/dist/resources/model-fit/native-probe/engines/llama.js +19 -123
  126. package/dist/resources/model-fit/native-probe/engines/nmt.d.ts +8 -0
  127. package/dist/resources/model-fit/native-probe/engines/nmt.js +11 -0
  128. package/dist/resources/model-fit/native-probe/engines/ocr.d.ts +9 -0
  129. package/dist/resources/model-fit/native-probe/engines/ocr.js +13 -0
  130. package/dist/resources/model-fit/native-probe/engines/tts.d.ts +2 -0
  131. package/dist/resources/model-fit/native-probe/engines/tts.js +49 -3
  132. package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +5 -3
  133. package/dist/resources/model-fit/native-probe/run-in-process-fit.js +38 -23
  134. package/dist/resources/model-fit/platform.d.ts +4 -4
  135. package/dist/resources/model-fit/platform.js +6 -8
  136. package/dist/resources/model-fit/types.d.ts +3 -117
  137. package/dist/rpc/inventory.js +5 -7
  138. package/dist/runtime/env.d.ts +0 -2
  139. package/dist/runtime/env.js +2 -9
  140. package/dist/runtime/model-config-utils.js +6 -1
  141. package/dist/runtime/request-context.d.ts +1 -1
  142. package/dist/runtime/request-context.js +8 -0
  143. package/dist/runtime/state.js +5 -0
  144. package/dist/schemas/assess-model-fit.d.ts +100 -43
  145. package/dist/schemas/assess-model-fit.js +66 -29
  146. package/dist/schemas/audio-gen.d.ts +186 -18
  147. package/dist/schemas/audio-gen.js +180 -48
  148. package/dist/schemas/batch-completion-stream.d.ts +202 -34
  149. package/dist/schemas/bci-config.d.ts +2 -0
  150. package/dist/schemas/bci.d.ts +23 -2
  151. package/dist/schemas/cancel.d.ts +5 -0
  152. package/dist/schemas/cancel.js +1 -0
  153. package/dist/schemas/common.d.ts +1083 -158
  154. package/dist/schemas/common.js +5 -2
  155. package/dist/schemas/completion-stream.d.ts +232 -37
  156. package/dist/schemas/completion-stream.js +64 -20
  157. package/dist/schemas/config.d.ts +477 -12
  158. package/dist/schemas/config.js +21 -1
  159. package/dist/schemas/decision.d.ts +602 -0
  160. package/dist/schemas/decision.js +205 -0
  161. package/dist/schemas/download-asset.d.ts +7 -3
  162. package/dist/schemas/engine-addon-map.d.ts +7 -0
  163. package/dist/schemas/engine-addon-map.js +23 -3
  164. package/dist/schemas/errors.d.ts +1 -0
  165. package/dist/schemas/errors.js +5 -0
  166. package/dist/schemas/index.d.ts +2 -0
  167. package/dist/schemas/index.js +2 -0
  168. package/dist/schemas/llamacpp-config.d.ts +42 -2
  169. package/dist/schemas/llamacpp-config.js +27 -0
  170. package/dist/schemas/load-model.d.ts +5247 -1309
  171. package/dist/schemas/load-model.js +88 -9
  172. package/dist/schemas/model-config-schemas.d.ts +233 -8
  173. package/dist/schemas/model-config-schemas.js +4 -1
  174. package/dist/schemas/model-fit-policy.d.ts +17 -0
  175. package/dist/schemas/model-fit-policy.js +17 -0
  176. package/dist/schemas/model-info.d.ts +20 -2
  177. package/dist/schemas/model-info.js +2 -1
  178. package/dist/schemas/model-resource-profile.d.ts +3 -4
  179. package/dist/schemas/model-resource-profile.js +1 -4
  180. package/dist/schemas/model-src-utils.d.ts +10 -2
  181. package/dist/schemas/model-types.d.ts +10 -1
  182. package/dist/schemas/model-types.js +3 -0
  183. package/dist/schemas/ocr.d.ts +74 -0
  184. package/dist/schemas/ocr.js +42 -0
  185. package/dist/schemas/plugin.d.ts +8 -2
  186. package/dist/schemas/plugin.js +6 -0
  187. package/dist/schemas/profiling.d.ts +4 -4
  188. package/dist/schemas/registry.d.ts +26 -0
  189. package/dist/schemas/registry.js +3 -0
  190. package/dist/schemas/reload-config.d.ts +10 -2
  191. package/dist/schemas/rpc-server.js +3 -3
  192. package/dist/schemas/sdcpp-config.d.ts +26 -0
  193. package/dist/schemas/sdcpp-config.js +8 -7
  194. package/dist/schemas/system-resources.d.ts +4 -4
  195. package/dist/schemas/text-to-speech.d.ts +402 -3
  196. package/dist/schemas/text-to-speech.js +129 -11
  197. package/dist/schemas/tools.d.ts +13 -4
  198. package/dist/schemas/tools.js +18 -6
  199. package/dist/schemas/transcription-config.d.ts +15 -0
  200. package/dist/schemas/transcription-config.js +38 -3
  201. package/dist/schemas/transcription.d.ts +46 -4
  202. package/dist/schemas/transcription.js +51 -4
  203. package/dist/schemas/translation-config.d.ts +30 -10
  204. package/dist/schemas/vla.d.ts +9 -2
  205. package/dist/schemas/vla.js +9 -5
  206. package/dist/surface.d.ts +1 -1
  207. package/dist/surface.js +1 -1
  208. package/dist/utils/addon-responses.d.ts +1 -0
  209. package/dist/utils/asr-events.d.ts +1 -0
  210. package/dist/utils/asr-events.js +2 -1
  211. package/dist/utils/cache/types.d.ts +7 -0
  212. package/dist/utils/mcp-adapter.js +12 -8
  213. package/dist/utils/tool-helpers.d.ts +1 -1
  214. package/dist/utils/tool-helpers.js +20 -52
  215. package/dist/utils/tools/parsers/dsml.js +8 -5
  216. package/dist/utils/tools/parsers/qwen35.js +7 -4
  217. package/dist/utils/tools/shared.d.ts +3 -0
  218. package/dist/utils/tools/shared.js +62 -0
  219. package/dist/utils/transcribe-metadata.d.ts +8 -1
  220. package/dist/utils/transcribe-metadata.js +13 -3
  221. package/dist/utils/tts-stats.d.ts +1 -0
  222. package/package.json +41 -32
  223. package/dist/resources/model-fit/calibration/darwin-arm64.d.ts +0 -8
  224. package/dist/resources/model-fit/calibration/darwin-arm64.js +0 -36
  225. package/dist/resources/model-fit/calibration/darwin-x64.d.ts +0 -8
  226. package/dist/resources/model-fit/calibration/darwin-x64.js +0 -38
  227. package/dist/resources/model-fit/calibration/fit.d.ts +0 -73
  228. package/dist/resources/model-fit/calibration/fit.js +0 -152
  229. package/dist/resources/model-fit/calibration/index.d.ts +0 -37
  230. package/dist/resources/model-fit/calibration/index.js +0 -75
  231. package/dist/resources/model-fit/calibration/linux-arm64.d.ts +0 -8
  232. package/dist/resources/model-fit/calibration/linux-arm64.js +0 -38
  233. package/dist/resources/model-fit/calibration/linux-x64-vulkan.d.ts +0 -8
  234. package/dist/resources/model-fit/calibration/linux-x64-vulkan.js +0 -39
  235. package/dist/resources/model-fit/calibration/linux-x64.d.ts +0 -8
  236. package/dist/resources/model-fit/calibration/linux-x64.js +0 -38
  237. package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.d.ts +0 -8
  238. package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.js +0 -39
  239. package/dist/resources/model-fit/calibration/win32-x64-vulkan.d.ts +0 -8
  240. package/dist/resources/model-fit/calibration/win32-x64-vulkan.js +0 -36
  241. package/dist/resources/model-fit/calibration/win32-x64.d.ts +0 -8
  242. package/dist/resources/model-fit/calibration/win32-x64.js +0 -38
  243. package/dist/resources/model-fit/estimators/llm.d.ts +0 -46
  244. package/dist/resources/model-fit/estimators/llm.js +0 -222
  245. package/dist/resources/model-fit/estimators/whisper.d.ts +0 -10
  246. package/dist/resources/model-fit/estimators/whisper.js +0 -76
@@ -4,12 +4,20 @@ import { audioInputSchema } from './transcription.js';
4
4
  import { inferenceBackendDiagnosticsSchema } from './system-resources.js';
5
5
  import { encodeBase64 } from '../utils/encoding.js';
6
6
  const base64Schema = z.string().min(1);
7
- // Mirrors requireMinimaxInferenceSteps and requireMinimaxCfgScale in @qvac/audiogen-ggml.
8
- const MINIMAX_MAX_INFERENCE_STEPS = 1000;
9
- const MINIMAX_CFG_SCALE_MAX = 3.4028234663852886e38;
10
- const MINIMAX_CFG_SCALE_MIN_POSITIVE = 1.401298464324817e-45;
7
+ // Mirror requireInferenceSteps, requireAcestepShift, requireMinimaxCfgScale and
8
+ // optionalFloat32 in @qvac/audiogen-ggml, so a value the addon would refuse
9
+ // fails here instead.
10
+ const MAX_INFERENCE_STEPS = 1000;
11
+ const FLOAT32_MAX = 3.4028234663852886e38;
12
+ const FLOAT32_MIN_POSITIVE = 1.401298464324817e-45;
11
13
  export const AUDIOGEN_ENGINES = ['acestep', 'minimax'];
12
14
  export const audioGenEngineSchema = z.enum(AUDIOGEN_ENGINES);
15
+ /**
16
+ * Where MiniMax-Music3 runs: `gpu` requires a usable GPU and fails the load
17
+ * otherwise, `auto` (what `useGPU: true` means) falls back to the CPU, `cpu`
18
+ * never touches a GPU.
19
+ */
20
+ const MINIMAX_DEVICES = ['cpu', 'gpu', 'auto'];
13
21
  /** Sample rate the ACE-Step engine expects for reference and source audio. */
14
22
  export const AUDIOGEN_INPUT_SAMPLE_RATE = 48000;
15
23
  /** Channel count (interleaved stereo) the ACE-Step engine expects for input audio. */
@@ -82,21 +90,30 @@ const commonAudioGenRuntimeConfigShape = {
82
90
  .string()
83
91
  .min(1)
84
92
  .optional()
85
- .describe('Advanced: override the prebuilds root scanned for dlopen’d ggml backend modules. Defaults to `<addon>/prebuilds`; needed on arm64, where the CPU backend ships as per-microarch module `.so` files.')
93
+ .describe('Advanced: directory scanned as given for dlopen’d ggml backend modules. Unset scans the directory the native module was loaded from. Needed on arm64, where the CPU backend ships as per-microarch module `.so` files.')
86
94
  };
95
+ /** Diffusion steps, 0..1000. `0` lets the engine pick its default. */
96
+ const inferenceStepsSchema = z.number().int().min(0).max(MAX_INFERENCE_STEPS);
97
+ /**
98
+ * ACE-Step DiT timestep shift. `0` lets the engine pick; a positive value too
99
+ * small for a float32 would round to `0` and silently select that default, so
100
+ * it is refused instead.
101
+ */
102
+ const acestepShiftSchema = z
103
+ .number()
104
+ .min(0)
105
+ .max(FLOAT32_MAX)
106
+ .refine((value) => value === 0 || value >= FLOAT32_MIN_POSITIVE, {
107
+ message: 'shift must be 0 or a positive float32 value'
108
+ });
87
109
  const acestepRuntimeConfigShape = {
88
110
  ...commonAudioGenRuntimeConfigShape,
89
- inferenceSteps: z
90
- .number()
91
- .int()
92
- .nonnegative()
111
+ inferenceSteps: inferenceStepsSchema
93
112
  .optional()
94
- .describe('DiT sampling steps; `0` (default) lets ACE-Step auto-pick per DiT architecture (turbo 8 / sft 50).'),
95
- shift: z
96
- .number()
97
- .nonnegative()
113
+ .describe('DiT sampling steps, 0..1000; `0` (default) lets ACE-Step auto-pick per DiT architecture (turbo 8 / base and sft 50). A generation or edit can override it per run.'),
114
+ shift: acestepShiftSchema
98
115
  .optional()
99
- .describe('Flow-matching time-shift; `0` (default) lets ACE-Step auto-pick per DiT architecture (turbo 3.0 / sft 1.0).'),
116
+ .describe('Flow-matching time-shift; `0` (default) lets ACE-Step auto-pick per DiT architecture (turbo 3.0 / base and sft 1.0). A generation or edit can override it per run.'),
100
117
  nGpuLayers: z
101
118
  .number()
102
119
  .int()
@@ -107,34 +124,46 @@ const acestepRuntimeConfigShape = {
107
124
  const minimaxCfgScaleSchema = z
108
125
  .number()
109
126
  .min(0)
110
- .max(MINIMAX_CFG_SCALE_MAX)
111
- .refine((value) => value === 0 || value >= MINIMAX_CFG_SCALE_MIN_POSITIVE, {
127
+ .max(FLOAT32_MAX)
128
+ .refine((value) => value === 0 || value >= FLOAT32_MIN_POSITIVE, {
112
129
  message: 'cfgScale must be 0 or a positive float32 value'
113
130
  })
114
131
  .describe('MiniMax flow classifier-free guidance scale; `0` uses the model default.');
115
132
  const minimaxRuntimeConfigShape = {
116
133
  ...commonAudioGenRuntimeConfigShape,
117
- inferenceSteps: z
118
- .number()
119
- .int()
120
- .min(0)
121
- .max(MINIMAX_MAX_INFERENCE_STEPS)
134
+ device: z
135
+ .enum(MINIMAX_DEVICES)
136
+ .optional()
137
+ .describe("Where MiniMax runs, instead of `useGPU`: `'gpu'` requires a usable GPU and fails the load otherwise, `'auto'` (what `useGPU: true` means) falls back to the CPU, `'cpu'` never touches a GPU. Cannot be combined with `useGPU`."),
138
+ inferenceSteps: inferenceStepsSchema
122
139
  .optional()
123
140
  .describe('MiniMax flow sampling steps; `0` uses the engine default (20).'),
124
141
  cfgScale: minimaxCfgScaleSchema.optional()
125
142
  };
143
+ /** The addon takes `useGPU` or `device` for MiniMax, never both. */
144
+ function refineMinimaxDevice(value, ctx) {
145
+ if (value.device === undefined || value.useGPU === undefined)
146
+ return;
147
+ ctx.addIssue({
148
+ code: 'custom',
149
+ path: ['device'],
150
+ message: 'MiniMax accepts either useGPU or device, not both'
151
+ });
152
+ }
126
153
  const acestepAudioGenRuntimeConfigSchema = z
127
154
  .object({
128
155
  engine: z.literal('acestep').optional().describe('Use the ACE-Step music-generation engine.'),
129
156
  ...acestepRuntimeConfigShape
130
157
  })
131
158
  .strict();
132
- const minimaxAudioGenRuntimeConfigSchema = z
133
- .object({
159
+ const minimaxEngineShape = {
134
160
  engine: z.literal('minimax').describe('Use the MiniMax-Music3 generation engine.'),
135
161
  ...minimaxRuntimeConfigShape
136
- })
137
- .strict();
162
+ };
163
+ const minimaxAudioGenRuntimeConfigSchema = z
164
+ .object(minimaxEngineShape)
165
+ .strict()
166
+ .superRefine(refineMinimaxDevice);
138
167
  export const audioGenRuntimeConfigSchema = z.discriminatedUnion('engine', [
139
168
  acestepAudioGenRuntimeConfigSchema,
140
169
  minimaxAudioGenRuntimeConfigSchema
@@ -147,12 +176,16 @@ const acestepAudioGenConfigSchema = acestepAudioGenRuntimeConfigSchema
147
176
  vaeModelSrc: modelSrcInputSchema.describe('VAE model source; decodes the latent into the output waveform.')
148
177
  })
149
178
  .strict();
150
- const minimaxAudioGenConfigSchema = minimaxAudioGenRuntimeConfigSchema
151
- .extend({
179
+ // Built from the shape rather than extended from the runtime schema, so the
180
+ // device refinement is declared on the schema that carries it.
181
+ const minimaxAudioGenConfigSchema = z
182
+ .object({
183
+ ...minimaxEngineShape,
152
184
  lmModelSrc: modelSrcInputSchema.describe('MiniMax language-model source; generates semantic music tokens.'),
153
185
  synthModelSrc: modelSrcInputSchema.describe('MiniMax synthesis-model source; converts semantic tokens into the output waveform.')
154
186
  })
155
- .strict();
187
+ .strict()
188
+ .superRefine(refineMinimaxDevice);
156
189
  export const audioGenConfigSchema = z.discriminatedUnion('engine', [
157
190
  acestepAudioGenConfigSchema,
158
191
  minimaxAudioGenConfigSchema
@@ -207,13 +240,26 @@ export const audioGenClientAudioCodesSchema = z
207
240
  z.array(z.number())
208
241
  ])
209
242
  .pipe(audioCodesWireSchema);
243
+ /**
244
+ * The ACE-Step controls a generation and an edit session's run share. The
245
+ * addon validates them identically on both paths: `seed` a safe integer (which
246
+ * zod's `int()` already is), `bpm` a non-negative int32, the DCW scalers
247
+ * within the float32 range.
248
+ */
249
+ const seedSchema = z.number().int();
250
+ const bpmSchema = z.number().int().min(0).max(INT32_MAX);
251
+ const dcwScalerSchema = z.number().nonnegative().max(FLOAT32_MAX);
210
252
  const audioGenGenerationShape = {
211
253
  modelId: z.string().min(1),
212
254
  caption: z.string().trim().min(1, 'caption must not be empty or whitespace-only'),
213
255
  lyrics: z.string().optional(),
214
- seed: z.number().int().optional(),
256
+ seed: seedSchema
257
+ .optional()
258
+ .describe('RNG seed; omit for a random one. An ACE-Step run reports the seed it used in `stats.metadata.seed`, so an unseeded take can be replayed.'),
215
259
  vocalLanguage: z.string().min(1).optional(),
216
- bpm: z.number().int().positive().optional(),
260
+ bpm: bpmSchema
261
+ .optional()
262
+ .describe('Beats per minute; `0` or unset lets the LM infer it. ACE-Step only.'),
217
263
  keyscale: z.string().min(1).optional(),
218
264
  timesignature: z.string().min(1).optional(),
219
265
  augmentCaptionWithMetadata: z
@@ -260,13 +306,12 @@ const audioGenGenerationShape = {
260
306
  .max(Number.MAX_SAFE_INTEGER)
261
307
  .optional()
262
308
  .describe('MiniMax semantic-frame cap. Cannot be combined with duration. MiniMax only; rejected by ACE-Step.'),
263
- inferenceSteps: z
264
- .number()
265
- .int()
266
- .min(0)
267
- .max(MINIMAX_MAX_INFERENCE_STEPS)
309
+ inferenceSteps: inferenceStepsSchema
310
+ .optional()
311
+ .describe('Diffusion steps for this generation, 0..1000; omit to keep the load-time `inferenceSteps`. 0 is the engine default: MiniMax 20 flow steps, ACE-Step 8 on turbo DiTs and 50 on base/sft.'),
312
+ shift: acestepShiftSchema
268
313
  .optional()
269
- .describe('MiniMax flow steps for this generation; 0 uses the engine default (20). MiniMax only; rejected by ACE-Step.'),
314
+ .describe('DiT timestep shift for this generation; omit to keep the load-time `shift`. 0 is the engine default: 3.0 on turbo DiTs, 1.0 on base/sft. ACE-Step only; rejected by MiniMax.'),
270
315
  cfgScale: minimaxCfgScaleSchema
271
316
  .optional()
272
317
  .describe('MiniMax flow classifier-free guidance scale for this generation. MiniMax only; rejected by ACE-Step.'),
@@ -290,14 +335,10 @@ const audioGenGenerationShape = {
290
335
  .boolean()
291
336
  .optional()
292
337
  .describe('Apply the official ACE-Step Haar DCW correction during DiT sampling (default: true).'),
293
- dcwScaler: z
294
- .number()
295
- .nonnegative()
338
+ dcwScaler: dcwScalerSchema
296
339
  .optional()
297
340
  .describe('DCW low-frequency correction strength (official default: 0.05).'),
298
- dcwHighScaler: z
299
- .number()
300
- .nonnegative()
341
+ dcwHighScaler: dcwScalerSchema
301
342
  .optional()
302
343
  .describe('DCW high-frequency correction strength (official default: 0.02).'),
303
344
  taskType: audioGenTaskTypeSchema
@@ -506,11 +547,47 @@ const audioEditShape = {
506
547
  .array(audioEditOperationSchema)
507
548
  .min(1)
508
549
  .describe('Ordered edit pipeline: operations run in array order and may repeat or mix. Flow-Edit is supported on turbo DiT variants only.'),
509
- seed: z
510
- .number()
511
- .int()
550
+ seed: seedSchema
551
+ .optional()
552
+ .describe('Seeds the first operation; each following operation uses seed + its index. Omit for a random one, reported in `stats.metadata.seed`.'),
553
+ vocalLanguage: z
554
+ .string()
555
+ .min(1)
556
+ .optional()
557
+ .describe("Vocal language hint for the edit prompts; the engine defaults to 'unknown'."),
558
+ bpm: bpmSchema
559
+ .optional()
560
+ .describe('Beats per minute for the edit prompts; `0` or unset leaves it unset.'),
561
+ keyscale: z
562
+ .string()
563
+ .min(1)
564
+ .optional()
565
+ .describe("Key and scale for the edit prompts, e.g. 'C minor'."),
566
+ timesignature: z
567
+ .string()
568
+ .min(1)
569
+ .optional()
570
+ .describe("Time signature for the edit prompts, e.g. '4/4'."),
571
+ augmentCaptionWithMetadata: z
572
+ .boolean()
512
573
  .optional()
513
- .describe('Seeds the first operation; each following operation uses seed + its index.')
574
+ .describe("Append BPM, time signature and key to each operation's conditioning caption (default: false)."),
575
+ dcwEnabled: z
576
+ .boolean()
577
+ .optional()
578
+ .describe('Repaint only: apply the Haar DCW sampler correction (default: true).'),
579
+ dcwScaler: dcwScalerSchema
580
+ .optional()
581
+ .describe('Repaint only: DCW low-frequency correction strength (default: 0.05).'),
582
+ dcwHighScaler: dcwScalerSchema
583
+ .optional()
584
+ .describe('Repaint only: DCW high-frequency correction strength (default: 0.02).'),
585
+ inferenceSteps: inferenceStepsSchema
586
+ .optional()
587
+ .describe('DiT steps for every operation, 0..1000; omit to keep the load-time `inferenceSteps`.'),
588
+ shift: acestepShiftSchema
589
+ .optional()
590
+ .describe('DiT timestep shift for every operation; omit to keep the load-time `shift`.')
514
591
  };
515
592
  /**
516
593
  * Cross-field rules the addon enforces per operation, checked up front so a
@@ -540,7 +617,8 @@ function validateAudioEditOperations(value, ctx) {
540
617
  export const audioEditClientParamsSchema = z
541
618
  .object({
542
619
  ...audioEditShape,
543
- sourceAudio: audioGenClientAudioInputSchema
620
+ sourceAudio: audioGenClientAudioInputSchema,
621
+ referenceAudio: audioGenClientAudioInputSchema.optional()
544
622
  })
545
623
  .strict()
546
624
  .superRefine(validateAudioEditOperations);
@@ -548,6 +626,9 @@ export const audioEditStreamRequestSchema = z
548
626
  .object({
549
627
  ...audioEditShape,
550
628
  sourceAudio: audioGenAudioInputSchema.describe('Recording to edit: a file path decoded server-side, or raw interleaved stereo 48 kHz Float32 LE PCM in [-1, 1].'),
629
+ referenceAudio: audioGenAudioInputSchema
630
+ .optional()
631
+ .describe('Optional timbre reference for every operation: a file path decoded server-side, or raw interleaved stereo 48 kHz Float32 LE PCM.'),
551
632
  type: z.literal('audioEditStream'),
552
633
  requestId: z.string().min(1).optional()
553
634
  })
@@ -575,7 +656,38 @@ export const audioGenUnderstandResultSchema = z.object({
575
656
  audioCodes: z
576
657
  .array(z.number().int().min(INT32_MIN).max(INT32_MAX))
577
658
  .max(AUDIOGEN_MAX_AUDIO_CODES)
578
- .describe("FSQ semantic codes recovered from the clip, reusable as a generation's `audioCodes` input.")
659
+ .describe("FSQ semantic codes recovered from the clip, reusable as a generation's `audioCodes` input."),
660
+ seed: z
661
+ .number()
662
+ .describe('The seed the LM decode used: the engine-drawn one when none was passed, so the description can be reproduced.')
663
+ });
664
+ /**
665
+ * What an ACE-Step run rendered, after the LM filled in what the caller left
666
+ * unset (Phase 1, Simple Mode, Query Rewriting). An edit reports its base seed
667
+ * and the prompt metadata it was given, with an empty caption and no code
668
+ * frames. MiniMax runs report none.
669
+ */
670
+ export const audioGenGenerationMetadataSchema = z.object({
671
+ caption: z
672
+ .string()
673
+ .describe('The caption the run used: the LM-composed one under Simple Mode or Query Rewriting. With `augmentCaptionWithMetadata` it is always your own caption, before the metadata suffix. Empty for an edit, whose operations carry their own.'),
674
+ lyrics: z.string().describe('The lyrics the run used (LM-written under Simple Mode).'),
675
+ bpm: z.number().describe('Beats per minute; 0 when unresolved.'),
676
+ keyscale: z.string(),
677
+ beatsPerBar: z
678
+ .number()
679
+ .describe('Beats per bar, the time-signature numerator (4 for 4/4); 0 when unresolved.'),
680
+ vocalLanguage: z.string(),
681
+ seed: z
682
+ .number()
683
+ .describe('The seed the run used: the engine-drawn one when none was passed, so the take can be replayed.'),
684
+ codeFrames: z
685
+ .number()
686
+ .describe('5 Hz semantic code frames the DiT was conditioned on; 0 when the LM and detokenizer are skipped (cover-nofsq, lego, edits).'),
687
+ qualityReport: z
688
+ .string()
689
+ .optional()
690
+ .describe('Per-condition breakdown of `qualityScore`. Present only when the run set `computeQualityScore`.')
579
691
  });
580
692
  export const audioGenStatsSchema = z.object({
581
693
  audioDurationMs: z.number().optional(),
@@ -583,6 +695,26 @@ export const audioGenStatsSchema = z.object({
583
695
  realTimeFactor: z.number().optional(),
584
696
  backendDevice: z.number().optional(),
585
697
  backendId: z.number().optional(),
698
+ emittedFrames: z
699
+ .number()
700
+ .optional()
701
+ .describe('MiniMax only: semantic frames the autoregressive stage emitted (25 per second). Below the frame cap when the song ended at the model end-of-sequence.'),
702
+ arMs: z
703
+ .number()
704
+ .optional()
705
+ .describe('MiniMax only: engine time in the autoregressive semantic stage, in ms.'),
706
+ conditionMs: z
707
+ .number()
708
+ .optional()
709
+ .describe('MiniMax only: engine time building the flow conditioning, in ms.'),
710
+ flowMs: z
711
+ .number()
712
+ .optional()
713
+ .describe('MiniMax only: engine time in the flow-matching DiT, in ms.'),
714
+ vocoderMs: z.number().optional().describe('MiniMax only: engine time in the vocoder, in ms.'),
715
+ metadata: audioGenGenerationMetadataSchema
716
+ .optional()
717
+ .describe('What an ACE-Step generation or edit rendered: the caption, lyrics and musical metadata after the LM filled in what was left unset, and the seed the run used. Absent for MiniMax and for `audioUnderstand()`.'),
586
718
  lyricsScore: z
587
719
  .number()
588
720
  .optional()