@qvac/audiogen-ggml 0.5.1 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +68 -0
- package/README.md +75 -5
- package/audiogen.d.ts +3 -0
- package/index.d.ts +99 -1
- package/index.js +123 -37
- package/lib/fit.d.ts +35 -7
- package/lib/fit.js +3 -3
- package/package.json +7 -7
- package/test/utils/runAudioGen.js +9 -3
package/CHANGELOG.md
CHANGED
|
@@ -7,6 +7,74 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
+
## [0.6.0] - 2026-10-09
|
|
11
|
+
|
|
12
|
+
### Added
|
|
13
|
+
|
|
14
|
+
- `assessFit({ engine: 'minimax', ... })` projects a MiniMax-Music3 model pair
|
|
15
|
+
on desktop builds instead of returning `unsupported-engine`. It reads GGUF
|
|
16
|
+
metadata only, takes `device`, `maxFrames` or `durationSeconds`, and
|
|
17
|
+
`promptTokens`, and reports `lm`, `depth`, `cond`, `dit` and `vocoder` rows.
|
|
18
|
+
- Engine options and results that the speech fabric's audiogen engines already
|
|
19
|
+
provided but the addon did not expose:
|
|
20
|
+
- ACE-Step generation metadata: a run reports what it rendered on the PCM
|
|
21
|
+
item and as `stats.metadata` — the caption and lyrics (LM-composed
|
|
22
|
+
under Simple Mode / Query Rewriting), `bpm`, `keyscale`, `beatsPerBar` (the
|
|
23
|
+
time-signature numerator), `vocalLanguage`, `codeFrames`, the `seed` it
|
|
24
|
+
used (so an unseeded take or edit can be replayed), and `qualityReport`
|
|
25
|
+
with `computeQualityScore`.
|
|
26
|
+
- ACE-Step per-run `inferenceSteps` and `shift` on `run()` and on an edit
|
|
27
|
+
session's `run()`; both used to be fixed at load time.
|
|
28
|
+
- Edit sessions: `run()` takes a timbre `referenceAudio`, the `vocalLanguage`
|
|
29
|
+
/ `bpm` / `keyscale` / `timesignature` / `augmentCaptionWithMetadata`
|
|
30
|
+
prompt metadata, and the Repaint DCW controls (`dcwEnabled`, `dcwScaler`,
|
|
31
|
+
`dcwHighScaler`). The engine read all of them; only the seed reached it.
|
|
32
|
+
- `understand()` reports the `seed` the LM decode used.
|
|
33
|
+
- MiniMax-Music3: the engine's stage timings (`arMs`, `conditionMs`,
|
|
34
|
+
`flowMs`, `vocoderMs`) and `emittedFrames` in the stats, and
|
|
35
|
+
`config.device: 'gpu'` to fail `load()` instead of falling back to the CPU
|
|
36
|
+
when no GPU is usable.
|
|
37
|
+
|
|
38
|
+
### Changed
|
|
39
|
+
|
|
40
|
+
- Raise the `speech-cpp` floor to `2026-10-07#2` and the `ggml-speech` floor to
|
|
41
|
+
`2026-10-07#1`, keeping the speech packages on one engine stack. The new
|
|
42
|
+
revisions add MOSS transcription memory fit and ggml memory queries that this
|
|
43
|
+
package does not use, so published behavior is unchanged.
|
|
44
|
+
- Raise the `speech-cpp` floor to `2026-10-07` for the MiniMax-Music3 memory
|
|
45
|
+
fit. Generation is unchanged.
|
|
46
|
+
- Raise the `speech-cpp` and `ggml-speech` floors to `2026-10-06`. Same models,
|
|
47
|
+
same backends, no API change; ACE-Step and MiniMax-Music3 output is
|
|
48
|
+
unchanged.
|
|
49
|
+
- Raise the `ggml-speech` floor to `2026-10-02`. The speech ggml now includes
|
|
50
|
+
the ggml changes of the QVAC LLM stack, so both build from the same backend
|
|
51
|
+
code. Same models, same backends, no API change.
|
|
52
|
+
- Raise the `speech-cpp` floor to `2026-10-01` and the `ggml-speech` floor to
|
|
53
|
+
`2026-09-30#1`. MiniMax-Music3 generates faster on CUDA and Vulkan: a
|
|
54
|
+
2-minute f16 song on an RTX 5090 takes about 74 s on CUDA (was 102 s) and
|
|
55
|
+
84 s on Vulkan (was 105 s). CUDA decoding now keeps f32 activations, matching
|
|
56
|
+
the CPU on 99.7% of tokens.
|
|
57
|
+
- ACE-Step `config.inferenceSteps` must be an integer from 0 to 1000 and
|
|
58
|
+
`config.shift` must be non-negative, the same rules as the new per-run
|
|
59
|
+
values; out-of-range values used to load. The engine already read 0 and
|
|
60
|
+
below as "auto".
|
|
61
|
+
- The ACE-Step options `run()` shares with edit runs are validated the same way
|
|
62
|
+
on both: `seed` must be a safe integer, `bpm` a non-negative int32,
|
|
63
|
+
`vocalLanguage` / `keyscale` / `timesignature` strings and the DCW scalers
|
|
64
|
+
within the float32 range. The native parser range-checks `seed`, `bpm` and
|
|
65
|
+
the DCW scalers before casting them.
|
|
66
|
+
- A positive `shift` too small for a float32 is rejected instead of rounding to
|
|
67
|
+
0 and silently selecting the automatic shift.
|
|
68
|
+
|
|
69
|
+
### Fixed
|
|
70
|
+
|
|
71
|
+
- `understand()` no longer repeats the previous run's `lyricsScore` in its
|
|
72
|
+
stats after a `generateLrc` generation.
|
|
73
|
+
- Raise the `ggml-speech` floor to `2026-09-30`. On Metal, `assessFit` no longer
|
|
74
|
+
reports more free device memory than total once the process has allocated
|
|
75
|
+
past the GPU's recommended working set, which made a model that does not fit
|
|
76
|
+
report `fits`. Generation is unchanged.
|
|
77
|
+
|
|
10
78
|
## [0.5.1] - 2026-09-29
|
|
11
79
|
|
|
12
80
|
### Fixed
|
package/README.md
CHANGED
|
@@ -8,6 +8,14 @@ on desktop CPUs and GPUs and returns stereo 44.1 kHz audio.
|
|
|
8
8
|
|
|
9
9
|
## How it works
|
|
10
10
|
|
|
11
|
+
The C++ CI carve-out builds the native addon on the Ubuntu 22.04 CPU runner,
|
|
12
|
+
reusing vcpkg and compiler caches with two build workers. Addon-level C++ tests
|
|
13
|
+
remain a stub. Only trusted default-branch builds save shared caches; pushes
|
|
14
|
+
affecting AudioGen or its cache configuration and manual dispatches warm them.
|
|
15
|
+
Non-PR runs build the triggering commit and reject inputs selecting another
|
|
16
|
+
repository or revision. PR events retain caller-selected PR-head checkouts.
|
|
17
|
+
See [C++ CI configuration](../../docs/ci/nx-ci-consolidation.md#optionsci-cheat-sheet).
|
|
18
|
+
|
|
11
19
|
Under the hood the model runs a small pipeline, and the addon just drives it and
|
|
12
20
|
hands you the audio:
|
|
13
21
|
|
|
@@ -148,7 +156,11 @@ controls such as BPM, DiT shift, frozen semantic codes, cover audio, and
|
|
|
148
156
|
`nGpuLayers`. Set `config.useGPU: true` to run the whole model pair on a GPU
|
|
149
157
|
backend (CUDA, Vulkan, Metal — ~22 GB of device memory for the f16 pair); the
|
|
150
158
|
engine falls back to CPU when no usable GPU exists and `stats.backendDevice`
|
|
151
|
-
reports the backend actually used.
|
|
159
|
+
reports the backend actually used. Pass `config.device: 'gpu'` instead to make
|
|
160
|
+
`load()` fail when no GPU is usable rather than run on the CPU. The stats split
|
|
161
|
+
the engine time into `arMs`, `conditionMs`, `flowMs` and `vocoderMs`, and
|
|
162
|
+
`emittedFrames` counts the semantic frames generated: fewer than the cap when
|
|
163
|
+
the song ended on its own. See
|
|
152
164
|
[`examples/generate-music-minimax.js`](examples/generate-music-minimax.js).
|
|
153
165
|
|
|
154
166
|
### 1. Simplest case — an instrumental
|
|
@@ -188,6 +200,7 @@ const stats = await response.await()
|
|
|
188
200
|
// gpuFallbackReason: 0 = none, 1 = not requested, 2 = no devices, 3 = init failed
|
|
189
201
|
// lyricsScore + lrc: alignment confidence and LRC text, only with generateLrc
|
|
190
202
|
// qualityScore: [0, 1], present only when the run set computeQualityScore
|
|
203
|
+
// metadata: what the run rendered (see below), ACE-Step only
|
|
191
204
|
|
|
192
205
|
await gen.destroy()
|
|
193
206
|
```
|
|
@@ -205,6 +218,24 @@ await gen.destroy()
|
|
|
205
218
|
> `audiogenGpuFallbackReason(stats.gpuFallbackReason)`.
|
|
206
219
|
> [`examples/generate-music.js`](examples/generate-music.js) shows the pattern.
|
|
207
220
|
|
|
221
|
+
An ACE-Step run also reports what it rendered once the LM filled in whatever
|
|
222
|
+
the request left unset, on the PCM item and again as `stats.metadata`:
|
|
223
|
+
|
|
224
|
+
```js
|
|
225
|
+
// { caption, lyrics, bpm, keyscale, beatsPerBar, vocalLanguage, seed,
|
|
226
|
+
// codeFrames, qualityReport? }
|
|
227
|
+
const { seed, lyrics } = stats.metadata
|
|
228
|
+
```
|
|
229
|
+
|
|
230
|
+
`seed` is the one the run used, so an unseeded take can be replayed by passing
|
|
231
|
+
it back. Under Simple Mode or Query Rewriting, `caption` and `lyrics` are the
|
|
232
|
+
LM-composed ones (with `augmentCaptionWithMetadata`, `caption` stays yours).
|
|
233
|
+
`beatsPerBar` is the time-signature numerator (`4` for `4/4`), the only part
|
|
234
|
+
the engine reports. `qualityReport` is the per-condition breakdown of
|
|
235
|
+
`stats.qualityScore` and is present only with `computeQualityScore`. An edit
|
|
236
|
+
run reports its base seed and prompt metadata; its `caption` is empty and
|
|
237
|
+
`codeFrames` is 0.
|
|
238
|
+
|
|
208
239
|
### 2. A song with lyrics + rhythm
|
|
209
240
|
|
|
210
241
|
Pass `lyrics` for vocals, and steer the LM with musical hints — `bpm`,
|
|
@@ -425,14 +456,15 @@ layout `sourceAudio` uses:
|
|
|
425
456
|
const response = await gen.understand(pcm, { seed: 42 })
|
|
426
457
|
const stats = await response.await()
|
|
427
458
|
const heard = stats.understand
|
|
428
|
-
// { caption, bpm, duration, keyscale, timesignature, vocalLanguage, audioCodes }
|
|
459
|
+
// { caption, bpm, duration, keyscale, timesignature, vocalLanguage, audioCodes, seed }
|
|
429
460
|
```
|
|
430
461
|
|
|
431
462
|
The description streams as an `understand` output item (progress ticks report
|
|
432
463
|
the `source`, `tok`, and `understand` stages) and is repeated on the terminal
|
|
433
464
|
stats. `audioCodes` are the recovered semantic codes — pass them back as a
|
|
434
465
|
generation's `audioCodes` to re-synthesize or remix the clip. A
|
|
435
|
-
`vocalLanguage` hint forces the language field instead of the LM's guess
|
|
466
|
+
`vocalLanguage` hint forces the language field instead of the LM's guess, and
|
|
467
|
+
`seed` is the one the LM decode used.
|
|
436
468
|
End to end from the repo (generates a clip, then describes it):
|
|
437
469
|
|
|
438
470
|
```bash
|
|
@@ -489,6 +521,12 @@ the end of the source. Flow-Edit is turbo DiT only (`turbo-q4`, `turbo-q8`) and
|
|
|
489
521
|
changes musical/lyrical conditioning over its `nMin`/`nMax` diffusion window.
|
|
490
522
|
Repainting an entire track is supported with `start: 0` and no `end`.
|
|
491
523
|
|
|
524
|
+
`run()` also takes the conditioning shared by every operation: a timbre
|
|
525
|
+
`referenceAudio` (same layout as the source), the `vocalLanguage` / `bpm` /
|
|
526
|
+
`keyscale` / `timesignature` / `augmentCaptionWithMetadata` prompt metadata, the
|
|
527
|
+
Repaint DCW sampler controls (`dcwEnabled`, `dcwScaler`, `dcwHighScaler`), and a
|
|
528
|
+
per-run `inferenceSteps` / `shift`.
|
|
529
|
+
|
|
492
530
|
### Turning PCM into a file
|
|
493
531
|
|
|
494
532
|
Convert each `Int16Array` view to bytes using its `byteOffset` and `byteLength`,
|
|
@@ -578,7 +616,8 @@ runnable end-to-end script (`npm run example`).
|
|
|
578
616
|
| Option | Meaning |
|
|
579
617
|
|--------|---------|
|
|
580
618
|
| `useGPU` | Run on GPU (Metal / CUDA / Vulkan, including Android Mali); falls back to CPU. |
|
|
581
|
-
| `
|
|
619
|
+
| `device` | MiniMax only, instead of `useGPU`: `gpu` fails `load()` without a usable GPU, `auto` falls back to the CPU, `cpu` never uses a GPU. |
|
|
620
|
+
| `inferenceSteps` / `shift` | Advanced; leave unset to auto-tune per DiT. `run()` can override both per call. |
|
|
582
621
|
| `cfgScale` | Default MiniMax flow guidance scale; `0` uses the model default. |
|
|
583
622
|
| `nGpuLayers` | GPU layers to offload when `useGPU` is set (99 = all). |
|
|
584
623
|
| `threads` | CPU thread count (0 / unset = hardware default). |
|
|
@@ -599,7 +638,9 @@ wrapped by a level-gated `QvacLogger`.
|
|
|
599
638
|
| `augmentCaptionWithMetadata` | Append BPM/tempo, time signature, and key guidance to the internal conditioning caption; defaults to `false`. |
|
|
600
639
|
| `duration` | Target length in seconds; omit to let the LM decide. |
|
|
601
640
|
| `maxFrames` | MiniMax semantic-frame cap; cannot be combined with `duration`. |
|
|
602
|
-
| `inferenceSteps`
|
|
641
|
+
| `inferenceSteps` | Per-run diffusion steps (`0`–`1000`); omit to keep `config.inferenceSteps`. |
|
|
642
|
+
| `shift` | ACE-Step per-run DiT timestep shift; omit to keep `config.shift`. |
|
|
643
|
+
| `cfgScale` | Per-run MiniMax flow guidance scale. |
|
|
603
644
|
| `seed` | RNG seed for reproducible generation. |
|
|
604
645
|
| `lmTemperature` / `lmTopP` / `lmTopK` / `lmCfgScale` | LM sampling controls. |
|
|
605
646
|
| `lmPhase1` | Allow the LM to infer missing metadata before generating semantic codes. |
|
|
@@ -707,6 +748,35 @@ and `vaePath` name them individually and win over it.
|
|
|
707
748
|
`deviceSharesHostMemory` reports that the device pool is system RAM, so host
|
|
708
749
|
bytes compete with device bytes.
|
|
709
750
|
|
|
751
|
+
### MiniMax-Music3
|
|
752
|
+
|
|
753
|
+
`engine: 'minimax'` projects a MiniMax-Music3 pair on desktop builds, with the
|
|
754
|
+
same result shape. `modelsDir` holds the pair; `lmPath` and `synthPath` name the
|
|
755
|
+
two GGUFs and win over it. The weights and the synthesis graphs stay resident
|
|
756
|
+
between runs and the LM's cache is rebuilt by each run, so `deviceBytes` is the
|
|
757
|
+
peak of repeated generations, and the `stages` rows are `lm`, `depth`, `cond`,
|
|
758
|
+
`dit` and `vocoder`.
|
|
759
|
+
|
|
760
|
+
```js
|
|
761
|
+
const fit = assessFit({
|
|
762
|
+
engine: 'minimax',
|
|
763
|
+
modelsDir: '/models/minimax-music3',
|
|
764
|
+
device: 'gpu',
|
|
765
|
+
durationSeconds: 120,
|
|
766
|
+
promptTokens: 900
|
|
767
|
+
})
|
|
768
|
+
```
|
|
769
|
+
|
|
770
|
+
| Option | Description |
|
|
771
|
+
| --- | --- |
|
|
772
|
+
| `device` | `'cpu'`, `'gpu'` or `'auto'`, as `config.device` takes it. Without it, `gpuLayers` greater than 0 projects `'auto'` and anything else `'cpu'`. |
|
|
773
|
+
| `durationSeconds`, `maxFrames` | The generation length, as `run()` takes it (25 frames per second); one or the other. Defaults to 300 frames. |
|
|
774
|
+
| `promptTokens` | Tokenized prompt length: caption, lyrics and template. Defaults to 1024. A prompt above the checkpoint's limit, or one that leaves too little context for the frames, is `workload-too-large`. |
|
|
775
|
+
| `threads`, `backendsDir`, `marginBytes` | As for ACE-Step. |
|
|
776
|
+
|
|
777
|
+
On Android and iOS, where MiniMax-Music3 is unavailable, `engine: 'minimax'`
|
|
778
|
+
comes back as `status: "error"` with `reason: "unsupported-engine"`.
|
|
779
|
+
|
|
710
780
|
A model the engine cannot read is `status: "error"`; a broken request, or a
|
|
711
781
|
host with no native binding, throws.
|
|
712
782
|
|
package/audiogen.d.ts
CHANGED
|
@@ -9,6 +9,8 @@ export interface AudioGenConfigurationParams {
|
|
|
9
9
|
inferenceSteps?: number;
|
|
10
10
|
shift?: number;
|
|
11
11
|
useGPU?: boolean;
|
|
12
|
+
/** MiniMax compute device ('cpu' | 'gpu' | 'auto'); overrides useGPU. */
|
|
13
|
+
device?: string;
|
|
12
14
|
nGpuLayers?: number;
|
|
13
15
|
threads?: number;
|
|
14
16
|
/**
|
|
@@ -79,6 +81,7 @@ export interface AudioGenJobData {
|
|
|
79
81
|
coverNoiseStrength?: number;
|
|
80
82
|
maxFrames?: number;
|
|
81
83
|
inferenceSteps?: number;
|
|
84
|
+
shift?: number;
|
|
82
85
|
cfgScale?: number;
|
|
83
86
|
editOperations?: AudioEditOperationJobData[];
|
|
84
87
|
}
|
package/index.d.ts
CHANGED
|
@@ -8,6 +8,8 @@ export declare const ENGINE_MINIMAX = "minimax";
|
|
|
8
8
|
export declare const MINIMAX_FRAMES_PER_SECOND = 25;
|
|
9
9
|
export declare const MINIMAX_DEFAULT_MAX_FRAMES = 300;
|
|
10
10
|
export type AudioGenEngine = typeof ENGINE_ACESTEP | typeof ENGINE_MINIMAX;
|
|
11
|
+
/** MiniMax compute device, see `AudioGenRuntimeConfig.device`. */
|
|
12
|
+
export type AudiogenMinimaxDevice = 'cpu' | 'gpu' | 'auto';
|
|
11
13
|
/** Model file paths for ACE-Step or MiniMax-Music3. */
|
|
12
14
|
export interface AudioGenFiles {
|
|
13
15
|
/** Directory holding the four ACE-Step GGUFs (engine auto-classifies them). */
|
|
@@ -39,6 +41,12 @@ export interface AudioGenRuntimeConfig {
|
|
|
39
41
|
* in use. MiniMax puts the whole model pair on the device (~22 GB for f16).
|
|
40
42
|
*/
|
|
41
43
|
useGPU?: boolean;
|
|
44
|
+
/**
|
|
45
|
+
* MiniMax only, instead of `useGPU`: `'gpu'` requires a usable GPU and fails
|
|
46
|
+
* `load()` otherwise, `'auto'` (what `useGPU: true` means) falls back to the
|
|
47
|
+
* CPU, `'cpu'` never touches a GPU.
|
|
48
|
+
*/
|
|
49
|
+
device?: AudiogenMinimaxDevice;
|
|
42
50
|
/** ACE-Step only: GPU layers to offload when `useGPU` is set (99 = all). */
|
|
43
51
|
nGpuLayers?: number;
|
|
44
52
|
/** 0 = engine auto-picks. */
|
|
@@ -79,10 +87,20 @@ export interface GenerateOptions {
|
|
|
79
87
|
duration?: number;
|
|
80
88
|
/** MiniMax semantic-frame cap. Cannot be combined with `duration`. */
|
|
81
89
|
maxFrames?: number;
|
|
82
|
-
/**
|
|
90
|
+
/**
|
|
91
|
+
* Diffusion steps for this generation, 0..1000; omit to keep
|
|
92
|
+
* `config.inferenceSteps`. 0 is the engine default: MiniMax 20 flow steps,
|
|
93
|
+
* ACE-Step 8 on turbo DiTs and 50 on base/sft.
|
|
94
|
+
*/
|
|
83
95
|
inferenceSteps?: number;
|
|
84
96
|
/** MiniMax flow classifier-free guidance scale for this generation. */
|
|
85
97
|
cfgScale?: number;
|
|
98
|
+
/**
|
|
99
|
+
* ACE-Step DiT timestep shift for this generation; omit to keep
|
|
100
|
+
* `config.shift`. 0 is the engine default: 3.0 on turbo DiTs, 1.0 on
|
|
101
|
+
* base/sft.
|
|
102
|
+
*/
|
|
103
|
+
shift?: number;
|
|
86
104
|
/** LM sampling temperature (ACE-Step default: 0.85). */
|
|
87
105
|
lmTemperature?: number;
|
|
88
106
|
/** LM nucleus-sampling probability (ACE-Step default: 0.9). */
|
|
@@ -230,6 +248,31 @@ export interface RepaintOptions extends AudioEditPrompt {
|
|
|
230
248
|
export interface AudioEditRunOptions {
|
|
231
249
|
/** Seeds the first operation; each following operation uses seed + its index. */
|
|
232
250
|
seed?: number;
|
|
251
|
+
/**
|
|
252
|
+
* Optional timbre reference for every operation: interleaved stereo float
|
|
253
|
+
* PCM at 48 kHz, the same layout `GenerateOptions.referenceAudio` uses.
|
|
254
|
+
*/
|
|
255
|
+
referenceAudio?: Float32Array;
|
|
256
|
+
/** Vocal language hint for the edit prompts; the engine defaults to `'unknown'`. */
|
|
257
|
+
vocalLanguage?: string;
|
|
258
|
+
/** Beats per minute for the edit prompts; 0/undefined leaves it unset. */
|
|
259
|
+
bpm?: number;
|
|
260
|
+
/** Key + scale for the edit prompts, e.g. "C minor". */
|
|
261
|
+
keyscale?: string;
|
|
262
|
+
/** Time signature for the edit prompts, e.g. "4/4". */
|
|
263
|
+
timesignature?: string;
|
|
264
|
+
/** Append BPM, time signature and key to each operation's conditioning caption. */
|
|
265
|
+
augmentCaptionWithMetadata?: boolean;
|
|
266
|
+
/** Repaint only: apply the Haar DCW sampler correction (default: true). */
|
|
267
|
+
dcwEnabled?: boolean;
|
|
268
|
+
/** Repaint only: DCW low-frequency correction strength (default: 0.05). */
|
|
269
|
+
dcwScaler?: number;
|
|
270
|
+
/** Repaint only: DCW high-frequency correction strength (default: 0.02). */
|
|
271
|
+
dcwHighScaler?: number;
|
|
272
|
+
/** DiT steps for every operation, 0..1000; omit to keep `config.inferenceSteps`. */
|
|
273
|
+
inferenceSteps?: number;
|
|
274
|
+
/** DiT timestep shift for every operation; omit to keep `config.shift`. */
|
|
275
|
+
shift?: number;
|
|
233
276
|
}
|
|
234
277
|
interface NativeFlowEditOperation {
|
|
235
278
|
type: AudioEditOperationType.FlowEdit;
|
|
@@ -257,6 +300,38 @@ export interface AudiogenProgress {
|
|
|
257
300
|
step: number;
|
|
258
301
|
total: number;
|
|
259
302
|
}
|
|
303
|
+
/**
|
|
304
|
+
* The request an ACE-Step run actually rendered, after the LM filled in what
|
|
305
|
+
* the caller left unset (Phase 1, Simple Mode, Query Rewriting). An edit run
|
|
306
|
+
* reports its base seed and the prompt metadata it was given; its caption is
|
|
307
|
+
* empty (each operation carries its own) and `codeFrames` is 0. MiniMax runs
|
|
308
|
+
* report none.
|
|
309
|
+
*/
|
|
310
|
+
export interface AudiogenGenerationMetadata {
|
|
311
|
+
/**
|
|
312
|
+
* The caption the run used: the LM-composed one under Simple Mode or Query
|
|
313
|
+
* Rewriting. With `augmentCaptionWithMetadata` set it is always the caller's
|
|
314
|
+
* own caption (before the metadata suffix), even under those modes.
|
|
315
|
+
*/
|
|
316
|
+
caption: string;
|
|
317
|
+
/** The lyrics the run used (LM-written under Simple Mode). */
|
|
318
|
+
lyrics: string;
|
|
319
|
+
/** Beats per minute; 0 when unresolved. */
|
|
320
|
+
bpm: number;
|
|
321
|
+
keyscale: string;
|
|
322
|
+
/** Beats per bar, the time-signature numerator (4 for "4/4"); 0 when unresolved. */
|
|
323
|
+
beatsPerBar: number;
|
|
324
|
+
vocalLanguage: string;
|
|
325
|
+
/** The seed the run used: the engine-drawn one when no `seed` was passed. */
|
|
326
|
+
seed: number;
|
|
327
|
+
/**
|
|
328
|
+
* 5 Hz semantic code frames the DiT was conditioned on; 0 when the LM and
|
|
329
|
+
* detokenizer are skipped (cover-nofsq, lego, edits).
|
|
330
|
+
*/
|
|
331
|
+
codeFrames: number;
|
|
332
|
+
/** Per-condition breakdown of `stats.qualityScore`; present only with `computeQualityScore`. */
|
|
333
|
+
qualityReport?: string;
|
|
334
|
+
}
|
|
260
335
|
/** One interleaved-Int16 PCM chunk emitted by the engine. */
|
|
261
336
|
export interface AudiogenPcmChunk {
|
|
262
337
|
outputArray: Int16Array;
|
|
@@ -264,6 +339,8 @@ export interface AudiogenPcmChunk {
|
|
|
264
339
|
channels: number;
|
|
265
340
|
/** LRC-formatted lyric timestamps; present only when the run set `generateLrc`. */
|
|
266
341
|
lrc?: string;
|
|
342
|
+
/** What an ACE-Step run rendered; absent for MiniMax. */
|
|
343
|
+
metadata?: AudiogenGenerationMetadata;
|
|
267
344
|
}
|
|
268
345
|
/** A progress tick delivered through the run's output stream. */
|
|
269
346
|
export interface AudiogenProgressChunk {
|
|
@@ -283,6 +360,8 @@ export interface AudiogenUnderstandResult {
|
|
|
283
360
|
timesignature: string;
|
|
284
361
|
vocalLanguage: string;
|
|
285
362
|
audioCodes: Int32Array;
|
|
363
|
+
/** The seed the LM decode used: the engine-drawn one when no `seed` was passed. */
|
|
364
|
+
seed: number;
|
|
286
365
|
}
|
|
287
366
|
/** The understand result delivered through the response's output stream. */
|
|
288
367
|
export interface AudiogenUnderstandChunk {
|
|
@@ -311,6 +390,19 @@ export interface AudiogenStats {
|
|
|
311
390
|
backendId?: number;
|
|
312
391
|
/** 0 = none, 1 = not requested, 2 = no devices, 3 = init failed. */
|
|
313
392
|
gpuFallbackReason?: number;
|
|
393
|
+
/**
|
|
394
|
+
* MiniMax only: semantic frames the AR stage emitted (25 per second). Below
|
|
395
|
+
* the frame cap when the song ended at the model's end-of-sequence.
|
|
396
|
+
*/
|
|
397
|
+
emittedFrames?: number;
|
|
398
|
+
/** MiniMax only: engine time in the autoregressive semantic stage. */
|
|
399
|
+
arMs?: number;
|
|
400
|
+
/** MiniMax only: engine time building the flow conditioning. */
|
|
401
|
+
conditionMs?: number;
|
|
402
|
+
/** MiniMax only: engine time in the flow-matching DiT. */
|
|
403
|
+
flowMs?: number;
|
|
404
|
+
/** MiniMax only: engine time in the vocoder. */
|
|
405
|
+
vocoderMs?: number;
|
|
314
406
|
/**
|
|
315
407
|
* Lyric-to-audio alignment confidence in [0, 1]. Present only when the run
|
|
316
408
|
* set `generateLrc`; the LRC text itself rides on the PCM chunk (`lrc`) and
|
|
@@ -330,6 +422,11 @@ export interface AudiogenStats {
|
|
|
330
422
|
* by an `understand()` response; also streamed as an output item.
|
|
331
423
|
*/
|
|
332
424
|
understand?: AudiogenUnderstandResult;
|
|
425
|
+
/**
|
|
426
|
+
* What an ACE-Step run rendered, repeated from the PCM chunk
|
|
427
|
+
* (`AudiogenPcmChunk.metadata`). Absent for `understand()` and MiniMax.
|
|
428
|
+
*/
|
|
429
|
+
metadata?: AudiogenGenerationMetadata;
|
|
333
430
|
}
|
|
334
431
|
/** Options accepted by `understand()`. */
|
|
335
432
|
export interface UnderstandOptions {
|
|
@@ -407,6 +504,7 @@ export declare class AudioGen {
|
|
|
407
504
|
private _lastLrc;
|
|
408
505
|
private _cancelTerminalResolve;
|
|
409
506
|
private _lastUnderstand;
|
|
507
|
+
private _lastMetadata;
|
|
410
508
|
constructor(options?: AudioGenOptions);
|
|
411
509
|
/** Create the native engine and load its GGUF files. Idempotent. */
|
|
412
510
|
load(): Promise<void>;
|
package/index.js
CHANGED
|
@@ -31,10 +31,11 @@ const SUPPORTED_ENGINES = [exports.ENGINE_ACESTEP, exports.ENGINE_MINIMAX];
|
|
|
31
31
|
exports.MINIMAX_FRAMES_PER_SECOND = 25;
|
|
32
32
|
exports.MINIMAX_DEFAULT_MAX_FRAMES = 300;
|
|
33
33
|
const MINIMAX_MIN_FRAMES = 1;
|
|
34
|
-
const
|
|
34
|
+
const MAX_INFERENCE_STEPS = 1000;
|
|
35
35
|
const INT32_MAX = 2147483647;
|
|
36
36
|
const FLOAT32_MAX = 3.4028234663852886e38;
|
|
37
37
|
const FLOAT32_MIN_POSITIVE = 1.401298464324817e-45;
|
|
38
|
+
const MINIMAX_DEVICES = ['cpu', 'gpu', 'auto'];
|
|
38
39
|
/** `AudiogenStats.backendId` codes, named. Codes match @qvac/tts-ggml. */
|
|
39
40
|
exports.AUDIOGEN_BACKEND_NAMES = {
|
|
40
41
|
0: 'cpu',
|
|
@@ -96,13 +97,43 @@ function requireSafeInteger(value, name) {
|
|
|
96
97
|
}
|
|
97
98
|
return value;
|
|
98
99
|
}
|
|
99
|
-
function
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
100
|
+
function optionalFloat32(value, name) {
|
|
101
|
+
if (value === undefined)
|
|
102
|
+
return undefined;
|
|
103
|
+
requireFiniteNumber(value, name);
|
|
104
|
+
if (Math.abs(value) > FLOAT32_MAX) {
|
|
105
|
+
throw invalidInput(`${name} must be within the float32 range, got ${value}`);
|
|
106
|
+
}
|
|
107
|
+
return value;
|
|
108
|
+
}
|
|
109
|
+
function optionalBoolean(value, name) {
|
|
110
|
+
if (value !== undefined && typeof value !== 'boolean') {
|
|
111
|
+
throw invalidInput(`${name} must be a boolean`);
|
|
112
|
+
}
|
|
113
|
+
return value;
|
|
114
|
+
}
|
|
115
|
+
function optionalString(value, name) {
|
|
116
|
+
if (value !== undefined && typeof value !== 'string') {
|
|
117
|
+
throw invalidInput(`${name} must be a string`);
|
|
118
|
+
}
|
|
119
|
+
return value;
|
|
120
|
+
}
|
|
121
|
+
function requireInferenceSteps(value, name = 'inferenceSteps') {
|
|
122
|
+
const steps = requireSafeInteger(value, name);
|
|
123
|
+
if (steps < 0 || steps > MAX_INFERENCE_STEPS) {
|
|
124
|
+
throw invalidInput(`${name} must be between 0 and ${MAX_INFERENCE_STEPS}`);
|
|
103
125
|
}
|
|
104
126
|
return steps;
|
|
105
127
|
}
|
|
128
|
+
// ACE-Step DiT timestep shift; 0 = auto (turbo 3.0, base/sft 1.0). A positive
|
|
129
|
+
// value that rounds to a float32 zero would silently select auto instead.
|
|
130
|
+
function requireAcestepShift(value, name = 'shift') {
|
|
131
|
+
const shift = requireFiniteNumber(value, name);
|
|
132
|
+
if (shift < 0 || shift > FLOAT32_MAX || (shift > 0 && shift < FLOAT32_MIN_POSITIVE)) {
|
|
133
|
+
throw invalidInput(`${name} must be 0 or a positive float32 value`);
|
|
134
|
+
}
|
|
135
|
+
return shift;
|
|
136
|
+
}
|
|
106
137
|
function requireNonNegativeInt32(value, name) {
|
|
107
138
|
const integer = requireSafeInteger(value, name);
|
|
108
139
|
if (integer < 0 || integer > INT32_MAX) {
|
|
@@ -281,7 +312,8 @@ const ACESTEP_GENERATE_KEYS = [
|
|
|
281
312
|
'coverNoiseStrength',
|
|
282
313
|
'generateLrc',
|
|
283
314
|
'computeQualityScore',
|
|
284
|
-
'rewriteQuery'
|
|
315
|
+
'rewriteQuery',
|
|
316
|
+
'shift'
|
|
285
317
|
];
|
|
286
318
|
function hasAnyFile(files, keys) {
|
|
287
319
|
return keys.some((key) => files[key] !== undefined);
|
|
@@ -325,6 +357,9 @@ function validateAcestepOptions(files, config) {
|
|
|
325
357
|
if (config.cfgScale !== undefined) {
|
|
326
358
|
throw invalidInput('ACE-Step does not accept cfgScale');
|
|
327
359
|
}
|
|
360
|
+
if (config.device !== undefined) {
|
|
361
|
+
throw invalidInput('ACE-Step does not accept device (use useGPU and nGpuLayers)');
|
|
362
|
+
}
|
|
328
363
|
}
|
|
329
364
|
function validateMinimaxConfig(config) {
|
|
330
365
|
if (config.useGPU !== undefined && typeof config.useGPU !== 'boolean') {
|
|
@@ -333,6 +368,14 @@ function validateMinimaxConfig(config) {
|
|
|
333
368
|
if (config.shift !== undefined || config.nGpuLayers !== undefined) {
|
|
334
369
|
throw invalidInput('MiniMax does not accept shift or nGpuLayers');
|
|
335
370
|
}
|
|
371
|
+
if (config.device !== undefined) {
|
|
372
|
+
if (typeof config.device !== 'string' || !MINIMAX_DEVICES.includes(config.device)) {
|
|
373
|
+
throw invalidInput("device must be 'cpu', 'gpu' or 'auto'");
|
|
374
|
+
}
|
|
375
|
+
if (config.useGPU !== undefined) {
|
|
376
|
+
throw invalidInput('MiniMax accepts either useGPU or device, not both');
|
|
377
|
+
}
|
|
378
|
+
}
|
|
336
379
|
}
|
|
337
380
|
function assertNoAcestepGenerateOptions(options) {
|
|
338
381
|
for (const key of ACESTEP_GENERATE_KEYS) {
|
|
@@ -342,10 +385,8 @@ function assertNoAcestepGenerateOptions(options) {
|
|
|
342
385
|
}
|
|
343
386
|
}
|
|
344
387
|
function assertNoMinimaxGenerateOptions(options) {
|
|
345
|
-
if (options.maxFrames !== undefined ||
|
|
346
|
-
|
|
347
|
-
options.cfgScale !== undefined) {
|
|
348
|
-
throw invalidInput('ACE-Step does not accept maxFrames, inferenceSteps, or cfgScale per run');
|
|
388
|
+
if (options.maxFrames !== undefined || options.cfgScale !== undefined) {
|
|
389
|
+
throw invalidInput('ACE-Step does not accept maxFrames or cfgScale per run');
|
|
349
390
|
}
|
|
350
391
|
}
|
|
351
392
|
function resolveMinimaxMaxFrames(options) {
|
|
@@ -371,6 +412,31 @@ function isMobilePlatform() {
|
|
|
371
412
|
const platform = os.platform();
|
|
372
413
|
return platform === 'android' || platform === 'ios';
|
|
373
414
|
}
|
|
415
|
+
/** One validation for the options both paths hand the engine; `prefix` names the path. */
|
|
416
|
+
function requireAcestepSharedOptions(options, prefix = '') {
|
|
417
|
+
const key = (name) => `${prefix}${name}`;
|
|
418
|
+
return {
|
|
419
|
+
seed: options.seed === undefined ? undefined : requireSafeInteger(options.seed, key('seed')),
|
|
420
|
+
vocalLanguage: optionalString(options.vocalLanguage, key('vocalLanguage')),
|
|
421
|
+
bpm: options.bpm === undefined ? undefined : requireNonNegativeInt32(options.bpm, key('bpm')),
|
|
422
|
+
keyscale: optionalString(options.keyscale, key('keyscale')),
|
|
423
|
+
timesignature: optionalString(options.timesignature, key('timesignature')),
|
|
424
|
+
augmentCaptionWithMetadata: optionalBoolean(options.augmentCaptionWithMetadata, key('augmentCaptionWithMetadata')),
|
|
425
|
+
dcwEnabled: optionalBoolean(options.dcwEnabled, key('dcwEnabled')),
|
|
426
|
+
dcwScaler: optionalFloat32(options.dcwScaler, key('dcwScaler')),
|
|
427
|
+
dcwHighScaler: optionalFloat32(options.dcwHighScaler, key('dcwHighScaler')),
|
|
428
|
+
inferenceSteps: options.inferenceSteps === undefined
|
|
429
|
+
? undefined
|
|
430
|
+
: requireInferenceSteps(options.inferenceSteps, key('inferenceSteps')),
|
|
431
|
+
shift: options.shift === undefined ? undefined : requireAcestepShift(options.shift, key('shift'))
|
|
432
|
+
};
|
|
433
|
+
}
|
|
434
|
+
function requireEditRunOptions(options) {
|
|
435
|
+
return {
|
|
436
|
+
...requireAcestepSharedOptions(options, 'edit.'),
|
|
437
|
+
referenceAudio: optionalStereoPcm(options.referenceAudio, 'edit.referenceAudio')
|
|
438
|
+
};
|
|
439
|
+
}
|
|
374
440
|
/**
|
|
375
441
|
* Fluent, ordered edit pipeline. Every call appends one operation; operations
|
|
376
442
|
* may be repeated in any order before the session is submitted with `run()`.
|
|
@@ -463,9 +529,9 @@ class AudioEditSession {
|
|
|
463
529
|
if (typeof options !== 'object' || options === null) {
|
|
464
530
|
throw invalidInput('edit session run options must be an object');
|
|
465
531
|
}
|
|
466
|
-
const
|
|
532
|
+
const runOptions = requireEditRunOptions(options);
|
|
467
533
|
this._started = true;
|
|
468
|
-
return this._runner(this._source, this._operations,
|
|
534
|
+
return this._runner(this._source, this._operations, runOptions);
|
|
469
535
|
}
|
|
470
536
|
}
|
|
471
537
|
exports.AudioEditSession = AudioEditSession;
|
|
@@ -496,6 +562,7 @@ class AudioGen {
|
|
|
496
562
|
_lastLrc;
|
|
497
563
|
_cancelTerminalResolve;
|
|
498
564
|
_lastUnderstand;
|
|
565
|
+
_lastMetadata;
|
|
499
566
|
constructor(options = {}) {
|
|
500
567
|
this._logger = new QvacLogger(options.logger);
|
|
501
568
|
const files = options.files ?? {};
|
|
@@ -510,7 +577,7 @@ class AudioGen {
|
|
|
510
577
|
}
|
|
511
578
|
validateMinimaxFiles(files);
|
|
512
579
|
validateMinimaxConfig(config);
|
|
513
|
-
this._defaultInferenceSteps =
|
|
580
|
+
this._defaultInferenceSteps = requireInferenceSteps(config.inferenceSteps ?? 0);
|
|
514
581
|
this._defaultCfgScale = requireMinimaxCfgScale(config.cfgScale ?? 0);
|
|
515
582
|
this._configuration = {
|
|
516
583
|
engineType: exports.ENGINE_MINIMAX,
|
|
@@ -519,12 +586,13 @@ class AudioGen {
|
|
|
519
586
|
synthModelPath: files.synthModel,
|
|
520
587
|
threads,
|
|
521
588
|
useGPU: config.useGPU ?? false,
|
|
589
|
+
device: config.device,
|
|
522
590
|
backendsDir
|
|
523
591
|
};
|
|
524
592
|
}
|
|
525
593
|
else {
|
|
526
594
|
validateAcestepOptions(files, config);
|
|
527
|
-
this._defaultInferenceSteps =
|
|
595
|
+
this._defaultInferenceSteps = requireInferenceSteps(config.inferenceSteps ?? 0);
|
|
528
596
|
this._defaultCfgScale = 0;
|
|
529
597
|
const ditModelPath = (0, models_1.resolveDitModelPath)({
|
|
530
598
|
modelDir: files.modelDir,
|
|
@@ -539,7 +607,7 @@ class AudioGen {
|
|
|
539
607
|
ditModelPath,
|
|
540
608
|
vaeModelPath: files.vaeModel,
|
|
541
609
|
inferenceSteps: this._defaultInferenceSteps,
|
|
542
|
-
shift:
|
|
610
|
+
shift: requireAcestepShift(config.shift ?? 0),
|
|
543
611
|
useGPU: config.useGPU ?? false,
|
|
544
612
|
nGpuLayers: requireFiniteNumber(config.nGpuLayers ?? 99, 'nGpuLayers', true),
|
|
545
613
|
threads,
|
|
@@ -666,7 +734,18 @@ class AudioGen {
|
|
|
666
734
|
input: '',
|
|
667
735
|
sourceAudio,
|
|
668
736
|
editOperations: [...operations],
|
|
669
|
-
seed: options.seed
|
|
737
|
+
seed: options.seed,
|
|
738
|
+
referenceAudio: options.referenceAudio,
|
|
739
|
+
vocalLanguage: options.vocalLanguage,
|
|
740
|
+
bpm: options.bpm,
|
|
741
|
+
keyscale: options.keyscale,
|
|
742
|
+
timesignature: options.timesignature,
|
|
743
|
+
augmentCaptionWithMetadata: options.augmentCaptionWithMetadata,
|
|
744
|
+
dcwEnabled: options.dcwEnabled,
|
|
745
|
+
dcwScaler: options.dcwScaler,
|
|
746
|
+
dcwHighScaler: options.dcwHighScaler,
|
|
747
|
+
inferenceSteps: options.inferenceSteps,
|
|
748
|
+
shift: options.shift
|
|
670
749
|
};
|
|
671
750
|
const revision = this._lifecycleRevision;
|
|
672
751
|
return new Promise((resolve, reject) => {
|
|
@@ -681,6 +760,7 @@ class AudioGen {
|
|
|
681
760
|
const addon = this._requireAddon();
|
|
682
761
|
this._lastLrc = undefined;
|
|
683
762
|
this._lastUnderstand = undefined;
|
|
763
|
+
this._lastMetadata = undefined;
|
|
684
764
|
const response = this._job.start();
|
|
685
765
|
let accepted;
|
|
686
766
|
try {
|
|
@@ -728,7 +808,7 @@ class AudioGen {
|
|
|
728
808
|
maxFrames: resolveMinimaxMaxFrames(opts),
|
|
729
809
|
inferenceSteps: opts.inferenceSteps === undefined
|
|
730
810
|
? this._defaultInferenceSteps
|
|
731
|
-
:
|
|
811
|
+
: requireInferenceSteps(opts.inferenceSteps),
|
|
732
812
|
cfgScale: opts.cfgScale === undefined ? this._defaultCfgScale : requireMinimaxCfgScale(opts.cfgScale)
|
|
733
813
|
};
|
|
734
814
|
}
|
|
@@ -737,13 +817,7 @@ class AudioGen {
|
|
|
737
817
|
if (opts.lmPhase1 !== undefined && typeof opts.lmPhase1 !== 'boolean') {
|
|
738
818
|
throw invalidInput('lmPhase1 must be a boolean');
|
|
739
819
|
}
|
|
740
|
-
|
|
741
|
-
typeof opts.augmentCaptionWithMetadata !== 'boolean') {
|
|
742
|
-
throw invalidInput('augmentCaptionWithMetadata must be a boolean');
|
|
743
|
-
}
|
|
744
|
-
if (opts.dcwEnabled !== undefined && typeof opts.dcwEnabled !== 'boolean') {
|
|
745
|
-
throw invalidInput('dcwEnabled must be a boolean');
|
|
746
|
-
}
|
|
820
|
+
const shared = requireAcestepSharedOptions(opts);
|
|
747
821
|
if (opts.audioCodes !== undefined && !(opts.audioCodes instanceof Int32Array)) {
|
|
748
822
|
throw invalidInput('audioCodes must be an Int32Array');
|
|
749
823
|
}
|
|
@@ -833,21 +907,21 @@ class AudioGen {
|
|
|
833
907
|
normalizeLoudness: opts.normalizeLoudness,
|
|
834
908
|
generateLrc: opts.generateLrc,
|
|
835
909
|
computeQualityScore: opts.computeQualityScore,
|
|
836
|
-
seed:
|
|
837
|
-
vocalLanguage:
|
|
838
|
-
bpm:
|
|
839
|
-
keyscale:
|
|
840
|
-
timesignature:
|
|
841
|
-
augmentCaptionWithMetadata:
|
|
910
|
+
seed: shared.seed,
|
|
911
|
+
vocalLanguage: shared.vocalLanguage,
|
|
912
|
+
bpm: shared.bpm,
|
|
913
|
+
keyscale: shared.keyscale,
|
|
914
|
+
timesignature: shared.timesignature,
|
|
915
|
+
augmentCaptionWithMetadata: shared.augmentCaptionWithMetadata,
|
|
842
916
|
duration: optionalFiniteNumber(opts.duration, 'duration'),
|
|
843
917
|
lmTemperature: optionalFiniteNumber(opts.lmTemperature, 'lmTemperature'),
|
|
844
918
|
lmTopP: optionalFiniteNumber(opts.lmTopP, 'lmTopP'),
|
|
845
919
|
lmTopK: optionalFiniteNumber(opts.lmTopK, 'lmTopK', true),
|
|
846
920
|
lmCfgScale: optionalFiniteNumber(opts.lmCfgScale, 'lmCfgScale'),
|
|
847
921
|
lmPhase1: opts.lmPhase1,
|
|
848
|
-
dcwEnabled:
|
|
849
|
-
dcwScaler:
|
|
850
|
-
dcwHighScaler:
|
|
922
|
+
dcwEnabled: shared.dcwEnabled,
|
|
923
|
+
dcwScaler: shared.dcwScaler,
|
|
924
|
+
dcwHighScaler: shared.dcwHighScaler,
|
|
851
925
|
audioCodes: opts.audioCodes,
|
|
852
926
|
referenceAudio,
|
|
853
927
|
sourceAudio,
|
|
@@ -855,7 +929,9 @@ class AudioGen {
|
|
|
855
929
|
track: opts.track,
|
|
856
930
|
guidanceScale,
|
|
857
931
|
audioCoverStrength: optionalFiniteNumber(opts.audioCoverStrength, 'audioCoverStrength'),
|
|
858
|
-
coverNoiseStrength: optionalFiniteNumber(opts.coverNoiseStrength, 'coverNoiseStrength')
|
|
932
|
+
coverNoiseStrength: optionalFiniteNumber(opts.coverNoiseStrength, 'coverNoiseStrength'),
|
|
933
|
+
inferenceSteps: shared.inferenceSteps,
|
|
934
|
+
shift: shared.shift
|
|
859
935
|
};
|
|
860
936
|
}
|
|
861
937
|
async cancel() {
|
|
@@ -1000,11 +1076,14 @@ class AudioGen {
|
|
|
1000
1076
|
}
|
|
1001
1077
|
if (d.outputArray) {
|
|
1002
1078
|
this._lastLrc = typeof d.lrc === 'string' ? d.lrc : undefined;
|
|
1079
|
+
this._lastMetadata =
|
|
1080
|
+
typeof d.metadata === 'object' && d.metadata !== null ? d.metadata : undefined;
|
|
1003
1081
|
this._job.output({
|
|
1004
1082
|
outputArray: d.outputArray,
|
|
1005
1083
|
sampleRate: d.sampleRate ?? 0,
|
|
1006
1084
|
channels: d.channels ?? 0,
|
|
1007
|
-
...(this._lastLrc !== undefined ? { lrc: this._lastLrc } : {})
|
|
1085
|
+
...(this._lastLrc !== undefined ? { lrc: this._lastLrc } : {}),
|
|
1086
|
+
...(this._lastMetadata !== undefined ? { metadata: this._lastMetadata } : {})
|
|
1008
1087
|
});
|
|
1009
1088
|
return;
|
|
1010
1089
|
}
|
|
@@ -1016,7 +1095,8 @@ class AudioGen {
|
|
|
1016
1095
|
keyscale: d.keyscale ?? '',
|
|
1017
1096
|
timesignature: d.timesignature ?? '',
|
|
1018
1097
|
vocalLanguage: d.vocalLanguage ?? '',
|
|
1019
|
-
audioCodes: d.audioCodes
|
|
1098
|
+
audioCodes: d.audioCodes,
|
|
1099
|
+
seed: d.seed ?? 0
|
|
1020
1100
|
};
|
|
1021
1101
|
this._lastUnderstand = understood;
|
|
1022
1102
|
this._job.output({ understand: understood });
|
|
@@ -1032,10 +1112,16 @@ class AudioGen {
|
|
|
1032
1112
|
...(typeof d.gpuFallbackReason === 'number'
|
|
1033
1113
|
? { gpuFallbackReason: d.gpuFallbackReason }
|
|
1034
1114
|
: {}),
|
|
1115
|
+
...(typeof d.emittedFrames === 'number' ? { emittedFrames: d.emittedFrames } : {}),
|
|
1116
|
+
...(typeof d.arMs === 'number' ? { arMs: d.arMs } : {}),
|
|
1117
|
+
...(typeof d.conditionMs === 'number' ? { conditionMs: d.conditionMs } : {}),
|
|
1118
|
+
...(typeof d.flowMs === 'number' ? { flowMs: d.flowMs } : {}),
|
|
1119
|
+
...(typeof d.vocoderMs === 'number' ? { vocoderMs: d.vocoderMs } : {}),
|
|
1035
1120
|
...(typeof d.lyricsScore === 'number' ? { lyricsScore: d.lyricsScore } : {}),
|
|
1036
1121
|
...(this._lastLrc !== undefined ? { lrc: this._lastLrc } : {}),
|
|
1037
1122
|
...(typeof d.qualityScore === 'number' ? { qualityScore: d.qualityScore } : {}),
|
|
1038
|
-
...(this._lastUnderstand !== undefined ? { understand: this._lastUnderstand } : {})
|
|
1123
|
+
...(this._lastUnderstand !== undefined ? { understand: this._lastUnderstand } : {}),
|
|
1124
|
+
...(this._lastMetadata !== undefined ? { metadata: this._lastMetadata } : {})
|
|
1039
1125
|
};
|
|
1040
1126
|
this._job.end(stats, stats);
|
|
1041
1127
|
}
|
package/lib/fit.d.ts
CHANGED
|
@@ -1,22 +1,43 @@
|
|
|
1
1
|
export type AudiogenFitEngine = 'acestep' | 'minimax'
|
|
2
2
|
|
|
3
3
|
export interface AudiogenFitRequest {
|
|
4
|
-
/**
|
|
4
|
+
/**
|
|
5
|
+
* Engine that would run the load. Defaults to `acestep`. `minimax` is
|
|
6
|
+
* projected on desktop builds; on Android and iOS it is `unsupported-engine`.
|
|
7
|
+
*/
|
|
5
8
|
engine?: AudiogenFitEngine
|
|
6
|
-
/**
|
|
9
|
+
/**
|
|
10
|
+
* Directory holding the model GGUFs (the four ACE-Step stages, or the MiniMax
|
|
11
|
+
* LM and synth pair); explicit paths win over it.
|
|
12
|
+
*/
|
|
7
13
|
modelsDir?: string
|
|
8
14
|
textEncoderPath?: string
|
|
15
|
+
/** The ACE-Step or MiniMax LM GGUF. */
|
|
9
16
|
lmPath?: string
|
|
10
17
|
ditPath?: string
|
|
11
18
|
vaePath?: string
|
|
19
|
+
/** MiniMax synthesis GGUF; pairs with `lmPath`. */
|
|
20
|
+
synthPath?: string
|
|
12
21
|
|
|
13
22
|
/** > 0 requests the GPU stack, with the fallbacks a real load applies. */
|
|
14
23
|
gpuLayers?: number
|
|
24
|
+
/**
|
|
25
|
+
* MiniMax compute device, as `config.device` takes it. Without it, `gpuLayers`
|
|
26
|
+
* > 0 projects `auto` (a GPU when one is usable) and anything else `cpu`.
|
|
27
|
+
*/
|
|
28
|
+
device?: 'cpu' | 'gpu' | 'auto'
|
|
15
29
|
threads?: number
|
|
16
30
|
backendsDir?: string
|
|
17
31
|
|
|
18
|
-
/**
|
|
32
|
+
/**
|
|
33
|
+
* Longest single generation the projection must accommodate. MiniMax converts
|
|
34
|
+
* it to 25 semantic frames per second, as `run()` does.
|
|
35
|
+
*/
|
|
19
36
|
durationSeconds?: number
|
|
37
|
+
/** MiniMax semantic-frame cap. Cannot be combined with `durationSeconds`. Defaults to 300. */
|
|
38
|
+
maxFrames?: number
|
|
39
|
+
/** MiniMax tokenized prompt length: caption, lyrics and template. Defaults to 1024. */
|
|
40
|
+
promptTokens?: number
|
|
20
41
|
textTokens?: number
|
|
21
42
|
lyricTokens?: number
|
|
22
43
|
/** 0 derives it from the text and lyric budgets. */
|
|
@@ -38,7 +59,10 @@ export type AudiogenFitStatus = 'fits' | 'does-not-fit' | 'error'
|
|
|
38
59
|
|
|
39
60
|
/** One pipeline stage, as the fitter placed it. */
|
|
40
61
|
export interface AudiogenFitStage {
|
|
41
|
-
/**
|
|
62
|
+
/**
|
|
63
|
+
* ACE-Step: `textenc` | `lm` | `cond` | `detok` | `dit` | `vae`.
|
|
64
|
+
* MiniMax: `lm` | `depth` | `cond` | `dit` | `vocoder`.
|
|
65
|
+
*/
|
|
42
66
|
name: string
|
|
43
67
|
/** Backend the stage loads on, after placement. */
|
|
44
68
|
deviceName: string
|
|
@@ -46,11 +70,11 @@ export interface AudiogenFitStage {
|
|
|
46
70
|
weightsBytes: number
|
|
47
71
|
/** The portion mapped in place from the file, on the host. */
|
|
48
72
|
weightsMmapBytes: number
|
|
49
|
-
/** The LM
|
|
73
|
+
/** The KV cache of the LM, or of the MiniMax depth decoder. */
|
|
50
74
|
stateBytes: number
|
|
51
75
|
/** Largest compute arena the stage holds at once. */
|
|
52
76
|
computeBytes: number
|
|
53
|
-
/** Stage-phase host buffers: masks, latents, PCM. */
|
|
77
|
+
/** Stage-phase host buffers: masks, latents, PCM, staged graph inputs. */
|
|
54
78
|
hostBytes: number
|
|
55
79
|
}
|
|
56
80
|
|
|
@@ -63,6 +87,7 @@ export interface AudiogenFitResult {
|
|
|
63
87
|
*/
|
|
64
88
|
reason: string
|
|
65
89
|
modelName: string
|
|
90
|
+
/** ACE-Step turbo checkpoint; always false for MiniMax. */
|
|
66
91
|
isTurbo: boolean
|
|
67
92
|
deviceName: string
|
|
68
93
|
deviceIsCpu: boolean
|
|
@@ -70,7 +95,10 @@ export interface AudiogenFitResult {
|
|
|
70
95
|
deviceSharesHostMemory: boolean
|
|
71
96
|
deviceFreeBytes: number
|
|
72
97
|
deviceTotalBytes: number
|
|
73
|
-
/**
|
|
98
|
+
/**
|
|
99
|
+
* Peak across the pipeline phases under the projected residency mode. MiniMax
|
|
100
|
+
* keeps its stages resident, so this is the steady state of repeated runs.
|
|
101
|
+
*/
|
|
74
102
|
deviceBytes: number
|
|
75
103
|
hostBytes: number
|
|
76
104
|
/** Host capacity, which is a budget of its own where the device has its own memory. */
|
package/lib/fit.js
CHANGED
|
@@ -8,9 +8,9 @@ const { resolveBackendsDir } = require('./backends')
|
|
|
8
8
|
* answers the same as the stage itself, so this can run before anything is
|
|
9
9
|
* downloaded.
|
|
10
10
|
*
|
|
11
|
-
* `engine` defaults to `acestep
|
|
12
|
-
*
|
|
13
|
-
* `reason: "unsupported-engine"`.
|
|
11
|
+
* `engine` defaults to `acestep`; `minimax` projects a MiniMax-Music3 pair on
|
|
12
|
+
* desktop builds. An engine the build carries no fitter for comes back as
|
|
13
|
+
* `status: "error"` with `reason: "unsupported-engine"`.
|
|
14
14
|
*
|
|
15
15
|
* A model set the engine cannot read comes back as `status: "error"` with its
|
|
16
16
|
* own reason; a broken request, or a host with no native binding, throws.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@qvac/audiogen-ggml",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.6.0",
|
|
4
4
|
"description": "Audio generation (music) addon for qvac",
|
|
5
5
|
"addon": true,
|
|
6
6
|
"engines": {
|
|
@@ -144,7 +144,7 @@
|
|
|
144
144
|
],
|
|
145
145
|
"devDependencies": {
|
|
146
146
|
"@qvac/registry-client": "^0.6.1",
|
|
147
|
-
"bare-pack": "^
|
|
147
|
+
"bare-pack": "^2.2.3",
|
|
148
148
|
"brittle": "^3.17.0",
|
|
149
149
|
"cmake-bare": "^1.7.5",
|
|
150
150
|
"cmake-vcpkg": "^1.1.0",
|
|
@@ -173,10 +173,10 @@
|
|
|
173
173
|
},
|
|
174
174
|
"types": "index.d.ts",
|
|
175
175
|
"optionalDependencies": {
|
|
176
|
-
"@qvac/audiogen-ggml-linux-x64": "0.
|
|
177
|
-
"@qvac/audiogen-ggml-linux-arm64": "0.
|
|
178
|
-
"@qvac/audiogen-ggml-darwin-arm64": "0.
|
|
179
|
-
"@qvac/audiogen-ggml-darwin-x64": "0.
|
|
180
|
-
"@qvac/audiogen-ggml-win32-x64": "0.
|
|
176
|
+
"@qvac/audiogen-ggml-linux-x64": "0.6.0",
|
|
177
|
+
"@qvac/audiogen-ggml-linux-arm64": "0.6.0",
|
|
178
|
+
"@qvac/audiogen-ggml-darwin-arm64": "0.6.0",
|
|
179
|
+
"@qvac/audiogen-ggml-darwin-x64": "0.6.0",
|
|
180
|
+
"@qvac/audiogen-ggml-win32-x64": "0.6.0"
|
|
181
181
|
}
|
|
182
182
|
}
|
|
@@ -38,8 +38,9 @@ async function loadAudioGen({
|
|
|
38
38
|
}
|
|
39
39
|
|
|
40
40
|
// Drive one generation. Returns { data: { sampleCount, sampleRate, channels,
|
|
41
|
-
// durationMs, chunkCount, stages, stats } }. `stages` is the set of
|
|
42
|
-
// stage names seen (lm/dit/vae); `
|
|
41
|
+
// durationMs, chunkCount, stages, stageTotals, stats } }. `stages` is the set of
|
|
42
|
+
// progress stage names seen (lm/dit/vae); `stageTotals` maps each stage to the
|
|
43
|
+
// last total it reported; `stats` is the terminal run stats.
|
|
43
44
|
async function runAudioGen(gen, { caption, opts = {} } = {}) {
|
|
44
45
|
const response = await gen.run(caption, opts)
|
|
45
46
|
return collectAudioGenResponse(response)
|
|
@@ -48,12 +49,16 @@ async function runAudioGen(gen, { caption, opts = {} } = {}) {
|
|
|
48
49
|
async function collectAudioGenResponse(response) {
|
|
49
50
|
const chunks = []
|
|
50
51
|
const stages = new Set()
|
|
52
|
+
const stageTotals = {}
|
|
51
53
|
let sampleRate = 0
|
|
52
54
|
let channels = 0
|
|
53
55
|
|
|
54
56
|
for await (const item of response.iterate()) {
|
|
55
57
|
if (item && item.progress) {
|
|
56
|
-
if (item.progress.stage)
|
|
58
|
+
if (item.progress.stage) {
|
|
59
|
+
stages.add(item.progress.stage)
|
|
60
|
+
stageTotals[item.progress.stage] = item.progress.total
|
|
61
|
+
}
|
|
57
62
|
continue
|
|
58
63
|
}
|
|
59
64
|
if (item && item.outputArray) {
|
|
@@ -94,6 +99,7 @@ async function collectAudioGenResponse(response) {
|
|
|
94
99
|
durationMs,
|
|
95
100
|
chunkCount: chunks.length,
|
|
96
101
|
stages: [...stages],
|
|
102
|
+
stageTotals,
|
|
97
103
|
peak: peakAbs,
|
|
98
104
|
rms,
|
|
99
105
|
chunks,
|