@qvac/audiogen-ggml 0.9.1 → 0.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +48 -0
- package/README.md +88 -27
- package/audiogen.d.ts +4 -0
- package/examples/generate-with-backends.js +58 -0
- package/index.d.ts +15 -4
- package/index.js +24 -2
- package/lib/fit.d.ts +31 -49
- package/lib/fit.js +4 -5
- package/package.json +13 -10
package/CHANGELOG.md
CHANGED
|
@@ -5,6 +5,54 @@ All notable changes to this project will be documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
+
## [Unreleased]
|
|
9
|
+
|
|
10
|
+
## [0.10.1] - 2026-11-10
|
|
11
|
+
|
|
12
|
+
### Added
|
|
13
|
+
|
|
14
|
+
- ACE-Step `backend`, `lmBackend`, and `dumpStagesDir` controls, with matching
|
|
15
|
+
backend selection in `assessFit()` and opt-in WebGPU source builds.
|
|
16
|
+
- Apple Core ML VAE/conditioner support and MiniMax `coremlConditionerPath`
|
|
17
|
+
for generation and preflight; MiniMax Core ML memory projections remain `unknown`.
|
|
18
|
+
|
|
19
|
+
### Fixed
|
|
20
|
+
|
|
21
|
+
- ACE-Step `normalizeLoudness: false` returns the engine's raw output. The
|
|
22
|
+
addon rescaled every non-edit result to a 0.9 peak whatever the option said,
|
|
23
|
+
so the flag only switched off the engine's percentile pass. Samples the VAE
|
|
24
|
+
leaves outside `[-1, 1]` now saturate at int16 full scale on that path.
|
|
25
|
+
- ACE-Step lego stems keep their mix gain relative to the source; the same
|
|
26
|
+
rescale raised or lowered each stem to a 0.9 peak.
|
|
27
|
+
- Include independent LM device pools in `assessFit()` when the primary
|
|
28
|
+
pipeline runs on CPU, counting the host peak once.
|
|
29
|
+
|
|
30
|
+
## [0.10.0] - 2026-11-05
|
|
31
|
+
|
|
32
|
+
### Breaking
|
|
33
|
+
|
|
34
|
+
- `assessFit` returns `{ status, devices, stats }`: each device the pipeline
|
|
35
|
+
uses, then the host CPU. `status` is `fits`, `does-not-fit` or `unknown`. An
|
|
36
|
+
unreadable model or an invalid request rejects
|
|
37
|
+
([#5474](https://github.com/tetherto/qvac/pull/5474)).
|
|
38
|
+
- On an iOS device, `assessFit` checks the load and its margin against the
|
|
39
|
+
app's memory limit ([#5474](https://github.com/tetherto/qvac/pull/5474)).
|
|
40
|
+
|
|
41
|
+
### Changed
|
|
42
|
+
|
|
43
|
+
- Update the native ACE-Step dependency and reject unusable automatic GPU backends.
|
|
44
|
+
|
|
45
|
+
- Update the shared `speech-cpp` native dependency to `2026-10-29`
|
|
46
|
+
([#5420](https://github.com/tetherto/qvac/pull/5420),
|
|
47
|
+
[#5447](https://github.com/tetherto/qvac/pull/5447)).
|
|
48
|
+
- Update the shared `speech-cpp` native dependency to `2026-11-03`
|
|
49
|
+
([registry #474](https://github.com/tetherto/qvac-registry-vcpkg/pull/474)).
|
|
50
|
+
- `assessFit` names the device and each stage's device by its ggml device name
|
|
51
|
+
(`CPU`, `CUDA0`, `MTL0`, `GPUOpenCL`), counts Android's OpenCL GPU as sharing
|
|
52
|
+
host memory, and gives a device that reports no memory of its own the
|
|
53
|
+
host's free and total figures
|
|
54
|
+
([#5447](https://github.com/tetherto/qvac/pull/5447)).
|
|
55
|
+
|
|
8
56
|
## [0.9.1] - 2026-10-28
|
|
9
57
|
|
|
10
58
|
### Added
|
package/README.md
CHANGED
|
@@ -165,6 +165,51 @@ the engine time into `arMs`, `conditionMs`, `flowMs` and `vocoderMs`, and
|
|
|
165
165
|
the song ended on its own. See
|
|
166
166
|
[`examples/generate-music-minimax.js`](examples/generate-music-minimax.js).
|
|
167
167
|
|
|
168
|
+
### ACE-Step backend selection and diagnostics
|
|
169
|
+
|
|
170
|
+
`config.backend` selects the primary native backend by name or device name,
|
|
171
|
+
for example `cpu`, `MTL0`, `CUDA0`, or `WebGPU`. `config.lmBackend` selects
|
|
172
|
+
an independent LM backend. Empty or `auto` keeps native automatic placement
|
|
173
|
+
and the existing `useGPU`/`nGpuLayers` behavior. An explicit unavailable
|
|
174
|
+
backend fails instead of silently selecting another one. Use the same
|
|
175
|
+
`backend` and `lmBackend` values in `assessFit()` to project that placement.
|
|
176
|
+
The returned device rows include the independent LM pool even when the
|
|
177
|
+
primary backend is CPU; the host CPU remains the last row and appears once.
|
|
178
|
+
|
|
179
|
+
`config.dumpStagesDir` enables native intermediate tensor dumps in an existing
|
|
180
|
+
directory. It is disabled when omitted or empty. The directory is not created
|
|
181
|
+
by the addon. See
|
|
182
|
+
[`examples/generate-with-backends.js`](examples/generate-with-backends.js).
|
|
183
|
+
|
|
184
|
+
WebGPU is an opt-in native Dawn backend. Build from source with
|
|
185
|
+
`bare-make generate -D AUDIOGEN_WEBGPU=ON`, then `bare-make build`,
|
|
186
|
+
`bare-make install`, and `npm run link:platform`. It is not included in the
|
|
187
|
+
default prebuilds. Select it with `backend: 'webgpu'` after building;
|
|
188
|
+
`backendId: 5` maps to `webgpu`.
|
|
189
|
+
Native WebGPU support is unmeasured; its automatic placement keeps the LM
|
|
190
|
+
and detokenizer on CPU. Independent backend selection does not override
|
|
191
|
+
native operation or model compatibility checks. JavaScript packages do not
|
|
192
|
+
include Hexagon.
|
|
193
|
+
|
|
194
|
+
### Apple Core ML sidecars
|
|
195
|
+
|
|
196
|
+
Apple builds enable the native Core ML sidecar support. An ACE-Step VAE
|
|
197
|
+
sidecar is discovered next to its GGUF: `vae-BF16.gguf` uses
|
|
198
|
+
`vae-decoder.mlmodelc` (the native resolver removes the quantization suffix). Absent
|
|
199
|
+
sidecars retain the native VAE route. Its existing native failure/fallback
|
|
200
|
+
policy is unchanged. ACE-Step preflight sizes the native ggml VAE route;
|
|
201
|
+
it does not include the additional Core ML sidecar allocations.
|
|
202
|
+
|
|
203
|
+
MiniMax can instead receive `config.coremlConditionerPath`, the absolute path
|
|
204
|
+
to a compiled conditioner `.mlmodelc` on the inference host. Only its
|
|
205
|
+
conditioner convolution uses Core ML with CPU and ANE allowed; the native
|
|
206
|
+
layer mix and resampling remain on CPU, while the other stages retain
|
|
207
|
+
`device`/`useGPU` placement. This option alone does not enable the full Core ML
|
|
208
|
+
profile. An explicit invalid or unsupported conditioner fails to load.
|
|
209
|
+
Forward the same `coremlConditionerPath` to `assessFit()`; it returns
|
|
210
|
+
`{ status: 'unknown', devices: [], stats: {} }` because Core ML resident
|
|
211
|
+
memory cannot be projected.
|
|
212
|
+
|
|
168
213
|
### MiniMax full Core ML on macOS
|
|
169
214
|
|
|
170
215
|
On macOS 15+, `speech-cpp[audiogen]` includes the full Core ML runtime. Use
|
|
@@ -192,7 +237,9 @@ must accommodate the prompt and generation; no request is truncated to fit it.
|
|
|
192
237
|
The programs stay resident between generations and are released on unload.
|
|
193
238
|
Core ML returns `backendId: 99` (`other`); native-only `backendDevice` and
|
|
194
239
|
`gpuFallbackReason` are omitted. Memory preflight returns
|
|
195
|
-
`
|
|
240
|
+
`{ status: 'unknown', devices: [], stats: {} }` for this route. The full
|
|
241
|
+
Core ML route remains experimental; exposing its controls does not establish
|
|
242
|
+
audio parity or a performance improvement for a profile.
|
|
196
243
|
|
|
197
244
|
CPU/ANE initial-logit parity against the native reference is still under
|
|
198
245
|
validation; the measured device policies are not a claim of identical sampled
|
|
@@ -663,6 +710,9 @@ runnable end-to-end script (`npm run example`).
|
|
|
663
710
|
| `inferenceSteps` / `shift` | Advanced; leave unset to auto-tune per DiT. `run()` can override both per call. |
|
|
664
711
|
| `cfgScale` | Default MiniMax flow guidance scale; `0` uses the model default. |
|
|
665
712
|
| `nGpuLayers` | GPU layers to offload when `useGPU` is set (99 = all). |
|
|
713
|
+
| `backend`, `lmBackend` | ACE-Step primary and independent LM backend names; empty/auto preserves native selection. |
|
|
714
|
+
| `dumpStagesDir` | ACE-Step existing directory for intermediate tensor dumps; omitted/empty disables dumps. |
|
|
715
|
+
| `coremlConditionerPath` | MiniMax compiled conditioner `.mlmodelc`; leaves other stages on the selected native device. |
|
|
666
716
|
| `threads` | CPU thread count (0 / unset = hardware default). |
|
|
667
717
|
| `backendsDir` | Advanced; directory scanned, as given, for dlopen'd ggml backend modules. Unset scans the addon's own, next to its native module, which is where the platform package and `bare-link` put them. |
|
|
668
718
|
|
|
@@ -689,7 +739,7 @@ wrapped by a level-gated `QvacLogger`.
|
|
|
689
739
|
| `lmPhase1` | Allow the LM to infer missing metadata before generating semantic codes. |
|
|
690
740
|
| `simpleMode` | Expand the caption query into a full request (caption, lyrics, unset metadata) before synthesis. |
|
|
691
741
|
| `rewriteQuery` | LM FORMAT pass: rewrite the caption into a detailed description, preserving the lyric content. Requires real `lyrics` and `taskType: 'text2music'`; mutually exclusive with `simpleMode`. |
|
|
692
|
-
| `normalizeLoudness` | Percentile loudness normalization of generated audio (default `true`);
|
|
742
|
+
| `normalizeLoudness` | Percentile loudness normalization of generated audio, then a 0.9 peak (default `true`); `false` returns the raw engine output. Edits and lego stems are never normalized. |
|
|
693
743
|
| `generateLrc` | Synchronized lyric timestamps: `stats.lrc` (LRC text) + `stats.lyricsScore`. Requires lyrics and `taskType: 'text2music'`. |
|
|
694
744
|
| `computeQualityScore` | Teacher-forced LM quality score of the generated codes; `stats.qualityScore` in `[0, 1]`. Requires `taskType: 'text2music'`. |
|
|
695
745
|
| `dcwEnabled` / `dcwScaler` / `dcwHighScaler` | Haar DCW correction controls. |
|
|
@@ -762,18 +812,30 @@ const fit = await assessFit({
|
|
|
762
812
|
lyricTokens: 256
|
|
763
813
|
})
|
|
764
814
|
|
|
765
|
-
fit.status // 'fits' | 'does-not-fit' | '
|
|
766
|
-
fit.
|
|
767
|
-
fit.
|
|
768
|
-
fit.isTurbo
|
|
769
|
-
fit.deviceName
|
|
770
|
-
fit.deviceBytes // peak across the pipeline phases under the projected residency mode
|
|
771
|
-
fit.hostBytes
|
|
772
|
-
fit.hostFreeBytes // host capacity, a budget of its own where the device has its own memory
|
|
773
|
-
fit.stagesResident
|
|
774
|
-
fit.report
|
|
815
|
+
fit.status // 'fits' | 'does-not-fit' | 'unknown'
|
|
816
|
+
fit.devices // [{ id, name, unified, usedBytes, freeBytes, totalBytes }]
|
|
817
|
+
fit.stats // {}
|
|
775
818
|
```
|
|
776
819
|
|
|
820
|
+
`devices` holds each device the pipeline uses, then the host CPU, which holds
|
|
821
|
+
the CPU-placed stages, the mapped weights and the host-side buffers. A load on
|
|
822
|
+
the CPU is one row. `usedBytes` is the load's peak on that device across the
|
|
823
|
+
pipeline phases, under the projected residency mode. `id` is the PCI bus ID when
|
|
824
|
+
the backend reports one, so the same GPU matches across addons built on
|
|
825
|
+
different backends, and the device name otherwise. `unified` is true for memory
|
|
826
|
+
the host owns: the CPU, an integrated GPU, a GPU that shares the host's memory,
|
|
827
|
+
Android's OpenCL GPU, and a device that reports no memory of its own, which
|
|
828
|
+
takes the host's figures.
|
|
829
|
+
|
|
830
|
+
On an iOS device every unified row shares the app's own memory limit,
|
|
831
|
+
`os_proc_available_memory()`. Each such row's free memory is capped at that
|
|
832
|
+
limit, and a load whose unified rows together take more than the limit less
|
|
833
|
+
the margin does not fit. A limit of 0 means the app has none left.
|
|
834
|
+
|
|
835
|
+
`status` is `unknown`, with no rows, for a load the fitter cannot size:
|
|
836
|
+
MiniMax-Music3 with Core ML stages, which have no memory estimate, or a load
|
|
837
|
+
whose memory the engine fails to measure.
|
|
838
|
+
|
|
777
839
|
`modelsDir` holds the four stage GGUFs; `textEncoderPath`, `lmPath`, `ditPath`
|
|
778
840
|
and `vaePath` name them individually and win over it.
|
|
779
841
|
|
|
@@ -787,20 +849,18 @@ and `vaePath` name them individually and win over it.
|
|
|
787
849
|
| `withSourceAudio` | Projects the extra VAE-encoder phase a cover or reference request loads. |
|
|
788
850
|
| `keepStages` | `-1` mirrors the engine, `0` forces the staged projection, `1` all-resident. |
|
|
789
851
|
| `gpuLayers` | Greater than 0 requests the GPU stack, with the fallbacks a real load applies. |
|
|
852
|
+
| `backend`, `lmBackend` | ACE-Step placement selectors, as the generation config takes them. |
|
|
853
|
+
| `coremlConditionerPath` | MiniMax conditioner path; projects as `unknown` with no device rows. |
|
|
790
854
|
| `threads`, `backendsDir` | As the load takes them. |
|
|
791
855
|
| `marginBytes` | Free memory that must remain for the projection to count as fitting. Defaults to 256 MiB. |
|
|
792
856
|
|
|
793
|
-
`deviceSharesHostMemory` reports that the device pool is system RAM, so host
|
|
794
|
-
bytes compete with device bytes.
|
|
795
|
-
|
|
796
857
|
### MiniMax-Music3
|
|
797
858
|
|
|
798
859
|
`engine: 'minimax'` projects a MiniMax-Music3 pair on desktop builds, with the
|
|
799
860
|
same result shape. `modelsDir` holds the pair; `lmPath` and `synthPath` name the
|
|
800
861
|
two GGUFs and win over it. The weights and the synthesis graphs stay resident
|
|
801
|
-
between runs and the LM's cache is rebuilt by each run, so `
|
|
802
|
-
peak of repeated generations
|
|
803
|
-
`dit` and `vocoder`.
|
|
862
|
+
between runs and the LM's cache is rebuilt by each run, so `usedBytes` is the
|
|
863
|
+
peak of repeated generations.
|
|
804
864
|
|
|
805
865
|
```js
|
|
806
866
|
const fit = await assessFit({
|
|
@@ -812,18 +872,19 @@ const fit = await assessFit({
|
|
|
812
872
|
})
|
|
813
873
|
```
|
|
814
874
|
|
|
815
|
-
| Option | Description
|
|
816
|
-
| --------------------------------------- |
|
|
817
|
-
| `device` | `'cpu'`, `'gpu'` or `'auto'`, as `config.device` takes it. Without it, `gpuLayers` greater than 0 projects `'auto'` and anything else `'cpu'`.
|
|
818
|
-
| `durationSeconds`, `maxFrames` | The generation length, as `run()` takes it (25 frames per second); one or the other. Defaults to 300 frames.
|
|
819
|
-
| `promptTokens` | Tokenized prompt length: caption, lyrics and template. Defaults to 1024. A prompt above the checkpoint's limit, or one that leaves too little context for the frames,
|
|
820
|
-
| `threads`, `backendsDir`, `marginBytes` | As for ACE-Step.
|
|
875
|
+
| Option | Description |
|
|
876
|
+
| --------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
877
|
+
| `device` | `'cpu'`, `'gpu'` or `'auto'`, as `config.device` takes it. Without it, `gpuLayers` greater than 0 projects `'auto'` and anything else `'cpu'`. |
|
|
878
|
+
| `durationSeconds`, `maxFrames` | The generation length, as `run()` takes it (25 frames per second); one or the other. Defaults to 300 frames. |
|
|
879
|
+
| `promptTokens` | Tokenized prompt length: caption, lyrics and template. Defaults to 1024. A prompt above the checkpoint's limit, or one that leaves too little context for the frames, rejects with `workload-too-large`. |
|
|
880
|
+
| `threads`, `backendsDir`, `marginBytes` | As for ACE-Step. |
|
|
821
881
|
|
|
822
882
|
On Android and iOS, where MiniMax-Music3 is unavailable, `engine: 'minimax'`
|
|
823
|
-
|
|
883
|
+
rejects.
|
|
824
884
|
|
|
825
|
-
A model the engine cannot read
|
|
826
|
-
host with no native binding
|
|
885
|
+
A model the engine cannot read, an engine with no fitter, or a broken request
|
|
886
|
+
rejects with an `InvalidArgument` error, and a host with no native binding
|
|
887
|
+
rejects as well.
|
|
827
888
|
|
|
828
889
|
## Lifecycle
|
|
829
890
|
|
package/audiogen.d.ts
CHANGED
|
@@ -11,6 +11,10 @@ export interface AudioGenConfigurationParams {
|
|
|
11
11
|
useGPU?: boolean;
|
|
12
12
|
/** MiniMax compute device ('cpu' | 'gpu' | 'auto'); overrides useGPU. */
|
|
13
13
|
device?: string;
|
|
14
|
+
backend?: string;
|
|
15
|
+
lmBackend?: string;
|
|
16
|
+
dumpStagesDir?: string;
|
|
17
|
+
coremlConditionerPath?: string;
|
|
14
18
|
coremlProfileDir?: string;
|
|
15
19
|
coremlComputeUnits?: string;
|
|
16
20
|
nGpuLayers?: number;
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
'use strict'
|
|
2
|
+
|
|
3
|
+
const fs = require('bare-fs')
|
|
4
|
+
const proc = require('bare-process')
|
|
5
|
+
const { AudioGen, assessFit } = require('@qvac/audiogen-ggml')
|
|
6
|
+
|
|
7
|
+
async function collectAudio(response) {
|
|
8
|
+
const chunks = []
|
|
9
|
+
let sampleRate = 0
|
|
10
|
+
let channels = 0
|
|
11
|
+
for await (const chunk of response.iterate()) {
|
|
12
|
+
if (!chunk.outputArray) continue
|
|
13
|
+
chunks.push(
|
|
14
|
+
Buffer.from(
|
|
15
|
+
chunk.outputArray.buffer,
|
|
16
|
+
chunk.outputArray.byteOffset,
|
|
17
|
+
chunk.outputArray.byteLength
|
|
18
|
+
)
|
|
19
|
+
)
|
|
20
|
+
sampleRate = chunk.sampleRate
|
|
21
|
+
channels = chunk.channels
|
|
22
|
+
}
|
|
23
|
+
return { pcm: Buffer.concat(chunks), sampleRate, channels, stats: await response.await() }
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
async function main() {
|
|
27
|
+
const [modelDir, backend, lmBackend, dumpStagesDir, outputPath] = proc.argv.slice(2)
|
|
28
|
+
if (!modelDir || !backend || !lmBackend || !dumpStagesDir || !outputPath) {
|
|
29
|
+
throw new Error(
|
|
30
|
+
'Usage: bare examples/generate-with-backends.js MODEL_DIR BACKEND LM_BACKEND EXISTING_DUMP_DIR OUTPUT.wav'
|
|
31
|
+
)
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
console.log(await assessFit({ modelsDir: modelDir, backend, lmBackend }))
|
|
35
|
+
const generator = new AudioGen({
|
|
36
|
+
files: { modelDir },
|
|
37
|
+
config: { backend, lmBackend, dumpStagesDir }
|
|
38
|
+
})
|
|
39
|
+
try {
|
|
40
|
+
await generator.load()
|
|
41
|
+
const response = await generator.run('A warm acoustic instrumental with gentle guitar', {
|
|
42
|
+
lyrics: '[Instrumental]',
|
|
43
|
+
duration: 6,
|
|
44
|
+
seed: 7
|
|
45
|
+
})
|
|
46
|
+
const { pcm, sampleRate, channels, stats } = await collectAudio(response)
|
|
47
|
+
const encoded = AudioGen.encode(pcm, 'wav', { sampleRate, channels })
|
|
48
|
+
fs.writeFileSync(outputPath, encoded.data)
|
|
49
|
+
console.log(stats)
|
|
50
|
+
} finally {
|
|
51
|
+
await generator.destroy()
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
main().catch((error) => {
|
|
56
|
+
console.error(error)
|
|
57
|
+
proc.exitCode = 1
|
|
58
|
+
})
|
package/index.d.ts
CHANGED
|
@@ -53,6 +53,14 @@ export interface AudioGenRuntimeConfig {
|
|
|
53
53
|
coremlProfileDir?: string;
|
|
54
54
|
/** MiniMax only: explicit Core ML device policy; requires coremlProfileDir. */
|
|
55
55
|
coremlComputeUnits?: AudiogenMinimaxCoreMLComputeUnits;
|
|
56
|
+
/** MiniMax only: compiled conditioner .mlmodelc; the remaining stages use device/useGPU. */
|
|
57
|
+
coremlConditionerPath?: string;
|
|
58
|
+
/** ACE-Step primary backend name or device name; empty/auto preserves native selection. */
|
|
59
|
+
backend?: string;
|
|
60
|
+
/** ACE-Step LM backend, selected independently from the primary backend. */
|
|
61
|
+
lmBackend?: string;
|
|
62
|
+
/** ACE-Step diagnostics: existing directory receiving native intermediate tensor dumps. */
|
|
63
|
+
dumpStagesDir?: string;
|
|
56
64
|
/** ACE-Step only: GPU layers to offload when `useGPU` is set (99 = all). */
|
|
57
65
|
nGpuLayers?: number;
|
|
58
66
|
/** 0 = engine auto-picks. */
|
|
@@ -136,8 +144,11 @@ export interface GenerateOptions {
|
|
|
136
144
|
/**
|
|
137
145
|
* Percentile loudness normalization on the generated audio (default true):
|
|
138
146
|
* the 99.999th-percentile sample scales to full scale and the tiny tail
|
|
139
|
-
* above it clips, matching the reference loudness
|
|
140
|
-
*
|
|
147
|
+
* above it clips, matching the reference loudness; the result is then
|
|
148
|
+
* scaled to a 0.9 peak for int16 headroom. Set false for the raw engine
|
|
149
|
+
* output at its own gain: samples the VAE leaves outside [-1, 1] saturate
|
|
150
|
+
* at int16 full scale. Audio edits and lego stems are never normalized, so
|
|
151
|
+
* a stem keeps its mix gain relative to the source.
|
|
141
152
|
*/
|
|
142
153
|
normalizeLoudness?: boolean;
|
|
143
154
|
/**
|
|
@@ -445,7 +456,7 @@ export interface UnderstandOptions {
|
|
|
445
456
|
lmTopK?: number;
|
|
446
457
|
}
|
|
447
458
|
/** Name of a backend `AudiogenStats.backendId` can resolve to. */
|
|
448
|
-
export type AudiogenBackendName = 'cpu' | 'metal' | 'cuda' | 'vulkan' | 'opencl' | 'other';
|
|
459
|
+
export type AudiogenBackendName = 'cpu' | 'metal' | 'cuda' | 'vulkan' | 'opencl' | 'webgpu' | 'other';
|
|
449
460
|
/** `AudiogenStats.backendId` codes, named. Codes match @qvac/tts-ggml. */
|
|
450
461
|
export declare const AUDIOGEN_BACKEND_NAMES: Readonly<Record<number, AudiogenBackendName>>;
|
|
451
462
|
/** `undefined` for an unset or unrecognised id, never a guessed name. */
|
|
@@ -560,7 +571,7 @@ export type { DitVariant, ModelManifest, ModelSources, ResolveDitModelPathOption
|
|
|
560
571
|
export { encodePcm, pcmToWav, SUPPORTED_FORMATS as OUTPUT_FORMATS } from './lib/audio-format';
|
|
561
572
|
export type { OutputFormat, EncodeOptions, EncodedAudio } from './lib/audio-format';
|
|
562
573
|
export { assessFit } from './lib/fit';
|
|
563
|
-
export type { AudiogenFitRequest, AudiogenFitResult, AudiogenFitStatus } from './lib/fit';
|
|
574
|
+
export type { AudiogenFitRequest, AudiogenFitResult, AudiogenFitStatus, AudiogenFitDevice, AudiogenFitStats } from './lib/fit';
|
|
564
575
|
export { ERR_CODE_RANGE, ERR_CODES, QvacErrorAudioGen } from './error';
|
|
565
576
|
export { AudioEditOperationType, RepaintMode } from './audiogen';
|
|
566
577
|
export type { AudioGenConfigurationParams, AudioGenJobData, AudioGenBinding, AudioGenOutputCallback } from './audiogen';
|
package/index.js
CHANGED
|
@@ -48,6 +48,7 @@ exports.AUDIOGEN_BACKEND_NAMES = {
|
|
|
48
48
|
2: 'cuda',
|
|
49
49
|
3: 'vulkan',
|
|
50
50
|
4: 'opencl',
|
|
51
|
+
5: 'webgpu',
|
|
51
52
|
99: 'other'
|
|
52
53
|
};
|
|
53
54
|
/** `undefined` for an unset or unrecognised id, never a guessed name. */
|
|
@@ -365,8 +366,19 @@ function validateAcestepOptions(files, config) {
|
|
|
365
366
|
if (config.device !== undefined) {
|
|
366
367
|
throw invalidInput('ACE-Step does not accept device (use useGPU and nGpuLayers)');
|
|
367
368
|
}
|
|
368
|
-
|
|
369
|
-
|
|
369
|
+
validateRuntimeStrings(config);
|
|
370
|
+
if (config.coremlConditionerPath !== undefined ||
|
|
371
|
+
config.coremlProfileDir !== undefined ||
|
|
372
|
+
config.coremlComputeUnits !== undefined) {
|
|
373
|
+
throw invalidInput('coremlConditionerPath, coremlProfileDir and coremlComputeUnits are MiniMax-only options');
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
function validateRuntimeStrings(config) {
|
|
377
|
+
for (const key of ['backend', 'lmBackend', 'dumpStagesDir', 'coremlConditionerPath']) {
|
|
378
|
+
const value = config[key];
|
|
379
|
+
if (value !== undefined && (typeof value !== 'string' || value.includes('\0'))) {
|
|
380
|
+
throw invalidInput(`${key} must be a string without NUL characters`);
|
|
381
|
+
}
|
|
370
382
|
}
|
|
371
383
|
}
|
|
372
384
|
function validateMinimaxCoreMLConfig(config) {
|
|
@@ -384,6 +396,12 @@ function validateMinimaxCoreMLConfig(config) {
|
|
|
384
396
|
}
|
|
385
397
|
}
|
|
386
398
|
function validateMinimaxConfig(config) {
|
|
399
|
+
validateRuntimeStrings(config);
|
|
400
|
+
if (config.backend !== undefined ||
|
|
401
|
+
config.lmBackend !== undefined ||
|
|
402
|
+
config.dumpStagesDir !== undefined) {
|
|
403
|
+
throw invalidInput('MiniMax does not accept backend, lmBackend or dumpStagesDir');
|
|
404
|
+
}
|
|
387
405
|
validateMinimaxCoreMLConfig(config);
|
|
388
406
|
if (config.useGPU !== undefined && typeof config.useGPU !== 'boolean') {
|
|
389
407
|
throw invalidInput('useGPU must be a boolean');
|
|
@@ -610,6 +628,7 @@ class AudioGen {
|
|
|
610
628
|
threads,
|
|
611
629
|
useGPU: config.useGPU ?? false,
|
|
612
630
|
device: config.device,
|
|
631
|
+
coremlConditionerPath: config.coremlConditionerPath,
|
|
613
632
|
coremlProfileDir: config.coremlProfileDir,
|
|
614
633
|
coremlComputeUnits: config.coremlComputeUnits,
|
|
615
634
|
backendsDir
|
|
@@ -634,6 +653,9 @@ class AudioGen {
|
|
|
634
653
|
inferenceSteps: this._defaultInferenceSteps,
|
|
635
654
|
shift: requireAcestepShift(config.shift ?? 0),
|
|
636
655
|
useGPU: config.useGPU ?? false,
|
|
656
|
+
backend: config.backend,
|
|
657
|
+
lmBackend: config.lmBackend,
|
|
658
|
+
dumpStagesDir: config.dumpStagesDir,
|
|
637
659
|
nGpuLayers: requireFiniteNumber(config.nGpuLayers ?? 99, 'nGpuLayers', true),
|
|
638
660
|
threads,
|
|
639
661
|
backendsDir
|
package/lib/fit.d.ts
CHANGED
|
@@ -3,7 +3,7 @@ export type AudiogenFitEngine = 'acestep' | 'minimax'
|
|
|
3
3
|
export interface AudiogenFitRequest {
|
|
4
4
|
/**
|
|
5
5
|
* Engine that would run the load. Defaults to `acestep`. `minimax` is
|
|
6
|
-
* projected on desktop builds; on Android and iOS it
|
|
6
|
+
* projected on desktop builds; on Android and iOS it rejects.
|
|
7
7
|
*/
|
|
8
8
|
engine?: AudiogenFitEngine
|
|
9
9
|
/**
|
|
@@ -19,6 +19,10 @@ export interface AudiogenFitRequest {
|
|
|
19
19
|
/** MiniMax synthesis GGUF; pairs with `lmPath`. */
|
|
20
20
|
synthPath?: string
|
|
21
21
|
|
|
22
|
+
/** ACE-Step primary backend name/device; empty/auto preserves native selection. */
|
|
23
|
+
backend?: string
|
|
24
|
+
/** ACE-Step LM backend, independent of the primary backend. */
|
|
25
|
+
lmBackend?: string
|
|
22
26
|
/** > 0 requests the GPU stack, with the fallbacks a real load applies. */
|
|
23
27
|
gpuLayers?: number
|
|
24
28
|
/**
|
|
@@ -26,7 +30,9 @@ export interface AudiogenFitRequest {
|
|
|
26
30
|
* > 0 projects `auto` (a GPU when one is usable) and anything else `cpu`.
|
|
27
31
|
*/
|
|
28
32
|
device?: 'cpu' | 'gpu' | 'auto'
|
|
29
|
-
/** MiniMax
|
|
33
|
+
/** MiniMax compiled conditioner. Core ML projection returns status unknown with no device rows. */
|
|
34
|
+
coremlConditionerPath?: string
|
|
35
|
+
/** MiniMax full Core ML profile. Projection returns status unknown with no device rows. */
|
|
30
36
|
coremlProfileDir?: string
|
|
31
37
|
coremlComputeUnits?: 'CPU_ONLY' | 'CPU_AND_GPU' | 'CPU_AND_NE' | 'ALL'
|
|
32
38
|
threads?: number
|
|
@@ -58,62 +64,38 @@ export interface AudiogenFitRequest {
|
|
|
58
64
|
marginBytes?: number
|
|
59
65
|
}
|
|
60
66
|
|
|
61
|
-
|
|
67
|
+
/**
|
|
68
|
+
* `unknown` is a load the fitter cannot size: MiniMax with Core ML stages, or
|
|
69
|
+
* a failed measurement.
|
|
70
|
+
*/
|
|
71
|
+
export type AudiogenFitStatus = 'fits' | 'does-not-fit' | 'unknown'
|
|
62
72
|
|
|
63
|
-
|
|
64
|
-
|
|
73
|
+
export interface AudiogenFitDevice {
|
|
74
|
+
/** The PCI bus ID when the backend reports one, the device name otherwise. */
|
|
75
|
+
id: string
|
|
76
|
+
name: string
|
|
77
|
+
/** Memory the host owns: the host CPU, and a GPU that shares the host's memory. */
|
|
78
|
+
unified: boolean
|
|
65
79
|
/**
|
|
66
|
-
*
|
|
67
|
-
*
|
|
80
|
+
* The load's peak on this device across the pipeline phases. MiniMax keeps
|
|
81
|
+
* its stages resident, so its peak is the steady state of repeated runs.
|
|
68
82
|
*/
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
onGpu: boolean
|
|
73
|
-
weightsBytes: number
|
|
74
|
-
/** The portion mapped in place from the file, on the host. */
|
|
75
|
-
weightsMmapBytes: number
|
|
76
|
-
/** The KV cache of the LM, or of the MiniMax depth decoder. */
|
|
77
|
-
stateBytes: number
|
|
78
|
-
/** Largest compute arena the stage holds at once. */
|
|
79
|
-
computeBytes: number
|
|
80
|
-
/** Stage-phase host buffers: masks, latents, PCM, staged graph inputs. */
|
|
81
|
-
hostBytes: number
|
|
83
|
+
usedBytes: number
|
|
84
|
+
freeBytes: number
|
|
85
|
+
totalBytes: number
|
|
82
86
|
}
|
|
83
87
|
|
|
88
|
+
export type AudiogenFitStats = Record<string, never>
|
|
89
|
+
|
|
84
90
|
export interface AudiogenFitResult {
|
|
85
91
|
status: AudiogenFitStatus
|
|
86
92
|
/**
|
|
87
|
-
*
|
|
88
|
-
*
|
|
89
|
-
*
|
|
90
|
-
*/
|
|
91
|
-
reason: string
|
|
92
|
-
modelName: string
|
|
93
|
-
/** ACE-Step turbo checkpoint; always false for MiniMax. */
|
|
94
|
-
isTurbo: boolean
|
|
95
|
-
deviceName: string
|
|
96
|
-
deviceIsCpu: boolean
|
|
97
|
-
/** The device pool is system RAM, so host bytes compete with device bytes. */
|
|
98
|
-
deviceSharesHostMemory: boolean
|
|
99
|
-
deviceFreeBytes: number
|
|
100
|
-
deviceTotalBytes: number
|
|
101
|
-
/**
|
|
102
|
-
* Peak across the pipeline phases under the projected residency mode. MiniMax
|
|
103
|
-
* keeps its stages resident, so this is the steady state of repeated runs.
|
|
104
|
-
*/
|
|
105
|
-
deviceBytes: number
|
|
106
|
-
hostBytes: number
|
|
107
|
-
/** Host capacity, which is a budget of its own where the device has its own memory. */
|
|
108
|
-
hostFreeBytes: number
|
|
109
|
-
hostTotalBytes: number
|
|
110
|
-
stagesResident: boolean
|
|
111
|
-
/**
|
|
112
|
-
* One row per pipeline stage. `deviceBytes` is a peak across phases, so the
|
|
113
|
-
* rows are where the weights, state and compute split apart.
|
|
93
|
+
* Each device the pipeline uses, then the host CPU, which holds the
|
|
94
|
+
* CPU-placed stages, the mapped weights and the host-side buffers. One CPU
|
|
95
|
+
* row for a load on the CPU. Empty when `unknown`.
|
|
114
96
|
*/
|
|
115
|
-
|
|
116
|
-
|
|
97
|
+
devices: AudiogenFitDevice[]
|
|
98
|
+
stats: AudiogenFitStats
|
|
117
99
|
}
|
|
118
100
|
|
|
119
101
|
export function assessFit(request: AudiogenFitRequest): Promise<AudiogenFitResult>
|
package/lib/fit.js
CHANGED
|
@@ -7,12 +7,11 @@
|
|
|
7
7
|
* downloaded.
|
|
8
8
|
*
|
|
9
9
|
* `engine` defaults to `acestep`; `minimax` projects a MiniMax-Music3 pair on
|
|
10
|
-
* desktop builds.
|
|
11
|
-
* `status: "error"` with `reason: "unsupported-engine"`.
|
|
10
|
+
* desktop builds.
|
|
12
11
|
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
12
|
+
* An engine the build carries no fitter for, a model set the engine cannot
|
|
13
|
+
* read, a broken request, or a host with no native binding rejects. Fits run
|
|
14
|
+
* one at a time within the addon.
|
|
16
15
|
*/
|
|
17
16
|
async function assessFit(request) {
|
|
18
17
|
const binding = require('../binding.js')
|
package/package.json
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@qvac/audiogen-ggml",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.10.1",
|
|
4
4
|
"description": "Audio generation (music) addon for qvac",
|
|
5
5
|
"qvac": {
|
|
6
|
-
"ggml-speech": "2026-10-27"
|
|
6
|
+
"ggml-speech": "2026-10-27#1"
|
|
7
7
|
},
|
|
8
8
|
"engines": {
|
|
9
9
|
"bare": ">=1.19.0"
|
|
@@ -57,6 +57,8 @@
|
|
|
57
57
|
"test:package": "npm run build:ts && brittle test/package/package-contract.test.js",
|
|
58
58
|
"test:node": "node --test scripts/__tests__/*.test.js",
|
|
59
59
|
"test:integration": "npm run build:ts && brittle-bare test/integration/**/*.test.js",
|
|
60
|
+
"test:cpp:build": "bare-make generate -D BUILD_TESTING=ON -D BUILD_FUZZING=OFF && bare-make build --target audiogen_ggml_tests",
|
|
61
|
+
"test:cpp": "npm run test:cpp:build && ./build/audiogen_ggml_tests",
|
|
60
62
|
"test:benchmark:rtf": "npm run build:ts && brittle-bare test/benchmark/rtf-benchmark.test.js",
|
|
61
63
|
"test:benchmark:rtf:matrix": "node scripts/run-rtf-benchmark-matrix.js",
|
|
62
64
|
"fuzz:build": "bare-make generate -D BUILD_FUZZING=ON -D BUILD_TESTING=OFF -D FUZZTEST_FUZZING_MODE=OFF && bare-make build --target audiogen-config-parse-fuzz",
|
|
@@ -111,7 +113,8 @@
|
|
|
111
113
|
"README.md",
|
|
112
114
|
"CHANGELOG.md",
|
|
113
115
|
"NOTICE",
|
|
114
|
-
"LICENSE"
|
|
116
|
+
"LICENSE",
|
|
117
|
+
"examples/generate-with-backends.js"
|
|
115
118
|
],
|
|
116
119
|
"license": "Apache-2.0",
|
|
117
120
|
"repository": {
|
|
@@ -156,19 +159,19 @@
|
|
|
156
159
|
"bare-process": "^4.2.2"
|
|
157
160
|
},
|
|
158
161
|
"optionalDependencies": {
|
|
159
|
-
"@qvac/audiogen-ggml-linux-x64": "0.
|
|
160
|
-
"@qvac/audiogen-ggml-linux-arm64": "0.
|
|
161
|
-
"@qvac/audiogen-ggml-darwin-arm64": "0.
|
|
162
|
-
"@qvac/audiogen-ggml-darwin-x64": "0.
|
|
163
|
-
"@qvac/audiogen-ggml-win32-x64": "0.
|
|
162
|
+
"@qvac/audiogen-ggml-linux-x64": "0.10.1",
|
|
163
|
+
"@qvac/audiogen-ggml-linux-arm64": "0.10.1",
|
|
164
|
+
"@qvac/audiogen-ggml-darwin-arm64": "0.10.1",
|
|
165
|
+
"@qvac/audiogen-ggml-darwin-x64": "0.10.1",
|
|
166
|
+
"@qvac/audiogen-ggml-win32-x64": "0.10.1"
|
|
164
167
|
},
|
|
165
168
|
"overrides": {
|
|
166
169
|
"bare-module-lexer": "1.6.3"
|
|
167
170
|
},
|
|
168
171
|
"types": "index.d.ts",
|
|
169
172
|
"peerDependencies": {
|
|
170
|
-
"@qvac/audiogen-ggml-android-arm64": "0.
|
|
171
|
-
"@qvac/audiogen-ggml-ios": "0.
|
|
173
|
+
"@qvac/audiogen-ggml-android-arm64": "0.10.1",
|
|
174
|
+
"@qvac/audiogen-ggml-ios": "0.10.1"
|
|
172
175
|
},
|
|
173
176
|
"peerDependenciesMeta": {
|
|
174
177
|
"@qvac/audiogen-ggml-android-arm64": {
|