@qvac/inference 0.19.0 → 0.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (150) hide show
  1. package/NOTICE +95 -39
  2. package/dist/api/audio-gen.d.ts +42 -1
  3. package/dist/api/audio-gen.js +137 -51
  4. package/dist/api/delete-cache.d.ts +7 -1
  5. package/dist/api/delete-cache.js +11 -1
  6. package/dist/api/index.d.ts +2 -1
  7. package/dist/api/index.js +2 -1
  8. package/dist/api/text-to-speech.d.ts +31 -4
  9. package/dist/api/text-to-speech.js +117 -21
  10. package/dist/api/vector-index.d.ts +142 -0
  11. package/dist/api/vector-index.js +168 -0
  12. package/dist/api/world.d.ts +2 -2
  13. package/dist/errors/index.d.ts +12 -0
  14. package/dist/errors/index.js +21 -0
  15. package/dist/handlers/delete-cache.js +3 -0
  16. package/dist/handlers/vector-index.d.ts +2 -0
  17. package/dist/handlers/vector-index.js +40 -0
  18. package/dist/index.d.ts +1 -1
  19. package/dist/index.js +1 -1
  20. package/dist/models/registry/models.d.ts +351 -12
  21. package/dist/models/registry/models.js +2271 -1957
  22. package/dist/models/registry/resource-profiles.js +135 -0
  23. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +2 -0
  24. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +65 -0
  25. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +9 -1
  26. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +24 -7
  27. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +46 -0
  28. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +218 -0
  29. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +1 -1
  30. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +56 -225
  31. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +8 -0
  32. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +28 -0
  33. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +248 -0
  34. package/dist/plugins/builtin/audiogen-ggml/plugin.js +17 -1
  35. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +5 -2
  36. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +1 -2
  37. package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +1 -1
  38. package/dist/plugins/builtin/ggml-ocr/plugin.js +1 -2
  39. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +6 -4
  40. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.d.ts +1 -2
  41. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +0 -4
  42. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +2 -0
  43. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +6 -0
  44. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +9 -0
  45. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +96 -89
  46. package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.d.ts +0 -1
  47. package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.js +0 -31
  48. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +48 -27
  49. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +239 -135
  50. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +11 -4
  51. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +18 -16
  52. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +11 -4
  53. package/dist/plugins/builtin/llamacpp-completion/plugin.js +1 -2
  54. package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +1 -2
  55. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -38
  56. package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +9 -0
  57. package/dist/plugins/builtin/llamacpp-embedding/transform.js +42 -0
  58. package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +1 -1
  59. package/dist/plugins/builtin/nmtcpp-translation/plugin.js +1 -2
  60. package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +1 -1
  61. package/dist/plugins/builtin/parakeet-transcription/plugin.js +1 -2
  62. package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +1 -0
  63. package/dist/plugins/builtin/sdcpp-generation/ops/video.js +8 -2
  64. package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +9 -4
  65. package/dist/plugins/builtin/sdcpp-generation/plugin.js +51 -19
  66. package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +29 -0
  67. package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +66 -0
  68. package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +10 -0
  69. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +3 -5
  70. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +143 -28
  71. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +5 -2
  72. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +182 -66
  73. package/dist/plugins/builtin/tts-ggml/plugin.d.ts +267 -22
  74. package/dist/plugins/builtin/tts-ggml/plugin.js +94 -35
  75. package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +1 -1
  76. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +1 -2
  77. package/dist/plugins/index.d.ts +1 -1
  78. package/dist/plugins/index.js +1 -1
  79. package/dist/plugins/ops/load-model.js +22 -2
  80. package/dist/plugins/ops/translate.js +58 -12
  81. package/dist/plugins/ops/translatepsy-utils.d.ts +17 -0
  82. package/dist/plugins/ops/translatepsy-utils.js +32 -0
  83. package/dist/plugins/registry.d.ts +15 -0
  84. package/dist/plugins/registry.js +133 -15
  85. package/dist/plugins/turbovec-backend.d.ts +17 -0
  86. package/dist/plugins/turbovec-backend.js +99 -0
  87. package/dist/registry.js +4 -0
  88. package/dist/resources/model-fit/assess.js +169 -68
  89. package/dist/resources/model-fit/estimators/llm.d.ts +1 -0
  90. package/dist/resources/model-fit/estimators/llm.js +3 -1
  91. package/dist/resources/model-fit/floor.d.ts +31 -0
  92. package/dist/resources/model-fit/floor.js +65 -0
  93. package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +46 -0
  94. package/dist/resources/model-fit/native-probe/advisory-fit.js +214 -0
  95. package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +26 -0
  96. package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +171 -0
  97. package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +42 -0
  98. package/dist/resources/model-fit/native-probe/run-in-process-fit.js +192 -0
  99. package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +70 -0
  100. package/dist/resources/model-fit/native-probe/run-isolated-fit.js +372 -0
  101. package/dist/resources/native.js +5 -4
  102. package/dist/resources/types.d.ts +1 -3
  103. package/dist/runtime/env.d.ts +2 -0
  104. package/dist/runtime/env.js +9 -2
  105. package/dist/runtime/lifecycle.js +2 -0
  106. package/dist/runtime/model-registry.d.ts +3 -1
  107. package/dist/runtime/model-registry.js +2 -1
  108. package/dist/runtime/request-context.js +17 -0
  109. package/dist/runtime/vector-index-backend.d.ts +25 -0
  110. package/dist/runtime/vector-index-backend.js +1 -0
  111. package/dist/runtime/vector-index-registry.d.ts +76 -0
  112. package/dist/runtime/vector-index-registry.js +161 -0
  113. package/dist/schemas/assess-model-fit.d.ts +84 -0
  114. package/dist/schemas/assess-model-fit.js +72 -3
  115. package/dist/schemas/audio-gen.d.ts +482 -7
  116. package/dist/schemas/audio-gen.js +376 -17
  117. package/dist/schemas/batch-completion-stream.d.ts +10 -0
  118. package/dist/schemas/batch-completion-stream.js +5 -3
  119. package/dist/schemas/bci.d.ts +1 -1
  120. package/dist/schemas/common.d.ts +782 -19
  121. package/dist/schemas/common.js +8 -1
  122. package/dist/schemas/completion-event.d.ts +14 -1
  123. package/dist/schemas/completion-event.js +4 -0
  124. package/dist/schemas/completion-stream.d.ts +22 -0
  125. package/dist/schemas/completion-stream.js +41 -4
  126. package/dist/schemas/config.d.ts +48 -30
  127. package/dist/schemas/errors.d.ts +4 -0
  128. package/dist/schemas/errors.js +22 -0
  129. package/dist/schemas/index.d.ts +1 -0
  130. package/dist/schemas/index.js +1 -0
  131. package/dist/schemas/llamacpp-config.d.ts +4 -8
  132. package/dist/schemas/llamacpp-config.js +4 -4
  133. package/dist/schemas/load-model.d.ts +2604 -180
  134. package/dist/schemas/model-config-schemas.d.ts +228 -5
  135. package/dist/schemas/model-ops.d.ts +3 -0
  136. package/dist/schemas/model-ops.js +6 -0
  137. package/dist/schemas/plugin.d.ts +21 -11
  138. package/dist/schemas/sdcpp-config.d.ts +221 -4
  139. package/dist/schemas/sdcpp-config.js +128 -22
  140. package/dist/schemas/text-to-speech.d.ts +780 -19
  141. package/dist/schemas/text-to-speech.js +298 -44
  142. package/dist/schemas/transcription-config.js +7 -4
  143. package/dist/schemas/vector-index.d.ts +194 -0
  144. package/dist/schemas/vector-index.js +185 -0
  145. package/dist/utils/addon-responses.d.ts +30 -3
  146. package/dist/utils/aggregate-events.d.ts +3 -1
  147. package/dist/utils/aggregate-events.js +7 -1
  148. package/dist/utils/tts-stats.d.ts +15 -11
  149. package/dist/utils/tts-stats.js +33 -11
  150. package/package.json +29 -26
@@ -1,232 +1,63 @@
1
- var __addDisposableResource = (this && this.__addDisposableResource) || function (env, value, async) {
2
- if (value !== null && value !== void 0) {
3
- if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
4
- var dispose, inner;
5
- if (async) {
6
- if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
7
- dispose = value[Symbol.asyncDispose];
8
- }
9
- if (dispose === void 0) {
10
- if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
11
- dispose = value[Symbol.dispose];
12
- if (async) inner = dispose;
13
- }
14
- if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
15
- if (inner) dispose = function() { try { inner.call(this); } catch (e) { return Promise.reject(e); } };
16
- env.stack.push({ value: value, dispose: dispose, async: async });
17
- }
18
- else if (async) {
19
- env.stack.push({ async: true });
20
- }
21
- return value;
22
- };
23
- var __disposeResources = (this && this.__disposeResources) || (function (SuppressedError) {
24
- return function (env) {
25
- function fail(e) {
26
- env.error = env.hasError ? new SuppressedError(e, env.error, "An error was suppressed during disposal.") : e;
27
- env.hasError = true;
28
- }
29
- var r, s = 0;
30
- function next() {
31
- while (r = env.stack.pop()) {
32
- try {
33
- if (!r.async && s === 1) return s = 0, env.stack.push(r), Promise.resolve().then(next);
34
- if (r.dispose) {
35
- var result = r.dispose.call(r.value);
36
- if (r.async) return s |= 2, Promise.resolve(result).then(next, function(e) { fail(e); return next(); });
37
- }
38
- else s |= 1;
39
- }
40
- catch (e) {
41
- fail(e);
42
- }
43
- }
44
- if (s === 1) return env.hasError ? Promise.reject(env.error) : Promise.resolve();
45
- if (env.hasError) throw env.error;
46
- }
47
- return next();
48
- };
49
- })(typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
50
- var e = new Error(message);
51
- return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
52
- });
53
- import { AudioGen, audiogenBackendName, audiogenGpuFallbackReason } from '@qvac/audiogen-ggml';
54
- import { z } from 'zod';
55
- import { audioGenStatsSchema } from '../../../../schemas/audio-gen.js';
56
- import { graphicsDriverSchema } from '../../../../schemas/index.js';
57
- import { getEngineLogger } from '../../../../logging/index.js';
58
- import { attachBackendDiagnostics } from '../../../../profiling/backend-diagnostics.js';
59
1
  import { resolveAudioGenPcm } from './audio-gen-input.js';
60
- import { getModel } from '../../../../runtime/model-registry.js';
61
- import { getRequestRegistry, withRequestContext } from '../../../../runtime/index.js';
62
- import { generateRequestId } from '../../../../runtime/request-id.js';
63
- import { ModelOperationNotSupportedError } from '../../../../errors/index.js';
64
- export async function* audioGenStream(request) {
65
- const env_1 = { stack: [], error: void 0, hasError: false };
66
- try {
67
- const ctx = __addDisposableResource(env_1, await getRequestRegistry().begin({
68
- requestId: request.requestId ?? generateRequestId(),
69
- kind: 'audiogen',
70
- modelId: request.modelId
71
- }), true);
72
- const logger = withRequestContext(getEngineLogger(), ctx);
73
- const candidate = getModel(request.modelId);
74
- if (!(candidate instanceof AudioGen)) {
75
- throw new ModelOperationNotSupportedError(request.modelId, 'audiogen-ggml', 'audioGenStream', [], []);
76
- }
77
- const model = candidate;
78
- // A queued request can resume from begin() already aborted. It never owned
79
- // the model slot, so calling the model-scoped cancel here would interrupt
80
- // the earlier same-model generation that still owns it.
81
- if (ctx.signal.aborted) {
82
- yield {
83
- type: 'audioGenStream',
84
- done: true,
85
- stopReason: 'cancelled'
86
- };
87
- return;
88
- }
89
- let cancelPromise;
90
- const onAbort = () => {
91
- cancelPromise ??= model.cancel().catch((error) => {
92
- logger.warn(`[cancel] model.cancel() rejected during abort for modelId=${request.modelId}: ${error instanceof Error ? error.message : String(error)}`);
93
- });
94
- };
95
- ctx.signal.addEventListener('abort', onAbort, { once: true });
96
- if (ctx.signal.aborted)
97
- onAbort();
98
- ctx.scope.defer(async () => {
99
- ctx.signal.removeEventListener('abort', onAbort);
100
- await cancelPromise;
101
- });
102
- let response;
103
- try {
2
+ import { streamAudioGenRun } from './audio-gen-run.js';
3
+ export function audioGenStream(request) {
4
+ return streamAudioGenRun({
5
+ type: 'audioGenStream',
6
+ request,
7
+ async start(model, ctx, logger) {
104
8
  // Reference/source audio is decoded before the run is admitted so the
105
9
  // model slot is never held by a request that fails on input decoding.
106
- // Decoding can take real time, so the abort state is re-checked after it:
107
- // an abort that fired mid-decode found no active native job to cancel, and
108
- // starting the run afterwards would leave a ghost generation on the slot.
109
- const { referenceAudio, sourceAudio } = ctx.signal.aborted
110
- ? {}
111
- : await resolveAudioInputs(request, logger);
112
- if (!ctx.signal.aborted) {
113
- response = await model.run(request.caption, {
114
- ...(request.lyrics !== undefined && { lyrics: request.lyrics }),
115
- ...(request.seed !== undefined && { seed: request.seed }),
116
- ...(request.vocalLanguage !== undefined && { vocalLanguage: request.vocalLanguage }),
117
- ...(request.bpm !== undefined && { bpm: request.bpm }),
118
- ...(request.keyscale !== undefined && { keyscale: request.keyscale }),
119
- ...(request.timesignature !== undefined && { timesignature: request.timesignature }),
120
- ...(request.duration !== undefined && { duration: request.duration }),
121
- ...(request.maxFrames !== undefined && { maxFrames: request.maxFrames }),
122
- ...(request.inferenceSteps !== undefined && { inferenceSteps: request.inferenceSteps }),
123
- ...(request.cfgScale !== undefined && { cfgScale: request.cfgScale }),
124
- ...(request.lmTemperature !== undefined && { lmTemperature: request.lmTemperature }),
125
- ...(request.lmTopP !== undefined && { lmTopP: request.lmTopP }),
126
- ...(request.lmTopK !== undefined && { lmTopK: request.lmTopK }),
127
- ...(request.lmCfgScale !== undefined && { lmCfgScale: request.lmCfgScale }),
128
- ...(request.lmPhase1 !== undefined && { lmPhase1: request.lmPhase1 }),
129
- ...(request.dcwEnabled !== undefined && { dcwEnabled: request.dcwEnabled }),
130
- ...(request.dcwScaler !== undefined && { dcwScaler: request.dcwScaler }),
131
- ...(request.dcwHighScaler !== undefined && { dcwHighScaler: request.dcwHighScaler }),
132
- ...(request.taskType !== undefined && { taskType: request.taskType }),
133
- ...(request.audioCoverStrength !== undefined && {
134
- audioCoverStrength: request.audioCoverStrength
135
- }),
136
- ...(request.coverNoiseStrength !== undefined && {
137
- coverNoiseStrength: request.coverNoiseStrength
138
- }),
139
- ...(referenceAudio && { referenceAudio }),
140
- ...(sourceAudio && { sourceAudio })
141
- });
142
- for await (const chunk of response.iterate()) {
143
- if (ctx.signal.aborted)
144
- break;
145
- if ('progress' in chunk) {
146
- yield {
147
- type: 'audioGenStream',
148
- progress: chunk.progress,
149
- done: false
150
- };
151
- continue;
152
- }
153
- const pcm = new Uint8Array(chunk.outputArray.buffer, chunk.outputArray.byteOffset, chunk.outputArray.byteLength);
154
- yield {
155
- type: 'audioGenStream',
156
- data: Buffer.from(pcm).toString('base64'),
157
- sampleRate: chunk.sampleRate,
158
- channels: chunk.channels,
159
- bitsPerSample: Int16Array.BYTES_PER_ELEMENT * 8,
160
- done: false
161
- };
162
- }
163
- }
164
- }
165
- catch (error) {
166
- if (!ctx.signal.aborted)
167
- throw error;
168
- }
169
- if (ctx.signal.aborted) {
170
- yield {
171
- type: 'audioGenStream',
172
- done: true,
173
- stopReason: 'cancelled'
174
- };
175
- return;
10
+ const { referenceAudio, sourceAudio } = await resolveAudioInputs(request, logger);
11
+ if (ctx.signal.aborted)
12
+ return undefined;
13
+ return model.run(request.caption, {
14
+ ...(request.lyrics !== undefined && { lyrics: request.lyrics }),
15
+ ...(request.seed !== undefined && { seed: request.seed }),
16
+ ...(request.vocalLanguage !== undefined && { vocalLanguage: request.vocalLanguage }),
17
+ ...(request.bpm !== undefined && { bpm: request.bpm }),
18
+ ...(request.keyscale !== undefined && { keyscale: request.keyscale }),
19
+ ...(request.timesignature !== undefined && { timesignature: request.timesignature }),
20
+ ...(request.augmentCaptionWithMetadata !== undefined && {
21
+ augmentCaptionWithMetadata: request.augmentCaptionWithMetadata
22
+ }),
23
+ ...(request.simpleMode !== undefined && { simpleMode: request.simpleMode }),
24
+ ...(request.rewriteQuery !== undefined && { rewriteQuery: request.rewriteQuery }),
25
+ ...(request.generateLrc !== undefined && { generateLrc: request.generateLrc }),
26
+ ...(request.computeQualityScore !== undefined && {
27
+ computeQualityScore: request.computeQualityScore
28
+ }),
29
+ ...(request.normalizeLoudness !== undefined && {
30
+ normalizeLoudness: request.normalizeLoudness
31
+ }),
32
+ ...(request.guidanceScale !== undefined && { guidanceScale: request.guidanceScale }),
33
+ ...(request.track !== undefined && { track: request.track }),
34
+ ...(request.duration !== undefined && { duration: request.duration }),
35
+ ...(request.maxFrames !== undefined && { maxFrames: request.maxFrames }),
36
+ ...(request.inferenceSteps !== undefined && { inferenceSteps: request.inferenceSteps }),
37
+ ...(request.cfgScale !== undefined && { cfgScale: request.cfgScale }),
38
+ ...(request.lmTemperature !== undefined && { lmTemperature: request.lmTemperature }),
39
+ ...(request.lmTopP !== undefined && { lmTopP: request.lmTopP }),
40
+ ...(request.lmTopK !== undefined && { lmTopK: request.lmTopK }),
41
+ ...(request.lmCfgScale !== undefined && { lmCfgScale: request.lmCfgScale }),
42
+ ...(request.lmPhase1 !== undefined && { lmPhase1: request.lmPhase1 }),
43
+ ...(request.dcwEnabled !== undefined && { dcwEnabled: request.dcwEnabled }),
44
+ ...(request.dcwScaler !== undefined && { dcwScaler: request.dcwScaler }),
45
+ ...(request.dcwHighScaler !== undefined && { dcwHighScaler: request.dcwHighScaler }),
46
+ ...(request.audioCodes !== undefined && {
47
+ audioCodes: Int32Array.from(request.audioCodes)
48
+ }),
49
+ ...(request.taskType !== undefined && { taskType: request.taskType }),
50
+ ...(request.audioCoverStrength !== undefined && {
51
+ audioCoverStrength: request.audioCoverStrength
52
+ }),
53
+ ...(request.coverNoiseStrength !== undefined && {
54
+ coverNoiseStrength: request.coverNoiseStrength
55
+ }),
56
+ ...(referenceAudio && { referenceAudio }),
57
+ ...(sourceAudio && { sourceAudio })
58
+ });
176
59
  }
177
- const raw = await response.await();
178
- const stats = audioGenStatsSchema.parse(raw);
179
- const diagnostics = buildBackendDiagnostics(audiogenStats.parse(raw));
180
- const terminal = {
181
- type: 'audioGenStream',
182
- done: true,
183
- stopReason: 'completed',
184
- stats,
185
- ...(diagnostics && { diagnostics })
186
- };
187
- yield diagnostics ? attachBackendDiagnostics(terminal, diagnostics) : terminal;
188
- }
189
- catch (e_1) {
190
- env_1.error = e_1;
191
- env_1.hasError = true;
192
- }
193
- finally {
194
- const result_1 = __disposeResources(env_1);
195
- if (result_1)
196
- await result_1;
197
- }
198
- }
199
- // The addon reports gpuFallbackReason, which the wire shape deliberately does
200
- // not carry: it reaches callers named, on diagnostics.fallback.reason.
201
- const audiogenStats = audioGenStatsSchema.extend({
202
- gpuFallbackReason: z.number().optional()
203
- });
204
- /** An unrecognized GPU id yields no diagnostics rather than a guessed backend name. */
205
- function buildBackendDiagnostics(stats) {
206
- if (stats.backendDevice === undefined)
207
- return undefined;
208
- if (stats.backendDevice !== 1) {
209
- const fallback = gpuFallbackDetail(stats);
210
- return { selectedBackend: 'cpu', selectedDevice: 'cpu', ...(fallback && { fallback }) };
211
- }
212
- // A 'cpu' name against backendDevice 1 is the addon contradicting itself.
213
- const selectedBackend = audiogenBackendName(stats.backendId);
214
- if (selectedBackend === undefined || selectedBackend === 'cpu')
215
- return undefined;
216
- const graphicsApi = graphicsDriverSchema.safeParse(selectedBackend);
217
- return {
218
- selectedBackend,
219
- selectedDevice: 'gpu',
220
- ...(graphicsApi.success && { graphicsApi: graphicsApi.data })
221
- };
222
- }
223
- // `none` and `not-requested` describe a run that never lost a GPU, and an
224
- // unmapped code must not become a guessed reason.
225
- function gpuFallbackDetail(stats) {
226
- const reason = audiogenGpuFallbackReason(stats.gpuFallbackReason);
227
- if (reason === undefined || reason === 'none' || reason === 'not-requested')
228
- return undefined;
229
- return { requestedDevice: 'gpu', reason };
60
+ });
230
61
  }
231
62
  /**
232
63
  * Decode both optional audio inputs concurrently. When both fail, the first
@@ -0,0 +1,8 @@
1
+ import type { AudioUnderstandRequest, AudioUnderstandResponse } from '../../../../schemas/audio-gen.js';
2
+ /**
3
+ * ACE-Step's reverse pipeline: encode a recording, recover its FSQ semantic
4
+ * codes, and let the LM describe it. Shares the generation path's admission and
5
+ * cancellation, and streams the description as one `understand` frame before
6
+ * the terminal frame repeats it on `stats`.
7
+ */
8
+ export declare function audioUnderstand(request: AudioUnderstandRequest): AsyncGenerator<AudioUnderstandResponse>;
@@ -0,0 +1,28 @@
1
+ import { resolveAudioGenPcm } from './audio-gen-input.js';
2
+ import { streamAudioGenRun } from './audio-gen-run.js';
3
+ /**
4
+ * ACE-Step's reverse pipeline: encode a recording, recover its FSQ semantic
5
+ * codes, and let the LM describe it. Shares the generation path's admission and
6
+ * cancellation, and streams the description as one `understand` frame before
7
+ * the terminal frame repeats it on `stats`.
8
+ */
9
+ export function audioUnderstand(request) {
10
+ return streamAudioGenRun({
11
+ type: 'audioUnderstand',
12
+ request,
13
+ async start(model, ctx) {
14
+ // Decoded before the job is admitted, so the model slot is never held by
15
+ // a request that fails on input decoding.
16
+ const sourceAudio = await resolveAudioGenPcm(request.sourceAudio, 'sourceAudio');
17
+ if (ctx.signal.aborted)
18
+ return undefined;
19
+ return model.understand(sourceAudio, {
20
+ ...(request.seed !== undefined && { seed: request.seed }),
21
+ ...(request.vocalLanguage !== undefined && { vocalLanguage: request.vocalLanguage }),
22
+ ...(request.lmTemperature !== undefined && { lmTemperature: request.lmTemperature }),
23
+ ...(request.lmTopP !== undefined && { lmTopP: request.lmTopP }),
24
+ ...(request.lmTopK !== undefined && { lmTopK: request.lmTopK })
25
+ });
26
+ }
27
+ });
28
+ }
@@ -261,6 +261,7 @@ export declare const audioGenPlugin: {
261
261
  audioGenStream: import("../../../schemas/index.js").PluginHandlerDefinition<import("zod").ZodObject<{
262
262
  type: import("zod").ZodLiteral<"audioGenStream">;
263
263
  requestId: import("zod").ZodOptional<import("zod").ZodString>;
264
+ audioCodes: import("zod").ZodOptional<import("zod").ZodArray<import("zod").ZodNumber>>;
264
265
  referenceAudio: import("zod").ZodOptional<import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
265
266
  type: import("zod").ZodLiteral<"base64">;
266
267
  value: import("zod").ZodString;
@@ -283,6 +284,27 @@ export declare const audioGenPlugin: {
283
284
  bpm: import("zod").ZodOptional<import("zod").ZodNumber>;
284
285
  keyscale: import("zod").ZodOptional<import("zod").ZodString>;
285
286
  timesignature: import("zod").ZodOptional<import("zod").ZodString>;
287
+ augmentCaptionWithMetadata: import("zod").ZodOptional<import("zod").ZodBoolean>;
288
+ simpleMode: import("zod").ZodOptional<import("zod").ZodBoolean>;
289
+ rewriteQuery: import("zod").ZodOptional<import("zod").ZodBoolean>;
290
+ generateLrc: import("zod").ZodOptional<import("zod").ZodBoolean>;
291
+ computeQualityScore: import("zod").ZodOptional<import("zod").ZodBoolean>;
292
+ normalizeLoudness: import("zod").ZodOptional<import("zod").ZodBoolean>;
293
+ guidanceScale: import("zod").ZodOptional<import("zod").ZodNumber>;
294
+ track: import("zod").ZodOptional<import("zod").ZodEnum<{
295
+ vocals: "vocals";
296
+ backing_vocals: "backing_vocals";
297
+ drums: "drums";
298
+ bass: "bass";
299
+ guitar: "guitar";
300
+ keyboard: "keyboard";
301
+ percussion: "percussion";
302
+ strings: "strings";
303
+ synth: "synth";
304
+ fx: "fx";
305
+ brass: "brass";
306
+ woodwinds: "woodwinds";
307
+ }>>;
286
308
  duration: import("zod").ZodOptional<import("zod").ZodNumber>;
287
309
  maxFrames: import("zod").ZodOptional<import("zod").ZodNumber>;
288
310
  inferenceSteps: import("zod").ZodOptional<import("zod").ZodNumber>;
@@ -298,11 +320,126 @@ export declare const audioGenPlugin: {
298
320
  taskType: import("zod").ZodOptional<import("zod").ZodEnum<{
299
321
  text2music: "text2music";
300
322
  "cover-nofsq": "cover-nofsq";
323
+ lego: "lego";
301
324
  }>>;
302
325
  audioCoverStrength: import("zod").ZodOptional<import("zod").ZodNumber>;
303
326
  coverNoiseStrength: import("zod").ZodOptional<import("zod").ZodNumber>;
304
327
  }, import("zod/v4/core").$strict>, import("zod").ZodObject<{
328
+ progress: import("zod").ZodOptional<import("zod").ZodObject<{
329
+ stage: import("zod").ZodString;
330
+ step: import("zod").ZodNumber;
331
+ total: import("zod").ZodNumber;
332
+ }, import("zod/v4/core").$strip>>;
333
+ data: import("zod").ZodOptional<import("zod").ZodString>;
334
+ sampleRate: import("zod").ZodOptional<import("zod").ZodNumber>;
335
+ channels: import("zod").ZodOptional<import("zod").ZodNumber>;
336
+ bitsPerSample: import("zod").ZodOptional<import("zod").ZodNumber>;
337
+ done: import("zod").ZodDefault<import("zod").ZodBoolean>;
338
+ stopReason: import("zod").ZodOptional<import("zod").ZodEnum<{
339
+ cancelled: "cancelled";
340
+ completed: "completed";
341
+ }>>;
342
+ stats: import("zod").ZodOptional<import("zod").ZodObject<{
343
+ audioDurationMs: import("zod").ZodOptional<import("zod").ZodNumber>;
344
+ totalTimeMs: import("zod").ZodOptional<import("zod").ZodNumber>;
345
+ realTimeFactor: import("zod").ZodOptional<import("zod").ZodNumber>;
346
+ backendDevice: import("zod").ZodOptional<import("zod").ZodNumber>;
347
+ backendId: import("zod").ZodOptional<import("zod").ZodNumber>;
348
+ lyricsScore: import("zod").ZodOptional<import("zod").ZodNumber>;
349
+ lrc: import("zod").ZodOptional<import("zod").ZodString>;
350
+ qualityScore: import("zod").ZodOptional<import("zod").ZodNumber>;
351
+ understand: import("zod").ZodOptional<import("zod").ZodObject<{
352
+ caption: import("zod").ZodString;
353
+ bpm: import("zod").ZodNumber;
354
+ duration: import("zod").ZodNumber;
355
+ keyscale: import("zod").ZodString;
356
+ timesignature: import("zod").ZodString;
357
+ vocalLanguage: import("zod").ZodString;
358
+ audioCodes: import("zod").ZodArray<import("zod").ZodNumber>;
359
+ }, import("zod/v4/core").$strip>>;
360
+ }, import("zod/v4/core").$strip>>;
361
+ diagnostics: import("zod").ZodOptional<import("zod").ZodObject<{
362
+ selectedBackend: import("zod").ZodString;
363
+ selectedDevice: import("zod").ZodEnum<{
364
+ gpu: "gpu";
365
+ cpu: "cpu";
366
+ }>;
367
+ graphicsApi: import("zod").ZodOptional<import("zod").ZodEnum<{
368
+ vulkan: "vulkan";
369
+ metal: "metal";
370
+ opencl: "opencl";
371
+ cuda: "cuda";
372
+ opengl: "opengl";
373
+ webgpu: "webgpu";
374
+ direct3d11: "direct3d11";
375
+ direct3d12: "direct3d12";
376
+ levelZero: "levelZero";
377
+ rocm: "rocm";
378
+ }>>;
379
+ driver: import("zod").ZodOptional<import("zod").ZodObject<{
380
+ name: import("zod").ZodString;
381
+ version: import("zod").ZodOptional<import("zod").ZodString>;
382
+ }, import("zod/v4/core").$strip>>;
383
+ gpuId: import("zod").ZodOptional<import("zod").ZodString>;
384
+ fallback: import("zod").ZodOptional<import("zod").ZodObject<{
385
+ requestedBackend: import("zod").ZodOptional<import("zod").ZodString>;
386
+ requestedDevice: import("zod").ZodOptional<import("zod").ZodEnum<{
387
+ gpu: "gpu";
388
+ cpu: "cpu";
389
+ }>>;
390
+ reason: import("zod").ZodString;
391
+ }, import("zod/v4/core").$strip>>;
392
+ probe: import("zod").ZodOptional<import("zod").ZodObject<{
393
+ status: import("zod").ZodEnum<{
394
+ unknown: "unknown";
395
+ compatible: "compatible";
396
+ incompatible: "incompatible";
397
+ }>;
398
+ backend: import("zod").ZodString;
399
+ reason: import("zod").ZodOptional<import("zod").ZodString>;
400
+ }, import("zod/v4/core").$strip>>;
401
+ }, import("zod/v4/core").$strip>>;
305
402
  type: import("zod").ZodLiteral<"audioGenStream">;
403
+ }, import("zod/v4/core").$strict>>;
404
+ audioEditStream: import("../../../schemas/index.js").PluginHandlerDefinition<import("zod").ZodObject<{
405
+ sourceAudio: import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
406
+ type: import("zod").ZodLiteral<"base64">;
407
+ value: import("zod").ZodString;
408
+ }, import("zod/v4/core").$strip>, import("zod").ZodObject<{
409
+ type: import("zod").ZodLiteral<"filePath">;
410
+ value: import("zod").ZodString;
411
+ }, import("zod/v4/core").$strip>], "type">;
412
+ type: import("zod").ZodLiteral<"audioEditStream">;
413
+ requestId: import("zod").ZodOptional<import("zod").ZodString>;
414
+ modelId: import("zod").ZodString;
415
+ operations: import("zod").ZodArray<import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
416
+ type: import("zod").ZodLiteral<"flow-edit">;
417
+ from: import("zod").ZodObject<{
418
+ caption: import("zod").ZodString;
419
+ lyrics: import("zod").ZodOptional<import("zod").ZodString>;
420
+ }, import("zod/v4/core").$strict>;
421
+ to: import("zod").ZodObject<{
422
+ caption: import("zod").ZodString;
423
+ lyrics: import("zod").ZodOptional<import("zod").ZodString>;
424
+ }, import("zod/v4/core").$strict>;
425
+ nMin: import("zod").ZodOptional<import("zod").ZodNumber>;
426
+ nMax: import("zod").ZodOptional<import("zod").ZodNumber>;
427
+ nAvg: import("zod").ZodOptional<import("zod").ZodNumber>;
428
+ }, import("zod/v4/core").$strict>, import("zod").ZodObject<{
429
+ type: import("zod").ZodLiteral<"repaint">;
430
+ caption: import("zod").ZodString;
431
+ lyrics: import("zod").ZodOptional<import("zod").ZodString>;
432
+ start: import("zod").ZodNumber;
433
+ end: import("zod").ZodOptional<import("zod").ZodNumber>;
434
+ mode: import("zod").ZodOptional<import("zod").ZodEnum<{
435
+ conservative: "conservative";
436
+ balanced: "balanced";
437
+ aggressive: "aggressive";
438
+ }>>;
439
+ strength: import("zod").ZodOptional<import("zod").ZodNumber>;
440
+ }, import("zod/v4/core").$strict>], "type">>;
441
+ seed: import("zod").ZodOptional<import("zod").ZodNumber>;
442
+ }, import("zod/v4/core").$strict>, import("zod").ZodObject<{
306
443
  progress: import("zod").ZodOptional<import("zod").ZodObject<{
307
444
  stage: import("zod").ZodString;
308
445
  step: import("zod").ZodNumber;
@@ -323,6 +460,117 @@ export declare const audioGenPlugin: {
323
460
  realTimeFactor: import("zod").ZodOptional<import("zod").ZodNumber>;
324
461
  backendDevice: import("zod").ZodOptional<import("zod").ZodNumber>;
325
462
  backendId: import("zod").ZodOptional<import("zod").ZodNumber>;
463
+ lyricsScore: import("zod").ZodOptional<import("zod").ZodNumber>;
464
+ lrc: import("zod").ZodOptional<import("zod").ZodString>;
465
+ qualityScore: import("zod").ZodOptional<import("zod").ZodNumber>;
466
+ understand: import("zod").ZodOptional<import("zod").ZodObject<{
467
+ caption: import("zod").ZodString;
468
+ bpm: import("zod").ZodNumber;
469
+ duration: import("zod").ZodNumber;
470
+ keyscale: import("zod").ZodString;
471
+ timesignature: import("zod").ZodString;
472
+ vocalLanguage: import("zod").ZodString;
473
+ audioCodes: import("zod").ZodArray<import("zod").ZodNumber>;
474
+ }, import("zod/v4/core").$strip>>;
475
+ }, import("zod/v4/core").$strip>>;
476
+ diagnostics: import("zod").ZodOptional<import("zod").ZodObject<{
477
+ selectedBackend: import("zod").ZodString;
478
+ selectedDevice: import("zod").ZodEnum<{
479
+ gpu: "gpu";
480
+ cpu: "cpu";
481
+ }>;
482
+ graphicsApi: import("zod").ZodOptional<import("zod").ZodEnum<{
483
+ vulkan: "vulkan";
484
+ metal: "metal";
485
+ opencl: "opencl";
486
+ cuda: "cuda";
487
+ opengl: "opengl";
488
+ webgpu: "webgpu";
489
+ direct3d11: "direct3d11";
490
+ direct3d12: "direct3d12";
491
+ levelZero: "levelZero";
492
+ rocm: "rocm";
493
+ }>>;
494
+ driver: import("zod").ZodOptional<import("zod").ZodObject<{
495
+ name: import("zod").ZodString;
496
+ version: import("zod").ZodOptional<import("zod").ZodString>;
497
+ }, import("zod/v4/core").$strip>>;
498
+ gpuId: import("zod").ZodOptional<import("zod").ZodString>;
499
+ fallback: import("zod").ZodOptional<import("zod").ZodObject<{
500
+ requestedBackend: import("zod").ZodOptional<import("zod").ZodString>;
501
+ requestedDevice: import("zod").ZodOptional<import("zod").ZodEnum<{
502
+ gpu: "gpu";
503
+ cpu: "cpu";
504
+ }>>;
505
+ reason: import("zod").ZodString;
506
+ }, import("zod/v4/core").$strip>>;
507
+ probe: import("zod").ZodOptional<import("zod").ZodObject<{
508
+ status: import("zod").ZodEnum<{
509
+ unknown: "unknown";
510
+ compatible: "compatible";
511
+ incompatible: "incompatible";
512
+ }>;
513
+ backend: import("zod").ZodString;
514
+ reason: import("zod").ZodOptional<import("zod").ZodString>;
515
+ }, import("zod/v4/core").$strip>>;
516
+ }, import("zod/v4/core").$strip>>;
517
+ type: import("zod").ZodLiteral<"audioEditStream">;
518
+ }, import("zod/v4/core").$strict>>;
519
+ audioUnderstand: import("../../../schemas/index.js").PluginHandlerDefinition<import("zod").ZodObject<{
520
+ sourceAudio: import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
521
+ type: import("zod").ZodLiteral<"base64">;
522
+ value: import("zod").ZodString;
523
+ }, import("zod/v4/core").$strip>, import("zod").ZodObject<{
524
+ type: import("zod").ZodLiteral<"filePath">;
525
+ value: import("zod").ZodString;
526
+ }, import("zod/v4/core").$strip>], "type">;
527
+ type: import("zod").ZodLiteral<"audioUnderstand">;
528
+ requestId: import("zod").ZodOptional<import("zod").ZodString>;
529
+ modelId: import("zod").ZodString;
530
+ seed: import("zod").ZodOptional<import("zod").ZodNumber>;
531
+ vocalLanguage: import("zod").ZodOptional<import("zod").ZodString>;
532
+ lmTemperature: import("zod").ZodOptional<import("zod").ZodNumber>;
533
+ lmTopP: import("zod").ZodOptional<import("zod").ZodNumber>;
534
+ lmTopK: import("zod").ZodOptional<import("zod").ZodNumber>;
535
+ }, import("zod/v4/core").$strict>, import("zod").ZodObject<{
536
+ type: import("zod").ZodLiteral<"audioUnderstand">;
537
+ progress: import("zod").ZodOptional<import("zod").ZodObject<{
538
+ stage: import("zod").ZodString;
539
+ step: import("zod").ZodNumber;
540
+ total: import("zod").ZodNumber;
541
+ }, import("zod/v4/core").$strip>>;
542
+ understand: import("zod").ZodOptional<import("zod").ZodObject<{
543
+ caption: import("zod").ZodString;
544
+ bpm: import("zod").ZodNumber;
545
+ duration: import("zod").ZodNumber;
546
+ keyscale: import("zod").ZodString;
547
+ timesignature: import("zod").ZodString;
548
+ vocalLanguage: import("zod").ZodString;
549
+ audioCodes: import("zod").ZodArray<import("zod").ZodNumber>;
550
+ }, import("zod/v4/core").$strip>>;
551
+ done: import("zod").ZodDefault<import("zod").ZodBoolean>;
552
+ stopReason: import("zod").ZodOptional<import("zod").ZodEnum<{
553
+ cancelled: "cancelled";
554
+ completed: "completed";
555
+ }>>;
556
+ stats: import("zod").ZodOptional<import("zod").ZodObject<{
557
+ audioDurationMs: import("zod").ZodOptional<import("zod").ZodNumber>;
558
+ totalTimeMs: import("zod").ZodOptional<import("zod").ZodNumber>;
559
+ realTimeFactor: import("zod").ZodOptional<import("zod").ZodNumber>;
560
+ backendDevice: import("zod").ZodOptional<import("zod").ZodNumber>;
561
+ backendId: import("zod").ZodOptional<import("zod").ZodNumber>;
562
+ lyricsScore: import("zod").ZodOptional<import("zod").ZodNumber>;
563
+ lrc: import("zod").ZodOptional<import("zod").ZodString>;
564
+ qualityScore: import("zod").ZodOptional<import("zod").ZodNumber>;
565
+ understand: import("zod").ZodOptional<import("zod").ZodObject<{
566
+ caption: import("zod").ZodString;
567
+ bpm: import("zod").ZodNumber;
568
+ duration: import("zod").ZodNumber;
569
+ keyscale: import("zod").ZodString;
570
+ timesignature: import("zod").ZodString;
571
+ vocalLanguage: import("zod").ZodString;
572
+ audioCodes: import("zod").ZodArray<import("zod").ZodNumber>;
573
+ }, import("zod/v4/core").$strip>>;
326
574
  }, import("zod/v4/core").$strip>>;
327
575
  diagnostics: import("zod").ZodOptional<import("zod").ZodObject<{
328
576
  selectedBackend: import("zod").ZodString;
@@ -1,8 +1,10 @@
1
1
  import { AudioGen, ENGINE_ACESTEP, ENGINE_MINIMAX } from '@qvac/audiogen-ggml';
2
- import { ADDON_AUDIOGEN, ModelType, audioGenConfigSchema, audioGenStreamRequestSchema, audioGenStreamResponseSchema, defineHandler, definePlugin } from '../../../schemas/index.js';
2
+ import { ADDON_AUDIOGEN, ModelType, audioEditStreamRequestSchema, audioEditStreamResponseSchema, audioGenConfigSchema, audioGenStreamRequestSchema, audioGenStreamResponseSchema, audioUnderstandRequestSchema, audioUnderstandResponseSchema, defineHandler, definePlugin } from '../../../schemas/index.js';
3
3
  import { createStreamLogger, registerAddonLogger } from '../../../logging/index.js';
4
4
  import { resolveAudioGenConfig } from './config.js';
5
+ import { audioEditStream } from './ops/audio-edit-stream.js';
5
6
  import { audioGenStream } from './ops/audio-gen-stream.js';
7
+ import { audioUnderstand } from './ops/audio-understand.js';
6
8
  import { ModelLoadFailedError } from '../../../errors/index.js';
7
9
  export const audioGenPlugin = definePlugin({
8
10
  modelType: ModelType.audiogenGgml,
@@ -35,6 +37,20 @@ export const audioGenPlugin = definePlugin({
35
37
  streaming: true,
36
38
  cancel: { scope: 'model', hard: true },
37
39
  handler: audioGenStream
40
+ }),
41
+ audioEditStream: defineHandler({
42
+ requestSchema: audioEditStreamRequestSchema,
43
+ responseSchema: audioEditStreamResponseSchema,
44
+ streaming: true,
45
+ cancel: { scope: 'model', hard: true },
46
+ handler: audioEditStream
47
+ }),
48
+ audioUnderstand: defineHandler({
49
+ requestSchema: audioUnderstandRequestSchema,
50
+ responseSchema: audioUnderstandResponseSchema,
51
+ streaming: true,
52
+ cancel: { scope: 'model', hard: true },
53
+ handler: audioUnderstand
38
54
  })
39
55
  }
40
56
  });