@qvac/inference 0.17.1 → 0.18.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/completion-stream.d.ts +1 -1
- package/dist/api/completion-stream.js +1 -1
- package/dist/api/index.d.ts +1 -1
- package/dist/api/index.js +1 -1
- package/dist/api/text-to-speech.js +22 -2
- package/dist/api/transcribe.d.ts +19 -2
- package/dist/api/transcribe.js +66 -27
- package/dist/api/translate.js +6 -17
- package/dist/api/vla.d.ts +42 -1
- package/dist/api/vla.js +50 -1
- package/dist/constants/audio.d.ts +2 -1
- package/dist/constants/audio.js +17 -1
- package/dist/errors/index.d.ts +3 -0
- package/dist/errors/index.js +5 -0
- package/dist/handlers/load-model/handler.js +2 -2
- package/dist/handlers/load-model/resolve-session.d.ts +1 -1
- package/dist/handlers/load-model/resolve-session.js +3 -3
- package/dist/handlers/load-model/resolve.d.ts +12 -2
- package/dist/handlers/load-model/resolve.js +90 -12
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/models/registry/models.d.ts +782 -6
- package/dist/models/registry/models.js +2664 -1859
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +19 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +153 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +54 -4
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +28 -0
- package/dist/plugins/builtin/ggml-ocr/plugin.js +3 -1
- package/dist/plugins/builtin/ggml-vla/ops/vla-set-embodiment.d.ts +2 -0
- package/dist/plugins/builtin/ggml-vla/ops/vla-set-embodiment.js +9 -0
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +37 -0
- package/dist/plugins/builtin/ggml-vla/plugin.js +21 -4
- package/dist/plugins/builtin/index.d.ts +1 -0
- package/dist/plugins/builtin/index.js +1 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +16 -15
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.d.ts +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +2 -2
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +6 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +10 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +1 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +131 -123
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +110 -20
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +50 -23
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +308 -143
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +1 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +2 -2
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +6 -6
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +56 -36
- package/dist/plugins/builtin/llamacpp-completion/transform.js +0 -7
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -3
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +6 -5
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +27 -16
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +23 -3
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +302 -4
- package/dist/plugins/builtin/tts-ggml/plugin.js +131 -0
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +48 -39
- package/dist/plugins/ops/cancel.d.ts +1 -1
- package/dist/plugins/ops/cancel.js +1 -1
- package/dist/plugins/ops/kv-cache-retention.js +11 -0
- package/dist/plugins/ops/kv-cache-utils.d.ts +1 -1
- package/dist/plugins/ops/kv-cache-utils.js +40 -12
- package/dist/plugins/ops/transcribe.d.ts +2 -2
- package/dist/plugins/ops/transcribe.js +54 -48
- package/dist/plugins/ops/translate-stats.d.ts +4 -0
- package/dist/plugins/ops/translate-stats.js +11 -0
- package/dist/plugins/ops/translate.js +66 -38
- package/dist/plugins/ops/unload-model.js +30 -24
- package/dist/runtime/model-registry.d.ts +1 -0
- package/dist/runtime/request-context.js +29 -14
- package/dist/runtime/request-registry.d.ts +27 -0
- package/dist/runtime/request-registry.js +246 -76
- package/dist/schemas/audio-gen.d.ts +103 -0
- package/dist/schemas/audio-gen.js +135 -4
- package/dist/schemas/common.d.ts +550 -11
- package/dist/schemas/config.d.ts +60 -24
- package/dist/schemas/errors.d.ts +1 -0
- package/dist/schemas/errors.js +5 -0
- package/dist/schemas/index.d.ts +1 -1
- package/dist/schemas/index.js +1 -1
- package/dist/schemas/llamacpp-config.d.ts +10 -10
- package/dist/schemas/llamacpp-config.js +168 -82
- package/dist/schemas/load-model.d.ts +3691 -336
- package/dist/schemas/load-model.js +37 -19
- package/dist/schemas/text-to-speech.d.ts +1086 -110
- package/dist/schemas/text-to-speech.js +197 -7
- package/dist/schemas/tools.d.ts +0 -16
- package/dist/schemas/tools.js +0 -15
- package/dist/schemas/transcription-config.d.ts +2 -0
- package/dist/schemas/transcription-config.js +6 -0
- package/dist/schemas/transcription.d.ts +3 -0
- package/dist/schemas/vla.d.ts +83 -0
- package/dist/schemas/vla.js +105 -2
- package/dist/surface.d.ts +1 -1
- package/dist/surface.js +1 -1
- package/dist/utils/addon-responses.d.ts +4 -12
- package/dist/utils/audio/decoder.d.ts +37 -1
- package/dist/utils/audio/decoder.js +82 -52
- package/dist/utils/config-transform.d.ts +3 -0
- package/dist/utils/config-transform.js +6 -0
- package/dist/utils/index.d.ts +0 -1
- package/dist/utils/index.js +0 -1
- package/dist/utils/shard-utils.d.ts +1 -0
- package/dist/utils/shard-utils.js +14 -0
- package/dist/utils/tool-integration.d.ts +3 -10
- package/dist/utils/tool-integration.js +3 -12
- package/package.json +26 -22
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-cancelled.d.ts +0 -22
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-cancelled.js +0 -30
- package/dist/utils/expand-gguf-shards.d.ts +0 -2
- package/dist/utils/expand-gguf-shards.js +0 -22
|
@@ -49,6 +49,26 @@ export function createRequestRegistry(options) {
|
|
|
49
49
|
* FIFO in O(1) without scanning every `KeyState`.
|
|
50
50
|
*/
|
|
51
51
|
const waitersById = new Map();
|
|
52
|
+
/**
|
|
53
|
+
* Ids reserved synchronously at `begin(...)` start and cleared once the entry
|
|
54
|
+
* lands in `entries` (or the begin throws). A same-tick second `begin(...)`
|
|
55
|
+
* with the same id runs before the first reaches `entries`/`waitersById`, so
|
|
56
|
+
* without this it would slip past the duplicate check and both would execute.
|
|
57
|
+
*/
|
|
58
|
+
const reservedIds = new Set();
|
|
59
|
+
// Models being torn down: a begin(...) for one starts aborted. The barrier
|
|
60
|
+
// half of cancelAndDrain — catches begins its scan can't see yet.
|
|
61
|
+
const drainingModels = new Set();
|
|
62
|
+
// Entries whose disposal has started but not finished. disposeEntry removes
|
|
63
|
+
// from `entries` before unwinding, so a drain must consult this too or it
|
|
64
|
+
// would return while a request is still tearing down natively.
|
|
65
|
+
const disposingEntries = new Set();
|
|
66
|
+
// Request ids whose entry is mid-disposal. disposeEntry drops the id from
|
|
67
|
+
// `entries` before its async scope unwinds; without this the id looks free
|
|
68
|
+
// during that gap, so a same-id begin could activate concurrently with the
|
|
69
|
+
// still-tearing-down request and a cancel would land as a cancel-before-begin
|
|
70
|
+
// tripwire that aborts a later reuse. Held until teardown completes.
|
|
71
|
+
const disposingIds = new Set();
|
|
52
72
|
function logLifecycle(event, ctx, durationMs) {
|
|
53
73
|
const modelId = ctx.modelId !== undefined ? ctx.modelId : '-';
|
|
54
74
|
const base = `[request-lifecycle] ${event} requestId=${ctx.requestId} kind=${ctx.kind} modelId=${modelId} state=${ctx.state}`;
|
|
@@ -133,7 +153,7 @@ export function createRequestRegistry(options) {
|
|
|
133
153
|
* removal resolves (granted / graceful cancel) or rejects (timeout /
|
|
134
154
|
* teardown).
|
|
135
155
|
*/
|
|
136
|
-
function removeWaiter(key, waiter) {
|
|
156
|
+
function removeWaiter(key, waiter, drainAfter = true) {
|
|
137
157
|
if (waiter.timer !== undefined) {
|
|
138
158
|
clearTimeout(waiter.timer);
|
|
139
159
|
waiter.timer = undefined;
|
|
@@ -145,8 +165,14 @@ export function createRequestRegistry(options) {
|
|
|
145
165
|
const i = st.waiters.indexOf(waiter);
|
|
146
166
|
if (i >= 0)
|
|
147
167
|
st.waiters.splice(i, 1);
|
|
148
|
-
|
|
168
|
+
// Removing a queued writer can unblock the readers behind it, so drain
|
|
169
|
+
// (which also deletes the KeyState once idle). `cancelAll` passes false:
|
|
170
|
+
// it's tearing every waiter down, so granting one mid-sweep is wrong.
|
|
171
|
+
if (drainAfter)
|
|
172
|
+
drain(st, key);
|
|
173
|
+
else if (st.active <= 0 && !st.exclusiveActive && st.waiters.length === 0) {
|
|
149
174
|
keyStates.delete(key);
|
|
175
|
+
}
|
|
150
176
|
}
|
|
151
177
|
/**
|
|
152
178
|
* Admission gate. Resolves to the slot key the caller must release on
|
|
@@ -164,29 +190,50 @@ export function createRequestRegistry(options) {
|
|
|
164
190
|
*/
|
|
165
191
|
async function acquireSlot(opts) {
|
|
166
192
|
if (opts.modelId === undefined)
|
|
167
|
-
return { slotKey: undefined };
|
|
193
|
+
return { slotKey: undefined, exclusive: false };
|
|
168
194
|
const policy = policies.get(opts.kind);
|
|
169
|
-
if (!policy
|
|
170
|
-
return { slotKey: undefined };
|
|
195
|
+
if (!policy)
|
|
196
|
+
return { slotKey: undefined, exclusive: false };
|
|
197
|
+
// The per-request override (the model's own `parallel`) wins over the
|
|
198
|
+
// per-kind default. A non-finite effective cap disables gating entirely.
|
|
199
|
+
const requestedMax = opts.maxConcurrentPerModel ?? policy.maxConcurrent;
|
|
200
|
+
if (!Number.isFinite(requestedMax)) {
|
|
201
|
+
return { slotKey: undefined, exclusive: false };
|
|
171
202
|
}
|
|
203
|
+
const maxConcurrent = clampSlotCount(requestedMax);
|
|
172
204
|
// A parent (process-shutdown) signal that's already aborted: don't
|
|
173
205
|
// queue behind live work that may never drain — let begin() proceed
|
|
174
206
|
// and abort immediately via the parentSignal path.
|
|
175
207
|
if (opts.parentSignal?.aborted)
|
|
176
|
-
return { slotKey: undefined };
|
|
208
|
+
return { slotKey: undefined, exclusive: false };
|
|
177
209
|
const modelId = opts.modelId;
|
|
178
|
-
//
|
|
179
|
-
// contend for one slot pool per model
|
|
180
|
-
// them on a single native context.
|
|
210
|
+
// Lane precedence: the policy's shared group → the kind itself. A shared
|
|
211
|
+
// lane lets unrelated kinds contend for one slot pool per model.
|
|
181
212
|
const key = slotKey(policy.slotGroup ?? opts.kind, modelId);
|
|
182
213
|
let st = keyStates.get(key);
|
|
183
214
|
if (!st) {
|
|
184
|
-
st = { active: 0, waiters: [] };
|
|
215
|
+
st = { active: 0, waiters: [], exclusiveActive: false };
|
|
185
216
|
keyStates.set(key, st);
|
|
186
217
|
}
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
218
|
+
const exclusive = policy.exclusive;
|
|
219
|
+
// A reader admits below the lane's pinned cap (see `readerCap`); an exclusive
|
|
220
|
+
// writer needs the lane empty. A queued waiter (of any kind) blocks admission
|
|
221
|
+
// so a writer isn't starved by a steady reader stream. For reader-only lanes
|
|
222
|
+
// `waiters` is empty whenever `active < cap`, so this guard is a no-op there.
|
|
223
|
+
const readerCap = st.readerCap ?? maxConcurrent;
|
|
224
|
+
const canAdmit = st.waiters.length === 0 &&
|
|
225
|
+
!st.exclusiveActive &&
|
|
226
|
+
(exclusive ? st.active === 0 : st.active < readerCap);
|
|
227
|
+
if (canAdmit) {
|
|
228
|
+
if (exclusive) {
|
|
229
|
+
st.exclusiveActive = true;
|
|
230
|
+
}
|
|
231
|
+
else {
|
|
232
|
+
st.active++;
|
|
233
|
+
if (st.readerCap === undefined)
|
|
234
|
+
st.readerCap = maxConcurrent;
|
|
235
|
+
}
|
|
236
|
+
return { slotKey: key, exclusive };
|
|
190
237
|
}
|
|
191
238
|
if (policy.onOverflow === 'reject') {
|
|
192
239
|
throw new RequestRejectedByPolicyError(opts.requestId, opts.kind, modelId, `another ${opts.kind} request is already running on this model`);
|
|
@@ -203,7 +250,9 @@ export function createRequestRegistry(options) {
|
|
|
203
250
|
enqueuedAt: Date.now(),
|
|
204
251
|
resolve,
|
|
205
252
|
reject,
|
|
206
|
-
timer: undefined
|
|
253
|
+
timer: undefined,
|
|
254
|
+
exclusive,
|
|
255
|
+
cap: maxConcurrent
|
|
207
256
|
};
|
|
208
257
|
if (policy.queueTimeoutMs !== undefined) {
|
|
209
258
|
const timeoutMs = policy.queueTimeoutMs;
|
|
@@ -215,32 +264,59 @@ export function createRequestRegistry(options) {
|
|
|
215
264
|
queue.waiters.push(waiter);
|
|
216
265
|
waitersById.set(opts.requestId, { key, waiter });
|
|
217
266
|
});
|
|
218
|
-
return granted ? { slotKey: key } : { slotKey: undefined };
|
|
267
|
+
return granted ? { slotKey: key, exclusive } : { slotKey: undefined, exclusive: false };
|
|
219
268
|
}
|
|
220
269
|
/**
|
|
221
|
-
*
|
|
222
|
-
*
|
|
223
|
-
*
|
|
224
|
-
*
|
|
270
|
+
* Grant the lane to as many FIFO-head waiters as now fit: an exclusive writer
|
|
271
|
+
* only when the lane is idle, readers up to each one's own cap. Stops at the
|
|
272
|
+
* first head that can't be admitted, so FIFO order holds. Deletes the
|
|
273
|
+
* `KeyState` once fully idle.
|
|
225
274
|
*/
|
|
226
|
-
function
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
clearTimeout(next.timer);
|
|
236
|
-
next.timer = undefined;
|
|
275
|
+
function drain(st, key) {
|
|
276
|
+
while (st.waiters.length > 0) {
|
|
277
|
+
const head = st.waiters[0];
|
|
278
|
+
if (head === undefined)
|
|
279
|
+
break;
|
|
280
|
+
if (head.exclusive) {
|
|
281
|
+
if (st.exclusiveActive || st.active > 0)
|
|
282
|
+
break;
|
|
283
|
+
st.exclusiveActive = true;
|
|
237
284
|
}
|
|
238
|
-
|
|
239
|
-
|
|
285
|
+
else {
|
|
286
|
+
const readerCap = st.readerCap ?? head.cap;
|
|
287
|
+
if (st.exclusiveActive || st.active >= readerCap)
|
|
288
|
+
break;
|
|
289
|
+
st.active++;
|
|
290
|
+
if (st.readerCap === undefined)
|
|
291
|
+
st.readerCap = head.cap;
|
|
292
|
+
}
|
|
293
|
+
st.waiters.shift();
|
|
294
|
+
waitersById.delete(head.requestId);
|
|
295
|
+
if (head.timer !== undefined) {
|
|
296
|
+
clearTimeout(head.timer);
|
|
297
|
+
head.timer = undefined;
|
|
298
|
+
}
|
|
299
|
+
head.resolve(true);
|
|
240
300
|
}
|
|
241
|
-
st.active
|
|
242
|
-
if (st.active <= 0 && st.waiters.length === 0)
|
|
301
|
+
if (st.active <= 0 && !st.exclusiveActive && st.waiters.length === 0) {
|
|
243
302
|
keyStates.delete(key);
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
/**
|
|
306
|
+
* Release a held slot, then hand the freed capacity to waiters. Called from
|
|
307
|
+
* `disposeEntry` after the scope has fully unwound, so its deferred teardown
|
|
308
|
+
* (for llama.cpp completion, disposing the run response that stops native
|
|
309
|
+
* decode) has run before the next same-model request is admitted.
|
|
310
|
+
*/
|
|
311
|
+
function releaseSlot(key, wasExclusive) {
|
|
312
|
+
const st = keyStates.get(key);
|
|
313
|
+
if (!st)
|
|
314
|
+
return;
|
|
315
|
+
if (wasExclusive)
|
|
316
|
+
st.exclusiveActive = false;
|
|
317
|
+
else
|
|
318
|
+
st.active--;
|
|
319
|
+
drain(st, key);
|
|
244
320
|
}
|
|
245
321
|
function cancelEntry(entry, reason) {
|
|
246
322
|
if (entry.controller.signal.aborted)
|
|
@@ -251,8 +327,13 @@ export function createRequestRegistry(options) {
|
|
|
251
327
|
return true;
|
|
252
328
|
}
|
|
253
329
|
async function disposeEntry(entry, outcome) {
|
|
254
|
-
if (entry.scope.disposed)
|
|
330
|
+
if (entry.scope.disposed || entry.disposing)
|
|
255
331
|
return;
|
|
332
|
+
entry.disposing = true;
|
|
333
|
+
disposingEntries.add(entry);
|
|
334
|
+
// Keep the id reserved across async teardown so a same-id begin can't
|
|
335
|
+
// activate concurrently and a cancel can't strand a tripwire for a reuse.
|
|
336
|
+
disposingIds.add(entry.ctx.requestId);
|
|
256
337
|
entry.ctx.state = outcome;
|
|
257
338
|
entry.detachParent();
|
|
258
339
|
logLifecycle('end', entry.ctx, Date.now() - entry.startedAt);
|
|
@@ -270,46 +351,81 @@ export function createRequestRegistry(options) {
|
|
|
270
351
|
// guarantees the slot is freed even if a deferred cleanup throws,
|
|
271
352
|
// otherwise a throwing teardown would strand the queue forever.
|
|
272
353
|
if (entry.slotKey !== undefined)
|
|
273
|
-
releaseSlot(entry.slotKey);
|
|
354
|
+
releaseSlot(entry.slotKey, entry.exclusive);
|
|
355
|
+
disposingEntries.delete(entry);
|
|
356
|
+
// The id is free for reuse only now that native teardown is complete.
|
|
357
|
+
disposingIds.delete(entry.ctx.requestId);
|
|
358
|
+
// Signal any `cancelAndDrain` waiting on this request that its native
|
|
359
|
+
// teardown is complete.
|
|
360
|
+
entry.resolveDisposed();
|
|
274
361
|
}
|
|
275
362
|
}
|
|
276
363
|
async function begin(opts) {
|
|
277
|
-
// A request id is reserved for its whole lifecycle
|
|
278
|
-
//
|
|
279
|
-
// still-queued begins (
|
|
280
|
-
//
|
|
281
|
-
//
|
|
282
|
-
//
|
|
283
|
-
|
|
364
|
+
// A request id is reserved for its whole lifecycle: the time it spends
|
|
365
|
+
// *queued* for an admission slot, while it is active, and while it is
|
|
366
|
+
// tearing down. `waitersById` holds the still-queued begins (not yet in
|
|
367
|
+
// `entries`) and `disposingIds` holds the ones whose async teardown hasn't
|
|
368
|
+
// finished, so a duplicate id must be rejected against all of them —
|
|
369
|
+
// otherwise a second begin with the same id would enqueue behind the first
|
|
370
|
+
// and overwrite its `waitersById` index (unreachable by `cancel(...)`), or
|
|
371
|
+
// activate while the prior request is still releasing its native context.
|
|
372
|
+
if (entries.has(opts.requestId) ||
|
|
373
|
+
waitersById.has(opts.requestId) ||
|
|
374
|
+
reservedIds.has(opts.requestId) ||
|
|
375
|
+
disposingIds.has(opts.requestId)) {
|
|
284
376
|
throw new RequestIdConflictError(opts.requestId);
|
|
285
377
|
}
|
|
286
|
-
//
|
|
287
|
-
//
|
|
288
|
-
|
|
289
|
-
//
|
|
290
|
-
|
|
291
|
-
//
|
|
292
|
-
|
|
293
|
-
//
|
|
294
|
-
//
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
378
|
+
// Reserve synchronously so a same-tick duplicate begin is rejected before
|
|
379
|
+
// this one reaches `entries`/`waitersById`. Cleared on every exit below.
|
|
380
|
+
reservedIds.add(opts.requestId);
|
|
381
|
+
// A request cancelled BEFORE begin must not queue for a slot on a saturated
|
|
382
|
+
// lane: consume its marker first and skip admission, so it resolves at once
|
|
383
|
+
// into an aborted context holding no slot instead of waiting for the holder.
|
|
384
|
+
let preCancel = consumeCancelBeforeBegin(opts.requestId);
|
|
385
|
+
// Admission control runs before allocation so a rejected begin leaves no
|
|
386
|
+
// controller / scope behind. This may await: a gated `(kind, modelId)` at
|
|
387
|
+
// capacity queues the begin FIFO and resolves once a slot frees (or rejects
|
|
388
|
+
// on overflow / queue-depth cap / timeout).
|
|
389
|
+
let slotKey;
|
|
390
|
+
let exclusive = false;
|
|
391
|
+
if (!preCancel) {
|
|
392
|
+
// Plain try/catch, not `.catch()`, so admission adds no extra microtask
|
|
393
|
+
// and observers still see the request registered on the same tick.
|
|
394
|
+
let acquired;
|
|
395
|
+
try {
|
|
396
|
+
acquired = await acquireSlot(opts);
|
|
397
|
+
}
|
|
398
|
+
catch (err) {
|
|
399
|
+
reservedIds.delete(opts.requestId);
|
|
400
|
+
throw err;
|
|
401
|
+
}
|
|
402
|
+
slotKey = acquired.slotKey;
|
|
403
|
+
exclusive = acquired.exclusive;
|
|
404
|
+
// The only interleaving point in this otherwise synchronous body is the
|
|
405
|
+
// await above. A duplicate id could (astronomically unlikely) have landed
|
|
406
|
+
// while we were queued — re-check and hand the just-acquired slot back
|
|
407
|
+
// rather than stranding it on the conflicting throw.
|
|
408
|
+
if (entries.has(opts.requestId)) {
|
|
409
|
+
if (slotKey !== undefined)
|
|
410
|
+
releaseSlot(slotKey, exclusive);
|
|
411
|
+
reservedIds.delete(opts.requestId);
|
|
412
|
+
throw new RequestIdConflictError(opts.requestId);
|
|
413
|
+
}
|
|
414
|
+
// A cancel that landed WHILE we were queued resolved acquireSlot with no
|
|
415
|
+
// slot and recorded a marker; consume it so the context starts aborted.
|
|
416
|
+
preCancel = consumeCancelBeforeBegin(opts.requestId);
|
|
299
417
|
}
|
|
418
|
+
// A begin arriving while its model is being torn down starts aborted — no
|
|
419
|
+
// native work against a model about to be freed.
|
|
420
|
+
const drainReason = opts.modelId !== undefined && drainingModels.has(opts.modelId) ? 'modelUnload' : undefined;
|
|
300
421
|
const controller = new AbortController();
|
|
301
422
|
const scope = createDisposableScope();
|
|
302
|
-
// Stop-button race close. If a
|
|
303
|
-
// `cancel({ requestId })` already arrived for this id, abort the
|
|
304
|
-
// new controller before observers can subscribe to it. The
|
|
305
|
-
// tripwire entry is consumed so a later, separate `begin(...)`
|
|
306
|
-
// with the same id is unaffected (in practice ids are UUIDv4 and
|
|
307
|
-
// never reused; this guard just keeps the contract self-
|
|
308
|
-
// consistent under retries).
|
|
309
|
-
const preCancel = consumeCancelBeforeBegin(opts.requestId);
|
|
310
423
|
if (preCancel) {
|
|
311
424
|
controller.abort(preCancel.reason);
|
|
312
425
|
}
|
|
426
|
+
else if (drainReason) {
|
|
427
|
+
controller.abort(drainReason);
|
|
428
|
+
}
|
|
313
429
|
let detachParent = () => { };
|
|
314
430
|
if (opts.parentSignal) {
|
|
315
431
|
const parent = opts.parentSignal;
|
|
@@ -334,17 +450,26 @@ export function createRequestRegistry(options) {
|
|
|
334
450
|
// already aborted at begin time. Both branches abort the
|
|
335
451
|
// controller above, so without this guard observers would see a
|
|
336
452
|
// momentarily-`running` context with an already-aborted signal.
|
|
337
|
-
state: preCancel || opts.parentSignal?.aborted ? 'cancelling' : 'running'
|
|
453
|
+
state: preCancel || drainReason || opts.parentSignal?.aborted ? 'cancelling' : 'running'
|
|
338
454
|
};
|
|
455
|
+
let resolveDisposed = () => { };
|
|
456
|
+
const disposed = new Promise((resolve) => {
|
|
457
|
+
resolveDisposed = resolve;
|
|
458
|
+
});
|
|
339
459
|
const entry = {
|
|
340
460
|
ctx,
|
|
341
461
|
controller,
|
|
342
462
|
scope,
|
|
343
463
|
detachParent,
|
|
344
464
|
startedAt: Date.now(),
|
|
345
|
-
slotKey
|
|
465
|
+
slotKey,
|
|
466
|
+
exclusive,
|
|
467
|
+
disposing: false,
|
|
468
|
+
disposed,
|
|
469
|
+
resolveDisposed
|
|
346
470
|
};
|
|
347
471
|
entries.set(opts.requestId, entry);
|
|
472
|
+
reservedIds.delete(opts.requestId);
|
|
348
473
|
logLifecycle('begin', ctx);
|
|
349
474
|
return {
|
|
350
475
|
get requestId() {
|
|
@@ -411,6 +536,12 @@ export function createRequestRegistry(options) {
|
|
|
411
536
|
cancelQueuedWaiterGracefully(queued.key, queued.waiter, target.reason);
|
|
412
537
|
return cancelled + 1;
|
|
413
538
|
}
|
|
539
|
+
// Mid-disposal: the request already reached a terminal outcome and is
|
|
540
|
+
// releasing its native context. There is nothing to cancel, and a
|
|
541
|
+
// cancel-before-begin marker would wrongly abort a later begin that
|
|
542
|
+
// reuses this id once teardown frees it.
|
|
543
|
+
if (disposingIds.has(target.requestId))
|
|
544
|
+
return cancelled;
|
|
414
545
|
// Stop-button race: the cancel beat its own
|
|
415
546
|
// `begin(...)`. Record the cancel so the next matching `begin`
|
|
416
547
|
// aborts immediately. The return value stays 0 — no in-flight
|
|
@@ -453,15 +584,48 @@ export function createRequestRegistry(options) {
|
|
|
453
584
|
// so there is nothing left for them to run. Snapshot first —
|
|
454
585
|
// `removeWaiter` mutates `waitersById`.
|
|
455
586
|
for (const { key, waiter } of Array.from(waitersById.values())) {
|
|
456
|
-
removeWaiter(key, waiter);
|
|
587
|
+
removeWaiter(key, waiter, false);
|
|
457
588
|
waiter.reject(new RequestRejectedByPolicyError(waiter.requestId, waiter.kind, waiter.modelId, `queued request cancelled (${reason})`));
|
|
458
589
|
}
|
|
459
|
-
//
|
|
460
|
-
// async sweep that awaits per-handler scope unwinding (e.g. join on
|
|
461
|
-
// the disposers). Today every handler unwinds on its own dispose
|
|
462
|
-
// path, so the function only needs to fire-and-forget the abort.
|
|
590
|
+
// Fire-and-forget the abort; each handler unwinds on its own dispose path.
|
|
463
591
|
return Promise.resolve();
|
|
464
592
|
}
|
|
593
|
+
// Cancel and drain every request for a model, then run `teardown` with the
|
|
594
|
+
// admission barrier held — any begin for the model (including one still
|
|
595
|
+
// suspended in admission) starts aborted — lifting it afterward. Scoped so the
|
|
596
|
+
// barrier can't be left raised.
|
|
597
|
+
async function withModelDraining(modelId, teardown) {
|
|
598
|
+
drainingModels.add(modelId);
|
|
599
|
+
try {
|
|
600
|
+
// Abort each active request and collect its disposal promise, so we wait
|
|
601
|
+
// for every handler's scope to fully unwind (native context freed, slot
|
|
602
|
+
// released) before teardown. Snapshot first — `disposeEntry` mutates
|
|
603
|
+
// `entries`.
|
|
604
|
+
const draining = [];
|
|
605
|
+
for (const entry of entries.values()) {
|
|
606
|
+
if (entry.ctx.modelId !== modelId)
|
|
607
|
+
continue;
|
|
608
|
+
cancelEntry(entry, 'modelUnload');
|
|
609
|
+
draining.push(entry.disposed);
|
|
610
|
+
}
|
|
611
|
+
// Requests already mid-disposal are out of `entries` but may still be
|
|
612
|
+
// tearing down natively — wait for them too.
|
|
613
|
+
for (const entry of disposingEntries) {
|
|
614
|
+
if (entry.ctx.modelId === modelId)
|
|
615
|
+
draining.push(entry.disposed);
|
|
616
|
+
}
|
|
617
|
+
// Queued waiters never held a slot, so just abort their begins.
|
|
618
|
+
for (const { key, waiter } of Array.from(waitersById.values())) {
|
|
619
|
+
if (waiter.modelId === modelId)
|
|
620
|
+
cancelQueuedWaiterGracefully(key, waiter, 'modelUnload');
|
|
621
|
+
}
|
|
622
|
+
await Promise.all(draining);
|
|
623
|
+
await teardown();
|
|
624
|
+
}
|
|
625
|
+
finally {
|
|
626
|
+
drainingModels.delete(modelId);
|
|
627
|
+
}
|
|
628
|
+
}
|
|
465
629
|
async function end(requestId, outcome) {
|
|
466
630
|
const entry = entries.get(requestId);
|
|
467
631
|
if (!entry)
|
|
@@ -477,6 +641,7 @@ export function createRequestRegistry(options) {
|
|
|
477
641
|
list,
|
|
478
642
|
cancel,
|
|
479
643
|
cancelAll,
|
|
644
|
+
withModelDraining,
|
|
480
645
|
end,
|
|
481
646
|
policy,
|
|
482
647
|
// Test-only: lets the registry race tests assert the bound
|
|
@@ -491,6 +656,12 @@ export function createRequestRegistry(options) {
|
|
|
491
656
|
__keyStateSize: () => keyStates.size
|
|
492
657
|
};
|
|
493
658
|
}
|
|
659
|
+
// A finite slot count floored to an integer >= 1: a value below 1 would gate
|
|
660
|
+
// every request forever and a fractional one could over-admit. Callers gate on
|
|
661
|
+
// `Number.isFinite` first — Infinity means "ungated" and must stay Infinity.
|
|
662
|
+
function clampSlotCount(count) {
|
|
663
|
+
return Math.max(1, Math.floor(count));
|
|
664
|
+
}
|
|
494
665
|
/**
|
|
495
666
|
* Resolve a `ConcurrencyPolicy` to concrete numbers, applying defaults and
|
|
496
667
|
* the deprecated `oneAtATimePerModel` alias. Done once at registration so
|
|
@@ -513,16 +684,15 @@ function normalizePolicy(opts) {
|
|
|
513
684
|
maxConcurrent = Infinity;
|
|
514
685
|
onOverflow = opts.onOverflow ?? 'queue';
|
|
515
686
|
}
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
if (Number.isFinite(maxConcurrent) && maxConcurrent < 1)
|
|
519
|
-
maxConcurrent = 1;
|
|
687
|
+
if (Number.isFinite(maxConcurrent))
|
|
688
|
+
maxConcurrent = clampSlotCount(maxConcurrent);
|
|
520
689
|
return {
|
|
521
690
|
maxConcurrent,
|
|
522
691
|
onOverflow,
|
|
523
692
|
maxQueueDepth: opts.maxQueueDepthPerModel ?? DEFAULT_MAX_QUEUE_DEPTH_PER_MODEL,
|
|
524
693
|
queueTimeoutMs: opts.queueTimeoutMs,
|
|
525
|
-
slotGroup: opts.sharedSlotGroup
|
|
694
|
+
slotGroup: opts.sharedSlotGroup,
|
|
695
|
+
exclusive: opts.exclusive === true
|
|
526
696
|
};
|
|
527
697
|
}
|
|
528
698
|
/**
|
|
@@ -1,4 +1,26 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
|
+
/** Sample rate the ACE-Step engine expects for reference and source audio. */
|
|
3
|
+
export declare const AUDIOGEN_INPUT_SAMPLE_RATE = 48000;
|
|
4
|
+
/** Channel count (interleaved stereo) the ACE-Step engine expects for input audio. */
|
|
5
|
+
export declare const AUDIOGEN_INPUT_CHANNELS = 2;
|
|
6
|
+
/**
|
|
7
|
+
* Longest reference/source clip the SDK accepts (10 minutes). Bounds the PCM
|
|
8
|
+
* the server materializes per input — a 48 kHz stereo Float32 clip weighs
|
|
9
|
+
* 384 KB per second — so a request cannot exhaust the inference process.
|
|
10
|
+
*/
|
|
11
|
+
export declare const AUDIOGEN_INPUT_MAX_SECONDS = 600;
|
|
12
|
+
/**
|
|
13
|
+
* ACE-Step task discriminators reachable through the SDK. `text2music` is the
|
|
14
|
+
* default caption-driven generation; `cover-nofsq` re-renders `sourceAudio`
|
|
15
|
+
* with a new caption while keeping its structure. The engine also reserves an
|
|
16
|
+
* FSQ-roundtrip `cover` task that is not implemented yet, so it is not offered
|
|
17
|
+
* here.
|
|
18
|
+
*/
|
|
19
|
+
export declare const AUDIOGEN_TASK_TYPES: readonly ["text2music", "cover-nofsq"];
|
|
20
|
+
export declare const audioGenTaskTypeSchema: z.ZodEnum<{
|
|
21
|
+
text2music: "text2music";
|
|
22
|
+
"cover-nofsq": "cover-nofsq";
|
|
23
|
+
}>;
|
|
2
24
|
export declare const audioGenRuntimeConfigSchema: z.ZodObject<{
|
|
3
25
|
useGPU: z.ZodOptional<z.ZodBoolean>;
|
|
4
26
|
inferenceSteps: z.ZodOptional<z.ZodNumber>;
|
|
@@ -171,7 +193,44 @@ export declare const audioGenConfigSchema: z.ZodObject<{
|
|
|
171
193
|
}>, z.ZodLiteral<"vad">]>>;
|
|
172
194
|
}, z.core.$strip>]>;
|
|
173
195
|
}, z.core.$strict>;
|
|
196
|
+
/**
|
|
197
|
+
* Wire form of a reference/source audio input. `filePath` inputs are decoded
|
|
198
|
+
* server-side (any format the SDK's audio decoder supports, plus raw PCM);
|
|
199
|
+
* `base64` inputs must already be interleaved stereo 48 kHz Float32 LE PCM.
|
|
200
|
+
*/
|
|
201
|
+
export declare const audioGenAudioInputSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
202
|
+
type: z.ZodLiteral<"base64">;
|
|
203
|
+
value: z.ZodString;
|
|
204
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
205
|
+
type: z.ZodLiteral<"filePath">;
|
|
206
|
+
value: z.ZodString;
|
|
207
|
+
}, z.core.$strip>], "type">;
|
|
208
|
+
/**
|
|
209
|
+
* Client form of a reference/source audio input: a file path, or raw
|
|
210
|
+
* interleaved stereo 48 kHz Float32 LE PCM bytes. Normalized to the wire form.
|
|
211
|
+
*/
|
|
212
|
+
export declare const audioGenClientAudioInputSchema: z.ZodPipe<z.ZodUnion<readonly [z.ZodString, z.ZodCustom<Uint8Array<ArrayBuffer>, Uint8Array<ArrayBuffer>>]>, z.ZodTransform<{
|
|
213
|
+
type: "base64";
|
|
214
|
+
value: string;
|
|
215
|
+
} | {
|
|
216
|
+
type: "filePath";
|
|
217
|
+
value: string;
|
|
218
|
+
}, string | Uint8Array<ArrayBuffer>>>;
|
|
174
219
|
export declare const audioGenClientParamsSchema: z.ZodObject<{
|
|
220
|
+
referenceAudio: z.ZodOptional<z.ZodPipe<z.ZodUnion<readonly [z.ZodString, z.ZodCustom<Uint8Array<ArrayBuffer>, Uint8Array<ArrayBuffer>>]>, z.ZodTransform<{
|
|
221
|
+
type: "base64";
|
|
222
|
+
value: string;
|
|
223
|
+
} | {
|
|
224
|
+
type: "filePath";
|
|
225
|
+
value: string;
|
|
226
|
+
}, string | Uint8Array<ArrayBuffer>>>>;
|
|
227
|
+
sourceAudio: z.ZodOptional<z.ZodPipe<z.ZodUnion<readonly [z.ZodString, z.ZodCustom<Uint8Array<ArrayBuffer>, Uint8Array<ArrayBuffer>>]>, z.ZodTransform<{
|
|
228
|
+
type: "base64";
|
|
229
|
+
value: string;
|
|
230
|
+
} | {
|
|
231
|
+
type: "filePath";
|
|
232
|
+
value: string;
|
|
233
|
+
}, string | Uint8Array<ArrayBuffer>>>>;
|
|
175
234
|
modelId: z.ZodString;
|
|
176
235
|
caption: z.ZodString;
|
|
177
236
|
lyrics: z.ZodOptional<z.ZodString>;
|
|
@@ -181,10 +240,38 @@ export declare const audioGenClientParamsSchema: z.ZodObject<{
|
|
|
181
240
|
keyscale: z.ZodOptional<z.ZodString>;
|
|
182
241
|
timesignature: z.ZodOptional<z.ZodString>;
|
|
183
242
|
duration: z.ZodOptional<z.ZodNumber>;
|
|
243
|
+
lmTemperature: z.ZodOptional<z.ZodNumber>;
|
|
244
|
+
lmTopP: z.ZodOptional<z.ZodNumber>;
|
|
245
|
+
lmTopK: z.ZodOptional<z.ZodNumber>;
|
|
246
|
+
lmCfgScale: z.ZodOptional<z.ZodNumber>;
|
|
247
|
+
lmPhase1: z.ZodOptional<z.ZodBoolean>;
|
|
248
|
+
dcwEnabled: z.ZodOptional<z.ZodBoolean>;
|
|
249
|
+
dcwScaler: z.ZodOptional<z.ZodNumber>;
|
|
250
|
+
dcwHighScaler: z.ZodOptional<z.ZodNumber>;
|
|
251
|
+
taskType: z.ZodOptional<z.ZodEnum<{
|
|
252
|
+
text2music: "text2music";
|
|
253
|
+
"cover-nofsq": "cover-nofsq";
|
|
254
|
+
}>>;
|
|
255
|
+
audioCoverStrength: z.ZodOptional<z.ZodNumber>;
|
|
256
|
+
coverNoiseStrength: z.ZodOptional<z.ZodNumber>;
|
|
184
257
|
}, z.core.$strict>;
|
|
185
258
|
export declare const audioGenStreamRequestSchema: z.ZodObject<{
|
|
186
259
|
type: z.ZodLiteral<"audioGenStream">;
|
|
187
260
|
requestId: z.ZodOptional<z.ZodString>;
|
|
261
|
+
referenceAudio: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
262
|
+
type: z.ZodLiteral<"base64">;
|
|
263
|
+
value: z.ZodString;
|
|
264
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
265
|
+
type: z.ZodLiteral<"filePath">;
|
|
266
|
+
value: z.ZodString;
|
|
267
|
+
}, z.core.$strip>], "type">>;
|
|
268
|
+
sourceAudio: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
269
|
+
type: z.ZodLiteral<"base64">;
|
|
270
|
+
value: z.ZodString;
|
|
271
|
+
}, z.core.$strip>, z.ZodObject<{
|
|
272
|
+
type: z.ZodLiteral<"filePath">;
|
|
273
|
+
value: z.ZodString;
|
|
274
|
+
}, z.core.$strip>], "type">>;
|
|
188
275
|
modelId: z.ZodString;
|
|
189
276
|
caption: z.ZodString;
|
|
190
277
|
lyrics: z.ZodOptional<z.ZodString>;
|
|
@@ -194,6 +281,20 @@ export declare const audioGenStreamRequestSchema: z.ZodObject<{
|
|
|
194
281
|
keyscale: z.ZodOptional<z.ZodString>;
|
|
195
282
|
timesignature: z.ZodOptional<z.ZodString>;
|
|
196
283
|
duration: z.ZodOptional<z.ZodNumber>;
|
|
284
|
+
lmTemperature: z.ZodOptional<z.ZodNumber>;
|
|
285
|
+
lmTopP: z.ZodOptional<z.ZodNumber>;
|
|
286
|
+
lmTopK: z.ZodOptional<z.ZodNumber>;
|
|
287
|
+
lmCfgScale: z.ZodOptional<z.ZodNumber>;
|
|
288
|
+
lmPhase1: z.ZodOptional<z.ZodBoolean>;
|
|
289
|
+
dcwEnabled: z.ZodOptional<z.ZodBoolean>;
|
|
290
|
+
dcwScaler: z.ZodOptional<z.ZodNumber>;
|
|
291
|
+
dcwHighScaler: z.ZodOptional<z.ZodNumber>;
|
|
292
|
+
taskType: z.ZodOptional<z.ZodEnum<{
|
|
293
|
+
text2music: "text2music";
|
|
294
|
+
"cover-nofsq": "cover-nofsq";
|
|
295
|
+
}>>;
|
|
296
|
+
audioCoverStrength: z.ZodOptional<z.ZodNumber>;
|
|
297
|
+
coverNoiseStrength: z.ZodOptional<z.ZodNumber>;
|
|
197
298
|
}, z.core.$strict>;
|
|
198
299
|
export declare const audioGenProgressSchema: z.ZodObject<{
|
|
199
300
|
stage: z.ZodString;
|
|
@@ -231,6 +332,8 @@ export declare const audioGenStreamResponseSchema: z.ZodObject<{
|
|
|
231
332
|
backendId: z.ZodOptional<z.ZodNumber>;
|
|
232
333
|
}, z.core.$strip>>;
|
|
233
334
|
}, z.core.$strict>;
|
|
335
|
+
export type AudioGenTaskType = z.infer<typeof audioGenTaskTypeSchema>;
|
|
336
|
+
export type AudioGenAudioInput = z.infer<typeof audioGenAudioInputSchema>;
|
|
234
337
|
export type AudioGenRuntimeConfig = z.infer<typeof audioGenRuntimeConfigSchema>;
|
|
235
338
|
export type AudioGenConfig = z.infer<typeof audioGenConfigSchema>;
|
|
236
339
|
export type AudioGenClientParams = z.input<typeof audioGenClientParamsSchema>;
|