@qvac/inference 0.17.1 → 0.18.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/dist/api/completion-stream.d.ts +1 -1
  2. package/dist/api/completion-stream.js +1 -1
  3. package/dist/api/index.d.ts +1 -1
  4. package/dist/api/index.js +1 -1
  5. package/dist/api/text-to-speech.js +22 -2
  6. package/dist/api/transcribe.d.ts +19 -2
  7. package/dist/api/transcribe.js +66 -27
  8. package/dist/api/translate.js +6 -17
  9. package/dist/api/vla.d.ts +42 -1
  10. package/dist/api/vla.js +50 -1
  11. package/dist/constants/audio.d.ts +2 -1
  12. package/dist/constants/audio.js +17 -1
  13. package/dist/errors/index.d.ts +3 -0
  14. package/dist/errors/index.js +5 -0
  15. package/dist/handlers/load-model/handler.js +2 -2
  16. package/dist/handlers/load-model/resolve-session.d.ts +1 -1
  17. package/dist/handlers/load-model/resolve-session.js +3 -3
  18. package/dist/handlers/load-model/resolve.d.ts +12 -2
  19. package/dist/handlers/load-model/resolve.js +90 -12
  20. package/dist/index.d.ts +1 -1
  21. package/dist/index.js +1 -1
  22. package/dist/models/registry/models.d.ts +782 -6
  23. package/dist/models/registry/models.js +2664 -1859
  24. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +19 -0
  25. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +153 -0
  26. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +54 -4
  27. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +28 -0
  28. package/dist/plugins/builtin/ggml-ocr/plugin.js +3 -1
  29. package/dist/plugins/builtin/ggml-vla/ops/vla-set-embodiment.d.ts +2 -0
  30. package/dist/plugins/builtin/ggml-vla/ops/vla-set-embodiment.js +9 -0
  31. package/dist/plugins/builtin/ggml-vla/plugin.d.ts +37 -0
  32. package/dist/plugins/builtin/ggml-vla/plugin.js +21 -4
  33. package/dist/plugins/builtin/index.d.ts +1 -0
  34. package/dist/plugins/builtin/index.js +1 -0
  35. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +16 -15
  36. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.d.ts +1 -1
  37. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +2 -2
  38. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +6 -0
  39. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +10 -0
  40. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +1 -0
  41. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +131 -123
  42. package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +110 -20
  43. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +50 -23
  44. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +308 -143
  45. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +1 -0
  46. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +2 -2
  47. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +6 -6
  48. package/dist/plugins/builtin/llamacpp-completion/plugin.js +56 -36
  49. package/dist/plugins/builtin/llamacpp-completion/transform.js +0 -7
  50. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -3
  51. package/dist/plugins/builtin/nmtcpp-translation/plugin.js +6 -5
  52. package/dist/plugins/builtin/parakeet-transcription/plugin.js +27 -16
  53. package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +23 -3
  54. package/dist/plugins/builtin/tts-ggml/plugin.d.ts +302 -4
  55. package/dist/plugins/builtin/tts-ggml/plugin.js +131 -0
  56. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +48 -39
  57. package/dist/plugins/ops/cancel.d.ts +1 -1
  58. package/dist/plugins/ops/cancel.js +1 -1
  59. package/dist/plugins/ops/kv-cache-retention.js +11 -0
  60. package/dist/plugins/ops/kv-cache-utils.d.ts +1 -1
  61. package/dist/plugins/ops/kv-cache-utils.js +40 -12
  62. package/dist/plugins/ops/transcribe.d.ts +2 -2
  63. package/dist/plugins/ops/transcribe.js +54 -48
  64. package/dist/plugins/ops/translate-stats.d.ts +4 -0
  65. package/dist/plugins/ops/translate-stats.js +11 -0
  66. package/dist/plugins/ops/translate.js +66 -38
  67. package/dist/plugins/ops/unload-model.js +30 -24
  68. package/dist/runtime/model-registry.d.ts +1 -0
  69. package/dist/runtime/request-context.js +29 -14
  70. package/dist/runtime/request-registry.d.ts +27 -0
  71. package/dist/runtime/request-registry.js +246 -76
  72. package/dist/schemas/audio-gen.d.ts +103 -0
  73. package/dist/schemas/audio-gen.js +135 -4
  74. package/dist/schemas/common.d.ts +550 -11
  75. package/dist/schemas/config.d.ts +60 -24
  76. package/dist/schemas/errors.d.ts +1 -0
  77. package/dist/schemas/errors.js +5 -0
  78. package/dist/schemas/index.d.ts +1 -1
  79. package/dist/schemas/index.js +1 -1
  80. package/dist/schemas/llamacpp-config.d.ts +10 -10
  81. package/dist/schemas/llamacpp-config.js +168 -82
  82. package/dist/schemas/load-model.d.ts +3691 -336
  83. package/dist/schemas/load-model.js +37 -19
  84. package/dist/schemas/text-to-speech.d.ts +1086 -110
  85. package/dist/schemas/text-to-speech.js +197 -7
  86. package/dist/schemas/tools.d.ts +0 -16
  87. package/dist/schemas/tools.js +0 -15
  88. package/dist/schemas/transcription-config.d.ts +2 -0
  89. package/dist/schemas/transcription-config.js +6 -0
  90. package/dist/schemas/transcription.d.ts +3 -0
  91. package/dist/schemas/vla.d.ts +83 -0
  92. package/dist/schemas/vla.js +105 -2
  93. package/dist/surface.d.ts +1 -1
  94. package/dist/surface.js +1 -1
  95. package/dist/utils/addon-responses.d.ts +4 -12
  96. package/dist/utils/audio/decoder.d.ts +37 -1
  97. package/dist/utils/audio/decoder.js +82 -52
  98. package/dist/utils/config-transform.d.ts +3 -0
  99. package/dist/utils/config-transform.js +6 -0
  100. package/dist/utils/index.d.ts +0 -1
  101. package/dist/utils/index.js +0 -1
  102. package/dist/utils/shard-utils.d.ts +1 -0
  103. package/dist/utils/shard-utils.js +14 -0
  104. package/dist/utils/tool-integration.d.ts +3 -10
  105. package/dist/utils/tool-integration.js +3 -12
  106. package/package.json +26 -22
  107. package/dist/plugins/builtin/llamacpp-completion/ops/batch-cancelled.d.ts +0 -22
  108. package/dist/plugins/builtin/llamacpp-completion/ops/batch-cancelled.js +0 -30
  109. package/dist/utils/expand-gguf-shards.d.ts +0 -2
  110. package/dist/utils/expand-gguf-shards.js +0 -22
@@ -49,6 +49,26 @@ export function createRequestRegistry(options) {
49
49
  * FIFO in O(1) without scanning every `KeyState`.
50
50
  */
51
51
  const waitersById = new Map();
52
+ /**
53
+ * Ids reserved synchronously at `begin(...)` start and cleared once the entry
54
+ * lands in `entries` (or the begin throws). A same-tick second `begin(...)`
55
+ * with the same id runs before the first reaches `entries`/`waitersById`, so
56
+ * without this it would slip past the duplicate check and both would execute.
57
+ */
58
+ const reservedIds = new Set();
59
+ // Models being torn down: a begin(...) for one starts aborted. The barrier
60
+ // half of cancelAndDrain — catches begins its scan can't see yet.
61
+ const drainingModels = new Set();
62
+ // Entries whose disposal has started but not finished. disposeEntry removes
63
+ // from `entries` before unwinding, so a drain must consult this too or it
64
+ // would return while a request is still tearing down natively.
65
+ const disposingEntries = new Set();
66
+ // Request ids whose entry is mid-disposal. disposeEntry drops the id from
67
+ // `entries` before its async scope unwinds; without this the id looks free
68
+ // during that gap, so a same-id begin could activate concurrently with the
69
+ // still-tearing-down request and a cancel would land as a cancel-before-begin
70
+ // tripwire that aborts a later reuse. Held until teardown completes.
71
+ const disposingIds = new Set();
52
72
  function logLifecycle(event, ctx, durationMs) {
53
73
  const modelId = ctx.modelId !== undefined ? ctx.modelId : '-';
54
74
  const base = `[request-lifecycle] ${event} requestId=${ctx.requestId} kind=${ctx.kind} modelId=${modelId} state=${ctx.state}`;
@@ -133,7 +153,7 @@ export function createRequestRegistry(options) {
133
153
  * removal resolves (granted / graceful cancel) or rejects (timeout /
134
154
  * teardown).
135
155
  */
136
- function removeWaiter(key, waiter) {
156
+ function removeWaiter(key, waiter, drainAfter = true) {
137
157
  if (waiter.timer !== undefined) {
138
158
  clearTimeout(waiter.timer);
139
159
  waiter.timer = undefined;
@@ -145,8 +165,14 @@ export function createRequestRegistry(options) {
145
165
  const i = st.waiters.indexOf(waiter);
146
166
  if (i >= 0)
147
167
  st.waiters.splice(i, 1);
148
- if (st.active <= 0 && st.waiters.length === 0)
168
+ // Removing a queued writer can unblock the readers behind it, so drain
169
+ // (which also deletes the KeyState once idle). `cancelAll` passes false:
170
+ // it's tearing every waiter down, so granting one mid-sweep is wrong.
171
+ if (drainAfter)
172
+ drain(st, key);
173
+ else if (st.active <= 0 && !st.exclusiveActive && st.waiters.length === 0) {
149
174
  keyStates.delete(key);
175
+ }
150
176
  }
151
177
  /**
152
178
  * Admission gate. Resolves to the slot key the caller must release on
@@ -164,29 +190,50 @@ export function createRequestRegistry(options) {
164
190
  */
165
191
  async function acquireSlot(opts) {
166
192
  if (opts.modelId === undefined)
167
- return { slotKey: undefined };
193
+ return { slotKey: undefined, exclusive: false };
168
194
  const policy = policies.get(opts.kind);
169
- if (!policy || !Number.isFinite(policy.maxConcurrent)) {
170
- return { slotKey: undefined };
195
+ if (!policy)
196
+ return { slotKey: undefined, exclusive: false };
197
+ // The per-request override (the model's own `parallel`) wins over the
198
+ // per-kind default. A non-finite effective cap disables gating entirely.
199
+ const requestedMax = opts.maxConcurrentPerModel ?? policy.maxConcurrent;
200
+ if (!Number.isFinite(requestedMax)) {
201
+ return { slotKey: undefined, exclusive: false };
171
202
  }
203
+ const maxConcurrent = clampSlotCount(requestedMax);
172
204
  // A parent (process-shutdown) signal that's already aborted: don't
173
205
  // queue behind live work that may never drain — let begin() proceed
174
206
  // and abort immediately via the parentSignal path.
175
207
  if (opts.parentSignal?.aborted)
176
- return { slotKey: undefined };
208
+ return { slotKey: undefined, exclusive: false };
177
209
  const modelId = opts.modelId;
178
- // A shared lane lets unrelated kinds (e.g. completion + batchCompletion)
179
- // contend for one slot pool per model, matching an addon that serializes
180
- // them on a single native context.
210
+ // Lane precedence: the policy's shared group → the kind itself. A shared
211
+ // lane lets unrelated kinds contend for one slot pool per model.
181
212
  const key = slotKey(policy.slotGroup ?? opts.kind, modelId);
182
213
  let st = keyStates.get(key);
183
214
  if (!st) {
184
- st = { active: 0, waiters: [] };
215
+ st = { active: 0, waiters: [], exclusiveActive: false };
185
216
  keyStates.set(key, st);
186
217
  }
187
- if (st.active < policy.maxConcurrent) {
188
- st.active++;
189
- return { slotKey: key };
218
+ const exclusive = policy.exclusive;
219
+ // A reader admits below the lane's pinned cap (see `readerCap`); an exclusive
220
+ // writer needs the lane empty. A queued waiter (of any kind) blocks admission
221
+ // so a writer isn't starved by a steady reader stream. For reader-only lanes
222
+ // `waiters` is empty whenever `active < cap`, so this guard is a no-op there.
223
+ const readerCap = st.readerCap ?? maxConcurrent;
224
+ const canAdmit = st.waiters.length === 0 &&
225
+ !st.exclusiveActive &&
226
+ (exclusive ? st.active === 0 : st.active < readerCap);
227
+ if (canAdmit) {
228
+ if (exclusive) {
229
+ st.exclusiveActive = true;
230
+ }
231
+ else {
232
+ st.active++;
233
+ if (st.readerCap === undefined)
234
+ st.readerCap = maxConcurrent;
235
+ }
236
+ return { slotKey: key, exclusive };
190
237
  }
191
238
  if (policy.onOverflow === 'reject') {
192
239
  throw new RequestRejectedByPolicyError(opts.requestId, opts.kind, modelId, `another ${opts.kind} request is already running on this model`);
@@ -203,7 +250,9 @@ export function createRequestRegistry(options) {
203
250
  enqueuedAt: Date.now(),
204
251
  resolve,
205
252
  reject,
206
- timer: undefined
253
+ timer: undefined,
254
+ exclusive,
255
+ cap: maxConcurrent
207
256
  };
208
257
  if (policy.queueTimeoutMs !== undefined) {
209
258
  const timeoutMs = policy.queueTimeoutMs;
@@ -215,32 +264,59 @@ export function createRequestRegistry(options) {
215
264
  queue.waiters.push(waiter);
216
265
  waitersById.set(opts.requestId, { key, waiter });
217
266
  });
218
- return granted ? { slotKey: key } : { slotKey: undefined };
267
+ return granted ? { slotKey: key, exclusive } : { slotKey: undefined, exclusive: false };
219
268
  }
220
269
  /**
221
- * Hand a freed slot to the next FIFO waiter, or free it outright when
222
- * none are queued. Called from `disposeEntry` after the scope has fully
223
- * unwound, so the native context is genuinely free before the next
224
- * same-model request is admitted.
270
+ * Grant the lane to as many FIFO-head waiters as now fit: an exclusive writer
271
+ * only when the lane is idle, readers up to each one's own cap. Stops at the
272
+ * first head that can't be admitted, so FIFO order holds. Deletes the
273
+ * `KeyState` once fully idle.
225
274
  */
226
- function releaseSlot(key) {
227
- const st = keyStates.get(key);
228
- if (!st)
229
- return;
230
- const next = st.waiters.shift();
231
- if (next) {
232
- // Hand off: the waiter inherits the slot, so `active` is unchanged.
233
- waitersById.delete(next.requestId);
234
- if (next.timer !== undefined) {
235
- clearTimeout(next.timer);
236
- next.timer = undefined;
275
+ function drain(st, key) {
276
+ while (st.waiters.length > 0) {
277
+ const head = st.waiters[0];
278
+ if (head === undefined)
279
+ break;
280
+ if (head.exclusive) {
281
+ if (st.exclusiveActive || st.active > 0)
282
+ break;
283
+ st.exclusiveActive = true;
237
284
  }
238
- next.resolve(true);
239
- return;
285
+ else {
286
+ const readerCap = st.readerCap ?? head.cap;
287
+ if (st.exclusiveActive || st.active >= readerCap)
288
+ break;
289
+ st.active++;
290
+ if (st.readerCap === undefined)
291
+ st.readerCap = head.cap;
292
+ }
293
+ st.waiters.shift();
294
+ waitersById.delete(head.requestId);
295
+ if (head.timer !== undefined) {
296
+ clearTimeout(head.timer);
297
+ head.timer = undefined;
298
+ }
299
+ head.resolve(true);
240
300
  }
241
- st.active--;
242
- if (st.active <= 0 && st.waiters.length === 0)
301
+ if (st.active <= 0 && !st.exclusiveActive && st.waiters.length === 0) {
243
302
  keyStates.delete(key);
303
+ }
304
+ }
305
+ /**
306
+ * Release a held slot, then hand the freed capacity to waiters. Called from
307
+ * `disposeEntry` after the scope has fully unwound, so its deferred teardown
308
+ * (for llama.cpp completion, disposing the run response that stops native
309
+ * decode) has run before the next same-model request is admitted.
310
+ */
311
+ function releaseSlot(key, wasExclusive) {
312
+ const st = keyStates.get(key);
313
+ if (!st)
314
+ return;
315
+ if (wasExclusive)
316
+ st.exclusiveActive = false;
317
+ else
318
+ st.active--;
319
+ drain(st, key);
244
320
  }
245
321
  function cancelEntry(entry, reason) {
246
322
  if (entry.controller.signal.aborted)
@@ -251,8 +327,13 @@ export function createRequestRegistry(options) {
251
327
  return true;
252
328
  }
253
329
  async function disposeEntry(entry, outcome) {
254
- if (entry.scope.disposed)
330
+ if (entry.scope.disposed || entry.disposing)
255
331
  return;
332
+ entry.disposing = true;
333
+ disposingEntries.add(entry);
334
+ // Keep the id reserved across async teardown so a same-id begin can't
335
+ // activate concurrently and a cancel can't strand a tripwire for a reuse.
336
+ disposingIds.add(entry.ctx.requestId);
256
337
  entry.ctx.state = outcome;
257
338
  entry.detachParent();
258
339
  logLifecycle('end', entry.ctx, Date.now() - entry.startedAt);
@@ -270,46 +351,81 @@ export function createRequestRegistry(options) {
270
351
  // guarantees the slot is freed even if a deferred cleanup throws,
271
352
  // otherwise a throwing teardown would strand the queue forever.
272
353
  if (entry.slotKey !== undefined)
273
- releaseSlot(entry.slotKey);
354
+ releaseSlot(entry.slotKey, entry.exclusive);
355
+ disposingEntries.delete(entry);
356
+ // The id is free for reuse only now that native teardown is complete.
357
+ disposingIds.delete(entry.ctx.requestId);
358
+ // Signal any `cancelAndDrain` waiting on this request that its native
359
+ // teardown is complete.
360
+ entry.resolveDisposed();
274
361
  }
275
362
  }
276
363
  async function begin(opts) {
277
- // A request id is reserved for its whole lifecycle, including the time
278
- // it spends *queued* for an admission slot. `waitersById` holds the
279
- // still-queued begins (they aren't in `entries` yet), so a duplicate id
280
- // must be rejected against both maps — otherwise a second begin with the
281
- // same id would enqueue behind the first and overwrite its `waitersById`
282
- // index, leaving the original waiter unreachable by `cancel({ requestId })`.
283
- if (entries.has(opts.requestId) || waitersById.has(opts.requestId)) {
364
+ // A request id is reserved for its whole lifecycle: the time it spends
365
+ // *queued* for an admission slot, while it is active, and while it is
366
+ // tearing down. `waitersById` holds the still-queued begins (not yet in
367
+ // `entries`) and `disposingIds` holds the ones whose async teardown hasn't
368
+ // finished, so a duplicate id must be rejected against all of them —
369
+ // otherwise a second begin with the same id would enqueue behind the first
370
+ // and overwrite its `waitersById` index (unreachable by `cancel(...)`), or
371
+ // activate while the prior request is still releasing its native context.
372
+ if (entries.has(opts.requestId) ||
373
+ waitersById.has(opts.requestId) ||
374
+ reservedIds.has(opts.requestId) ||
375
+ disposingIds.has(opts.requestId)) {
284
376
  throw new RequestIdConflictError(opts.requestId);
285
377
  }
286
- // Admission control runs before allocation so a rejected begin leaves
287
- // no controller / scope behind. This may await: a gated `(kind,
288
- // modelId)` at capacity queues the begin FIFO and resolves once a slot
289
- // frees (or rejects on overflow / queue-depth cap / timeout).
290
- const { slotKey } = await acquireSlot(opts);
291
- // The only interleaving point in this otherwise synchronous body is the
292
- // await above. A duplicate id could (astronomically unlikely) have
293
- // landed while we were queued — re-check and hand the just-acquired
294
- // slot back rather than stranding it on the conflicting throw.
295
- if (entries.has(opts.requestId)) {
296
- if (slotKey !== undefined)
297
- releaseSlot(slotKey);
298
- throw new RequestIdConflictError(opts.requestId);
378
+ // Reserve synchronously so a same-tick duplicate begin is rejected before
379
+ // this one reaches `entries`/`waitersById`. Cleared on every exit below.
380
+ reservedIds.add(opts.requestId);
381
+ // A request cancelled BEFORE begin must not queue for a slot on a saturated
382
+ // lane: consume its marker first and skip admission, so it resolves at once
383
+ // into an aborted context holding no slot instead of waiting for the holder.
384
+ let preCancel = consumeCancelBeforeBegin(opts.requestId);
385
+ // Admission control runs before allocation so a rejected begin leaves no
386
+ // controller / scope behind. This may await: a gated `(kind, modelId)` at
387
+ // capacity queues the begin FIFO and resolves once a slot frees (or rejects
388
+ // on overflow / queue-depth cap / timeout).
389
+ let slotKey;
390
+ let exclusive = false;
391
+ if (!preCancel) {
392
+ // Plain try/catch, not `.catch()`, so admission adds no extra microtask
393
+ // and observers still see the request registered on the same tick.
394
+ let acquired;
395
+ try {
396
+ acquired = await acquireSlot(opts);
397
+ }
398
+ catch (err) {
399
+ reservedIds.delete(opts.requestId);
400
+ throw err;
401
+ }
402
+ slotKey = acquired.slotKey;
403
+ exclusive = acquired.exclusive;
404
+ // The only interleaving point in this otherwise synchronous body is the
405
+ // await above. A duplicate id could (astronomically unlikely) have landed
406
+ // while we were queued — re-check and hand the just-acquired slot back
407
+ // rather than stranding it on the conflicting throw.
408
+ if (entries.has(opts.requestId)) {
409
+ if (slotKey !== undefined)
410
+ releaseSlot(slotKey, exclusive);
411
+ reservedIds.delete(opts.requestId);
412
+ throw new RequestIdConflictError(opts.requestId);
413
+ }
414
+ // A cancel that landed WHILE we were queued resolved acquireSlot with no
415
+ // slot and recorded a marker; consume it so the context starts aborted.
416
+ preCancel = consumeCancelBeforeBegin(opts.requestId);
299
417
  }
418
+ // A begin arriving while its model is being torn down starts aborted — no
419
+ // native work against a model about to be freed.
420
+ const drainReason = opts.modelId !== undefined && drainingModels.has(opts.modelId) ? 'modelUnload' : undefined;
300
421
  const controller = new AbortController();
301
422
  const scope = createDisposableScope();
302
- // Stop-button race close. If a
303
- // `cancel({ requestId })` already arrived for this id, abort the
304
- // new controller before observers can subscribe to it. The
305
- // tripwire entry is consumed so a later, separate `begin(...)`
306
- // with the same id is unaffected (in practice ids are UUIDv4 and
307
- // never reused; this guard just keeps the contract self-
308
- // consistent under retries).
309
- const preCancel = consumeCancelBeforeBegin(opts.requestId);
310
423
  if (preCancel) {
311
424
  controller.abort(preCancel.reason);
312
425
  }
426
+ else if (drainReason) {
427
+ controller.abort(drainReason);
428
+ }
313
429
  let detachParent = () => { };
314
430
  if (opts.parentSignal) {
315
431
  const parent = opts.parentSignal;
@@ -334,17 +450,26 @@ export function createRequestRegistry(options) {
334
450
  // already aborted at begin time. Both branches abort the
335
451
  // controller above, so without this guard observers would see a
336
452
  // momentarily-`running` context with an already-aborted signal.
337
- state: preCancel || opts.parentSignal?.aborted ? 'cancelling' : 'running'
453
+ state: preCancel || drainReason || opts.parentSignal?.aborted ? 'cancelling' : 'running'
338
454
  };
455
+ let resolveDisposed = () => { };
456
+ const disposed = new Promise((resolve) => {
457
+ resolveDisposed = resolve;
458
+ });
339
459
  const entry = {
340
460
  ctx,
341
461
  controller,
342
462
  scope,
343
463
  detachParent,
344
464
  startedAt: Date.now(),
345
- slotKey
465
+ slotKey,
466
+ exclusive,
467
+ disposing: false,
468
+ disposed,
469
+ resolveDisposed
346
470
  };
347
471
  entries.set(opts.requestId, entry);
472
+ reservedIds.delete(opts.requestId);
348
473
  logLifecycle('begin', ctx);
349
474
  return {
350
475
  get requestId() {
@@ -411,6 +536,12 @@ export function createRequestRegistry(options) {
411
536
  cancelQueuedWaiterGracefully(queued.key, queued.waiter, target.reason);
412
537
  return cancelled + 1;
413
538
  }
539
+ // Mid-disposal: the request already reached a terminal outcome and is
540
+ // releasing its native context. There is nothing to cancel, and a
541
+ // cancel-before-begin marker would wrongly abort a later begin that
542
+ // reuses this id once teardown frees it.
543
+ if (disposingIds.has(target.requestId))
544
+ return cancelled;
414
545
  // Stop-button race: the cancel beat its own
415
546
  // `begin(...)`. Record the cancel so the next matching `begin`
416
547
  // aborts immediately. The return value stays 0 — no in-flight
@@ -453,15 +584,48 @@ export function createRequestRegistry(options) {
453
584
  // so there is nothing left for them to run. Snapshot first —
454
585
  // `removeWaiter` mutates `waitersById`.
455
586
  for (const { key, waiter } of Array.from(waitersById.values())) {
456
- removeWaiter(key, waiter);
587
+ removeWaiter(key, waiter, false);
457
588
  waiter.reject(new RequestRejectedByPolicyError(waiter.requestId, waiter.kind, waiter.modelId, `queued request cancelled (${reason})`));
458
589
  }
459
- // The interface returns Promise<void> so we can later make this an
460
- // async sweep that awaits per-handler scope unwinding (e.g. join on
461
- // the disposers). Today every handler unwinds on its own dispose
462
- // path, so the function only needs to fire-and-forget the abort.
590
+ // Fire-and-forget the abort; each handler unwinds on its own dispose path.
463
591
  return Promise.resolve();
464
592
  }
593
+ // Cancel and drain every request for a model, then run `teardown` with the
594
+ // admission barrier held — any begin for the model (including one still
595
+ // suspended in admission) starts aborted — lifting it afterward. Scoped so the
596
+ // barrier can't be left raised.
597
+ async function withModelDraining(modelId, teardown) {
598
+ drainingModels.add(modelId);
599
+ try {
600
+ // Abort each active request and collect its disposal promise, so we wait
601
+ // for every handler's scope to fully unwind (native context freed, slot
602
+ // released) before teardown. Snapshot first — `disposeEntry` mutates
603
+ // `entries`.
604
+ const draining = [];
605
+ for (const entry of entries.values()) {
606
+ if (entry.ctx.modelId !== modelId)
607
+ continue;
608
+ cancelEntry(entry, 'modelUnload');
609
+ draining.push(entry.disposed);
610
+ }
611
+ // Requests already mid-disposal are out of `entries` but may still be
612
+ // tearing down natively — wait for them too.
613
+ for (const entry of disposingEntries) {
614
+ if (entry.ctx.modelId === modelId)
615
+ draining.push(entry.disposed);
616
+ }
617
+ // Queued waiters never held a slot, so just abort their begins.
618
+ for (const { key, waiter } of Array.from(waitersById.values())) {
619
+ if (waiter.modelId === modelId)
620
+ cancelQueuedWaiterGracefully(key, waiter, 'modelUnload');
621
+ }
622
+ await Promise.all(draining);
623
+ await teardown();
624
+ }
625
+ finally {
626
+ drainingModels.delete(modelId);
627
+ }
628
+ }
465
629
  async function end(requestId, outcome) {
466
630
  const entry = entries.get(requestId);
467
631
  if (!entry)
@@ -477,6 +641,7 @@ export function createRequestRegistry(options) {
477
641
  list,
478
642
  cancel,
479
643
  cancelAll,
644
+ withModelDraining,
480
645
  end,
481
646
  policy,
482
647
  // Test-only: lets the registry race tests assert the bound
@@ -491,6 +656,12 @@ export function createRequestRegistry(options) {
491
656
  __keyStateSize: () => keyStates.size
492
657
  };
493
658
  }
659
+ // A finite slot count floored to an integer >= 1: a value below 1 would gate
660
+ // every request forever and a fractional one could over-admit. Callers gate on
661
+ // `Number.isFinite` first — Infinity means "ungated" and must stay Infinity.
662
+ function clampSlotCount(count) {
663
+ return Math.max(1, Math.floor(count));
664
+ }
494
665
  /**
495
666
  * Resolve a `ConcurrencyPolicy` to concrete numbers, applying defaults and
496
667
  * the deprecated `oneAtATimePerModel` alias. Done once at registration so
@@ -513,16 +684,15 @@ function normalizePolicy(opts) {
513
684
  maxConcurrent = Infinity;
514
685
  onOverflow = opts.onOverflow ?? 'queue';
515
686
  }
516
- // A finite limit below 1 would gate every request forever; clamp to the
517
- // smallest sensible serial limit.
518
- if (Number.isFinite(maxConcurrent) && maxConcurrent < 1)
519
- maxConcurrent = 1;
687
+ if (Number.isFinite(maxConcurrent))
688
+ maxConcurrent = clampSlotCount(maxConcurrent);
520
689
  return {
521
690
  maxConcurrent,
522
691
  onOverflow,
523
692
  maxQueueDepth: opts.maxQueueDepthPerModel ?? DEFAULT_MAX_QUEUE_DEPTH_PER_MODEL,
524
693
  queueTimeoutMs: opts.queueTimeoutMs,
525
- slotGroup: opts.sharedSlotGroup
694
+ slotGroup: opts.sharedSlotGroup,
695
+ exclusive: opts.exclusive === true
526
696
  };
527
697
  }
528
698
  /**
@@ -1,4 +1,26 @@
1
1
  import { z } from 'zod';
2
+ /** Sample rate the ACE-Step engine expects for reference and source audio. */
3
+ export declare const AUDIOGEN_INPUT_SAMPLE_RATE = 48000;
4
+ /** Channel count (interleaved stereo) the ACE-Step engine expects for input audio. */
5
+ export declare const AUDIOGEN_INPUT_CHANNELS = 2;
6
+ /**
7
+ * Longest reference/source clip the SDK accepts (10 minutes). Bounds the PCM
8
+ * the server materializes per input — a 48 kHz stereo Float32 clip weighs
9
+ * 384 KB per second — so a request cannot exhaust the inference process.
10
+ */
11
+ export declare const AUDIOGEN_INPUT_MAX_SECONDS = 600;
12
+ /**
13
+ * ACE-Step task discriminators reachable through the SDK. `text2music` is the
14
+ * default caption-driven generation; `cover-nofsq` re-renders `sourceAudio`
15
+ * with a new caption while keeping its structure. The engine also reserves an
16
+ * FSQ-roundtrip `cover` task that is not implemented yet, so it is not offered
17
+ * here.
18
+ */
19
+ export declare const AUDIOGEN_TASK_TYPES: readonly ["text2music", "cover-nofsq"];
20
+ export declare const audioGenTaskTypeSchema: z.ZodEnum<{
21
+ text2music: "text2music";
22
+ "cover-nofsq": "cover-nofsq";
23
+ }>;
2
24
  export declare const audioGenRuntimeConfigSchema: z.ZodObject<{
3
25
  useGPU: z.ZodOptional<z.ZodBoolean>;
4
26
  inferenceSteps: z.ZodOptional<z.ZodNumber>;
@@ -171,7 +193,44 @@ export declare const audioGenConfigSchema: z.ZodObject<{
171
193
  }>, z.ZodLiteral<"vad">]>>;
172
194
  }, z.core.$strip>]>;
173
195
  }, z.core.$strict>;
196
+ /**
197
+ * Wire form of a reference/source audio input. `filePath` inputs are decoded
198
+ * server-side (any format the SDK's audio decoder supports, plus raw PCM);
199
+ * `base64` inputs must already be interleaved stereo 48 kHz Float32 LE PCM.
200
+ */
201
+ export declare const audioGenAudioInputSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
202
+ type: z.ZodLiteral<"base64">;
203
+ value: z.ZodString;
204
+ }, z.core.$strip>, z.ZodObject<{
205
+ type: z.ZodLiteral<"filePath">;
206
+ value: z.ZodString;
207
+ }, z.core.$strip>], "type">;
208
+ /**
209
+ * Client form of a reference/source audio input: a file path, or raw
210
+ * interleaved stereo 48 kHz Float32 LE PCM bytes. Normalized to the wire form.
211
+ */
212
+ export declare const audioGenClientAudioInputSchema: z.ZodPipe<z.ZodUnion<readonly [z.ZodString, z.ZodCustom<Uint8Array<ArrayBuffer>, Uint8Array<ArrayBuffer>>]>, z.ZodTransform<{
213
+ type: "base64";
214
+ value: string;
215
+ } | {
216
+ type: "filePath";
217
+ value: string;
218
+ }, string | Uint8Array<ArrayBuffer>>>;
174
219
  export declare const audioGenClientParamsSchema: z.ZodObject<{
220
+ referenceAudio: z.ZodOptional<z.ZodPipe<z.ZodUnion<readonly [z.ZodString, z.ZodCustom<Uint8Array<ArrayBuffer>, Uint8Array<ArrayBuffer>>]>, z.ZodTransform<{
221
+ type: "base64";
222
+ value: string;
223
+ } | {
224
+ type: "filePath";
225
+ value: string;
226
+ }, string | Uint8Array<ArrayBuffer>>>>;
227
+ sourceAudio: z.ZodOptional<z.ZodPipe<z.ZodUnion<readonly [z.ZodString, z.ZodCustom<Uint8Array<ArrayBuffer>, Uint8Array<ArrayBuffer>>]>, z.ZodTransform<{
228
+ type: "base64";
229
+ value: string;
230
+ } | {
231
+ type: "filePath";
232
+ value: string;
233
+ }, string | Uint8Array<ArrayBuffer>>>>;
175
234
  modelId: z.ZodString;
176
235
  caption: z.ZodString;
177
236
  lyrics: z.ZodOptional<z.ZodString>;
@@ -181,10 +240,38 @@ export declare const audioGenClientParamsSchema: z.ZodObject<{
181
240
  keyscale: z.ZodOptional<z.ZodString>;
182
241
  timesignature: z.ZodOptional<z.ZodString>;
183
242
  duration: z.ZodOptional<z.ZodNumber>;
243
+ lmTemperature: z.ZodOptional<z.ZodNumber>;
244
+ lmTopP: z.ZodOptional<z.ZodNumber>;
245
+ lmTopK: z.ZodOptional<z.ZodNumber>;
246
+ lmCfgScale: z.ZodOptional<z.ZodNumber>;
247
+ lmPhase1: z.ZodOptional<z.ZodBoolean>;
248
+ dcwEnabled: z.ZodOptional<z.ZodBoolean>;
249
+ dcwScaler: z.ZodOptional<z.ZodNumber>;
250
+ dcwHighScaler: z.ZodOptional<z.ZodNumber>;
251
+ taskType: z.ZodOptional<z.ZodEnum<{
252
+ text2music: "text2music";
253
+ "cover-nofsq": "cover-nofsq";
254
+ }>>;
255
+ audioCoverStrength: z.ZodOptional<z.ZodNumber>;
256
+ coverNoiseStrength: z.ZodOptional<z.ZodNumber>;
184
257
  }, z.core.$strict>;
185
258
  export declare const audioGenStreamRequestSchema: z.ZodObject<{
186
259
  type: z.ZodLiteral<"audioGenStream">;
187
260
  requestId: z.ZodOptional<z.ZodString>;
261
+ referenceAudio: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
262
+ type: z.ZodLiteral<"base64">;
263
+ value: z.ZodString;
264
+ }, z.core.$strip>, z.ZodObject<{
265
+ type: z.ZodLiteral<"filePath">;
266
+ value: z.ZodString;
267
+ }, z.core.$strip>], "type">>;
268
+ sourceAudio: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
269
+ type: z.ZodLiteral<"base64">;
270
+ value: z.ZodString;
271
+ }, z.core.$strip>, z.ZodObject<{
272
+ type: z.ZodLiteral<"filePath">;
273
+ value: z.ZodString;
274
+ }, z.core.$strip>], "type">>;
188
275
  modelId: z.ZodString;
189
276
  caption: z.ZodString;
190
277
  lyrics: z.ZodOptional<z.ZodString>;
@@ -194,6 +281,20 @@ export declare const audioGenStreamRequestSchema: z.ZodObject<{
194
281
  keyscale: z.ZodOptional<z.ZodString>;
195
282
  timesignature: z.ZodOptional<z.ZodString>;
196
283
  duration: z.ZodOptional<z.ZodNumber>;
284
+ lmTemperature: z.ZodOptional<z.ZodNumber>;
285
+ lmTopP: z.ZodOptional<z.ZodNumber>;
286
+ lmTopK: z.ZodOptional<z.ZodNumber>;
287
+ lmCfgScale: z.ZodOptional<z.ZodNumber>;
288
+ lmPhase1: z.ZodOptional<z.ZodBoolean>;
289
+ dcwEnabled: z.ZodOptional<z.ZodBoolean>;
290
+ dcwScaler: z.ZodOptional<z.ZodNumber>;
291
+ dcwHighScaler: z.ZodOptional<z.ZodNumber>;
292
+ taskType: z.ZodOptional<z.ZodEnum<{
293
+ text2music: "text2music";
294
+ "cover-nofsq": "cover-nofsq";
295
+ }>>;
296
+ audioCoverStrength: z.ZodOptional<z.ZodNumber>;
297
+ coverNoiseStrength: z.ZodOptional<z.ZodNumber>;
197
298
  }, z.core.$strict>;
198
299
  export declare const audioGenProgressSchema: z.ZodObject<{
199
300
  stage: z.ZodString;
@@ -231,6 +332,8 @@ export declare const audioGenStreamResponseSchema: z.ZodObject<{
231
332
  backendId: z.ZodOptional<z.ZodNumber>;
232
333
  }, z.core.$strip>>;
233
334
  }, z.core.$strict>;
335
+ export type AudioGenTaskType = z.infer<typeof audioGenTaskTypeSchema>;
336
+ export type AudioGenAudioInput = z.infer<typeof audioGenAudioInputSchema>;
234
337
  export type AudioGenRuntimeConfig = z.infer<typeof audioGenRuntimeConfigSchema>;
235
338
  export type AudioGenConfig = z.infer<typeof audioGenConfigSchema>;
236
339
  export type AudioGenClientParams = z.input<typeof audioGenClientParamsSchema>;