@qvac/inference 0.17.1 → 0.18.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/dist/api/completion-stream.d.ts +1 -1
  2. package/dist/api/completion-stream.js +1 -1
  3. package/dist/api/index.d.ts +1 -1
  4. package/dist/api/index.js +1 -1
  5. package/dist/api/text-to-speech.js +22 -2
  6. package/dist/api/transcribe.d.ts +19 -2
  7. package/dist/api/transcribe.js +66 -27
  8. package/dist/api/translate.js +6 -17
  9. package/dist/api/vla.d.ts +42 -1
  10. package/dist/api/vla.js +50 -1
  11. package/dist/constants/audio.d.ts +2 -1
  12. package/dist/constants/audio.js +17 -1
  13. package/dist/errors/index.d.ts +3 -0
  14. package/dist/errors/index.js +5 -0
  15. package/dist/handlers/load-model/handler.js +2 -2
  16. package/dist/handlers/load-model/resolve-session.d.ts +1 -1
  17. package/dist/handlers/load-model/resolve-session.js +3 -3
  18. package/dist/handlers/load-model/resolve.d.ts +12 -2
  19. package/dist/handlers/load-model/resolve.js +90 -12
  20. package/dist/index.d.ts +1 -1
  21. package/dist/index.js +1 -1
  22. package/dist/models/registry/models.d.ts +782 -6
  23. package/dist/models/registry/models.js +2664 -1859
  24. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +19 -0
  25. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +153 -0
  26. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +54 -4
  27. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +28 -0
  28. package/dist/plugins/builtin/ggml-ocr/plugin.js +3 -1
  29. package/dist/plugins/builtin/ggml-vla/ops/vla-set-embodiment.d.ts +2 -0
  30. package/dist/plugins/builtin/ggml-vla/ops/vla-set-embodiment.js +9 -0
  31. package/dist/plugins/builtin/ggml-vla/plugin.d.ts +37 -0
  32. package/dist/plugins/builtin/ggml-vla/plugin.js +21 -4
  33. package/dist/plugins/builtin/index.d.ts +1 -0
  34. package/dist/plugins/builtin/index.js +1 -0
  35. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +16 -15
  36. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.d.ts +1 -1
  37. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +2 -2
  38. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +6 -0
  39. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +10 -0
  40. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +1 -0
  41. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +131 -123
  42. package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +110 -20
  43. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +50 -23
  44. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +308 -143
  45. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +1 -0
  46. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +2 -2
  47. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +6 -6
  48. package/dist/plugins/builtin/llamacpp-completion/plugin.js +56 -36
  49. package/dist/plugins/builtin/llamacpp-completion/transform.js +0 -7
  50. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -3
  51. package/dist/plugins/builtin/nmtcpp-translation/plugin.js +6 -5
  52. package/dist/plugins/builtin/parakeet-transcription/plugin.js +27 -16
  53. package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +23 -3
  54. package/dist/plugins/builtin/tts-ggml/plugin.d.ts +302 -4
  55. package/dist/plugins/builtin/tts-ggml/plugin.js +131 -0
  56. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +48 -39
  57. package/dist/plugins/ops/cancel.d.ts +1 -1
  58. package/dist/plugins/ops/cancel.js +1 -1
  59. package/dist/plugins/ops/kv-cache-retention.js +11 -0
  60. package/dist/plugins/ops/kv-cache-utils.d.ts +1 -1
  61. package/dist/plugins/ops/kv-cache-utils.js +40 -12
  62. package/dist/plugins/ops/transcribe.d.ts +2 -2
  63. package/dist/plugins/ops/transcribe.js +54 -48
  64. package/dist/plugins/ops/translate-stats.d.ts +4 -0
  65. package/dist/plugins/ops/translate-stats.js +11 -0
  66. package/dist/plugins/ops/translate.js +66 -38
  67. package/dist/plugins/ops/unload-model.js +30 -24
  68. package/dist/runtime/model-registry.d.ts +1 -0
  69. package/dist/runtime/request-context.js +29 -14
  70. package/dist/runtime/request-registry.d.ts +27 -0
  71. package/dist/runtime/request-registry.js +246 -76
  72. package/dist/schemas/audio-gen.d.ts +103 -0
  73. package/dist/schemas/audio-gen.js +135 -4
  74. package/dist/schemas/common.d.ts +550 -11
  75. package/dist/schemas/config.d.ts +60 -24
  76. package/dist/schemas/errors.d.ts +1 -0
  77. package/dist/schemas/errors.js +5 -0
  78. package/dist/schemas/index.d.ts +1 -1
  79. package/dist/schemas/index.js +1 -1
  80. package/dist/schemas/llamacpp-config.d.ts +10 -10
  81. package/dist/schemas/llamacpp-config.js +168 -82
  82. package/dist/schemas/load-model.d.ts +3691 -336
  83. package/dist/schemas/load-model.js +37 -19
  84. package/dist/schemas/text-to-speech.d.ts +1086 -110
  85. package/dist/schemas/text-to-speech.js +197 -7
  86. package/dist/schemas/tools.d.ts +0 -16
  87. package/dist/schemas/tools.js +0 -15
  88. package/dist/schemas/transcription-config.d.ts +2 -0
  89. package/dist/schemas/transcription-config.js +6 -0
  90. package/dist/schemas/transcription.d.ts +3 -0
  91. package/dist/schemas/vla.d.ts +83 -0
  92. package/dist/schemas/vla.js +105 -2
  93. package/dist/surface.d.ts +1 -1
  94. package/dist/surface.js +1 -1
  95. package/dist/utils/addon-responses.d.ts +4 -12
  96. package/dist/utils/audio/decoder.d.ts +37 -1
  97. package/dist/utils/audio/decoder.js +82 -52
  98. package/dist/utils/config-transform.d.ts +3 -0
  99. package/dist/utils/config-transform.js +6 -0
  100. package/dist/utils/index.d.ts +0 -1
  101. package/dist/utils/index.js +0 -1
  102. package/dist/utils/shard-utils.d.ts +1 -0
  103. package/dist/utils/shard-utils.js +14 -0
  104. package/dist/utils/tool-integration.d.ts +3 -10
  105. package/dist/utils/tool-integration.js +3 -12
  106. package/package.json +26 -22
  107. package/dist/plugins/builtin/llamacpp-completion/ops/batch-cancelled.d.ts +0 -22
  108. package/dist/plugins/builtin/llamacpp-completion/ops/batch-cancelled.js +0 -30
  109. package/dist/utils/expand-gguf-shards.d.ts +0 -2
  110. package/dist/utils/expand-gguf-shards.js +0 -22
@@ -37,6 +37,7 @@ interface CacheCommitContext {
37
37
  producedTokens: boolean;
38
38
  generatedTokens?: number | undefined;
39
39
  predict?: number | undefined;
40
+ stoppedAtContextBoundary: boolean;
40
41
  }
41
42
  export declare function shouldCommitCachedTurn(context: CacheCommitContext): boolean;
42
43
  /**
@@ -15,12 +15,12 @@
15
15
  * slice-decision helper remains here.
16
16
  */
17
17
  export function shouldCommitCachedTurn(context) {
18
- const { aborted, producedTokens, generatedTokens, predict } = context;
18
+ const { aborted, producedTokens, generatedTokens, predict, stoppedAtContextBoundary } = context;
19
19
  const stoppedByBudget = predict !== undefined &&
20
20
  predict > 0 &&
21
21
  generatedTokens !== undefined &&
22
22
  generatedTokens >= predict;
23
- return !aborted && producedTokens && !stoppedByBudget;
23
+ return !aborted && producedTokens && !stoppedByBudget && !stoppedAtContextBoundary;
24
24
  }
25
25
  /**
26
26
  * Pure slice decision for `prepareMessagesForCache`.
@@ -28,10 +28,6 @@ export declare const llmPlugin: {
28
28
  n_discarded: import("zod").ZodOptional<import("zod").ZodNumber>;
29
29
  parallel: import("zod").ZodOptional<import("zod").ZodNumber>;
30
30
  tools: import("zod").ZodOptional<import("zod").ZodBoolean>;
31
- toolsMode: import("zod").ZodOptional<import("zod").ZodEnum<{
32
- static: "static";
33
- dynamic: "dynamic";
34
- }>>;
35
31
  'cache-type-k': import("zod").ZodOptional<import("zod").ZodString>;
36
32
  'cache-type-v': import("zod").ZodOptional<import("zod").ZodString>;
37
33
  'main-gpu': import("zod").ZodOptional<import("zod").ZodUnion<readonly [import("zod").ZodNumber, import("zod").ZodEnum<{
@@ -90,6 +86,10 @@ export declare const llmPlugin: {
90
86
  batched: "batched";
91
87
  sequential: "sequential";
92
88
  }>>;
89
+ image_no_upscale: import("zod").ZodOptional<import("zod").ZodEnum<{
90
+ on: "on";
91
+ off: "off";
92
+ }>>;
93
93
  'mmproj-use-gpu': import("zod").ZodOptional<import("zod").ZodBoolean>;
94
94
  }, import("zod/v4/core").$strip>;
95
95
  resolveConfig(cfg: LlmConfigInput, ctx: ResolveContext): Promise<{
@@ -113,7 +113,6 @@ export declare const llmPlugin: {
113
113
  n_discarded?: number | undefined;
114
114
  parallel?: number | undefined;
115
115
  tools?: boolean | undefined;
116
- toolsMode?: "static" | "dynamic" | undefined;
117
116
  'cache-type-k'?: string | undefined;
118
117
  'cache-type-v'?: string | undefined;
119
118
  'main-gpu'?: number | "integrated" | "dedicated" | undefined;
@@ -122,6 +121,7 @@ export declare const llmPlugin: {
122
121
  openclCacheDir?: string | undefined;
123
122
  reasoning_budget?: number | undefined;
124
123
  image_tile_mode?: "disabled" | "batched" | "sequential" | undefined;
124
+ image_no_upscale?: "on" | "off" | undefined;
125
125
  'mmproj-use-gpu'?: boolean | undefined;
126
126
  };
127
127
  artifacts?: never;
@@ -146,7 +146,6 @@ export declare const llmPlugin: {
146
146
  n_discarded?: number | undefined;
147
147
  parallel?: number | undefined;
148
148
  tools?: boolean | undefined;
149
- toolsMode?: "static" | "dynamic" | undefined;
150
149
  'cache-type-k'?: string | undefined;
151
150
  'cache-type-v'?: string | undefined;
152
151
  'main-gpu'?: number | "integrated" | "dedicated" | undefined;
@@ -155,6 +154,7 @@ export declare const llmPlugin: {
155
154
  openclCacheDir?: string | undefined;
156
155
  reasoning_budget?: number | undefined;
157
156
  image_tile_mode?: "disabled" | "batched" | "sequential" | undefined;
157
+ image_no_upscale?: "on" | "off" | undefined;
158
158
  'mmproj-use-gpu'?: boolean | undefined;
159
159
  };
160
160
  artifacts: {
@@ -54,7 +54,8 @@ import LlmLlamacpp from '@qvac/llm-llamacpp';
54
54
  import llmAddonLogging from '@qvac/llm-llamacpp/addonLogging';
55
55
  import { definePlugin, defineHandler, finetuneRequestSchema, batchCompletionStreamRequestSchema, batchCompletionStreamResponseSchema, completionStreamRequestSchema, completionStreamResponseSchema, finetuneResponseSchema, translateRequestSchema, translateResponseSchema, ModelType, llmConfigBaseSchema, ADDON_LLM } from '../../../schemas/index.js';
56
56
  import { createStreamLogger, registerAddonLogger, getEngineLogger } from '../../../logging/index.js';
57
- import { expandGGUFIntoShards } from '../../../utils/index.js';
57
+ import { getFirstShardPath } from '../../../utils/index.js';
58
+ import { getModelParallel } from '../../../utils/config-transform.js';
58
59
  import { completion } from '../../../plugins/builtin/llamacpp-completion/ops/completion-stream.js';
59
60
  import { batchCompletion } from '../../../plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js';
60
61
  import { finetune } from '../../../plugins/builtin/llamacpp-completion/ops/finetune.js';
@@ -68,7 +69,7 @@ import { getRequestRegistry, withRequestContext } from '../../../runtime/index.j
68
69
  import { generateRandomRequestId } from '../../../runtime/request-id.js';
69
70
  import { ContextOverflowError } from '../../../errors/index.js';
70
71
  import { isAddonContextOverflowError, parseContextOverflowMessage } from '../../../plugins/builtin/llamacpp-completion/ops/context-overflow.js';
71
- import { isAddonCancelledError } from '../../../plugins/builtin/llamacpp-completion/ops/batch-cancelled.js';
72
+ import { stoppedByLength } from '../../../plugins/builtin/llamacpp-completion/ops/completion-stats.js';
72
73
  import { isMobile } from '../../../runtime/state.js';
73
74
  import { stripMultiGpuKeys } from '../../../utils/multi-gpu-mobile.js';
74
75
  function createLlmModel(modelId, modelPath, llmConfig, projectionModelPath) {
@@ -81,15 +82,17 @@ function createLlmModel(modelId, modelPath, llmConfig, projectionModelPath) {
81
82
  getEngineLogger().warn(`[${ModelType.llamacppCompletion}:${modelId}] Multi-GPU parameters (${stripped.join(', ')}) are not supported on mobile (single-GPU device) — removing from config; model will load with single-GPU defaults`);
82
83
  }
83
84
  }
84
- const modelFiles = expandGGUFIntoShards(modelPath);
85
+ // Addon overflow flag: at a single slot the addon throws when driven over
86
+ // capacity, at parallel>1 it queues the surplus. Not on the SDK surface.
87
+ const rejectWhenBusy = getModelParallel(llmConfig) <= 1;
85
88
  const model = new LlmLlamacpp({
86
89
  files: {
87
- model: modelFiles,
90
+ model: [getFirstShardPath(modelPath)],
88
91
  ...(projectionModelPath && { projectionModel: projectionModelPath })
89
92
  },
90
93
  config: llmConfigStrings,
91
94
  logger,
92
- opts: { stats: true }
95
+ opts: { stats: true, rejectWhenBusy }
93
96
  });
94
97
  return { model };
95
98
  }
@@ -135,7 +138,8 @@ export const llmPlugin = definePlugin({
135
138
  requestSchema: batchCompletionStreamRequestSchema,
136
139
  responseSchema: batchCompletionStreamResponseSchema,
137
140
  streaming: true,
138
- cancel: { scope: 'model', hard: true },
141
+ // Request-scope: a model-wide cancel would kill concurrent batching peers.
142
+ cancel: { scope: 'request', hard: true },
139
143
  handler: async function* (request) {
140
144
  const env_1 = { stack: [], error: void 0, hasError: false };
141
145
  try {
@@ -161,10 +165,14 @@ export const llmPlugin = definePlugin({
161
165
  }
162
166
  return normalizer;
163
167
  }
168
+ // Batches share the completion lane at the `parallel` cap. The cap counts
169
+ // requests, not the prompts inside a batch (the addon schedules those).
170
+ const parallel = getModelParallel(modelCfg);
164
171
  const ctx = __addDisposableResource(env_1, await getRequestRegistry().begin({
165
172
  requestId: request.requestId ?? generateRandomRequestId(),
166
173
  kind: 'batchCompletion',
167
- modelId: request.modelId
174
+ modelId: request.modelId,
175
+ maxConcurrentPerModel: parallel
168
176
  }), true);
169
177
  const requestLogger = withRequestContext(getEngineLogger(), ctx);
170
178
  // Boolean(...) keeps `ctx.signal.aborted` a boolean so the later
@@ -240,14 +248,11 @@ export const llmPlugin = definePlugin({
240
248
  const { promptTokens, ctxSize } = parseContextOverflowMessage(err instanceof Error ? err.message : '');
241
249
  throw new ContextOverflowError(promptTokens, ctxSize, request.modelId, err);
242
250
  }
243
- // Cancelling a batch that still has prompts queued behind the
244
- // `parallel` slot limit fails the whole native batch group with
245
- // a `Cancelled` error (the addon rejects `run()` rather than
246
- // resolving with partial outputs — see continuous-batching docs).
247
- // Ride the same graceful "done" path the non-overflow cancel
248
- // uses: emit cancelled terminals per id so the client surfaces
249
- // `InferenceCancelledError`, not a generic `CompletionFailedError`.
250
- if (ctx.signal.aborted && isAddonCancelledError(err)) {
251
+ // Once the registry accepts cancellation, the request signal owns the
252
+ // terminal outcome. The addon may reject with different shapes
253
+ // depending on whether native sequences were active or queued; those
254
+ // transport details must not change the SDK cancellation contract.
255
+ if (ctx.signal.aborted) {
251
256
  const cancelledIds = ids.length > 0
252
257
  ? ids
253
258
  : request.prompts.map((prompt, index) => prompt.id ?? String(index));
@@ -283,7 +288,8 @@ export const llmPlugin = definePlugin({
283
288
  requestSchema: completionStreamRequestSchema,
284
289
  responseSchema: completionStreamResponseSchema,
285
290
  streaming: true,
286
- cancel: { scope: 'model', hard: true },
291
+ // Request-scope: a model-wide cancel would kill concurrent batching peers.
292
+ cancel: { scope: 'request', hard: true },
287
293
  handler: async function* (request) {
288
294
  const env_2 = { stack: [], error: void 0, hasError: false };
289
295
  try {
@@ -313,10 +319,14 @@ export const llmPlugin = definePlugin({
313
319
  // signal into `completion()` and expose `requestId` so the
314
320
  // caller can target this run with `cancel({ requestId })`.
315
321
  // Falls back to a generated id if the caller didn't send one.
322
+ // Admit up to `parallel` jobs. Disk-KV-cache turns share this lane too;
323
+ // same-file writes serialise per cache path inside the KV-cache session.
324
+ const parallel = getModelParallel(modelCfg);
316
325
  const ctx = __addDisposableResource(env_2, await getRequestRegistry().begin({
317
326
  requestId: request.requestId ?? generateRandomRequestId(),
318
327
  kind: 'completion',
319
- modelId: request.modelId
328
+ modelId: request.modelId,
329
+ maxConcurrentPerModel: parallel
320
330
  }), true);
321
331
  const requestLogger = withRequestContext(getEngineLogger(), ctx);
322
332
  // begin() can return already-aborted when the caller cancels while
@@ -358,26 +368,26 @@ export const llmPlugin = definePlugin({
358
368
  }
359
369
  result = await stream.next();
360
370
  }
361
- const { modelExecutionMs, stats, toolCalls } = result.value;
371
+ const { modelExecutionMs, stats, toolCalls, stoppedAtContextBoundary } = result.value;
362
372
  // Cancellation rides the done path: observable via the last event's
363
373
  // stopReason; client aggregates reject with InferenceCancelledError.
364
374
  const cancelled = ctx.signal.aborted;
365
375
  // EOS tokens are not decoded by llama_decode, so n_eval (and
366
376
  // therefore stats.generatedTokens) counts only real decode calls.
367
- // When generatedTokens >= effectivePredict the run exhausted its
368
- // token budget without hitting EOS — emit stopReason "length".
369
- // -1 (unlimited) and -2 (context fill) must never trigger this.
377
+ // Positive prediction-budget exhaustion and a full context window
378
+ // both map to stopReason "length"; cancellation takes precedence.
370
379
  const effectivePredict = request.generationParams?.predict ?? modelCfg.predict;
371
- const stoppedByBudget = !cancelled &&
372
- effectivePredict !== undefined &&
373
- effectivePredict > 0 &&
374
- stats?.generatedTokens !== undefined &&
375
- stats.generatedTokens >= effectivePredict;
380
+ const lengthStop = stoppedByLength({
381
+ cancelled,
382
+ effectivePredict,
383
+ generatedTokens: stats?.generatedTokens,
384
+ stoppedAtContextBoundary
385
+ });
376
386
  const terminalEvents = normalizer.finish({
377
387
  ...(stats && { stats }),
378
388
  ...(toolCalls.length > 0 && { toolCalls }),
379
389
  ...(cancelled && { stopReason: 'cancelled' }),
380
- ...(stoppedByBudget && { stopReason: 'length' })
390
+ ...(lengthStop && { stopReason: 'length' })
381
391
  });
382
392
  if (!request.stream) {
383
393
  batchedEvents.push(...terminalEvents);
@@ -390,18 +400,27 @@ export const llmPlugin = definePlugin({
390
400
  }, modelExecutionMs);
391
401
  }
392
402
  catch (err) {
393
- // The llama.cpp addon emits a structured `ContextOverflow` status
394
- // (LlmErrors.hpp::ContextOverflow = 14) when the prompt exceeds
395
- // the model's `ctx_size`. Bare's `js_throw_error(env, code, msg)`
396
- // surfaces it as a JS Error with `.code = "[ <addonId> :: ContextOverflow ]"`
397
- // and `.message` carrying the C++-formatted detail. Rethrow as
398
- // a typed `ContextOverflowError` so consumers can switch on the
399
- // class (and `err.code === ERROR_CODES.CONTEXT_OVERFLOW`)
400
- // instead of substring-matching on the raw addon message.
403
+ // Classify a structured ContextOverflow first, even under an aborted
404
+ // signal, so a real overflow racing a cancel surfaces as a typed
405
+ // ContextOverflowError.
401
406
  if (isAddonContextOverflowError(err)) {
402
407
  const { promptTokens, ctxSize } = parseContextOverflowMessage(err instanceof Error ? err.message : '');
403
408
  throw new ContextOverflowError(promptTokens, ctxSize, request.modelId, err);
404
409
  }
410
+ // Context overflow is classified above. For every other error after
411
+ // accepted cancellation, the request signal owns the terminal outcome
412
+ // regardless of the addon's active/queued error shape.
413
+ if (ctx.signal.aborted) {
414
+ yield attachModelExecutionMs({
415
+ type: 'completionStream',
416
+ done: true,
417
+ events: normalizer.finish({ stopReason: 'cancelled' })
418
+ },
419
+ // The error path has no model execution time to report; omit it
420
+ // rather than claim 0.
421
+ undefined);
422
+ return;
423
+ }
405
424
  throw err;
406
425
  }
407
426
  finally {
@@ -439,7 +458,8 @@ export const llmPlugin = definePlugin({
439
458
  requestSchema: translateRequestSchema,
440
459
  responseSchema: translateResponseSchema,
441
460
  streaming: true,
442
- cancel: { scope: 'model', hard: true },
461
+ // LLM translate cancels its own run (request-scoped), like completion.
462
+ cancel: { scope: 'request', hard: true },
443
463
  handler: async function* (request) {
444
464
  const stream = translate(request, request.requestId);
445
465
  try {
@@ -1,4 +1,3 @@
1
- import { TOOLS_MODE } from '../../../schemas/index.js';
2
1
  /**
3
2
  * Converts an LlmConfig into the flat string-keyed map the C++ addon expects.
4
3
  *
@@ -24,11 +23,5 @@ export function transformLlmConfig(llmConfig) {
24
23
  transformed['openclCacheDir'] = transformed['opencl_cache_dir'];
25
24
  delete transformed['opencl_cache_dir'];
26
25
  }
27
- if ('tools_mode' in transformed) {
28
- if (transformed['tools_mode'] === TOOLS_MODE.dynamic) {
29
- transformed['tools_compact'] = 'true';
30
- }
31
- delete transformed['tools_mode'];
32
- }
33
26
  return transformed;
34
27
  }
@@ -2,7 +2,7 @@ import EmbedLlamacpp from '@qvac/embed-llamacpp';
2
2
  import embedAddonLogging from '@qvac/embed-llamacpp/addonLogging';
3
3
  import { definePlugin, defineHandler, embedRequestSchema, embedResponseSchema, ModelType, embedConfigBaseSchema, ADDON_EMBEDDING } from '../../../schemas/index.js';
4
4
  import { createStreamLogger, registerAddonLogger, getEngineLogger } from '../../../logging/index.js';
5
- import { expandGGUFIntoShards } from '../../../utils/index.js';
5
+ import { getFirstShardPath } from '../../../utils/index.js';
6
6
  import { embed } from '../../../plugins/ops/embed.js';
7
7
  import { forwardModelExecution } from '../../../profiling/model-execution.js';
8
8
  import { isMobile } from '../../../runtime/state.js';
@@ -53,9 +53,8 @@ function createEmbeddingsModel(modelId, modelPath, embedConfig) {
53
53
  getEngineLogger().warn(`[${ModelType.llamacppEmbedding}:${modelId}] Multi-GPU parameters (${stripped.join(', ')}) are not supported on mobile (single-GPU device) — removing from config; model will load with single-GPU defaults`);
54
54
  }
55
55
  }
56
- const modelFiles = expandGGUFIntoShards(modelPath);
57
56
  const model = new EmbedLlamacpp({
58
- files: { model: modelFiles },
57
+ files: { model: [getFirstShardPath(modelPath)] },
59
58
  config,
60
59
  logger,
61
60
  opts: { stats: true }
@@ -31,7 +31,7 @@ function deriveColocatedBergamotVocabPaths(modelPath) {
31
31
  function createNmtModel(modelId, modelPath, nmtConfig, srcVocabPath, dstVocabPath, pivotModelPath, pivotSrcVocabPath, pivotDstVocabPath) {
32
32
  const logger = createStreamLogger(modelId, ModelType.nmtcppTranslation);
33
33
  registerAddonLogger(modelId, ModelType.nmtcppTranslation, logger);
34
- const { mode, from, to, engine, beamsize, lengthpenalty, maxlength, repetitionpenalty, norepeatngramsize, temperature, topk, topp } = nmtConfig;
34
+ const { from, to, engine, beamsize, lengthpenalty, maxlength, repetitionpenalty, norepeatngramsize, temperature, topk, topp } = nmtConfig;
35
35
  const files = {
36
36
  model: modelPath,
37
37
  ...(srcVocabPath && { srcVocab: srcVocabPath }),
@@ -69,7 +69,7 @@ function createNmtModel(modelId, modelPath, nmtConfig, srcVocabPath, dstVocabPat
69
69
  };
70
70
  const model = new TranslationNmtcpp({
71
71
  files,
72
- params: { mode, srcLang: from, dstLang: to },
72
+ params: { srcLang: from, dstLang: to },
73
73
  config,
74
74
  logger,
75
75
  opts: { stats: true }
@@ -107,9 +107,10 @@ export const nmtPlugin = definePlugin({
107
107
  requestSchema: translateRequestSchema,
108
108
  responseSchema: translateResponseSchema,
109
109
  streaming: true,
110
- // nmtcpp does not expose a cancel surface today — we fall
111
- // back to soft-cancel (stop yielding, drop result, skip
112
- // post-processing; the C++ work runs to completion).
110
+ // nmtcpp 0.7.0 exposes response.cancel(), but the translate op is not
111
+ // wired for addon-level cancellation yet — we keep soft-cancel (stop
112
+ // yielding, drop result, skip post-processing; the C++ work may run to
113
+ // completion).
113
114
  cancel: { scope: 'none' },
114
115
  handler: async function* (request) {
115
116
  const stream = translate(request, request.requestId);
@@ -100,26 +100,37 @@ export const parakeetPlugin = definePlugin({
100
100
  })
101
101
  };
102
102
  const iterator = transcribeStream(request.modelId, inputStream, undefined, false, streamOpts, request.requestId);
103
- for await (const value of iterator) {
104
- if (typeof value === 'object' && value !== null && 'type' in value) {
105
- if (value.type === 'endOfTurn') {
106
- yield {
107
- type: 'transcribeStream',
108
- endOfTurn: { source: 'parakeet' }
109
- };
103
+ try {
104
+ let result = await iterator.next();
105
+ while (!result.done) {
106
+ const value = result.value;
107
+ if (typeof value === 'object' && value !== null && 'type' in value) {
108
+ if (value.type === 'endOfTurn') {
109
+ yield {
110
+ type: 'transcribeStream',
111
+ endOfTurn: { source: 'parakeet' }
112
+ };
113
+ }
114
+ result = await iterator.next();
115
+ continue;
110
116
  }
111
- continue;
117
+ yield {
118
+ type: 'transcribeStream',
119
+ text: value
120
+ };
121
+ result = await iterator.next();
112
122
  }
113
- yield {
123
+ const { modelExecutionMs, stats } = result.value;
124
+ yield attachModelExecutionMs({
114
125
  type: 'transcribeStream',
115
- text: value
116
- };
126
+ text: '',
127
+ done: true,
128
+ ...(stats && { stats })
129
+ }, modelExecutionMs);
130
+ }
131
+ finally {
132
+ await iterator.return?.(undefined);
117
133
  }
118
- yield {
119
- type: 'transcribeStream',
120
- text: '',
121
- done: true
122
- };
123
134
  }
124
135
  })
125
136
  },
@@ -21,10 +21,30 @@ export function getParlerJobOptions(request) {
21
21
  ...(request.quality !== undefined ? { quality: request.quality } : {})
22
22
  };
23
23
  }
24
+ // Per-call conditioning keys shared by the engines that support them.
25
+ // CosyVoice3 accepts the cross-engine emotion/pace pair; everything else in
26
+ // the description/template surface remains Parler-only.
27
+ const CROSS_ENGINE_CONDITIONING_KEYS = ['emotion', 'pace'];
24
28
  export function assertParlerJobOptionsSupported(model, options, handlerName) {
25
- if (Object.keys(options).length === 0)
29
+ const setKeys = Object.keys(options);
30
+ if (setKeys.length === 0)
26
31
  return;
27
- if (hasEngineType(model) && model.getEngineType() === 'parler')
32
+ const engine = hasEngineType(model) ? model.getEngineType() : undefined;
33
+ if (engine === 'parler')
28
34
  return;
29
- throw new PluginRequestValidationFailedError(handlerName, 'description, voiceDescription, voice, emotion, pitch, pace, expressivity, noise, reverb, and quality are only supported by Parler TTS models');
35
+ if (engine === 'cosyvoice3') {
36
+ const parlerOnlyKeys = setKeys.filter((key) => !CROSS_ENGINE_CONDITIONING_KEYS.includes(key));
37
+ if (parlerOnlyKeys.length > 0) {
38
+ throw new PluginRequestValidationFailedError(handlerName, `${parlerOnlyKeys.join(', ')} ${parlerOnlyKeys.length === 1 ? 'is' : 'are'} only supported by Parler TTS models; CosyVoice3 supports emotion and pace`);
39
+ }
40
+ // CosyVoice3 is trained on one instruction per synthesis; a request that
41
+ // engages both channels always conflicts regardless of the loaded config.
42
+ if (options.emotion !== undefined &&
43
+ options.pace !== undefined &&
44
+ options.pace !== 'moderate') {
45
+ throw new PluginRequestValidationFailedError(handlerName, 'CosyVoice3 accepts one conditioning control per synthesis; set either emotion or a non-moderate pace, not both');
46
+ }
47
+ return;
48
+ }
49
+ throw new PluginRequestValidationFailedError(handlerName, 'description, voiceDescription, voice, emotion, pitch, pace, expressivity, noise, reverb, and quality are only supported by Parler TTS and CosyVoice3 models (CosyVoice3: emotion and pace only)');
30
50
  }