@qvac/sdk 0.17.0 → 0.18.0

package/dist/server/bare/plugins/llamacpp-completion/ops/kv-cache-state.d.ts

@@ -37,6 +37,7 @@ interface CacheCommitContext {
     producedTokens: boolean;
     generatedTokens?: number | undefined;
     predict?: number | undefined;
+    stoppedAtContextBoundary: boolean;
 }
 export declare function shouldCommitCachedTurn(context: CacheCommitContext): boolean;
 /**

package/dist/server/bare/plugins/llamacpp-completion/ops/kv-cache-state.js

@@ -15,12 +15,12 @@
  * slice-decision helper remains here.
  */
 export function shouldCommitCachedTurn(context) {
-    const { aborted, producedTokens, generatedTokens, predict } = context;
+    const { aborted, producedTokens, generatedTokens, predict, stoppedAtContextBoundary } = context;
     const stoppedByBudget = predict !== undefined &&
         predict > 0 &&
         generatedTokens !== undefined &&
         generatedTokens >= predict;
-    return !aborted && producedTokens && !stoppedByBudget;
+    return !aborted && producedTokens && !stoppedByBudget && !stoppedAtContextBoundary;
 }
 /**
  * Pure slice decision for `prepareMessagesForCache`.

package/dist/server/bare/plugins/llamacpp-completion/plugin.d.ts

@@ -28,10 +28,6 @@ export declare const llmPlugin: {
         n_discarded: import("zod").ZodOptional<import("zod").ZodNumber>;
         parallel: import("zod").ZodOptional<import("zod").ZodNumber>;
         tools: import("zod").ZodOptional<import("zod").ZodBoolean>;
-        toolsMode: import("zod").ZodOptional<import("zod").ZodEnum<{
-            static: "static";
-            dynamic: "dynamic";
-        }>>;
         'cache-type-k': import("zod").ZodOptional<import("zod").ZodString>;
         'cache-type-v': import("zod").ZodOptional<import("zod").ZodString>;
         'main-gpu': import("zod").ZodOptional<import("zod").ZodUnion<readonly [import("zod").ZodNumber, import("zod").ZodEnum<{
@@ -90,6 +86,10 @@ export declare const llmPlugin: {
             batched: "batched";
             sequential: "sequential";
         }>>;
+        image_no_upscale: import("zod").ZodOptional<import("zod").ZodEnum<{
+            on: "on";
+            off: "off";
+        }>>;
         'mmproj-use-gpu': import("zod").ZodOptional<import("zod").ZodBoolean>;
     }, import("zod/v4/core").$strip>;
     resolveConfig(cfg: LlmConfigInput, ctx: ResolveContext): Promise<{
@@ -113,7 +113,6 @@ export declare const llmPlugin: {
             n_discarded?: number | undefined;
             parallel?: number | undefined;
             tools?: boolean | undefined;
-            toolsMode?: "static" | "dynamic" | undefined;
             'cache-type-k'?: string | undefined;
             'cache-type-v'?: string | undefined;
             'main-gpu'?: number | "integrated" | "dedicated" | undefined;
@@ -122,6 +121,7 @@ export declare const llmPlugin: {
             openclCacheDir?: string | undefined;
             reasoning_budget?: number | undefined;
             image_tile_mode?: "disabled" | "batched" | "sequential" | undefined;
+            image_no_upscale?: "on" | "off" | undefined;
             'mmproj-use-gpu'?: boolean | undefined;
         };
         artifacts?: never;
@@ -146,7 +146,6 @@ export declare const llmPlugin: {
             n_discarded?: number | undefined;
             parallel?: number | undefined;
             tools?: boolean | undefined;
-            toolsMode?: "static" | "dynamic" | undefined;
             'cache-type-k'?: string | undefined;
             'cache-type-v'?: string | undefined;
             'main-gpu'?: number | "integrated" | "dedicated" | undefined;
@@ -155,6 +154,7 @@ export declare const llmPlugin: {
             openclCacheDir?: string | undefined;
             reasoning_budget?: number | undefined;
             image_tile_mode?: "disabled" | "batched" | "sequential" | undefined;
+            image_no_upscale?: "on" | "off" | undefined;
             'mmproj-use-gpu'?: boolean | undefined;
         };
         artifacts: {

package/dist/server/bare/plugins/llamacpp-completion/plugin.js

@@ -54,7 +54,7 @@ import LlmLlamacpp from '@qvac/llm-llamacpp';
 import llmAddonLogging from '@qvac/llm-llamacpp/addonLogging';
 import { definePlugin, defineHandler, finetuneRequestSchema, batchCompletionStreamRequestSchema, batchCompletionStreamResponseSchema, completionStreamRequestSchema, completionStreamResponseSchema, finetuneResponseSchema, translateRequestSchema, translateResponseSchema, ModelType, llmConfigBaseSchema, ADDON_LLM } from '../../../../schemas/index.js';
 import { createStreamLogger, registerAddonLogger } from '../../../../logging/index.js';
-import { expandGGUFIntoShards } from '../../../../server/utils/index.js';
+import { getFirstShardPath, getModelParallel } from '../../../../server/utils/index.js';
 import { completion } from '../../../../server/bare/plugins/llamacpp-completion/ops/completion-stream.js';
 import { batchCompletion } from '../../../../server/bare/plugins/llamacpp-completion/ops/batch-completion-stream.js';
 import { finetune } from '../../../../server/bare/plugins/llamacpp-completion/ops/finetune.js';
@@ -69,7 +69,7 @@ import { generateServerRequestId } from '../../../../server/bare/runtime/request
 import { getServerLogger } from '../../../../logging/index.js';
 import { ContextOverflowError } from '../../../../utils/errors-server.js';
 import { isAddonContextOverflowError, parseContextOverflowMessage } from '../../../../server/bare/plugins/llamacpp-completion/ops/context-overflow.js';
-import { isAddonCancelledError } from '../../../../server/bare/plugins/llamacpp-completion/ops/batch-cancelled.js';
+import { stoppedByLength } from '../../../../server/bare/plugins/llamacpp-completion/ops/completion-stats.js';
 import { isMobile } from '../../../../server/bare/registry/runtime-context-registry.js';
 import { stripMultiGpuKeys } from '../../../../server/utils/multi-gpu-mobile.js';
 function createLlmModel(modelId, modelPath, llmConfig, projectionModelPath) {
@@ -82,15 +82,17 @@ function createLlmModel(modelId, modelPath, llmConfig, projectionModelPath) {
             getServerLogger().warn(`[${ModelType.llamacppCompletion}:${modelId}] Multi-GPU parameters (${stripped.join(', ')}) are not supported on mobile (single-GPU device) — removing from config; model will load with single-GPU defaults`);
         }
     }
-    const modelFiles = expandGGUFIntoShards(modelPath);
+    // Addon overflow flag: at a single slot the addon throws when driven over
+    // capacity, at parallel>1 it queues the surplus. Not on the SDK surface.
+    const rejectWhenBusy = getModelParallel(llmConfig) <= 1;
     const model = new LlmLlamacpp({
         files: {
-            model: modelFiles,
+            model: [getFirstShardPath(modelPath)],
             ...(projectionModelPath && { projectionModel: projectionModelPath })
         },
         config: llmConfigStrings,
         logger,
-        opts: { stats: true }
+        opts: { stats: true, rejectWhenBusy }
     });
     return { model };
 }
@@ -136,7 +138,8 @@ export const llmPlugin = definePlugin({
             requestSchema: batchCompletionStreamRequestSchema,
             responseSchema: batchCompletionStreamResponseSchema,
             streaming: true,
-            cancel: { scope: 'model', hard: true },
+            // Request-scope: a model-wide cancel would kill concurrent batching peers.
+            cancel: { scope: 'request', hard: true },
             handler: async function* (request) {
                 const env_1 = { stack: [], error: void 0, hasError: false };
                 try {
@@ -162,10 +165,14 @@ export const llmPlugin = definePlugin({
                         }
                         return normalizer;
                     }
+                    // Batches share the completion lane at the `parallel` cap. The cap counts
+                    // requests, not the prompts inside a batch (the addon schedules those).
+                    const parallel = getModelParallel(modelCfg);
                     const ctx = __addDisposableResource(env_1, await getRequestRegistry().begin({
                         requestId: request.requestId ?? generateServerRequestId(),
                         kind: 'batchCompletion',
-                        modelId: request.modelId
+                        modelId: request.modelId,
+                        maxConcurrentPerModel: parallel
                     }), true);
                     const requestLogger = withRequestContext(getServerLogger(), ctx);
                     if (Boolean(ctx.signal.aborted)) {
@@ -238,14 +245,11 @@ export const llmPlugin = definePlugin({
                             const { promptTokens, ctxSize } = parseContextOverflowMessage(err instanceof Error ? err.message : '');
                             throw new ContextOverflowError(promptTokens, ctxSize, request.modelId, err);
                         }
-                        // Cancelling a batch that still has prompts queued behind the
-                        // `parallel` slot limit fails the whole native batch group with
-                        // a `Cancelled` error (the addon rejects `run()` rather than
-                        // resolving with partial outputs — see continuous-batching docs).
-                        // Ride the same graceful "done" path the non-overflow cancel
-                        // uses: emit cancelled terminals per id so the client surfaces
-                        // `InferenceCancelledError`, not a generic `CompletionFailedError`.
-                        if (ctx.signal.aborted && isAddonCancelledError(err)) {
+                        // Once the registry accepts cancellation, the request signal owns the
+                        // terminal outcome. The addon may reject with different shapes
+                        // depending on whether native sequences were active or queued; those
+                        // transport details must not change the SDK cancellation contract.
+                        if (ctx.signal.aborted) {
                             const cancelledIds = ids.length > 0
                                 ? ids
                                 : request.prompts.map((prompt, index) => prompt.id ?? String(index));
@@ -281,7 +285,8 @@ export const llmPlugin = definePlugin({
             requestSchema: completionStreamRequestSchema,
             responseSchema: completionStreamResponseSchema,
             streaming: true,
-            cancel: { scope: 'model', hard: true },
+            // Request-scope: a model-wide cancel would kill concurrent batching peers.
+            cancel: { scope: 'request', hard: true },
             handler: async function* (request) {
                 const env_2 = { stack: [], error: void 0, hasError: false };
                 try {
@@ -312,10 +317,14 @@ export const llmPlugin = definePlugin({
                     // client can target this run with `cancel({ requestId })`.
                     // Falls back to a server-generated id if the client (e.g. an
                     // older release) didn't send one.
+                    // Admit up to `parallel` jobs. Disk-KV-cache turns share this lane too;
+                    // same-file writes serialise per cache path inside the KV-cache session.
+                    const parallel = getModelParallel(modelCfg);
                     const ctx = __addDisposableResource(env_2, await getRequestRegistry().begin({
                         requestId: request.requestId ?? generateServerRequestId(),
                         kind: 'completion',
-                        modelId: request.modelId
+                        modelId: request.modelId,
+                        maxConcurrentPerModel: parallel
                     }), true);
                     const requestLogger = withRequestContext(getServerLogger(), ctx);
                     // begin() can return already-aborted when the client cancels while
@@ -361,26 +370,26 @@ export const llmPlugin = definePlugin({
                             }
                             result = await stream.next();
                         }
-                        const { modelExecutionMs, stats, toolCalls } = result.value;
+                        const { modelExecutionMs, stats, toolCalls, stoppedAtContextBoundary } = result.value;
                         // Cancellation rides the done path: observable via the last event's
                         // stopReason; client aggregates reject with InferenceCancelledError.
                         const cancelled = ctx.signal.aborted;
                         // EOS tokens are not decoded by llama_decode, so n_eval (and
                         // therefore stats.generatedTokens) counts only real decode calls.
-                        // When generatedTokens >= effectivePredict the run exhausted its
-                        // token budget without hitting EOS — emit stopReason "length".
-                        // -1 (unlimited) and -2 (context fill) must never trigger this.
+                        // Positive prediction-budget exhaustion and a full context window
+                        // both map to stopReason "length"; cancellation takes precedence.
                         const effectivePredict = request.generationParams?.predict ?? modelCfg.predict;
-                        const stoppedByBudget = !cancelled &&
-                            effectivePredict !== undefined &&
-                            effectivePredict > 0 &&
-                            stats?.generatedTokens !== undefined &&
-                            stats.generatedTokens >= effectivePredict;
+                        const lengthStop = stoppedByLength({
+                            cancelled,
+                            effectivePredict,
+                            generatedTokens: stats?.generatedTokens,
+                            stoppedAtContextBoundary
+                        });
                         const terminalEvents = normalizer.finish({
                             ...(stats && { stats }),
                             ...(toolCalls.length > 0 && { toolCalls }),
                             ...(cancelled && { stopReason: 'cancelled' }),
-                            ...(stoppedByBudget && { stopReason: 'length' })
+                            ...(lengthStop && { stopReason: 'length' })
                         });
                         if (!request.stream) {
                             batchedEvents.push(...terminalEvents);
@@ -393,18 +402,27 @@ export const llmPlugin = definePlugin({
                         }, modelExecutionMs);
                     }
                     catch (err) {
-                        // The llama.cpp addon emits a structured `ContextOverflow` status
-                        // (LlmErrors.hpp::ContextOverflow = 14) when the prompt exceeds
-                        // the model's `ctx_size`. Bare's `js_throw_error(env, code, msg)`
-                        // surfaces it as a JS Error with `.code = "[ <addonId> :: ContextOverflow ]"`
-                        // and `.message` carrying the C++-formatted detail. Rethrow as
-                        // a typed `ContextOverflowError` so consumers can switch on the
-                        // class (and `err.code === SDK_SERVER_ERROR_CODES.CONTEXT_OVERFLOW`)
-                        // instead of substring-matching on the raw addon message.
+                        // Classify a structured ContextOverflow first, even under an aborted
+                        // signal, so a real overflow racing a cancel surfaces as a typed
+                        // ContextOverflowError.
                         if (isAddonContextOverflowError(err)) {
                             const { promptTokens, ctxSize } = parseContextOverflowMessage(err instanceof Error ? err.message : '');
                             throw new ContextOverflowError(promptTokens, ctxSize, request.modelId, err);
                         }
+                        // Context overflow is classified above. For every other error after
+                        // accepted cancellation, the request signal owns the terminal outcome
+                        // regardless of the addon's active/queued error shape.
+                        if (ctx.signal.aborted) {
+                            yield attachModelExecutionMs({
+                                type: 'completionStream',
+                                done: true,
+                                events: normalizer.finish({ stopReason: 'cancelled' })
+                            }, 
+                            // The error path has no model execution time to report; omit it
+                            // rather than claim 0.
+                            undefined);
+                            return;
+                        }
                         throw err;
                     }
                     finally {
@@ -442,7 +460,8 @@ export const llmPlugin = definePlugin({
             requestSchema: translateRequestSchema,
             responseSchema: translateResponseSchema,
             streaming: true,
-            cancel: { scope: 'model', hard: true },
+            // LLM translate cancels its own run (request-scoped), like completion.
+            cancel: { scope: 'request', hard: true },
             handler: async function* (request) {
                 const stream = translate(request, request.requestId);
                 try {

package/dist/server/bare/plugins/llamacpp-completion/transform.js

@@ -1,4 +1,4 @@
-import { TOOLS_MODE } from '../../../../schemas/index.js';
+import {} from '../../../../schemas/index.js';
 /**
  * Converts an LlmConfig into the flat string-keyed map the C++ addon expects.
  *
@@ -24,11 +24,5 @@ export function transformLlmConfig(llmConfig) {
         transformed['openclCacheDir'] = transformed['opencl_cache_dir'];
         delete transformed['opencl_cache_dir'];
     }
-    if ('tools_mode' in transformed) {
-        if (transformed['tools_mode'] === TOOLS_MODE.dynamic) {
-            transformed['tools_compact'] = 'true';
-        }
-        delete transformed['tools_mode'];
-    }
     return transformed;
 }

package/dist/server/bare/plugins/llamacpp-embedding/plugin.js

@@ -2,7 +2,7 @@ import EmbedLlamacpp, {} from '@qvac/embed-llamacpp';
 import embedAddonLogging from '@qvac/embed-llamacpp/addonLogging';
 import { definePlugin, defineHandler, embedRequestSchema, embedResponseSchema, ModelType, embedConfigBaseSchema, ADDON_EMBEDDING } from '../../../../schemas/index.js';
 import { createStreamLogger, registerAddonLogger, getServerLogger } from '../../../../logging/index.js';
-import { expandGGUFIntoShards } from '../../../../server/utils/index.js';
+import { getFirstShardPath } from '../../../../server/utils/index.js';
 import { embed } from '../../../../server/bare/ops/embed.js';
 import { forwardModelExecution } from '../../../../profiling/model-execution.js';
 import { isMobile } from '../../../../server/bare/registry/runtime-context-registry.js';
@@ -53,9 +53,8 @@ function createEmbeddingsModel(modelId, modelPath, embedConfig) {
             getServerLogger().warn(`[${ModelType.llamacppEmbedding}:${modelId}] Multi-GPU parameters (${stripped.join(', ')}) are not supported on mobile (single-GPU device) — removing from config; model will load with single-GPU defaults`);
         }
     }
-    const modelFiles = expandGGUFIntoShards(modelPath);
     const model = new EmbedLlamacpp({
-        files: { model: modelFiles },
+        files: { model: [getFirstShardPath(modelPath)] },
         config,
         logger,
         opts: { stats: true }

package/dist/server/bare/plugins/nmtcpp-translation/plugin.js

@@ -31,7 +31,7 @@ function deriveColocatedBergamotVocabPaths(modelPath) {
 function createNmtModel(modelId, modelPath, nmtConfig, srcVocabPath, dstVocabPath, pivotModelPath, pivotSrcVocabPath, pivotDstVocabPath) {
     const logger = createStreamLogger(modelId, ModelType.nmtcppTranslation);
     registerAddonLogger(modelId, ModelType.nmtcppTranslation, logger);
-    const { mode, from, to, engine, beamsize, lengthpenalty, maxlength, repetitionpenalty, norepeatngramsize, temperature, topk, topp } = nmtConfig;
+    const { from, to, engine, beamsize, lengthpenalty, maxlength, repetitionpenalty, norepeatngramsize, temperature, topk, topp } = nmtConfig;
     const files = {
         model: modelPath,
         ...(srcVocabPath && { srcVocab: srcVocabPath }),
@@ -69,7 +69,7 @@ function createNmtModel(modelId, modelPath, nmtConfig, srcVocabPath, dstVocabPat
     };
     const model = new TranslationNmtcpp({
         files,
-        params: { mode, srcLang: from, dstLang: to },
+        params: { srcLang: from, dstLang: to },
         config,
         logger,
         opts: { stats: true }
@@ -107,9 +107,10 @@ export const nmtPlugin = definePlugin({
             requestSchema: translateRequestSchema,
             responseSchema: translateResponseSchema,
             streaming: true,
-            // nmtcpp does not expose a cancel surface today — SDK falls
-            // back to soft-cancel (stop yielding, drop result, skip
-            // post-processing; the C++ work runs to completion).
+            // nmtcpp 0.7.0 exposes response.cancel(), but the translate op is not
+            // wired for addon-level cancellation yet — SDK keeps soft-cancel (stop
+            // yielding, drop result, skip post-processing; the C++ work may run to
+            // completion).
             cancel: { scope: 'none' },
             handler: async function* (request) {
                 const stream = translate(request, request.requestId);

package/dist/server/bare/plugins/parakeet-transcription/plugin.js

@@ -100,26 +100,37 @@ export const parakeetPlugin = definePlugin({
                     })
                 };
                 const iterator = transcribeStream(request.modelId, inputStream, undefined, false, streamOpts, request.requestId);
-                for await (const value of iterator) {
-                    if (typeof value === 'object' && value !== null && 'type' in value) {
-                        if (value.type === 'endOfTurn') {
-                            yield {
-                                type: 'transcribeStream',
-                                endOfTurn: { source: 'parakeet' }
-                            };
+                try {
+                    let result = await iterator.next();
+                    while (!result.done) {
+                        const value = result.value;
+                        if (typeof value === 'object' && value !== null && 'type' in value) {
+                            if (value.type === 'endOfTurn') {
+                                yield {
+                                    type: 'transcribeStream',
+                                    endOfTurn: { source: 'parakeet' }
+                                };
+                            }
+                            result = await iterator.next();
+                            continue;
                         }
-                        continue;
+                        yield {
+                            type: 'transcribeStream',
+                            text: value
+                        };
+                        result = await iterator.next();
                     }
-                    yield {
+                    const { modelExecutionMs, stats } = result.value;
+                    yield attachModelExecutionMs({
                         type: 'transcribeStream',
-                        text: value
-                    };
+                        text: '',
+                        done: true,
+                        ...(stats && { stats })
+                    }, modelExecutionMs);
+                }
+                finally {
+                    await iterator.return?.(undefined);
                 }
-                yield {
-                    type: 'transcribeStream',
-                    text: '',
-                    done: true
-                };
             }
         })
     },

package/dist/server/bare/plugins/tts-ggml/ops/parler-options.js

@@ -21,10 +21,30 @@ export function getParlerJobOptions(request) {
         ...(request.quality !== undefined ? { quality: request.quality } : {})
     };
 }
+// Per-call conditioning keys shared by the engines that support them.
+// CosyVoice3 accepts the cross-engine emotion/pace pair; everything else in
+// the description/template surface remains Parler-only.
+const CROSS_ENGINE_CONDITIONING_KEYS = ['emotion', 'pace'];
 export function assertParlerJobOptionsSupported(model, options, handlerName) {
-    if (Object.keys(options).length === 0)
+    const setKeys = Object.keys(options);
+    if (setKeys.length === 0)
         return;
-    if (hasEngineType(model) && model.getEngineType() === 'parler')
+    const engine = hasEngineType(model) ? model.getEngineType() : undefined;
+    if (engine === 'parler')
         return;
-    throw new PluginRequestValidationFailedError(handlerName, 'description, voiceDescription, voice, emotion, pitch, pace, expressivity, noise, reverb, and quality are only supported by Parler TTS models');
+    if (engine === 'cosyvoice3') {
+        const parlerOnlyKeys = setKeys.filter((key) => !CROSS_ENGINE_CONDITIONING_KEYS.includes(key));
+        if (parlerOnlyKeys.length > 0) {
+            throw new PluginRequestValidationFailedError(handlerName, `${parlerOnlyKeys.join(', ')} ${parlerOnlyKeys.length === 1 ? 'is' : 'are'} only supported by Parler TTS models; CosyVoice3 supports emotion and pace`);
+        }
+        // CosyVoice3 is trained on one instruction per synthesis; a request that
+        // engages both channels always conflicts regardless of the loaded config.
+        if (options.emotion !== undefined &&
+            options.pace !== undefined &&
+            options.pace !== 'moderate') {
+            throw new PluginRequestValidationFailedError(handlerName, 'CosyVoice3 accepts one conditioning control per synthesis; set either emotion or a non-moderate pace, not both');
+        }
+        return;
+    }
+    throw new PluginRequestValidationFailedError(handlerName, 'description, voiceDescription, voice, emotion, pitch, pace, expressivity, noise, reverb, and quality are only supported by Parler TTS and CosyVoice3 models (CosyVoice3: emotion and pace only)');
 }