@qvac/inference 0.17.1 → 0.18.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/completion-stream.d.ts +1 -1
- package/dist/api/completion-stream.js +1 -1
- package/dist/api/index.d.ts +1 -1
- package/dist/api/index.js +1 -1
- package/dist/api/text-to-speech.js +22 -2
- package/dist/api/transcribe.d.ts +19 -2
- package/dist/api/transcribe.js +66 -27
- package/dist/api/translate.js +6 -17
- package/dist/api/vla.d.ts +42 -1
- package/dist/api/vla.js +50 -1
- package/dist/constants/audio.d.ts +2 -1
- package/dist/constants/audio.js +17 -1
- package/dist/errors/index.d.ts +3 -0
- package/dist/errors/index.js +5 -0
- package/dist/handlers/load-model/handler.js +2 -2
- package/dist/handlers/load-model/resolve-session.d.ts +1 -1
- package/dist/handlers/load-model/resolve-session.js +3 -3
- package/dist/handlers/load-model/resolve.d.ts +12 -2
- package/dist/handlers/load-model/resolve.js +90 -12
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/models/registry/models.d.ts +782 -6
- package/dist/models/registry/models.js +2664 -1859
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +19 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +153 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +54 -4
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +28 -0
- package/dist/plugins/builtin/ggml-ocr/plugin.js +3 -1
- package/dist/plugins/builtin/ggml-vla/ops/vla-set-embodiment.d.ts +2 -0
- package/dist/plugins/builtin/ggml-vla/ops/vla-set-embodiment.js +9 -0
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +37 -0
- package/dist/plugins/builtin/ggml-vla/plugin.js +21 -4
- package/dist/plugins/builtin/index.d.ts +1 -0
- package/dist/plugins/builtin/index.js +1 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +16 -15
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.d.ts +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +2 -2
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +6 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +10 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +1 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +131 -123
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +110 -20
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +50 -23
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +308 -143
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +1 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +2 -2
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +6 -6
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +56 -36
- package/dist/plugins/builtin/llamacpp-completion/transform.js +0 -7
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -3
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +6 -5
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +27 -16
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +23 -3
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +302 -4
- package/dist/plugins/builtin/tts-ggml/plugin.js +131 -0
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +48 -39
- package/dist/plugins/ops/cancel.d.ts +1 -1
- package/dist/plugins/ops/cancel.js +1 -1
- package/dist/plugins/ops/kv-cache-retention.js +11 -0
- package/dist/plugins/ops/kv-cache-utils.d.ts +1 -1
- package/dist/plugins/ops/kv-cache-utils.js +40 -12
- package/dist/plugins/ops/transcribe.d.ts +2 -2
- package/dist/plugins/ops/transcribe.js +54 -48
- package/dist/plugins/ops/translate-stats.d.ts +4 -0
- package/dist/plugins/ops/translate-stats.js +11 -0
- package/dist/plugins/ops/translate.js +66 -38
- package/dist/plugins/ops/unload-model.js +30 -24
- package/dist/runtime/model-registry.d.ts +1 -0
- package/dist/runtime/request-context.js +29 -14
- package/dist/runtime/request-registry.d.ts +27 -0
- package/dist/runtime/request-registry.js +246 -76
- package/dist/schemas/audio-gen.d.ts +103 -0
- package/dist/schemas/audio-gen.js +135 -4
- package/dist/schemas/common.d.ts +550 -11
- package/dist/schemas/config.d.ts +60 -24
- package/dist/schemas/errors.d.ts +1 -0
- package/dist/schemas/errors.js +5 -0
- package/dist/schemas/index.d.ts +1 -1
- package/dist/schemas/index.js +1 -1
- package/dist/schemas/llamacpp-config.d.ts +10 -10
- package/dist/schemas/llamacpp-config.js +168 -82
- package/dist/schemas/load-model.d.ts +3691 -336
- package/dist/schemas/load-model.js +37 -19
- package/dist/schemas/text-to-speech.d.ts +1086 -110
- package/dist/schemas/text-to-speech.js +197 -7
- package/dist/schemas/tools.d.ts +0 -16
- package/dist/schemas/tools.js +0 -15
- package/dist/schemas/transcription-config.d.ts +2 -0
- package/dist/schemas/transcription-config.js +6 -0
- package/dist/schemas/transcription.d.ts +3 -0
- package/dist/schemas/vla.d.ts +83 -0
- package/dist/schemas/vla.js +105 -2
- package/dist/surface.d.ts +1 -1
- package/dist/surface.js +1 -1
- package/dist/utils/addon-responses.d.ts +4 -12
- package/dist/utils/audio/decoder.d.ts +37 -1
- package/dist/utils/audio/decoder.js +82 -52
- package/dist/utils/config-transform.d.ts +3 -0
- package/dist/utils/config-transform.js +6 -0
- package/dist/utils/index.d.ts +0 -1
- package/dist/utils/index.js +0 -1
- package/dist/utils/shard-utils.d.ts +1 -0
- package/dist/utils/shard-utils.js +14 -0
- package/dist/utils/tool-integration.d.ts +3 -10
- package/dist/utils/tool-integration.js +3 -12
- package/package.json +26 -22
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-cancelled.d.ts +0 -22
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-cancelled.js +0 -30
- package/dist/utils/expand-gguf-shards.d.ts +0 -2
- package/dist/utils/expand-gguf-shards.js +0 -22
|
@@ -37,6 +37,7 @@ interface CacheCommitContext {
|
|
|
37
37
|
producedTokens: boolean;
|
|
38
38
|
generatedTokens?: number | undefined;
|
|
39
39
|
predict?: number | undefined;
|
|
40
|
+
stoppedAtContextBoundary: boolean;
|
|
40
41
|
}
|
|
41
42
|
export declare function shouldCommitCachedTurn(context: CacheCommitContext): boolean;
|
|
42
43
|
/**
|
|
@@ -15,12 +15,12 @@
|
|
|
15
15
|
* slice-decision helper remains here.
|
|
16
16
|
*/
|
|
17
17
|
export function shouldCommitCachedTurn(context) {
|
|
18
|
-
const { aborted, producedTokens, generatedTokens, predict } = context;
|
|
18
|
+
const { aborted, producedTokens, generatedTokens, predict, stoppedAtContextBoundary } = context;
|
|
19
19
|
const stoppedByBudget = predict !== undefined &&
|
|
20
20
|
predict > 0 &&
|
|
21
21
|
generatedTokens !== undefined &&
|
|
22
22
|
generatedTokens >= predict;
|
|
23
|
-
return !aborted && producedTokens && !stoppedByBudget;
|
|
23
|
+
return !aborted && producedTokens && !stoppedByBudget && !stoppedAtContextBoundary;
|
|
24
24
|
}
|
|
25
25
|
/**
|
|
26
26
|
* Pure slice decision for `prepareMessagesForCache`.
|
|
@@ -28,10 +28,6 @@ export declare const llmPlugin: {
|
|
|
28
28
|
n_discarded: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
29
29
|
parallel: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
30
30
|
tools: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
31
|
-
toolsMode: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
32
|
-
static: "static";
|
|
33
|
-
dynamic: "dynamic";
|
|
34
|
-
}>>;
|
|
35
31
|
'cache-type-k': import("zod").ZodOptional<import("zod").ZodString>;
|
|
36
32
|
'cache-type-v': import("zod").ZodOptional<import("zod").ZodString>;
|
|
37
33
|
'main-gpu': import("zod").ZodOptional<import("zod").ZodUnion<readonly [import("zod").ZodNumber, import("zod").ZodEnum<{
|
|
@@ -90,6 +86,10 @@ export declare const llmPlugin: {
|
|
|
90
86
|
batched: "batched";
|
|
91
87
|
sequential: "sequential";
|
|
92
88
|
}>>;
|
|
89
|
+
image_no_upscale: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
90
|
+
on: "on";
|
|
91
|
+
off: "off";
|
|
92
|
+
}>>;
|
|
93
93
|
'mmproj-use-gpu': import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
94
94
|
}, import("zod/v4/core").$strip>;
|
|
95
95
|
resolveConfig(cfg: LlmConfigInput, ctx: ResolveContext): Promise<{
|
|
@@ -113,7 +113,6 @@ export declare const llmPlugin: {
|
|
|
113
113
|
n_discarded?: number | undefined;
|
|
114
114
|
parallel?: number | undefined;
|
|
115
115
|
tools?: boolean | undefined;
|
|
116
|
-
toolsMode?: "static" | "dynamic" | undefined;
|
|
117
116
|
'cache-type-k'?: string | undefined;
|
|
118
117
|
'cache-type-v'?: string | undefined;
|
|
119
118
|
'main-gpu'?: number | "integrated" | "dedicated" | undefined;
|
|
@@ -122,6 +121,7 @@ export declare const llmPlugin: {
|
|
|
122
121
|
openclCacheDir?: string | undefined;
|
|
123
122
|
reasoning_budget?: number | undefined;
|
|
124
123
|
image_tile_mode?: "disabled" | "batched" | "sequential" | undefined;
|
|
124
|
+
image_no_upscale?: "on" | "off" | undefined;
|
|
125
125
|
'mmproj-use-gpu'?: boolean | undefined;
|
|
126
126
|
};
|
|
127
127
|
artifacts?: never;
|
|
@@ -146,7 +146,6 @@ export declare const llmPlugin: {
|
|
|
146
146
|
n_discarded?: number | undefined;
|
|
147
147
|
parallel?: number | undefined;
|
|
148
148
|
tools?: boolean | undefined;
|
|
149
|
-
toolsMode?: "static" | "dynamic" | undefined;
|
|
150
149
|
'cache-type-k'?: string | undefined;
|
|
151
150
|
'cache-type-v'?: string | undefined;
|
|
152
151
|
'main-gpu'?: number | "integrated" | "dedicated" | undefined;
|
|
@@ -155,6 +154,7 @@ export declare const llmPlugin: {
|
|
|
155
154
|
openclCacheDir?: string | undefined;
|
|
156
155
|
reasoning_budget?: number | undefined;
|
|
157
156
|
image_tile_mode?: "disabled" | "batched" | "sequential" | undefined;
|
|
157
|
+
image_no_upscale?: "on" | "off" | undefined;
|
|
158
158
|
'mmproj-use-gpu'?: boolean | undefined;
|
|
159
159
|
};
|
|
160
160
|
artifacts: {
|
|
@@ -54,7 +54,8 @@ import LlmLlamacpp from '@qvac/llm-llamacpp';
|
|
|
54
54
|
import llmAddonLogging from '@qvac/llm-llamacpp/addonLogging';
|
|
55
55
|
import { definePlugin, defineHandler, finetuneRequestSchema, batchCompletionStreamRequestSchema, batchCompletionStreamResponseSchema, completionStreamRequestSchema, completionStreamResponseSchema, finetuneResponseSchema, translateRequestSchema, translateResponseSchema, ModelType, llmConfigBaseSchema, ADDON_LLM } from '../../../schemas/index.js';
|
|
56
56
|
import { createStreamLogger, registerAddonLogger, getEngineLogger } from '../../../logging/index.js';
|
|
57
|
-
import {
|
|
57
|
+
import { getFirstShardPath } from '../../../utils/index.js';
|
|
58
|
+
import { getModelParallel } from '../../../utils/config-transform.js';
|
|
58
59
|
import { completion } from '../../../plugins/builtin/llamacpp-completion/ops/completion-stream.js';
|
|
59
60
|
import { batchCompletion } from '../../../plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js';
|
|
60
61
|
import { finetune } from '../../../plugins/builtin/llamacpp-completion/ops/finetune.js';
|
|
@@ -68,7 +69,7 @@ import { getRequestRegistry, withRequestContext } from '../../../runtime/index.j
|
|
|
68
69
|
import { generateRandomRequestId } from '../../../runtime/request-id.js';
|
|
69
70
|
import { ContextOverflowError } from '../../../errors/index.js';
|
|
70
71
|
import { isAddonContextOverflowError, parseContextOverflowMessage } from '../../../plugins/builtin/llamacpp-completion/ops/context-overflow.js';
|
|
71
|
-
import {
|
|
72
|
+
import { stoppedByLength } from '../../../plugins/builtin/llamacpp-completion/ops/completion-stats.js';
|
|
72
73
|
import { isMobile } from '../../../runtime/state.js';
|
|
73
74
|
import { stripMultiGpuKeys } from '../../../utils/multi-gpu-mobile.js';
|
|
74
75
|
function createLlmModel(modelId, modelPath, llmConfig, projectionModelPath) {
|
|
@@ -81,15 +82,17 @@ function createLlmModel(modelId, modelPath, llmConfig, projectionModelPath) {
|
|
|
81
82
|
getEngineLogger().warn(`[${ModelType.llamacppCompletion}:${modelId}] Multi-GPU parameters (${stripped.join(', ')}) are not supported on mobile (single-GPU device) — removing from config; model will load with single-GPU defaults`);
|
|
82
83
|
}
|
|
83
84
|
}
|
|
84
|
-
|
|
85
|
+
// Addon overflow flag: at a single slot the addon throws when driven over
|
|
86
|
+
// capacity, at parallel>1 it queues the surplus. Not on the SDK surface.
|
|
87
|
+
const rejectWhenBusy = getModelParallel(llmConfig) <= 1;
|
|
85
88
|
const model = new LlmLlamacpp({
|
|
86
89
|
files: {
|
|
87
|
-
model:
|
|
90
|
+
model: [getFirstShardPath(modelPath)],
|
|
88
91
|
...(projectionModelPath && { projectionModel: projectionModelPath })
|
|
89
92
|
},
|
|
90
93
|
config: llmConfigStrings,
|
|
91
94
|
logger,
|
|
92
|
-
opts: { stats: true }
|
|
95
|
+
opts: { stats: true, rejectWhenBusy }
|
|
93
96
|
});
|
|
94
97
|
return { model };
|
|
95
98
|
}
|
|
@@ -135,7 +138,8 @@ export const llmPlugin = definePlugin({
|
|
|
135
138
|
requestSchema: batchCompletionStreamRequestSchema,
|
|
136
139
|
responseSchema: batchCompletionStreamResponseSchema,
|
|
137
140
|
streaming: true,
|
|
138
|
-
|
|
141
|
+
// Request-scope: a model-wide cancel would kill concurrent batching peers.
|
|
142
|
+
cancel: { scope: 'request', hard: true },
|
|
139
143
|
handler: async function* (request) {
|
|
140
144
|
const env_1 = { stack: [], error: void 0, hasError: false };
|
|
141
145
|
try {
|
|
@@ -161,10 +165,14 @@ export const llmPlugin = definePlugin({
|
|
|
161
165
|
}
|
|
162
166
|
return normalizer;
|
|
163
167
|
}
|
|
168
|
+
// Batches share the completion lane at the `parallel` cap. The cap counts
|
|
169
|
+
// requests, not the prompts inside a batch (the addon schedules those).
|
|
170
|
+
const parallel = getModelParallel(modelCfg);
|
|
164
171
|
const ctx = __addDisposableResource(env_1, await getRequestRegistry().begin({
|
|
165
172
|
requestId: request.requestId ?? generateRandomRequestId(),
|
|
166
173
|
kind: 'batchCompletion',
|
|
167
|
-
modelId: request.modelId
|
|
174
|
+
modelId: request.modelId,
|
|
175
|
+
maxConcurrentPerModel: parallel
|
|
168
176
|
}), true);
|
|
169
177
|
const requestLogger = withRequestContext(getEngineLogger(), ctx);
|
|
170
178
|
// Boolean(...) keeps `ctx.signal.aborted` a boolean so the later
|
|
@@ -240,14 +248,11 @@ export const llmPlugin = definePlugin({
|
|
|
240
248
|
const { promptTokens, ctxSize } = parseContextOverflowMessage(err instanceof Error ? err.message : '');
|
|
241
249
|
throw new ContextOverflowError(promptTokens, ctxSize, request.modelId, err);
|
|
242
250
|
}
|
|
243
|
-
//
|
|
244
|
-
//
|
|
245
|
-
//
|
|
246
|
-
//
|
|
247
|
-
|
|
248
|
-
// uses: emit cancelled terminals per id so the client surfaces
|
|
249
|
-
// `InferenceCancelledError`, not a generic `CompletionFailedError`.
|
|
250
|
-
if (ctx.signal.aborted && isAddonCancelledError(err)) {
|
|
251
|
+
// Once the registry accepts cancellation, the request signal owns the
|
|
252
|
+
// terminal outcome. The addon may reject with different shapes
|
|
253
|
+
// depending on whether native sequences were active or queued; those
|
|
254
|
+
// transport details must not change the SDK cancellation contract.
|
|
255
|
+
if (ctx.signal.aborted) {
|
|
251
256
|
const cancelledIds = ids.length > 0
|
|
252
257
|
? ids
|
|
253
258
|
: request.prompts.map((prompt, index) => prompt.id ?? String(index));
|
|
@@ -283,7 +288,8 @@ export const llmPlugin = definePlugin({
|
|
|
283
288
|
requestSchema: completionStreamRequestSchema,
|
|
284
289
|
responseSchema: completionStreamResponseSchema,
|
|
285
290
|
streaming: true,
|
|
286
|
-
|
|
291
|
+
// Request-scope: a model-wide cancel would kill concurrent batching peers.
|
|
292
|
+
cancel: { scope: 'request', hard: true },
|
|
287
293
|
handler: async function* (request) {
|
|
288
294
|
const env_2 = { stack: [], error: void 0, hasError: false };
|
|
289
295
|
try {
|
|
@@ -313,10 +319,14 @@ export const llmPlugin = definePlugin({
|
|
|
313
319
|
// signal into `completion()` and expose `requestId` so the
|
|
314
320
|
// caller can target this run with `cancel({ requestId })`.
|
|
315
321
|
// Falls back to a generated id if the caller didn't send one.
|
|
322
|
+
// Admit up to `parallel` jobs. Disk-KV-cache turns share this lane too;
|
|
323
|
+
// same-file writes serialise per cache path inside the KV-cache session.
|
|
324
|
+
const parallel = getModelParallel(modelCfg);
|
|
316
325
|
const ctx = __addDisposableResource(env_2, await getRequestRegistry().begin({
|
|
317
326
|
requestId: request.requestId ?? generateRandomRequestId(),
|
|
318
327
|
kind: 'completion',
|
|
319
|
-
modelId: request.modelId
|
|
328
|
+
modelId: request.modelId,
|
|
329
|
+
maxConcurrentPerModel: parallel
|
|
320
330
|
}), true);
|
|
321
331
|
const requestLogger = withRequestContext(getEngineLogger(), ctx);
|
|
322
332
|
// begin() can return already-aborted when the caller cancels while
|
|
@@ -358,26 +368,26 @@ export const llmPlugin = definePlugin({
|
|
|
358
368
|
}
|
|
359
369
|
result = await stream.next();
|
|
360
370
|
}
|
|
361
|
-
const { modelExecutionMs, stats, toolCalls } = result.value;
|
|
371
|
+
const { modelExecutionMs, stats, toolCalls, stoppedAtContextBoundary } = result.value;
|
|
362
372
|
// Cancellation rides the done path: observable via the last event's
|
|
363
373
|
// stopReason; client aggregates reject with InferenceCancelledError.
|
|
364
374
|
const cancelled = ctx.signal.aborted;
|
|
365
375
|
// EOS tokens are not decoded by llama_decode, so n_eval (and
|
|
366
376
|
// therefore stats.generatedTokens) counts only real decode calls.
|
|
367
|
-
//
|
|
368
|
-
//
|
|
369
|
-
// -1 (unlimited) and -2 (context fill) must never trigger this.
|
|
377
|
+
// Positive prediction-budget exhaustion and a full context window
|
|
378
|
+
// both map to stopReason "length"; cancellation takes precedence.
|
|
370
379
|
const effectivePredict = request.generationParams?.predict ?? modelCfg.predict;
|
|
371
|
-
const
|
|
372
|
-
|
|
373
|
-
effectivePredict
|
|
374
|
-
stats?.generatedTokens
|
|
375
|
-
|
|
380
|
+
const lengthStop = stoppedByLength({
|
|
381
|
+
cancelled,
|
|
382
|
+
effectivePredict,
|
|
383
|
+
generatedTokens: stats?.generatedTokens,
|
|
384
|
+
stoppedAtContextBoundary
|
|
385
|
+
});
|
|
376
386
|
const terminalEvents = normalizer.finish({
|
|
377
387
|
...(stats && { stats }),
|
|
378
388
|
...(toolCalls.length > 0 && { toolCalls }),
|
|
379
389
|
...(cancelled && { stopReason: 'cancelled' }),
|
|
380
|
-
...(
|
|
390
|
+
...(lengthStop && { stopReason: 'length' })
|
|
381
391
|
});
|
|
382
392
|
if (!request.stream) {
|
|
383
393
|
batchedEvents.push(...terminalEvents);
|
|
@@ -390,18 +400,27 @@ export const llmPlugin = definePlugin({
|
|
|
390
400
|
}, modelExecutionMs);
|
|
391
401
|
}
|
|
392
402
|
catch (err) {
|
|
393
|
-
//
|
|
394
|
-
//
|
|
395
|
-
//
|
|
396
|
-
// surfaces it as a JS Error with `.code = "[ <addonId> :: ContextOverflow ]"`
|
|
397
|
-
// and `.message` carrying the C++-formatted detail. Rethrow as
|
|
398
|
-
// a typed `ContextOverflowError` so consumers can switch on the
|
|
399
|
-
// class (and `err.code === ERROR_CODES.CONTEXT_OVERFLOW`)
|
|
400
|
-
// instead of substring-matching on the raw addon message.
|
|
403
|
+
// Classify a structured ContextOverflow first, even under an aborted
|
|
404
|
+
// signal, so a real overflow racing a cancel surfaces as a typed
|
|
405
|
+
// ContextOverflowError.
|
|
401
406
|
if (isAddonContextOverflowError(err)) {
|
|
402
407
|
const { promptTokens, ctxSize } = parseContextOverflowMessage(err instanceof Error ? err.message : '');
|
|
403
408
|
throw new ContextOverflowError(promptTokens, ctxSize, request.modelId, err);
|
|
404
409
|
}
|
|
410
|
+
// Context overflow is classified above. For every other error after
|
|
411
|
+
// accepted cancellation, the request signal owns the terminal outcome
|
|
412
|
+
// regardless of the addon's active/queued error shape.
|
|
413
|
+
if (ctx.signal.aborted) {
|
|
414
|
+
yield attachModelExecutionMs({
|
|
415
|
+
type: 'completionStream',
|
|
416
|
+
done: true,
|
|
417
|
+
events: normalizer.finish({ stopReason: 'cancelled' })
|
|
418
|
+
},
|
|
419
|
+
// The error path has no model execution time to report; omit it
|
|
420
|
+
// rather than claim 0.
|
|
421
|
+
undefined);
|
|
422
|
+
return;
|
|
423
|
+
}
|
|
405
424
|
throw err;
|
|
406
425
|
}
|
|
407
426
|
finally {
|
|
@@ -439,7 +458,8 @@ export const llmPlugin = definePlugin({
|
|
|
439
458
|
requestSchema: translateRequestSchema,
|
|
440
459
|
responseSchema: translateResponseSchema,
|
|
441
460
|
streaming: true,
|
|
442
|
-
|
|
461
|
+
// LLM translate cancels its own run (request-scoped), like completion.
|
|
462
|
+
cancel: { scope: 'request', hard: true },
|
|
443
463
|
handler: async function* (request) {
|
|
444
464
|
const stream = translate(request, request.requestId);
|
|
445
465
|
try {
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { TOOLS_MODE } from '../../../schemas/index.js';
|
|
2
1
|
/**
|
|
3
2
|
* Converts an LlmConfig into the flat string-keyed map the C++ addon expects.
|
|
4
3
|
*
|
|
@@ -24,11 +23,5 @@ export function transformLlmConfig(llmConfig) {
|
|
|
24
23
|
transformed['openclCacheDir'] = transformed['opencl_cache_dir'];
|
|
25
24
|
delete transformed['opencl_cache_dir'];
|
|
26
25
|
}
|
|
27
|
-
if ('tools_mode' in transformed) {
|
|
28
|
-
if (transformed['tools_mode'] === TOOLS_MODE.dynamic) {
|
|
29
|
-
transformed['tools_compact'] = 'true';
|
|
30
|
-
}
|
|
31
|
-
delete transformed['tools_mode'];
|
|
32
|
-
}
|
|
33
26
|
return transformed;
|
|
34
27
|
}
|
|
@@ -2,7 +2,7 @@ import EmbedLlamacpp from '@qvac/embed-llamacpp';
|
|
|
2
2
|
import embedAddonLogging from '@qvac/embed-llamacpp/addonLogging';
|
|
3
3
|
import { definePlugin, defineHandler, embedRequestSchema, embedResponseSchema, ModelType, embedConfigBaseSchema, ADDON_EMBEDDING } from '../../../schemas/index.js';
|
|
4
4
|
import { createStreamLogger, registerAddonLogger, getEngineLogger } from '../../../logging/index.js';
|
|
5
|
-
import {
|
|
5
|
+
import { getFirstShardPath } from '../../../utils/index.js';
|
|
6
6
|
import { embed } from '../../../plugins/ops/embed.js';
|
|
7
7
|
import { forwardModelExecution } from '../../../profiling/model-execution.js';
|
|
8
8
|
import { isMobile } from '../../../runtime/state.js';
|
|
@@ -53,9 +53,8 @@ function createEmbeddingsModel(modelId, modelPath, embedConfig) {
|
|
|
53
53
|
getEngineLogger().warn(`[${ModelType.llamacppEmbedding}:${modelId}] Multi-GPU parameters (${stripped.join(', ')}) are not supported on mobile (single-GPU device) — removing from config; model will load with single-GPU defaults`);
|
|
54
54
|
}
|
|
55
55
|
}
|
|
56
|
-
const modelFiles = expandGGUFIntoShards(modelPath);
|
|
57
56
|
const model = new EmbedLlamacpp({
|
|
58
|
-
files: { model:
|
|
57
|
+
files: { model: [getFirstShardPath(modelPath)] },
|
|
59
58
|
config,
|
|
60
59
|
logger,
|
|
61
60
|
opts: { stats: true }
|
|
@@ -31,7 +31,7 @@ function deriveColocatedBergamotVocabPaths(modelPath) {
|
|
|
31
31
|
function createNmtModel(modelId, modelPath, nmtConfig, srcVocabPath, dstVocabPath, pivotModelPath, pivotSrcVocabPath, pivotDstVocabPath) {
|
|
32
32
|
const logger = createStreamLogger(modelId, ModelType.nmtcppTranslation);
|
|
33
33
|
registerAddonLogger(modelId, ModelType.nmtcppTranslation, logger);
|
|
34
|
-
const {
|
|
34
|
+
const { from, to, engine, beamsize, lengthpenalty, maxlength, repetitionpenalty, norepeatngramsize, temperature, topk, topp } = nmtConfig;
|
|
35
35
|
const files = {
|
|
36
36
|
model: modelPath,
|
|
37
37
|
...(srcVocabPath && { srcVocab: srcVocabPath }),
|
|
@@ -69,7 +69,7 @@ function createNmtModel(modelId, modelPath, nmtConfig, srcVocabPath, dstVocabPat
|
|
|
69
69
|
};
|
|
70
70
|
const model = new TranslationNmtcpp({
|
|
71
71
|
files,
|
|
72
|
-
params: {
|
|
72
|
+
params: { srcLang: from, dstLang: to },
|
|
73
73
|
config,
|
|
74
74
|
logger,
|
|
75
75
|
opts: { stats: true }
|
|
@@ -107,9 +107,10 @@ export const nmtPlugin = definePlugin({
|
|
|
107
107
|
requestSchema: translateRequestSchema,
|
|
108
108
|
responseSchema: translateResponseSchema,
|
|
109
109
|
streaming: true,
|
|
110
|
-
// nmtcpp
|
|
111
|
-
//
|
|
112
|
-
// post-processing; the C++ work
|
|
110
|
+
// nmtcpp 0.7.0 exposes response.cancel(), but the translate op is not
|
|
111
|
+
// wired for addon-level cancellation yet — we keep soft-cancel (stop
|
|
112
|
+
// yielding, drop result, skip post-processing; the C++ work may run to
|
|
113
|
+
// completion).
|
|
113
114
|
cancel: { scope: 'none' },
|
|
114
115
|
handler: async function* (request) {
|
|
115
116
|
const stream = translate(request, request.requestId);
|
|
@@ -100,26 +100,37 @@ export const parakeetPlugin = definePlugin({
|
|
|
100
100
|
})
|
|
101
101
|
};
|
|
102
102
|
const iterator = transcribeStream(request.modelId, inputStream, undefined, false, streamOpts, request.requestId);
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
103
|
+
try {
|
|
104
|
+
let result = await iterator.next();
|
|
105
|
+
while (!result.done) {
|
|
106
|
+
const value = result.value;
|
|
107
|
+
if (typeof value === 'object' && value !== null && 'type' in value) {
|
|
108
|
+
if (value.type === 'endOfTurn') {
|
|
109
|
+
yield {
|
|
110
|
+
type: 'transcribeStream',
|
|
111
|
+
endOfTurn: { source: 'parakeet' }
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
result = await iterator.next();
|
|
115
|
+
continue;
|
|
110
116
|
}
|
|
111
|
-
|
|
117
|
+
yield {
|
|
118
|
+
type: 'transcribeStream',
|
|
119
|
+
text: value
|
|
120
|
+
};
|
|
121
|
+
result = await iterator.next();
|
|
112
122
|
}
|
|
113
|
-
|
|
123
|
+
const { modelExecutionMs, stats } = result.value;
|
|
124
|
+
yield attachModelExecutionMs({
|
|
114
125
|
type: 'transcribeStream',
|
|
115
|
-
text:
|
|
116
|
-
|
|
126
|
+
text: '',
|
|
127
|
+
done: true,
|
|
128
|
+
...(stats && { stats })
|
|
129
|
+
}, modelExecutionMs);
|
|
130
|
+
}
|
|
131
|
+
finally {
|
|
132
|
+
await iterator.return?.(undefined);
|
|
117
133
|
}
|
|
118
|
-
yield {
|
|
119
|
-
type: 'transcribeStream',
|
|
120
|
-
text: '',
|
|
121
|
-
done: true
|
|
122
|
-
};
|
|
123
134
|
}
|
|
124
135
|
})
|
|
125
136
|
},
|
|
@@ -21,10 +21,30 @@ export function getParlerJobOptions(request) {
|
|
|
21
21
|
...(request.quality !== undefined ? { quality: request.quality } : {})
|
|
22
22
|
};
|
|
23
23
|
}
|
|
24
|
+
// Per-call conditioning keys shared by the engines that support them.
|
|
25
|
+
// CosyVoice3 accepts the cross-engine emotion/pace pair; everything else in
|
|
26
|
+
// the description/template surface remains Parler-only.
|
|
27
|
+
const CROSS_ENGINE_CONDITIONING_KEYS = ['emotion', 'pace'];
|
|
24
28
|
export function assertParlerJobOptionsSupported(model, options, handlerName) {
|
|
25
|
-
|
|
29
|
+
const setKeys = Object.keys(options);
|
|
30
|
+
if (setKeys.length === 0)
|
|
26
31
|
return;
|
|
27
|
-
|
|
32
|
+
const engine = hasEngineType(model) ? model.getEngineType() : undefined;
|
|
33
|
+
if (engine === 'parler')
|
|
28
34
|
return;
|
|
29
|
-
|
|
35
|
+
if (engine === 'cosyvoice3') {
|
|
36
|
+
const parlerOnlyKeys = setKeys.filter((key) => !CROSS_ENGINE_CONDITIONING_KEYS.includes(key));
|
|
37
|
+
if (parlerOnlyKeys.length > 0) {
|
|
38
|
+
throw new PluginRequestValidationFailedError(handlerName, `${parlerOnlyKeys.join(', ')} ${parlerOnlyKeys.length === 1 ? 'is' : 'are'} only supported by Parler TTS models; CosyVoice3 supports emotion and pace`);
|
|
39
|
+
}
|
|
40
|
+
// CosyVoice3 is trained on one instruction per synthesis; a request that
|
|
41
|
+
// engages both channels always conflicts regardless of the loaded config.
|
|
42
|
+
if (options.emotion !== undefined &&
|
|
43
|
+
options.pace !== undefined &&
|
|
44
|
+
options.pace !== 'moderate') {
|
|
45
|
+
throw new PluginRequestValidationFailedError(handlerName, 'CosyVoice3 accepts one conditioning control per synthesis; set either emotion or a non-moderate pace, not both');
|
|
46
|
+
}
|
|
47
|
+
return;
|
|
48
|
+
}
|
|
49
|
+
throw new PluginRequestValidationFailedError(handlerName, 'description, voiceDescription, voice, emotion, pitch, pace, expressivity, noise, reverb, and quality are only supported by Parler TTS and CosyVoice3 models (CosyVoice3: emotion and pace only)');
|
|
30
50
|
}
|