@qvac/inference 0.17.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +22 -80
- package/dist/api/completion-stream.d.ts +1 -1
- package/dist/api/completion-stream.js +1 -1
- package/dist/api/index.d.ts +1 -1
- package/dist/api/index.js +1 -1
- package/dist/api/text-to-speech.js +22 -2
- package/dist/api/transcribe.d.ts +19 -2
- package/dist/api/transcribe.js +66 -27
- package/dist/api/translate.js +6 -17
- package/dist/api/vla.d.ts +42 -1
- package/dist/api/vla.js +50 -1
- package/dist/constants/audio.d.ts +2 -1
- package/dist/constants/audio.js +17 -1
- package/dist/errors/index.d.ts +3 -0
- package/dist/errors/index.js +5 -0
- package/dist/handlers/load-model/handler.js +2 -2
- package/dist/handlers/load-model/resolve-session.d.ts +1 -1
- package/dist/handlers/load-model/resolve-session.js +3 -3
- package/dist/handlers/load-model/resolve.d.ts +12 -2
- package/dist/handlers/load-model/resolve.js +90 -12
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/models/registry/models.d.ts +782 -6
- package/dist/models/registry/models.js +2664 -1859
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +19 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +153 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +54 -4
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +28 -0
- package/dist/plugins/builtin/ggml-ocr/plugin.js +3 -1
- package/dist/plugins/builtin/ggml-vla/ops/vla-set-embodiment.d.ts +2 -0
- package/dist/plugins/builtin/ggml-vla/ops/vla-set-embodiment.js +9 -0
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +37 -0
- package/dist/plugins/builtin/ggml-vla/plugin.js +21 -4
- package/dist/plugins/builtin/index.d.ts +1 -0
- package/dist/plugins/builtin/index.js +1 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +16 -15
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.d.ts +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +2 -2
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +6 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +10 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +1 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +131 -123
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +110 -20
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +50 -23
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +308 -143
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +1 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +2 -2
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +6 -6
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +56 -36
- package/dist/plugins/builtin/llamacpp-completion/transform.js +0 -7
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -3
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +6 -5
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +27 -16
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +23 -3
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +302 -4
- package/dist/plugins/builtin/tts-ggml/plugin.js +131 -0
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +48 -39
- package/dist/plugins/ops/cancel.d.ts +1 -1
- package/dist/plugins/ops/cancel.js +1 -1
- package/dist/plugins/ops/kv-cache-retention.js +11 -0
- package/dist/plugins/ops/kv-cache-utils.d.ts +1 -1
- package/dist/plugins/ops/kv-cache-utils.js +40 -12
- package/dist/plugins/ops/transcribe.d.ts +2 -2
- package/dist/plugins/ops/transcribe.js +54 -48
- package/dist/plugins/ops/translate-stats.d.ts +4 -0
- package/dist/plugins/ops/translate-stats.js +11 -0
- package/dist/plugins/ops/translate.js +66 -38
- package/dist/plugins/ops/unload-model.js +30 -24
- package/dist/runtime/model-registry.d.ts +1 -0
- package/dist/runtime/request-context.js +29 -14
- package/dist/runtime/request-registry.d.ts +27 -0
- package/dist/runtime/request-registry.js +246 -76
- package/dist/schemas/audio-gen.d.ts +103 -0
- package/dist/schemas/audio-gen.js +135 -4
- package/dist/schemas/common.d.ts +550 -11
- package/dist/schemas/config.d.ts +60 -24
- package/dist/schemas/errors.d.ts +1 -0
- package/dist/schemas/errors.js +5 -0
- package/dist/schemas/index.d.ts +1 -1
- package/dist/schemas/index.js +1 -1
- package/dist/schemas/llamacpp-config.d.ts +10 -10
- package/dist/schemas/llamacpp-config.js +12 -5
- package/dist/schemas/load-model.d.ts +3691 -336
- package/dist/schemas/load-model.js +37 -19
- package/dist/schemas/text-to-speech.d.ts +1086 -110
- package/dist/schemas/text-to-speech.js +197 -7
- package/dist/schemas/tools.d.ts +0 -16
- package/dist/schemas/tools.js +0 -15
- package/dist/schemas/transcription-config.d.ts +2 -0
- package/dist/schemas/transcription-config.js +6 -0
- package/dist/schemas/transcription.d.ts +3 -0
- package/dist/schemas/vla.d.ts +83 -0
- package/dist/schemas/vla.js +105 -2
- package/dist/surface.d.ts +1 -1
- package/dist/surface.js +1 -1
- package/dist/utils/addon-responses.d.ts +4 -12
- package/dist/utils/audio/decoder.d.ts +37 -1
- package/dist/utils/audio/decoder.js +82 -52
- package/dist/utils/config-transform.d.ts +3 -0
- package/dist/utils/config-transform.js +6 -0
- package/dist/utils/index.d.ts +0 -1
- package/dist/utils/index.js +0 -1
- package/dist/utils/shard-utils.d.ts +1 -0
- package/dist/utils/shard-utils.js +14 -0
- package/dist/utils/tool-integration.d.ts +3 -10
- package/dist/utils/tool-integration.js +3 -12
- package/package.json +26 -22
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-cancelled.d.ts +0 -22
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-cancelled.js +0 -30
- package/dist/utils/expand-gguf-shards.d.ts +0 -2
- package/dist/utils/expand-gguf-shards.js +0 -22
|
@@ -5,10 +5,17 @@ import { getEngineLogger } from '../../logging/index.js';
|
|
|
5
5
|
const logger = getEngineLogger();
|
|
6
6
|
const AUTO_CACHE_MARKER_PREFIX = '.auto-cache-';
|
|
7
7
|
const AUTO_CACHE_MARKER_PATTERN = /^\.auto-cache-([a-f0-9]{16})$/;
|
|
8
|
+
// SDK-generated auto-cache keys are 16 hex chars. Marker ops gate on this so a
|
|
9
|
+
// caller-provided key never gets a marker and the raw-key marker path stays
|
|
10
|
+
// inside the cache root.
|
|
11
|
+
const AUTO_CACHE_KEY_PATTERN = /^[a-f0-9]{16}$/;
|
|
8
12
|
function getAutoCacheMarkerPath(cacheKey) {
|
|
9
13
|
return path.join(getKVCacheDir(), `${AUTO_CACHE_MARKER_PREFIX}${cacheKey}`);
|
|
10
14
|
}
|
|
11
15
|
export async function markAutoCacheKey(cacheKey) {
|
|
16
|
+
// Only generated 16-hex auto keys get markers (see AUTO_CACHE_KEY_PATTERN).
|
|
17
|
+
if (!AUTO_CACHE_KEY_PATTERN.test(cacheKey))
|
|
18
|
+
return;
|
|
12
19
|
try {
|
|
13
20
|
await fsPromises.writeFile(getAutoCacheMarkerPath(cacheKey), '');
|
|
14
21
|
}
|
|
@@ -17,6 +24,8 @@ export async function markAutoCacheKey(cacheKey) {
|
|
|
17
24
|
}
|
|
18
25
|
}
|
|
19
26
|
export async function removeAutoCacheMarker(cacheKey) {
|
|
27
|
+
if (!AUTO_CACHE_KEY_PATTERN.test(cacheKey))
|
|
28
|
+
return;
|
|
20
29
|
try {
|
|
21
30
|
await fsPromises.unlink(getAutoCacheMarkerPath(cacheKey));
|
|
22
31
|
}
|
|
@@ -25,6 +34,8 @@ export async function removeAutoCacheMarker(cacheKey) {
|
|
|
25
34
|
}
|
|
26
35
|
}
|
|
27
36
|
export async function removeAutoCacheMarkerIfMissing(cacheKey) {
|
|
37
|
+
if (!AUTO_CACHE_KEY_PATTERN.test(cacheKey))
|
|
38
|
+
return;
|
|
28
39
|
try {
|
|
29
40
|
await fsPromises.access(path.join(getKVCacheDir(), cacheKey));
|
|
30
41
|
}
|
|
@@ -12,7 +12,7 @@ export declare function getCurrentCacheInfo(modelId: string, configHash: string,
|
|
|
12
12
|
cachePath: string;
|
|
13
13
|
}>;
|
|
14
14
|
export declare function renameCacheFile(oldPath: string, newPath: string): Promise<boolean>;
|
|
15
|
-
export declare function pruneEmptyCacheDirectories(cacheFilePath: string): Promise<void>;
|
|
15
|
+
export declare function pruneEmptyCacheDirectories(cacheFilePath: string, activePaths?: readonly string[]): Promise<void>;
|
|
16
16
|
export declare function deleteCache(options: {
|
|
17
17
|
all: true;
|
|
18
18
|
} | {
|
|
@@ -3,6 +3,7 @@ import { promises as fsPromises } from 'bare-fs';
|
|
|
3
3
|
import path from 'bare-path';
|
|
4
4
|
import { getAutoCacheLookupHistory, getKVCacheDir, validateAndJoinPath } from '../../utils/index.js';
|
|
5
5
|
import { getEngineLogger } from '../../logging/index.js';
|
|
6
|
+
import { PathTraversalError } from '../../errors/index.js';
|
|
6
7
|
import { Buffer } from 'bare-buffer';
|
|
7
8
|
import { markAutoCacheKey } from '../../plugins/ops/kv-cache-retention.js';
|
|
8
9
|
const logger = getEngineLogger();
|
|
@@ -14,20 +15,28 @@ export function extractSystemPrompt(messages) {
|
|
|
14
15
|
const systemMessage = messages.find((msg) => msg.role === 'system');
|
|
15
16
|
return systemMessage ? systemMessage.content : null;
|
|
16
17
|
}
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
18
|
+
// Cache hash based on the system prompt + complete tool definitions.
|
|
19
|
+
// Callers pass tools only when the tool block is written into the cache and
|
|
20
|
+
// left there, so a different tool set gets its own cache instead of reusing a
|
|
21
|
+
// prefix that holds the old block. Every prompt-affecting field participates,
|
|
22
|
+
// not just the name: canonical serialization avoids cache misses caused only
|
|
23
|
+
// by object-key insertion order, while tool-array order is preserved because
|
|
24
|
+
// that is the order sent to the model.
|
|
25
|
+
function canonicalizeHashInput(value) {
|
|
26
|
+
if (Array.isArray(value))
|
|
27
|
+
return value.map(canonicalizeHashInput);
|
|
28
|
+
if (typeof value !== 'object' || value === null)
|
|
29
|
+
return value;
|
|
30
|
+
const canonical = {};
|
|
31
|
+
for (const key of Object.keys(value).sort()) {
|
|
32
|
+
canonical[key] = canonicalizeHashInput(value[key]);
|
|
33
|
+
}
|
|
34
|
+
return canonical;
|
|
24
35
|
}
|
|
25
|
-
// Cache hash based on system prompt + tool names
|
|
26
|
-
// Different tools = different cache (tools anchored in first message, protected from n_discarded)
|
|
27
36
|
export function generateConfigHash(systemPrompt, tools) {
|
|
28
37
|
const hash = crypto.createHash('sha-256');
|
|
29
|
-
const
|
|
30
|
-
hash.update(Buffer.from(
|
|
38
|
+
const canonicalConfig = JSON.stringify(canonicalizeHashInput({ systemPrompt, tools: Array.isArray(tools) ? tools : [] }));
|
|
39
|
+
hash.update(Buffer.from(canonicalConfig, 'utf8'));
|
|
31
40
|
return hash.digest('hex').substring(0, 16);
|
|
32
41
|
}
|
|
33
42
|
export function generateCacheKey(messages) {
|
|
@@ -88,11 +97,15 @@ export async function renameCacheFile(oldPath, newPath) {
|
|
|
88
97
|
return false;
|
|
89
98
|
}
|
|
90
99
|
}
|
|
91
|
-
export async function pruneEmptyCacheDirectories(cacheFilePath) {
|
|
100
|
+
export async function pruneEmptyCacheDirectories(cacheFilePath, activePaths = []) {
|
|
92
101
|
const cacheDir = getKVCacheDir();
|
|
93
102
|
const cacheDirPrefix = `${cacheDir}${path.sep}`;
|
|
94
103
|
let currentDirectory = path.dirname(cacheFilePath);
|
|
95
104
|
while (currentDirectory.startsWith(cacheDirPrefix)) {
|
|
105
|
+
// Keep a directory another in-flight turn still holds (its .bin isn't on disk yet).
|
|
106
|
+
const childPrefix = `${currentDirectory}${path.sep}`;
|
|
107
|
+
if (activePaths.some((p) => p.startsWith(childPrefix)))
|
|
108
|
+
return;
|
|
96
109
|
try {
|
|
97
110
|
await fsPromises.rmdir(currentDirectory);
|
|
98
111
|
}
|
|
@@ -113,8 +126,23 @@ export async function deleteCache(options) {
|
|
|
113
126
|
await fsPromises.mkdir(cacheDir, { recursive: true });
|
|
114
127
|
return cacheDir;
|
|
115
128
|
}
|
|
129
|
+
// validateAndJoinPath sanitizes the key (strips traversal) and rejects
|
|
130
|
+
// escapes, so nested and other in-root keys resolve normally. A key that
|
|
131
|
+
// resolves to the cache root would wipe every cache — reject it BEFORE joining
|
|
132
|
+
// any modelId, so an empty key plus a modelId can't delete a real key dir
|
|
133
|
+
// under the root.
|
|
134
|
+
const cacheRoot = path.resolve(cacheDir);
|
|
116
135
|
const kvCacheDir = validateAndJoinPath(cacheDir, options.kvCacheKey);
|
|
136
|
+
if (path.resolve(kvCacheDir) === cacheRoot) {
|
|
137
|
+
throw new PathTraversalError(options.kvCacheKey, cacheDir);
|
|
138
|
+
}
|
|
139
|
+
// Omitting modelId means "delete every model under the key"; a PROVIDED but
|
|
140
|
+
// empty/dot modelId collapses back to that same dir and would silently broaden
|
|
141
|
+
// the delete — reject it.
|
|
117
142
|
const targetDir = options.modelId !== undefined ? validateAndJoinPath(kvCacheDir, options.modelId) : kvCacheDir;
|
|
143
|
+
if (options.modelId !== undefined && path.resolve(targetDir) === path.resolve(kvCacheDir)) {
|
|
144
|
+
throw new PathTraversalError(options.modelId, kvCacheDir);
|
|
145
|
+
}
|
|
118
146
|
await fsPromises.rm(targetDir, { recursive: true, force: true });
|
|
119
147
|
return targetDir;
|
|
120
148
|
}
|
|
@@ -16,5 +16,5 @@ export interface TranscribeStreamOpts {
|
|
|
16
16
|
vadRunIntervalMs?: number;
|
|
17
17
|
parakeetStreamingConfig?: ParakeetStreamingRunConfig;
|
|
18
18
|
}
|
|
19
|
-
export declare function transcribeStream(modelId: string, audioInputStream: AsyncIterable<Buffer>, prompt: string | undefined, metadata: true, opts?: TranscribeStreamOpts, requestId?: string): AsyncGenerator<TranscribeSegment | TranscribeStreamEvent,
|
|
20
|
-
export declare function transcribeStream(modelId: string, audioInputStream: AsyncIterable<Buffer>, prompt?: string, metadata?: boolean, opts?: TranscribeStreamOpts, requestId?: string): AsyncGenerator<string | TranscribeStreamEvent,
|
|
19
|
+
export declare function transcribeStream(modelId: string, audioInputStream: AsyncIterable<Buffer>, prompt: string | undefined, metadata: true, opts?: TranscribeStreamOpts, requestId?: string): AsyncGenerator<TranscribeSegment | TranscribeStreamEvent, TranscribeReturn, void>;
|
|
20
|
+
export declare function transcribeStream(modelId: string, audioInputStream: AsyncIterable<Buffer>, prompt?: string, metadata?: boolean, opts?: TranscribeStreamOpts, requestId?: string): AsyncGenerator<string | TranscribeStreamEvent, TranscribeReturn, void>;
|
|
@@ -100,6 +100,55 @@ async function restorePrompt(modelId, originalConfig) {
|
|
|
100
100
|
return;
|
|
101
101
|
await model.reload(buildWhisperReloadConfig({ ...originalConfig, initial_prompt: '' }));
|
|
102
102
|
}
|
|
103
|
+
function mapTranscribeStats(stats) {
|
|
104
|
+
return {
|
|
105
|
+
...(stats?.audioDurationMs !== undefined && {
|
|
106
|
+
audioDuration: stats.audioDurationMs
|
|
107
|
+
}),
|
|
108
|
+
...(stats?.realTimeFactor !== undefined && {
|
|
109
|
+
realTimeFactor: stats.realTimeFactor
|
|
110
|
+
}),
|
|
111
|
+
...(stats?.tokensPerSecond !== undefined && {
|
|
112
|
+
tokensPerSecond: stats.tokensPerSecond
|
|
113
|
+
}),
|
|
114
|
+
...(stats?.totalTokens !== undefined && {
|
|
115
|
+
totalTokens: stats.totalTokens
|
|
116
|
+
}),
|
|
117
|
+
...(stats?.totalSegments !== undefined && {
|
|
118
|
+
totalSegments: stats.totalSegments
|
|
119
|
+
}),
|
|
120
|
+
...(stats?.whisperEncodeMs !== undefined && {
|
|
121
|
+
whisperEncodeTime: stats.whisperEncodeMs
|
|
122
|
+
}),
|
|
123
|
+
...(stats?.whisperDecodeMs !== undefined && {
|
|
124
|
+
whisperDecodeTime: stats.whisperDecodeMs
|
|
125
|
+
}),
|
|
126
|
+
...(stats?.encoderMs !== undefined && {
|
|
127
|
+
encoderTime: stats.encoderMs
|
|
128
|
+
}),
|
|
129
|
+
...(stats?.decoderMs !== undefined && {
|
|
130
|
+
decoderTime: stats.decoderMs
|
|
131
|
+
}),
|
|
132
|
+
...(stats?.melSpecMs !== undefined && {
|
|
133
|
+
melSpecTime: stats.melSpecMs
|
|
134
|
+
}),
|
|
135
|
+
...(stats?.backendDevice !== undefined && {
|
|
136
|
+
backendDevice: stats.backendDevice
|
|
137
|
+
}),
|
|
138
|
+
...(stats?.backendId !== undefined && {
|
|
139
|
+
backendId: stats.backendId
|
|
140
|
+
}),
|
|
141
|
+
...(stats?.gpuUnsupported !== undefined && {
|
|
142
|
+
gpuUnsupported: stats.gpuUnsupported
|
|
143
|
+
}),
|
|
144
|
+
...(stats?.gpuMemTotalMb !== undefined && {
|
|
145
|
+
gpuMemTotalMb: stats.gpuMemTotalMb
|
|
146
|
+
}),
|
|
147
|
+
...(stats?.gpuMemFreeMb !== undefined && {
|
|
148
|
+
gpuMemFreeMb: stats.gpuMemFreeMb
|
|
149
|
+
})
|
|
150
|
+
};
|
|
151
|
+
}
|
|
103
152
|
export async function* transcribe(params, requestId) {
|
|
104
153
|
const env_1 = { stack: [], error: void 0, hasError: false };
|
|
105
154
|
try {
|
|
@@ -173,54 +222,7 @@ export async function* transcribe(params, requestId) {
|
|
|
173
222
|
}
|
|
174
223
|
}
|
|
175
224
|
const modelExecutionMs = nowMs() - modelStart;
|
|
176
|
-
|
|
177
|
-
...(response.stats?.audioDurationMs !== undefined && {
|
|
178
|
-
audioDuration: response.stats.audioDurationMs
|
|
179
|
-
}),
|
|
180
|
-
...(response.stats?.realTimeFactor !== undefined && {
|
|
181
|
-
realTimeFactor: response.stats.realTimeFactor
|
|
182
|
-
}),
|
|
183
|
-
...(response.stats?.tokensPerSecond !== undefined && {
|
|
184
|
-
tokensPerSecond: response.stats.tokensPerSecond
|
|
185
|
-
}),
|
|
186
|
-
...(response.stats?.totalTokens !== undefined && {
|
|
187
|
-
totalTokens: response.stats.totalTokens
|
|
188
|
-
}),
|
|
189
|
-
...(response.stats?.totalSegments !== undefined && {
|
|
190
|
-
totalSegments: response.stats.totalSegments
|
|
191
|
-
}),
|
|
192
|
-
...(response.stats?.whisperEncodeMs !== undefined && {
|
|
193
|
-
whisperEncodeTime: response.stats.whisperEncodeMs
|
|
194
|
-
}),
|
|
195
|
-
...(response.stats?.whisperDecodeMs !== undefined && {
|
|
196
|
-
whisperDecodeTime: response.stats.whisperDecodeMs
|
|
197
|
-
}),
|
|
198
|
-
...(response.stats?.encoderMs !== undefined && {
|
|
199
|
-
encoderTime: response.stats.encoderMs
|
|
200
|
-
}),
|
|
201
|
-
...(response.stats?.decoderMs !== undefined && {
|
|
202
|
-
decoderTime: response.stats.decoderMs
|
|
203
|
-
}),
|
|
204
|
-
...(response.stats?.melSpecMs !== undefined && {
|
|
205
|
-
melSpecTime: response.stats.melSpecMs
|
|
206
|
-
}),
|
|
207
|
-
...(response.stats?.backendDevice !== undefined && {
|
|
208
|
-
backendDevice: response.stats.backendDevice
|
|
209
|
-
}),
|
|
210
|
-
...(response.stats?.backendId !== undefined && {
|
|
211
|
-
backendId: response.stats.backendId
|
|
212
|
-
}),
|
|
213
|
-
...(response.stats?.gpuUnsupported !== undefined && {
|
|
214
|
-
gpuUnsupported: response.stats.gpuUnsupported
|
|
215
|
-
}),
|
|
216
|
-
...(response.stats?.gpuMemTotalMb !== undefined && {
|
|
217
|
-
gpuMemTotalMb: response.stats.gpuMemTotalMb
|
|
218
|
-
}),
|
|
219
|
-
...(response.stats?.gpuMemFreeMb !== undefined && {
|
|
220
|
-
gpuMemFreeMb: response.stats.gpuMemFreeMb
|
|
221
|
-
})
|
|
222
|
-
};
|
|
223
|
-
return buildStreamResult(modelExecutionMs, stats);
|
|
225
|
+
return buildStreamResult(modelExecutionMs, mapTranscribeStats(response.stats));
|
|
224
226
|
}
|
|
225
227
|
catch (e_1) {
|
|
226
228
|
env_1.error = e_1;
|
|
@@ -286,6 +288,7 @@ export async function* transcribeStream(modelId, audioInputStream, prompt, metad
|
|
|
286
288
|
ctx.signal.removeEventListener('abort', onAbort);
|
|
287
289
|
});
|
|
288
290
|
const runOpts = buildRunStreamingOpts(engineType, opts);
|
|
291
|
+
const modelStart = nowMs();
|
|
289
292
|
const response = await model.runStreaming(audioInputStream, runOpts);
|
|
290
293
|
for await (const output of response.iterate()) {
|
|
291
294
|
if (ctx.signal.aborted)
|
|
@@ -316,6 +319,9 @@ export async function* transcribeStream(modelId, audioInputStream, prompt, metad
|
|
|
316
319
|
yield* emitSegment(segment, metadata, silenceMarker);
|
|
317
320
|
}
|
|
318
321
|
}
|
|
322
|
+
await response.await();
|
|
323
|
+
const modelExecutionMs = nowMs() - modelStart;
|
|
324
|
+
return buildStreamResult(modelExecutionMs, mapTranscribeStats(response.stats));
|
|
319
325
|
}
|
|
320
326
|
catch (e_2) {
|
|
321
327
|
env_2.error = e_2;
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import type { TranslationStats } from '../../schemas/index.js';
|
|
2
|
+
import type { NmtStats } from '../../utils/addon-responses.js';
|
|
3
|
+
export declare const NMT_SECONDS_TO_MS = 1000;
|
|
4
|
+
export declare function buildNmtTranslationStats(stats: Partial<NmtStats> | undefined): TranslationStats;
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export const NMT_SECONDS_TO_MS = 1000;
|
|
2
|
+
export function buildNmtTranslationStats(stats) {
|
|
3
|
+
return {
|
|
4
|
+
...(stats?.totalTime !== undefined && { totalTime: stats.totalTime * NMT_SECONDS_TO_MS }),
|
|
5
|
+
...(stats?.totalTokens !== undefined && { totalTokens: stats.totalTokens }),
|
|
6
|
+
...(stats?.decodeTime !== undefined && { decodeTime: stats.decodeTime * NMT_SECONDS_TO_MS }),
|
|
7
|
+
...(stats?.encodeTime !== undefined && { encodeTime: stats.encodeTime * NMT_SECONDS_TO_MS }),
|
|
8
|
+
...(stats?.TPS !== undefined && { tokensPerSecond: stats.TPS }),
|
|
9
|
+
...(stats?.TTFT !== undefined && { timeToFirstToken: stats.TTFT })
|
|
10
|
+
};
|
|
11
|
+
}
|
|
@@ -55,9 +55,12 @@ import { translateServerParamsSchema, normalizeModelType, ModelType, AFRICAN_LAN
|
|
|
55
55
|
import { getLangName, detectOne } from '@qvac/langdetect-text';
|
|
56
56
|
import { nowMs } from '../../profiling/index.js';
|
|
57
57
|
import { buildStreamResult } from '../../profiling/model-execution.js';
|
|
58
|
+
import { buildNmtTranslationStats } from '../../plugins/ops/translate-stats.js';
|
|
58
59
|
import { ModelIsDelegatedError, ModelNotFoundError, ModelTypeMismatchError, TranslationFailedError } from '../../errors/index.js';
|
|
59
60
|
import { getRequestRegistry, withRequestContext } from '../../runtime/index.js';
|
|
61
|
+
import { isAddonContextOverflowError } from '../../plugins/builtin/llamacpp-completion/ops/context-overflow.js';
|
|
60
62
|
import { generateRandomRequestId } from '../../runtime/request-id.js';
|
|
63
|
+
import { getModelParallel } from '../../utils/config-transform.js';
|
|
61
64
|
import { getEngineLogger } from '../../logging/index.js';
|
|
62
65
|
export function getLanguage(code) {
|
|
63
66
|
if (!code)
|
|
@@ -121,38 +124,47 @@ export async function* translate(params, requestId) {
|
|
|
121
124
|
const afriquePrompt = isLlm && (isAfrican(from) || isAfrican(to));
|
|
122
125
|
const fromLanguage = getLanguage(from);
|
|
123
126
|
const toLanguage = getLanguage(to);
|
|
124
|
-
// Open a request-scoped lifecycle for both engine branches. LLM
|
|
125
|
-
//
|
|
126
|
-
//
|
|
127
|
-
//
|
|
128
|
-
//
|
|
129
|
-
//
|
|
130
|
-
// to completion in the background — acceptable because the result
|
|
131
|
-
// is dropped either way.
|
|
127
|
+
// Open a request-scoped lifecycle for both engine branches. The LLM
|
|
128
|
+
// path cancels only its own run response (wired below), so a peer
|
|
129
|
+
// completion on the same model keeps running; NMT-translate has no
|
|
130
|
+
// addon cancel — the loop exits on `signal.aborted`, scope unwinds,
|
|
131
|
+
// and the addon may run to completion in the background, which is
|
|
132
|
+
// fine because the result is dropped either way.
|
|
132
133
|
const ctx = __addDisposableResource(env_1, await getRequestRegistry().begin({
|
|
133
134
|
requestId: requestId ?? generateRandomRequestId(),
|
|
134
135
|
kind: 'translate',
|
|
135
|
-
modelId
|
|
136
|
+
modelId,
|
|
137
|
+
// LLM translate joins the completion lane (its cap = the model's own
|
|
138
|
+
// `parallel`); NMT passes no cap and stays ungated on its own model.
|
|
139
|
+
...(isLlm && {
|
|
140
|
+
maxConcurrentPerModel: getModelParallel(entry.local.config)
|
|
141
|
+
})
|
|
136
142
|
}), true);
|
|
137
143
|
const requestLogger = withRequestContext(getEngineLogger(), ctx);
|
|
144
|
+
// Cancel only this run's response, not the addon's global cancel, so a peer
|
|
145
|
+
// completion keeps running.
|
|
146
|
+
let activeResponse = null;
|
|
147
|
+
const cancelActive = () => {
|
|
148
|
+
activeResponse?.cancel().catch((err) => {
|
|
149
|
+
requestLogger.warn(`[cancel] translate response.cancel() rejected during abort for modelId=${modelId}: ${err instanceof Error ? err.message : String(err)}`);
|
|
150
|
+
});
|
|
151
|
+
};
|
|
138
152
|
if (isLlm) {
|
|
139
|
-
|
|
140
|
-
const addon = model.addon;
|
|
141
|
-
if (addon?.cancel) {
|
|
142
|
-
addon.cancel.call(addon).catch((err) => {
|
|
143
|
-
requestLogger.warn(`[cancel] addon.cancel() rejected during abort for modelId=${modelId}: ${err instanceof Error ? err.message : String(err)}`);
|
|
144
|
-
});
|
|
145
|
-
}
|
|
146
|
-
};
|
|
147
|
-
ctx.signal.addEventListener('abort', onAbort, { once: true });
|
|
153
|
+
ctx.signal.addEventListener('abort', cancelActive, { once: true });
|
|
148
154
|
if (ctx.signal.aborted)
|
|
149
|
-
|
|
155
|
+
cancelActive();
|
|
150
156
|
ctx.scope.defer(() => {
|
|
151
|
-
ctx.signal.removeEventListener('abort',
|
|
157
|
+
ctx.signal.removeEventListener('abort', cancelActive);
|
|
158
|
+
activeResponse = null;
|
|
152
159
|
});
|
|
153
160
|
}
|
|
154
161
|
// Check if input is an array and model type is NMT
|
|
155
162
|
if (Array.isArray(text) && canonicalModelType === ModelType.nmtcppTranslation) {
|
|
163
|
+
// A cancel that landed before native work (e.g. the model is being unloaded)
|
|
164
|
+
// must not start the batch. Bail before runBatch, dropping the result.
|
|
165
|
+
if (ctx.signal.aborted) {
|
|
166
|
+
return { modelExecutionMs: 0 };
|
|
167
|
+
}
|
|
156
168
|
// Use runBatch for batch processing
|
|
157
169
|
const modelStart = nowMs();
|
|
158
170
|
const translations = await model.runBatch(text);
|
|
@@ -188,6 +200,15 @@ export async function* translate(params, requestId) {
|
|
|
188
200
|
: `${context ? `${context}. ` : ''}Translate the following text from ${fromLanguage} into ${toLanguage}. Only output the translation, nothing else.\n\n${fromLanguage}: ${singleText}\n${toLanguage}:`
|
|
189
201
|
}
|
|
190
202
|
];
|
|
203
|
+
// A translate cancelled before native work (queued-cancel, or the model being
|
|
204
|
+
// unloaded) must not call run(): LLM could decode against an exclusive finetune
|
|
205
|
+
// holding the lane, and either engine would run against a model being torn
|
|
206
|
+
// down. Bail with an empty soft-cancel result for both engines — the server
|
|
207
|
+
// never throws InferenceCancelledError (that error is client-constructed and
|
|
208
|
+
// would cross the RPC as a generic error).
|
|
209
|
+
if (ctx.signal.aborted) {
|
|
210
|
+
return { modelExecutionMs: 0 };
|
|
211
|
+
}
|
|
191
212
|
const modelStart = nowMs();
|
|
192
213
|
let response;
|
|
193
214
|
if (canonicalModelType === ModelType.llamacppCompletion &&
|
|
@@ -199,14 +220,34 @@ export async function* translate(params, requestId) {
|
|
|
199
220
|
else {
|
|
200
221
|
response = await model.run(input);
|
|
201
222
|
}
|
|
223
|
+
if (isLlm) {
|
|
224
|
+
activeResponse = response;
|
|
225
|
+
// A cancel that landed while `run(...)` was admitting still applies.
|
|
226
|
+
if (ctx.signal.aborted)
|
|
227
|
+
cancelActive();
|
|
228
|
+
}
|
|
202
229
|
// Check if the response has an iterate method (like LLM models)
|
|
203
230
|
if (canonicalModelType === ModelType.llamacppCompletion &&
|
|
204
231
|
typeof response.iterate === 'function') {
|
|
205
232
|
const llmResponse = response;
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
233
|
+
try {
|
|
234
|
+
for await (const token of llmResponse.iterate()) {
|
|
235
|
+
if (ctx.signal.aborted)
|
|
236
|
+
break;
|
|
237
|
+
yield token;
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
catch (err) {
|
|
241
|
+
// A context-overflow rejection is a real terminal condition, not a
|
|
242
|
+
// cancellation — surface it even under an aborted signal, as completion does.
|
|
243
|
+
if (isAddonContextOverflowError(err))
|
|
244
|
+
throw err;
|
|
245
|
+
// The request signal is the SDK-owned cancellation contract. Addon
|
|
246
|
+
// rejection shapes differ for active and queued native sequences, so once
|
|
247
|
+
// cancellation is accepted the translate ends cleanly regardless of that
|
|
248
|
+
// transport detail; without an abort, every error still propagates.
|
|
249
|
+
if (!ctx.signal.aborted)
|
|
250
|
+
throw err;
|
|
210
251
|
}
|
|
211
252
|
const modelExecutionMs = nowMs() - modelStart;
|
|
212
253
|
const stats = {
|
|
@@ -228,20 +269,7 @@ export async function* translate(params, requestId) {
|
|
|
228
269
|
yield token;
|
|
229
270
|
}
|
|
230
271
|
const modelExecutionMs = nowMs() - modelStart;
|
|
231
|
-
const stats =
|
|
232
|
-
...(nmtResponse.stats?.totalTime !== undefined && { totalTime: nmtResponse.stats.totalTime }),
|
|
233
|
-
...(nmtResponse.stats?.totalTokens !== undefined && {
|
|
234
|
-
totalTokens: nmtResponse.stats.totalTokens
|
|
235
|
-
}),
|
|
236
|
-
...(nmtResponse.stats?.decodeTime !== undefined && {
|
|
237
|
-
decodeTime: nmtResponse.stats.decodeTime
|
|
238
|
-
}),
|
|
239
|
-
...(nmtResponse.stats?.encodeTime !== undefined && {
|
|
240
|
-
encodeTime: nmtResponse.stats.encodeTime
|
|
241
|
-
}),
|
|
242
|
-
...(nmtResponse.stats?.TPS !== undefined && { tokensPerSecond: nmtResponse.stats.TPS }),
|
|
243
|
-
...(nmtResponse.stats?.TTFT !== undefined && { timeToFirstToken: nmtResponse.stats.TTFT })
|
|
244
|
-
};
|
|
272
|
+
const stats = buildNmtTranslationStats(nmtResponse.stats);
|
|
245
273
|
return buildStreamResult(modelExecutionMs, stats);
|
|
246
274
|
}
|
|
247
275
|
catch (e_1) {
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { promises as fsPromises } from 'bare-fs';
|
|
2
2
|
import path from 'bare-path';
|
|
3
3
|
import { unregisterModel } from '../../runtime/model-registry.js';
|
|
4
|
+
import { getRequestRegistry } from '../../runtime/index.js';
|
|
4
5
|
import { unregisterAllLoggingStreams } from '../../runtime/logging-stream-registry.js';
|
|
5
6
|
import { clearFinetuneRuntimeState } from '../../plugins/builtin/llamacpp-completion/ops/finetune.js';
|
|
6
7
|
import { unregisterAddonLogger, getEngineLogger } from '../../logging/index.js';
|
|
@@ -15,31 +16,36 @@ export async function unloadModel(params) {
|
|
|
15
16
|
if (!entry) {
|
|
16
17
|
throw new ModelNotLoadedError(modelId);
|
|
17
18
|
}
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
if (
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
const shardInfo = detectShardedModel(modelFileName);
|
|
27
|
-
if (shardInfo.isSharded) {
|
|
28
|
-
const shardDir = path.dirname(modelPath);
|
|
29
|
-
await fsPromises.rm(shardDir, { recursive: true, force: true });
|
|
30
|
-
logger.info(`Sharded model storage cleared: ${shardDir}`);
|
|
19
|
+
// Cancel and drain the model's requests, then tear down under the admission
|
|
20
|
+
// barrier so nothing goes active against a model being freed. The barrier is
|
|
21
|
+
// lifted for us when this resolves.
|
|
22
|
+
await getRequestRegistry().withModelDraining(modelId, async () => {
|
|
23
|
+
clearFinetuneRuntimeState(modelId);
|
|
24
|
+
if (!entry.isDelegated) {
|
|
25
|
+
if (entry.local.model.unload) {
|
|
26
|
+
await entry.local.model.unload();
|
|
31
27
|
}
|
|
32
|
-
|
|
33
|
-
const
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
28
|
+
if (clearStorage && entry.local.path) {
|
|
29
|
+
const modelPath = entry.local.path;
|
|
30
|
+
const modelFileName = path.basename(modelPath);
|
|
31
|
+
const shardInfo = detectShardedModel(modelFileName);
|
|
32
|
+
if (shardInfo.isSharded) {
|
|
33
|
+
const shardDir = path.dirname(modelPath);
|
|
34
|
+
await fsPromises.rm(shardDir, { recursive: true, force: true });
|
|
35
|
+
logger.info(`Sharded model storage cleared: ${shardDir}`);
|
|
36
|
+
}
|
|
37
|
+
else {
|
|
38
|
+
const target = getClearStorageTarget(modelPath);
|
|
39
|
+
await fsPromises.rm(target.path, {
|
|
40
|
+
recursive: target.kind === 'directory',
|
|
41
|
+
force: true
|
|
42
|
+
});
|
|
43
|
+
logger.info(`Model storage cleared (${target.kind}): ${target.path}`);
|
|
44
|
+
}
|
|
39
45
|
}
|
|
40
46
|
}
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
47
|
+
unregisterAddonLogger(modelId);
|
|
48
|
+
unregisterAllLoggingStreams(modelId);
|
|
49
|
+
logger.info(`Model ${modelId} unloaded`);
|
|
50
|
+
});
|
|
45
51
|
}
|
|
@@ -10,22 +10,19 @@ import { createRequestRegistry as createRegistry } from '../runtime/request-regi
|
|
|
10
10
|
* baseline concurrency policies.
|
|
11
11
|
*/
|
|
12
12
|
let registry = null;
|
|
13
|
-
//
|
|
14
|
-
//
|
|
15
|
-
//
|
|
16
|
-
//
|
|
17
|
-
//
|
|
18
|
-
//
|
|
19
|
-
//
|
|
20
|
-
//
|
|
13
|
+
// One admission lane per model for the llama.cpp addon: completion,
|
|
14
|
+
// batchCompletion, LLM translate, and finetune all key on
|
|
15
|
+
// (LLAMACPP_COMPLETION_SLOT_GROUP, modelId), so they compete for the model's
|
|
16
|
+
// `parallel` slots first-come-first-serve and share ONE bounded FIFO wait queue.
|
|
17
|
+
// The queue depth is the per-model cap below (64): once 64 requests across those
|
|
18
|
+
// kinds are waiting, the 65th begin is rejected with RequestRejectedByPolicyError.
|
|
19
|
+
// Disk-KV-cache turns ride the same lane and serialise same-file writes per cache
|
|
20
|
+
// path in the KV-cache session, so they need no separate group. NMT translate
|
|
21
|
+
// passes an infinite cap and never enters the lane, so it stays ungated.
|
|
21
22
|
const LLAMACPP_COMPLETION_SLOT_GROUP = 'llamacppCompletion';
|
|
22
23
|
function installDefaultPolicies(r) {
|
|
23
|
-
//
|
|
24
|
-
//
|
|
25
|
-
// rather than reject: the second waits FIFO. maxConcurrentPerModel: 1 is
|
|
26
|
-
// today's reality — raise it once continuous batching lands. The depth cap
|
|
27
|
-
// bounds queue memory. The shared slot group extends that serialization
|
|
28
|
-
// across `completion` + `batchCompletion` on the same model (see note above).
|
|
24
|
+
// Cap is the model's `parallel`, passed per request by the handlers; the value
|
|
25
|
+
// here is only the fallback when a caller supplies none.
|
|
29
26
|
r.policy({
|
|
30
27
|
kind: 'completion',
|
|
31
28
|
maxConcurrentPerModel: 1,
|
|
@@ -40,6 +37,24 @@ function installDefaultPolicies(r) {
|
|
|
40
37
|
maxQueueDepthPerModel: 64,
|
|
41
38
|
sharedSlotGroup: LLAMACPP_COMPLETION_SLOT_GROUP
|
|
42
39
|
});
|
|
40
|
+
// Finetune's only stop is the addon's global cancel, which would kill
|
|
41
|
+
// concurrent completions. Run it exclusively so nothing else is on the model.
|
|
42
|
+
r.policy({
|
|
43
|
+
kind: 'finetune',
|
|
44
|
+
maxConcurrentPerModel: 1,
|
|
45
|
+
onOverflow: 'queue',
|
|
46
|
+
maxQueueDepthPerModel: 64,
|
|
47
|
+
sharedSlotGroup: LLAMACPP_COMPLETION_SLOT_GROUP,
|
|
48
|
+
exclusive: true
|
|
49
|
+
});
|
|
50
|
+
// LLM translate shares the completion lane as a reader (capped by `parallel`),
|
|
51
|
+
// so a finetune blocks it too. NMT passes no cap and stays ungated.
|
|
52
|
+
r.policy({
|
|
53
|
+
kind: 'translate',
|
|
54
|
+
onOverflow: 'queue',
|
|
55
|
+
maxQueueDepthPerModel: 64,
|
|
56
|
+
sharedSlotGroup: LLAMACPP_COMPLETION_SLOT_GROUP
|
|
57
|
+
});
|
|
43
58
|
// ACE-Step owns one active job per model. Starting another run replaces the
|
|
44
59
|
// addon's active response, and model-scoped cancel targets that single active
|
|
45
60
|
// job. Keep the registry authoritative by admitting one AudioGen request per
|
|
@@ -19,6 +19,18 @@ export interface BeginOpts {
|
|
|
19
19
|
* the parent does not require iterating the registry.
|
|
20
20
|
*/
|
|
21
21
|
parentSignal?: AbortSignal;
|
|
22
|
+
/**
|
|
23
|
+
* Per-request override of the kind policy's `maxConcurrentPerModel`. The
|
|
24
|
+
* policy value is one number for every model of a kind, but handlers may
|
|
25
|
+
* use a loaded model's `parallel` as its top-level request-permit cap.
|
|
26
|
+
* This counts requests, not native sequences: a multi-prompt batch still
|
|
27
|
+
* consumes one permit and the addon schedules its sequences internally.
|
|
28
|
+
* Must be uniform across all requests routed to the same `(lane, modelId)`
|
|
29
|
+
* (a given model always yields the same `parallel`): the FIFO slot
|
|
30
|
+
* hand-off assumes one cap per lane and does not re-check it. A finite
|
|
31
|
+
* value below 1 is floored to 1.
|
|
32
|
+
*/
|
|
33
|
+
maxConcurrentPerModel?: number;
|
|
22
34
|
}
|
|
23
35
|
export interface CancelByRequestId {
|
|
24
36
|
requestId: string;
|
|
@@ -97,6 +109,13 @@ export interface ConcurrencyPolicy {
|
|
|
97
109
|
* internal queue.
|
|
98
110
|
*/
|
|
99
111
|
sharedSlotGroup?: string;
|
|
112
|
+
/**
|
|
113
|
+
* Admit at most one such request per lane AND only while no other request
|
|
114
|
+
* holds the lane — a writer in a reader/writer lock. Used for finetune,
|
|
115
|
+
* whose only cancel is the addon's global cancel: making it exclusive keeps
|
|
116
|
+
* completions out of the addon while it runs, so that cancel can't hit them.
|
|
117
|
+
*/
|
|
118
|
+
exclusive?: boolean;
|
|
100
119
|
}
|
|
101
120
|
/**
|
|
102
121
|
* `ManagedRequestContext` is the value `begin(...)` resolves to. It extends
|
|
@@ -169,6 +188,14 @@ export interface RequestRegistry {
|
|
|
169
188
|
* scope unwinding still happens on each handler's own dispose path.
|
|
170
189
|
*/
|
|
171
190
|
cancelAll(reason: 'shutdown' | 'modelUnload'): Promise<void>;
|
|
191
|
+
/**
|
|
192
|
+
* Cancel every request for a model and wait for each to fully dispose (native
|
|
193
|
+
* context freed, slot released) before running `teardown` with an admission
|
|
194
|
+
* barrier held — any begin for the model starts aborted — then lift it. Used
|
|
195
|
+
* by model unload so the model object isn't freed while a request is still
|
|
196
|
+
* mid-teardown, and no new request goes active against it during teardown.
|
|
197
|
+
*/
|
|
198
|
+
withModelDraining(modelId: string, teardown: () => Promise<void>): Promise<void>;
|
|
172
199
|
/**
|
|
173
200
|
* Mark a request finished and dispose its scope. Equivalent to
|
|
174
201
|
* `await ctx[Symbol.asyncDispose]()` with an explicit outcome.
|