@qvac/inference 0.17.1 → 0.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/dist/api/completion-stream.d.ts +1 -1
  2. package/dist/api/completion-stream.js +1 -1
  3. package/dist/api/index.d.ts +1 -1
  4. package/dist/api/index.js +1 -1
  5. package/dist/api/text-to-speech.js +22 -2
  6. package/dist/api/transcribe.d.ts +19 -2
  7. package/dist/api/transcribe.js +66 -27
  8. package/dist/api/translate.js +6 -17
  9. package/dist/api/vla.d.ts +42 -1
  10. package/dist/api/vla.js +50 -1
  11. package/dist/constants/audio.d.ts +2 -1
  12. package/dist/constants/audio.js +17 -1
  13. package/dist/errors/index.d.ts +3 -0
  14. package/dist/errors/index.js +5 -0
  15. package/dist/handlers/load-model/handler.js +2 -2
  16. package/dist/handlers/load-model/resolve-session.d.ts +1 -1
  17. package/dist/handlers/load-model/resolve-session.js +3 -3
  18. package/dist/handlers/load-model/resolve.d.ts +12 -2
  19. package/dist/handlers/load-model/resolve.js +90 -12
  20. package/dist/index.d.ts +1 -1
  21. package/dist/index.js +1 -1
  22. package/dist/models/registry/models.d.ts +782 -6
  23. package/dist/models/registry/models.js +2664 -1859
  24. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +19 -0
  25. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +153 -0
  26. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +54 -4
  27. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +28 -0
  28. package/dist/plugins/builtin/ggml-ocr/plugin.js +3 -1
  29. package/dist/plugins/builtin/ggml-vla/ops/vla-set-embodiment.d.ts +2 -0
  30. package/dist/plugins/builtin/ggml-vla/ops/vla-set-embodiment.js +9 -0
  31. package/dist/plugins/builtin/ggml-vla/plugin.d.ts +37 -0
  32. package/dist/plugins/builtin/ggml-vla/plugin.js +21 -4
  33. package/dist/plugins/builtin/index.d.ts +1 -0
  34. package/dist/plugins/builtin/index.js +1 -0
  35. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +16 -15
  36. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.d.ts +1 -1
  37. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +2 -2
  38. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +6 -0
  39. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +10 -0
  40. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +1 -0
  41. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +131 -123
  42. package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +110 -20
  43. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +50 -23
  44. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +308 -143
  45. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +1 -0
  46. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +2 -2
  47. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +6 -6
  48. package/dist/plugins/builtin/llamacpp-completion/plugin.js +56 -36
  49. package/dist/plugins/builtin/llamacpp-completion/transform.js +0 -7
  50. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -3
  51. package/dist/plugins/builtin/nmtcpp-translation/plugin.js +6 -5
  52. package/dist/plugins/builtin/parakeet-transcription/plugin.js +27 -16
  53. package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +23 -3
  54. package/dist/plugins/builtin/tts-ggml/plugin.d.ts +302 -4
  55. package/dist/plugins/builtin/tts-ggml/plugin.js +131 -0
  56. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +48 -39
  57. package/dist/plugins/ops/cancel.d.ts +1 -1
  58. package/dist/plugins/ops/cancel.js +1 -1
  59. package/dist/plugins/ops/kv-cache-retention.js +11 -0
  60. package/dist/plugins/ops/kv-cache-utils.d.ts +1 -1
  61. package/dist/plugins/ops/kv-cache-utils.js +40 -12
  62. package/dist/plugins/ops/transcribe.d.ts +2 -2
  63. package/dist/plugins/ops/transcribe.js +54 -48
  64. package/dist/plugins/ops/translate-stats.d.ts +4 -0
  65. package/dist/plugins/ops/translate-stats.js +11 -0
  66. package/dist/plugins/ops/translate.js +66 -38
  67. package/dist/plugins/ops/unload-model.js +30 -24
  68. package/dist/runtime/model-registry.d.ts +1 -0
  69. package/dist/runtime/request-context.js +29 -14
  70. package/dist/runtime/request-registry.d.ts +27 -0
  71. package/dist/runtime/request-registry.js +246 -76
  72. package/dist/schemas/audio-gen.d.ts +103 -0
  73. package/dist/schemas/audio-gen.js +135 -4
  74. package/dist/schemas/common.d.ts +550 -11
  75. package/dist/schemas/config.d.ts +60 -24
  76. package/dist/schemas/errors.d.ts +1 -0
  77. package/dist/schemas/errors.js +5 -0
  78. package/dist/schemas/index.d.ts +1 -1
  79. package/dist/schemas/index.js +1 -1
  80. package/dist/schemas/llamacpp-config.d.ts +10 -10
  81. package/dist/schemas/llamacpp-config.js +12 -5
  82. package/dist/schemas/load-model.d.ts +3691 -336
  83. package/dist/schemas/load-model.js +37 -19
  84. package/dist/schemas/text-to-speech.d.ts +1086 -110
  85. package/dist/schemas/text-to-speech.js +197 -7
  86. package/dist/schemas/tools.d.ts +0 -16
  87. package/dist/schemas/tools.js +0 -15
  88. package/dist/schemas/transcription-config.d.ts +2 -0
  89. package/dist/schemas/transcription-config.js +6 -0
  90. package/dist/schemas/transcription.d.ts +3 -0
  91. package/dist/schemas/vla.d.ts +83 -0
  92. package/dist/schemas/vla.js +105 -2
  93. package/dist/surface.d.ts +1 -1
  94. package/dist/surface.js +1 -1
  95. package/dist/utils/addon-responses.d.ts +4 -12
  96. package/dist/utils/audio/decoder.d.ts +37 -1
  97. package/dist/utils/audio/decoder.js +82 -52
  98. package/dist/utils/config-transform.d.ts +3 -0
  99. package/dist/utils/config-transform.js +6 -0
  100. package/dist/utils/index.d.ts +0 -1
  101. package/dist/utils/index.js +0 -1
  102. package/dist/utils/shard-utils.d.ts +1 -0
  103. package/dist/utils/shard-utils.js +14 -0
  104. package/dist/utils/tool-integration.d.ts +3 -10
  105. package/dist/utils/tool-integration.js +3 -12
  106. package/package.json +26 -22
  107. package/dist/plugins/builtin/llamacpp-completion/ops/batch-cancelled.d.ts +0 -22
  108. package/dist/plugins/builtin/llamacpp-completion/ops/batch-cancelled.js +0 -30
  109. package/dist/utils/expand-gguf-shards.d.ts +0 -2
  110. package/dist/utils/expand-gguf-shards.js +0 -22
@@ -5,10 +5,17 @@ import { getEngineLogger } from '../../logging/index.js';
5
5
  const logger = getEngineLogger();
6
6
  const AUTO_CACHE_MARKER_PREFIX = '.auto-cache-';
7
7
  const AUTO_CACHE_MARKER_PATTERN = /^\.auto-cache-([a-f0-9]{16})$/;
8
+ // SDK-generated auto-cache keys are 16 hex chars. Marker ops gate on this so a
9
+ // caller-provided key never gets a marker and the raw-key marker path stays
10
+ // inside the cache root.
11
+ const AUTO_CACHE_KEY_PATTERN = /^[a-f0-9]{16}$/;
8
12
  function getAutoCacheMarkerPath(cacheKey) {
9
13
  return path.join(getKVCacheDir(), `${AUTO_CACHE_MARKER_PREFIX}${cacheKey}`);
10
14
  }
11
15
  export async function markAutoCacheKey(cacheKey) {
16
+ // Only generated 16-hex auto keys get markers (see AUTO_CACHE_KEY_PATTERN).
17
+ if (!AUTO_CACHE_KEY_PATTERN.test(cacheKey))
18
+ return;
12
19
  try {
13
20
  await fsPromises.writeFile(getAutoCacheMarkerPath(cacheKey), '');
14
21
  }
@@ -17,6 +24,8 @@ export async function markAutoCacheKey(cacheKey) {
17
24
  }
18
25
  }
19
26
  export async function removeAutoCacheMarker(cacheKey) {
27
+ if (!AUTO_CACHE_KEY_PATTERN.test(cacheKey))
28
+ return;
20
29
  try {
21
30
  await fsPromises.unlink(getAutoCacheMarkerPath(cacheKey));
22
31
  }
@@ -25,6 +34,8 @@ export async function removeAutoCacheMarker(cacheKey) {
25
34
  }
26
35
  }
27
36
  export async function removeAutoCacheMarkerIfMissing(cacheKey) {
37
+ if (!AUTO_CACHE_KEY_PATTERN.test(cacheKey))
38
+ return;
28
39
  try {
29
40
  await fsPromises.access(path.join(getKVCacheDir(), cacheKey));
30
41
  }
@@ -12,7 +12,7 @@ export declare function getCurrentCacheInfo(modelId: string, configHash: string,
12
12
  cachePath: string;
13
13
  }>;
14
14
  export declare function renameCacheFile(oldPath: string, newPath: string): Promise<boolean>;
15
- export declare function pruneEmptyCacheDirectories(cacheFilePath: string): Promise<void>;
15
+ export declare function pruneEmptyCacheDirectories(cacheFilePath: string, activePaths?: readonly string[]): Promise<void>;
16
16
  export declare function deleteCache(options: {
17
17
  all: true;
18
18
  } | {
@@ -3,6 +3,7 @@ import { promises as fsPromises } from 'bare-fs';
3
3
  import path from 'bare-path';
4
4
  import { getAutoCacheLookupHistory, getKVCacheDir, validateAndJoinPath } from '../../utils/index.js';
5
5
  import { getEngineLogger } from '../../logging/index.js';
6
+ import { PathTraversalError } from '../../errors/index.js';
6
7
  import { Buffer } from 'bare-buffer';
7
8
  import { markAutoCacheKey } from '../../plugins/ops/kv-cache-retention.js';
8
9
  const logger = getEngineLogger();
@@ -14,20 +15,28 @@ export function extractSystemPrompt(messages) {
14
15
  const systemMessage = messages.find((msg) => msg.role === 'system');
15
16
  return systemMessage ? systemMessage.content : null;
16
17
  }
17
- function getToolNamesForHash(tools) {
18
- if (!Array.isArray(tools))
19
- return [];
20
- return tools
21
- .map((t) => t.name)
22
- .filter((n) => typeof n === 'string')
23
- .sort();
18
+ // Cache hash based on the system prompt + complete tool definitions.
19
+ // Callers pass tools only when the tool block is written into the cache and
20
+ // left there, so a different tool set gets its own cache instead of reusing a
21
+ // prefix that holds the old block. Every prompt-affecting field participates,
22
+ // not just the name: canonical serialization avoids cache misses caused only
23
+ // by object-key insertion order, while tool-array order is preserved because
24
+ // that is the order sent to the model.
25
+ function canonicalizeHashInput(value) {
26
+ if (Array.isArray(value))
27
+ return value.map(canonicalizeHashInput);
28
+ if (typeof value !== 'object' || value === null)
29
+ return value;
30
+ const canonical = {};
31
+ for (const key of Object.keys(value).sort()) {
32
+ canonical[key] = canonicalizeHashInput(value[key]);
33
+ }
34
+ return canonical;
24
35
  }
25
- // Cache hash based on system prompt + tool names
26
- // Different tools = different cache (tools anchored in first message, protected from n_discarded)
27
36
  export function generateConfigHash(systemPrompt, tools) {
28
37
  const hash = crypto.createHash('sha-256');
29
- const toolNames = getToolNamesForHash(tools);
30
- hash.update(Buffer.from(JSON.stringify({ systemPrompt, toolNames }), 'utf8'));
38
+ const canonicalConfig = JSON.stringify(canonicalizeHashInput({ systemPrompt, tools: Array.isArray(tools) ? tools : [] }));
39
+ hash.update(Buffer.from(canonicalConfig, 'utf8'));
31
40
  return hash.digest('hex').substring(0, 16);
32
41
  }
33
42
  export function generateCacheKey(messages) {
@@ -88,11 +97,15 @@ export async function renameCacheFile(oldPath, newPath) {
88
97
  return false;
89
98
  }
90
99
  }
91
- export async function pruneEmptyCacheDirectories(cacheFilePath) {
100
+ export async function pruneEmptyCacheDirectories(cacheFilePath, activePaths = []) {
92
101
  const cacheDir = getKVCacheDir();
93
102
  const cacheDirPrefix = `${cacheDir}${path.sep}`;
94
103
  let currentDirectory = path.dirname(cacheFilePath);
95
104
  while (currentDirectory.startsWith(cacheDirPrefix)) {
105
+ // Keep a directory another in-flight turn still holds (its .bin isn't on disk yet).
106
+ const childPrefix = `${currentDirectory}${path.sep}`;
107
+ if (activePaths.some((p) => p.startsWith(childPrefix)))
108
+ return;
96
109
  try {
97
110
  await fsPromises.rmdir(currentDirectory);
98
111
  }
@@ -113,8 +126,23 @@ export async function deleteCache(options) {
113
126
  await fsPromises.mkdir(cacheDir, { recursive: true });
114
127
  return cacheDir;
115
128
  }
129
+ // validateAndJoinPath sanitizes the key (strips traversal) and rejects
130
+ // escapes, so nested and other in-root keys resolve normally. A key that
131
+ // resolves to the cache root would wipe every cache — reject it BEFORE joining
132
+ // any modelId, so an empty key plus a modelId can't delete a real key dir
133
+ // under the root.
134
+ const cacheRoot = path.resolve(cacheDir);
116
135
  const kvCacheDir = validateAndJoinPath(cacheDir, options.kvCacheKey);
136
+ if (path.resolve(kvCacheDir) === cacheRoot) {
137
+ throw new PathTraversalError(options.kvCacheKey, cacheDir);
138
+ }
139
+ // Omitting modelId means "delete every model under the key"; a PROVIDED but
140
+ // empty/dot modelId collapses back to that same dir and would silently broaden
141
+ // the delete — reject it.
117
142
  const targetDir = options.modelId !== undefined ? validateAndJoinPath(kvCacheDir, options.modelId) : kvCacheDir;
143
+ if (options.modelId !== undefined && path.resolve(targetDir) === path.resolve(kvCacheDir)) {
144
+ throw new PathTraversalError(options.modelId, kvCacheDir);
145
+ }
118
146
  await fsPromises.rm(targetDir, { recursive: true, force: true });
119
147
  return targetDir;
120
148
  }
@@ -16,5 +16,5 @@ export interface TranscribeStreamOpts {
16
16
  vadRunIntervalMs?: number;
17
17
  parakeetStreamingConfig?: ParakeetStreamingRunConfig;
18
18
  }
19
- export declare function transcribeStream(modelId: string, audioInputStream: AsyncIterable<Buffer>, prompt: string | undefined, metadata: true, opts?: TranscribeStreamOpts, requestId?: string): AsyncGenerator<TranscribeSegment | TranscribeStreamEvent, void, void>;
20
- export declare function transcribeStream(modelId: string, audioInputStream: AsyncIterable<Buffer>, prompt?: string, metadata?: boolean, opts?: TranscribeStreamOpts, requestId?: string): AsyncGenerator<string | TranscribeStreamEvent, void, void>;
19
+ export declare function transcribeStream(modelId: string, audioInputStream: AsyncIterable<Buffer>, prompt: string | undefined, metadata: true, opts?: TranscribeStreamOpts, requestId?: string): AsyncGenerator<TranscribeSegment | TranscribeStreamEvent, TranscribeReturn, void>;
20
+ export declare function transcribeStream(modelId: string, audioInputStream: AsyncIterable<Buffer>, prompt?: string, metadata?: boolean, opts?: TranscribeStreamOpts, requestId?: string): AsyncGenerator<string | TranscribeStreamEvent, TranscribeReturn, void>;
@@ -100,6 +100,55 @@ async function restorePrompt(modelId, originalConfig) {
100
100
  return;
101
101
  await model.reload(buildWhisperReloadConfig({ ...originalConfig, initial_prompt: '' }));
102
102
  }
103
+ function mapTranscribeStats(stats) {
104
+ return {
105
+ ...(stats?.audioDurationMs !== undefined && {
106
+ audioDuration: stats.audioDurationMs
107
+ }),
108
+ ...(stats?.realTimeFactor !== undefined && {
109
+ realTimeFactor: stats.realTimeFactor
110
+ }),
111
+ ...(stats?.tokensPerSecond !== undefined && {
112
+ tokensPerSecond: stats.tokensPerSecond
113
+ }),
114
+ ...(stats?.totalTokens !== undefined && {
115
+ totalTokens: stats.totalTokens
116
+ }),
117
+ ...(stats?.totalSegments !== undefined && {
118
+ totalSegments: stats.totalSegments
119
+ }),
120
+ ...(stats?.whisperEncodeMs !== undefined && {
121
+ whisperEncodeTime: stats.whisperEncodeMs
122
+ }),
123
+ ...(stats?.whisperDecodeMs !== undefined && {
124
+ whisperDecodeTime: stats.whisperDecodeMs
125
+ }),
126
+ ...(stats?.encoderMs !== undefined && {
127
+ encoderTime: stats.encoderMs
128
+ }),
129
+ ...(stats?.decoderMs !== undefined && {
130
+ decoderTime: stats.decoderMs
131
+ }),
132
+ ...(stats?.melSpecMs !== undefined && {
133
+ melSpecTime: stats.melSpecMs
134
+ }),
135
+ ...(stats?.backendDevice !== undefined && {
136
+ backendDevice: stats.backendDevice
137
+ }),
138
+ ...(stats?.backendId !== undefined && {
139
+ backendId: stats.backendId
140
+ }),
141
+ ...(stats?.gpuUnsupported !== undefined && {
142
+ gpuUnsupported: stats.gpuUnsupported
143
+ }),
144
+ ...(stats?.gpuMemTotalMb !== undefined && {
145
+ gpuMemTotalMb: stats.gpuMemTotalMb
146
+ }),
147
+ ...(stats?.gpuMemFreeMb !== undefined && {
148
+ gpuMemFreeMb: stats.gpuMemFreeMb
149
+ })
150
+ };
151
+ }
103
152
  export async function* transcribe(params, requestId) {
104
153
  const env_1 = { stack: [], error: void 0, hasError: false };
105
154
  try {
@@ -173,54 +222,7 @@ export async function* transcribe(params, requestId) {
173
222
  }
174
223
  }
175
224
  const modelExecutionMs = nowMs() - modelStart;
176
- const stats = {
177
- ...(response.stats?.audioDurationMs !== undefined && {
178
- audioDuration: response.stats.audioDurationMs
179
- }),
180
- ...(response.stats?.realTimeFactor !== undefined && {
181
- realTimeFactor: response.stats.realTimeFactor
182
- }),
183
- ...(response.stats?.tokensPerSecond !== undefined && {
184
- tokensPerSecond: response.stats.tokensPerSecond
185
- }),
186
- ...(response.stats?.totalTokens !== undefined && {
187
- totalTokens: response.stats.totalTokens
188
- }),
189
- ...(response.stats?.totalSegments !== undefined && {
190
- totalSegments: response.stats.totalSegments
191
- }),
192
- ...(response.stats?.whisperEncodeMs !== undefined && {
193
- whisperEncodeTime: response.stats.whisperEncodeMs
194
- }),
195
- ...(response.stats?.whisperDecodeMs !== undefined && {
196
- whisperDecodeTime: response.stats.whisperDecodeMs
197
- }),
198
- ...(response.stats?.encoderMs !== undefined && {
199
- encoderTime: response.stats.encoderMs
200
- }),
201
- ...(response.stats?.decoderMs !== undefined && {
202
- decoderTime: response.stats.decoderMs
203
- }),
204
- ...(response.stats?.melSpecMs !== undefined && {
205
- melSpecTime: response.stats.melSpecMs
206
- }),
207
- ...(response.stats?.backendDevice !== undefined && {
208
- backendDevice: response.stats.backendDevice
209
- }),
210
- ...(response.stats?.backendId !== undefined && {
211
- backendId: response.stats.backendId
212
- }),
213
- ...(response.stats?.gpuUnsupported !== undefined && {
214
- gpuUnsupported: response.stats.gpuUnsupported
215
- }),
216
- ...(response.stats?.gpuMemTotalMb !== undefined && {
217
- gpuMemTotalMb: response.stats.gpuMemTotalMb
218
- }),
219
- ...(response.stats?.gpuMemFreeMb !== undefined && {
220
- gpuMemFreeMb: response.stats.gpuMemFreeMb
221
- })
222
- };
223
- return buildStreamResult(modelExecutionMs, stats);
225
+ return buildStreamResult(modelExecutionMs, mapTranscribeStats(response.stats));
224
226
  }
225
227
  catch (e_1) {
226
228
  env_1.error = e_1;
@@ -286,6 +288,7 @@ export async function* transcribeStream(modelId, audioInputStream, prompt, metad
286
288
  ctx.signal.removeEventListener('abort', onAbort);
287
289
  });
288
290
  const runOpts = buildRunStreamingOpts(engineType, opts);
291
+ const modelStart = nowMs();
289
292
  const response = await model.runStreaming(audioInputStream, runOpts);
290
293
  for await (const output of response.iterate()) {
291
294
  if (ctx.signal.aborted)
@@ -316,6 +319,9 @@ export async function* transcribeStream(modelId, audioInputStream, prompt, metad
316
319
  yield* emitSegment(segment, metadata, silenceMarker);
317
320
  }
318
321
  }
322
+ await response.await();
323
+ const modelExecutionMs = nowMs() - modelStart;
324
+ return buildStreamResult(modelExecutionMs, mapTranscribeStats(response.stats));
319
325
  }
320
326
  catch (e_2) {
321
327
  env_2.error = e_2;
@@ -0,0 +1,4 @@
1
+ import type { TranslationStats } from '../../schemas/index.js';
2
+ import type { NmtStats } from '../../utils/addon-responses.js';
3
+ export declare const NMT_SECONDS_TO_MS = 1000;
4
+ export declare function buildNmtTranslationStats(stats: Partial<NmtStats> | undefined): TranslationStats;
@@ -0,0 +1,11 @@
1
+ export const NMT_SECONDS_TO_MS = 1000;
2
+ export function buildNmtTranslationStats(stats) {
3
+ return {
4
+ ...(stats?.totalTime !== undefined && { totalTime: stats.totalTime * NMT_SECONDS_TO_MS }),
5
+ ...(stats?.totalTokens !== undefined && { totalTokens: stats.totalTokens }),
6
+ ...(stats?.decodeTime !== undefined && { decodeTime: stats.decodeTime * NMT_SECONDS_TO_MS }),
7
+ ...(stats?.encodeTime !== undefined && { encodeTime: stats.encodeTime * NMT_SECONDS_TO_MS }),
8
+ ...(stats?.TPS !== undefined && { tokensPerSecond: stats.TPS }),
9
+ ...(stats?.TTFT !== undefined && { timeToFirstToken: stats.TTFT })
10
+ };
11
+ }
@@ -55,9 +55,12 @@ import { translateServerParamsSchema, normalizeModelType, ModelType, AFRICAN_LAN
55
55
  import { getLangName, detectOne } from '@qvac/langdetect-text';
56
56
  import { nowMs } from '../../profiling/index.js';
57
57
  import { buildStreamResult } from '../../profiling/model-execution.js';
58
+ import { buildNmtTranslationStats } from '../../plugins/ops/translate-stats.js';
58
59
  import { ModelIsDelegatedError, ModelNotFoundError, ModelTypeMismatchError, TranslationFailedError } from '../../errors/index.js';
59
60
  import { getRequestRegistry, withRequestContext } from '../../runtime/index.js';
61
+ import { isAddonContextOverflowError } from '../../plugins/builtin/llamacpp-completion/ops/context-overflow.js';
60
62
  import { generateRandomRequestId } from '../../runtime/request-id.js';
63
+ import { getModelParallel } from '../../utils/config-transform.js';
61
64
  import { getEngineLogger } from '../../logging/index.js';
62
65
  export function getLanguage(code) {
63
66
  if (!code)
@@ -121,38 +124,47 @@ export async function* translate(params, requestId) {
121
124
  const afriquePrompt = isLlm && (isAfrican(from) || isAfrican(to));
122
125
  const fromLanguage = getLanguage(from);
123
126
  const toLanguage = getLanguage(to);
124
- // Open a request-scoped lifecycle for both engine branches. LLM-
125
- // translate inherits llamacpp-completion's `{ scope: "model",
126
- // hard: true }` cancel surface; NMT-translate inherits nmtcpp's
127
- // `{ scope: "none" }` — so the addon-cancel wiring below only
128
- // engages on the LLM path. NMT cancel is purely soft: the loop
129
- // exits on `signal.aborted`, scope unwinds, and the addon may run
130
- // to completion in the background — acceptable because the result
131
- // is dropped either way.
127
+ // Open a request-scoped lifecycle for both engine branches. The LLM
128
+ // path cancels only its own run response (wired below), so a peer
129
+ // completion on the same model keeps running; NMT-translate has no
130
+ // addon cancel — the loop exits on `signal.aborted`, scope unwinds,
131
+ // and the addon may run to completion in the background, which is
132
+ // fine because the result is dropped either way.
132
133
  const ctx = __addDisposableResource(env_1, await getRequestRegistry().begin({
133
134
  requestId: requestId ?? generateRandomRequestId(),
134
135
  kind: 'translate',
135
- modelId
136
+ modelId,
137
+ // LLM translate joins the completion lane (its cap = the model's own
138
+ // `parallel`); NMT passes no cap and stays ungated on its own model.
139
+ ...(isLlm && {
140
+ maxConcurrentPerModel: getModelParallel(entry.local.config)
141
+ })
136
142
  }), true);
137
143
  const requestLogger = withRequestContext(getEngineLogger(), ctx);
144
+ // Cancel only this run's response, not the addon's global cancel, so a peer
145
+ // completion keeps running.
146
+ let activeResponse = null;
147
+ const cancelActive = () => {
148
+ activeResponse?.cancel().catch((err) => {
149
+ requestLogger.warn(`[cancel] translate response.cancel() rejected during abort for modelId=${modelId}: ${err instanceof Error ? err.message : String(err)}`);
150
+ });
151
+ };
138
152
  if (isLlm) {
139
- const onAbort = () => {
140
- const addon = model.addon;
141
- if (addon?.cancel) {
142
- addon.cancel.call(addon).catch((err) => {
143
- requestLogger.warn(`[cancel] addon.cancel() rejected during abort for modelId=${modelId}: ${err instanceof Error ? err.message : String(err)}`);
144
- });
145
- }
146
- };
147
- ctx.signal.addEventListener('abort', onAbort, { once: true });
153
+ ctx.signal.addEventListener('abort', cancelActive, { once: true });
148
154
  if (ctx.signal.aborted)
149
- onAbort();
155
+ cancelActive();
150
156
  ctx.scope.defer(() => {
151
- ctx.signal.removeEventListener('abort', onAbort);
157
+ ctx.signal.removeEventListener('abort', cancelActive);
158
+ activeResponse = null;
152
159
  });
153
160
  }
154
161
  // Check if input is an array and model type is NMT
155
162
  if (Array.isArray(text) && canonicalModelType === ModelType.nmtcppTranslation) {
163
+ // A cancel that landed before native work (e.g. the model is being unloaded)
164
+ // must not start the batch. Bail before runBatch, dropping the result.
165
+ if (ctx.signal.aborted) {
166
+ return { modelExecutionMs: 0 };
167
+ }
156
168
  // Use runBatch for batch processing
157
169
  const modelStart = nowMs();
158
170
  const translations = await model.runBatch(text);
@@ -188,6 +200,15 @@ export async function* translate(params, requestId) {
188
200
  : `${context ? `${context}. ` : ''}Translate the following text from ${fromLanguage} into ${toLanguage}. Only output the translation, nothing else.\n\n${fromLanguage}: ${singleText}\n${toLanguage}:`
189
201
  }
190
202
  ];
203
+ // A translate cancelled before native work (queued-cancel, or the model being
204
+ // unloaded) must not call run(): LLM could decode against an exclusive finetune
205
+ // holding the lane, and either engine would run against a model being torn
206
+ // down. Bail with an empty soft-cancel result for both engines — the server
207
+ // never throws InferenceCancelledError (that error is client-constructed and
208
+ // would cross the RPC as a generic error).
209
+ if (ctx.signal.aborted) {
210
+ return { modelExecutionMs: 0 };
211
+ }
191
212
  const modelStart = nowMs();
192
213
  let response;
193
214
  if (canonicalModelType === ModelType.llamacppCompletion &&
@@ -199,14 +220,34 @@ export async function* translate(params, requestId) {
199
220
  else {
200
221
  response = await model.run(input);
201
222
  }
223
+ if (isLlm) {
224
+ activeResponse = response;
225
+ // A cancel that landed while `run(...)` was admitting still applies.
226
+ if (ctx.signal.aborted)
227
+ cancelActive();
228
+ }
202
229
  // Check if the response has an iterate method (like LLM models)
203
230
  if (canonicalModelType === ModelType.llamacppCompletion &&
204
231
  typeof response.iterate === 'function') {
205
232
  const llmResponse = response;
206
- for await (const token of llmResponse.iterate()) {
207
- if (ctx.signal.aborted)
208
- break;
209
- yield token;
233
+ try {
234
+ for await (const token of llmResponse.iterate()) {
235
+ if (ctx.signal.aborted)
236
+ break;
237
+ yield token;
238
+ }
239
+ }
240
+ catch (err) {
241
+ // A context-overflow rejection is a real terminal condition, not a
242
+ // cancellation — surface it even under an aborted signal, as completion does.
243
+ if (isAddonContextOverflowError(err))
244
+ throw err;
245
+ // The request signal is the SDK-owned cancellation contract. Addon
246
+ // rejection shapes differ for active and queued native sequences, so once
247
+ // cancellation is accepted the translate ends cleanly regardless of that
248
+ // transport detail; without an abort, every error still propagates.
249
+ if (!ctx.signal.aborted)
250
+ throw err;
210
251
  }
211
252
  const modelExecutionMs = nowMs() - modelStart;
212
253
  const stats = {
@@ -228,20 +269,7 @@ export async function* translate(params, requestId) {
228
269
  yield token;
229
270
  }
230
271
  const modelExecutionMs = nowMs() - modelStart;
231
- const stats = {
232
- ...(nmtResponse.stats?.totalTime !== undefined && { totalTime: nmtResponse.stats.totalTime }),
233
- ...(nmtResponse.stats?.totalTokens !== undefined && {
234
- totalTokens: nmtResponse.stats.totalTokens
235
- }),
236
- ...(nmtResponse.stats?.decodeTime !== undefined && {
237
- decodeTime: nmtResponse.stats.decodeTime
238
- }),
239
- ...(nmtResponse.stats?.encodeTime !== undefined && {
240
- encodeTime: nmtResponse.stats.encodeTime
241
- }),
242
- ...(nmtResponse.stats?.TPS !== undefined && { tokensPerSecond: nmtResponse.stats.TPS }),
243
- ...(nmtResponse.stats?.TTFT !== undefined && { timeToFirstToken: nmtResponse.stats.TTFT })
244
- };
272
+ const stats = buildNmtTranslationStats(nmtResponse.stats);
245
273
  return buildStreamResult(modelExecutionMs, stats);
246
274
  }
247
275
  catch (e_1) {
@@ -1,6 +1,7 @@
1
1
  import { promises as fsPromises } from 'bare-fs';
2
2
  import path from 'bare-path';
3
3
  import { unregisterModel } from '../../runtime/model-registry.js';
4
+ import { getRequestRegistry } from '../../runtime/index.js';
4
5
  import { unregisterAllLoggingStreams } from '../../runtime/logging-stream-registry.js';
5
6
  import { clearFinetuneRuntimeState } from '../../plugins/builtin/llamacpp-completion/ops/finetune.js';
6
7
  import { unregisterAddonLogger, getEngineLogger } from '../../logging/index.js';
@@ -15,31 +16,36 @@ export async function unloadModel(params) {
15
16
  if (!entry) {
16
17
  throw new ModelNotLoadedError(modelId);
17
18
  }
18
- clearFinetuneRuntimeState(modelId);
19
- if (!entry.isDelegated) {
20
- if (entry.local.model.unload) {
21
- await entry.local.model.unload();
22
- }
23
- if (clearStorage && entry.local.path) {
24
- const modelPath = entry.local.path;
25
- const modelFileName = path.basename(modelPath);
26
- const shardInfo = detectShardedModel(modelFileName);
27
- if (shardInfo.isSharded) {
28
- const shardDir = path.dirname(modelPath);
29
- await fsPromises.rm(shardDir, { recursive: true, force: true });
30
- logger.info(`Sharded model storage cleared: ${shardDir}`);
19
+ // Cancel and drain the model's requests, then tear down under the admission
20
+ // barrier so nothing goes active against a model being freed. The barrier is
21
+ // lifted for us when this resolves.
22
+ await getRequestRegistry().withModelDraining(modelId, async () => {
23
+ clearFinetuneRuntimeState(modelId);
24
+ if (!entry.isDelegated) {
25
+ if (entry.local.model.unload) {
26
+ await entry.local.model.unload();
31
27
  }
32
- else {
33
- const target = getClearStorageTarget(modelPath);
34
- await fsPromises.rm(target.path, {
35
- recursive: target.kind === 'directory',
36
- force: true
37
- });
38
- logger.info(`Model storage cleared (${target.kind}): ${target.path}`);
28
+ if (clearStorage && entry.local.path) {
29
+ const modelPath = entry.local.path;
30
+ const modelFileName = path.basename(modelPath);
31
+ const shardInfo = detectShardedModel(modelFileName);
32
+ if (shardInfo.isSharded) {
33
+ const shardDir = path.dirname(modelPath);
34
+ await fsPromises.rm(shardDir, { recursive: true, force: true });
35
+ logger.info(`Sharded model storage cleared: ${shardDir}`);
36
+ }
37
+ else {
38
+ const target = getClearStorageTarget(modelPath);
39
+ await fsPromises.rm(target.path, {
40
+ recursive: target.kind === 'directory',
41
+ force: true
42
+ });
43
+ logger.info(`Model storage cleared (${target.kind}): ${target.path}`);
44
+ }
39
45
  }
40
46
  }
41
- }
42
- unregisterAddonLogger(modelId);
43
- unregisterAllLoggingStreams(modelId);
44
- logger.info(`Model ${modelId} unloaded`);
47
+ unregisterAddonLogger(modelId);
48
+ unregisterAllLoggingStreams(modelId);
49
+ logger.info(`Model ${modelId} unloaded`);
50
+ });
45
51
  }
@@ -5,6 +5,7 @@ interface AddonInterface {
5
5
  interface ModelRunResponse {
6
6
  iterate(): AsyncIterable<unknown>;
7
7
  await(): Promise<unknown>;
8
+ cancel(): Promise<void>;
8
9
  }
9
10
  export interface AnyModel {
10
11
  load(force?: boolean): Promise<void>;
@@ -10,22 +10,19 @@ import { createRequestRegistry as createRegistry } from '../runtime/request-regi
10
10
  * baseline concurrency policies.
11
11
  */
12
12
  let registry = null;
13
- // `completion` and `batchCompletion` both run on the same `@qvac/llm-llamacpp`
14
- // instance, which funnels every `run()` (single-prompt and batch alike) through
15
- // one per-instance exclusive run queue plus a single-job native runner. They
16
- // therefore can't actually execute at once on the same model. Sharing one
17
- // admission lane makes the queue reflect that reality: a completion and a batch
18
- // on the same model serialize FIFO at the registry layer instead of both being
19
- // admitted and silently serializing inside the addon (which hides them from the
20
- // registry's queue-depth accounting, `requestId` diagnostics, and cancel).
13
+ // One admission lane per model for the llama.cpp addon: completion,
14
+ // batchCompletion, LLM translate, and finetune all key on
15
+ // (LLAMACPP_COMPLETION_SLOT_GROUP, modelId), so they compete for the model's
16
+ // `parallel` slots first-come-first-serve and share ONE bounded FIFO wait queue.
17
+ // The queue depth is the per-model cap below (64): once 64 requests across those
18
+ // kinds are waiting, the 65th begin is rejected with RequestRejectedByPolicyError.
19
+ // Disk-KV-cache turns ride the same lane and serialise same-file writes per cache
20
+ // path in the KV-cache session, so they need no separate group. NMT translate
21
+ // passes an infinite cap and never enters the lane, so it stays ungated.
21
22
  const LLAMACPP_COMPLETION_SLOT_GROUP = 'llamacppCompletion';
22
23
  function installDefaultPolicies(r) {
23
- // A loaded model is a single native context (one KV-cache, single-slot
24
- // decode), so two same-model completions can't run in parallel. Serialize
25
- // rather than reject: the second waits FIFO. maxConcurrentPerModel: 1 is
26
- // today's reality — raise it once continuous batching lands. The depth cap
27
- // bounds queue memory. The shared slot group extends that serialization
28
- // across `completion` + `batchCompletion` on the same model (see note above).
24
+ // Cap is the model's `parallel`, passed per request by the handlers; the value
25
+ // here is only the fallback when a caller supplies none.
29
26
  r.policy({
30
27
  kind: 'completion',
31
28
  maxConcurrentPerModel: 1,
@@ -40,6 +37,24 @@ function installDefaultPolicies(r) {
40
37
  maxQueueDepthPerModel: 64,
41
38
  sharedSlotGroup: LLAMACPP_COMPLETION_SLOT_GROUP
42
39
  });
40
+ // Finetune's only stop is the addon's global cancel, which would kill
41
+ // concurrent completions. Run it exclusively so nothing else is on the model.
42
+ r.policy({
43
+ kind: 'finetune',
44
+ maxConcurrentPerModel: 1,
45
+ onOverflow: 'queue',
46
+ maxQueueDepthPerModel: 64,
47
+ sharedSlotGroup: LLAMACPP_COMPLETION_SLOT_GROUP,
48
+ exclusive: true
49
+ });
50
+ // LLM translate shares the completion lane as a reader (capped by `parallel`),
51
+ // so a finetune blocks it too. NMT passes no cap and stays ungated.
52
+ r.policy({
53
+ kind: 'translate',
54
+ onOverflow: 'queue',
55
+ maxQueueDepthPerModel: 64,
56
+ sharedSlotGroup: LLAMACPP_COMPLETION_SLOT_GROUP
57
+ });
43
58
  // ACE-Step owns one active job per model. Starting another run replaces the
44
59
  // addon's active response, and model-scoped cancel targets that single active
45
60
  // job. Keep the registry authoritative by admitting one AudioGen request per
@@ -19,6 +19,18 @@ export interface BeginOpts {
19
19
  * the parent does not require iterating the registry.
20
20
  */
21
21
  parentSignal?: AbortSignal;
22
+ /**
23
+ * Per-request override of the kind policy's `maxConcurrentPerModel`. The
24
+ * policy value is one number for every model of a kind, but handlers may
25
+ * use a loaded model's `parallel` as its top-level request-permit cap.
26
+ * This counts requests, not native sequences: a multi-prompt batch still
27
+ * consumes one permit and the addon schedules its sequences internally.
28
+ * Must be uniform across all requests routed to the same `(lane, modelId)`
29
+ * (a given model always yields the same `parallel`): the FIFO slot
30
+ * hand-off assumes one cap per lane and does not re-check it. A finite
31
+ * value below 1 is floored to 1.
32
+ */
33
+ maxConcurrentPerModel?: number;
22
34
  }
23
35
  export interface CancelByRequestId {
24
36
  requestId: string;
@@ -97,6 +109,13 @@ export interface ConcurrencyPolicy {
97
109
  * internal queue.
98
110
  */
99
111
  sharedSlotGroup?: string;
112
+ /**
113
+ * Admit at most one such request per lane AND only while no other request
114
+ * holds the lane — a writer in a reader/writer lock. Used for finetune,
115
+ * whose only cancel is the addon's global cancel: making it exclusive keeps
116
+ * completions out of the addon while it runs, so that cancel can't hit them.
117
+ */
118
+ exclusive?: boolean;
100
119
  }
101
120
  /**
102
121
  * `ManagedRequestContext` is the value `begin(...)` resolves to. It extends
@@ -169,6 +188,14 @@ export interface RequestRegistry {
169
188
  * scope unwinding still happens on each handler's own dispose path.
170
189
  */
171
190
  cancelAll(reason: 'shutdown' | 'modelUnload'): Promise<void>;
191
+ /**
192
+ * Cancel every request for a model and wait for each to fully dispose (native
193
+ * context freed, slot released) before running `teardown` with an admission
194
+ * barrier held — any begin for the model starts aborted — then lift it. Used
195
+ * by model unload so the model object isn't freed while a request is still
196
+ * mid-teardown, and no new request goes active against it during teardown.
197
+ */
198
+ withModelDraining(modelId: string, teardown: () => Promise<void>): Promise<void>;
172
199
  /**
173
200
  * Mark a request finished and dispose its scope. Equivalent to
174
201
  * `await ctx[Symbol.asyncDispose]()` with an explicit outcome.