@qvac/inference 0.19.0 → 0.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (150) hide show
  1. package/NOTICE +95 -39
  2. package/dist/api/audio-gen.d.ts +42 -1
  3. package/dist/api/audio-gen.js +137 -51
  4. package/dist/api/delete-cache.d.ts +7 -1
  5. package/dist/api/delete-cache.js +11 -1
  6. package/dist/api/index.d.ts +2 -1
  7. package/dist/api/index.js +2 -1
  8. package/dist/api/text-to-speech.d.ts +31 -4
  9. package/dist/api/text-to-speech.js +117 -21
  10. package/dist/api/vector-index.d.ts +142 -0
  11. package/dist/api/vector-index.js +168 -0
  12. package/dist/api/world.d.ts +2 -2
  13. package/dist/errors/index.d.ts +12 -0
  14. package/dist/errors/index.js +21 -0
  15. package/dist/handlers/delete-cache.js +3 -0
  16. package/dist/handlers/vector-index.d.ts +2 -0
  17. package/dist/handlers/vector-index.js +40 -0
  18. package/dist/index.d.ts +1 -1
  19. package/dist/index.js +1 -1
  20. package/dist/models/registry/models.d.ts +351 -12
  21. package/dist/models/registry/models.js +2271 -1957
  22. package/dist/models/registry/resource-profiles.js +135 -0
  23. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +2 -0
  24. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +65 -0
  25. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +9 -1
  26. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +24 -7
  27. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +46 -0
  28. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +218 -0
  29. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +1 -1
  30. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +56 -225
  31. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +8 -0
  32. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +28 -0
  33. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +248 -0
  34. package/dist/plugins/builtin/audiogen-ggml/plugin.js +17 -1
  35. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +5 -2
  36. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +1 -2
  37. package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +1 -1
  38. package/dist/plugins/builtin/ggml-ocr/plugin.js +1 -2
  39. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +6 -4
  40. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.d.ts +1 -2
  41. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +0 -4
  42. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +2 -0
  43. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +6 -0
  44. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +9 -0
  45. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +96 -89
  46. package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.d.ts +0 -1
  47. package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.js +0 -31
  48. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +48 -27
  49. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +239 -135
  50. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +11 -4
  51. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +18 -16
  52. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +11 -4
  53. package/dist/plugins/builtin/llamacpp-completion/plugin.js +1 -2
  54. package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +1 -2
  55. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -38
  56. package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +9 -0
  57. package/dist/plugins/builtin/llamacpp-embedding/transform.js +42 -0
  58. package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +1 -1
  59. package/dist/plugins/builtin/nmtcpp-translation/plugin.js +1 -2
  60. package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +1 -1
  61. package/dist/plugins/builtin/parakeet-transcription/plugin.js +1 -2
  62. package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +1 -0
  63. package/dist/plugins/builtin/sdcpp-generation/ops/video.js +8 -2
  64. package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +9 -4
  65. package/dist/plugins/builtin/sdcpp-generation/plugin.js +51 -19
  66. package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +29 -0
  67. package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +66 -0
  68. package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +10 -0
  69. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +3 -5
  70. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +143 -28
  71. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +5 -2
  72. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +182 -66
  73. package/dist/plugins/builtin/tts-ggml/plugin.d.ts +267 -22
  74. package/dist/plugins/builtin/tts-ggml/plugin.js +94 -35
  75. package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +1 -1
  76. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +1 -2
  77. package/dist/plugins/index.d.ts +1 -1
  78. package/dist/plugins/index.js +1 -1
  79. package/dist/plugins/ops/load-model.js +22 -2
  80. package/dist/plugins/ops/translate.js +58 -12
  81. package/dist/plugins/ops/translatepsy-utils.d.ts +17 -0
  82. package/dist/plugins/ops/translatepsy-utils.js +32 -0
  83. package/dist/plugins/registry.d.ts +15 -0
  84. package/dist/plugins/registry.js +133 -15
  85. package/dist/plugins/turbovec-backend.d.ts +17 -0
  86. package/dist/plugins/turbovec-backend.js +99 -0
  87. package/dist/registry.js +4 -0
  88. package/dist/resources/model-fit/assess.js +169 -68
  89. package/dist/resources/model-fit/estimators/llm.d.ts +1 -0
  90. package/dist/resources/model-fit/estimators/llm.js +3 -1
  91. package/dist/resources/model-fit/floor.d.ts +31 -0
  92. package/dist/resources/model-fit/floor.js +65 -0
  93. package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +46 -0
  94. package/dist/resources/model-fit/native-probe/advisory-fit.js +214 -0
  95. package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +26 -0
  96. package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +171 -0
  97. package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +42 -0
  98. package/dist/resources/model-fit/native-probe/run-in-process-fit.js +192 -0
  99. package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +70 -0
  100. package/dist/resources/model-fit/native-probe/run-isolated-fit.js +372 -0
  101. package/dist/resources/native.js +5 -4
  102. package/dist/resources/types.d.ts +1 -3
  103. package/dist/runtime/env.d.ts +2 -0
  104. package/dist/runtime/env.js +9 -2
  105. package/dist/runtime/lifecycle.js +2 -0
  106. package/dist/runtime/model-registry.d.ts +3 -1
  107. package/dist/runtime/model-registry.js +2 -1
  108. package/dist/runtime/request-context.js +17 -0
  109. package/dist/runtime/vector-index-backend.d.ts +25 -0
  110. package/dist/runtime/vector-index-backend.js +1 -0
  111. package/dist/runtime/vector-index-registry.d.ts +76 -0
  112. package/dist/runtime/vector-index-registry.js +161 -0
  113. package/dist/schemas/assess-model-fit.d.ts +84 -0
  114. package/dist/schemas/assess-model-fit.js +72 -3
  115. package/dist/schemas/audio-gen.d.ts +482 -7
  116. package/dist/schemas/audio-gen.js +376 -17
  117. package/dist/schemas/batch-completion-stream.d.ts +10 -0
  118. package/dist/schemas/batch-completion-stream.js +5 -3
  119. package/dist/schemas/bci.d.ts +1 -1
  120. package/dist/schemas/common.d.ts +782 -19
  121. package/dist/schemas/common.js +8 -1
  122. package/dist/schemas/completion-event.d.ts +14 -1
  123. package/dist/schemas/completion-event.js +4 -0
  124. package/dist/schemas/completion-stream.d.ts +22 -0
  125. package/dist/schemas/completion-stream.js +41 -4
  126. package/dist/schemas/config.d.ts +48 -30
  127. package/dist/schemas/errors.d.ts +4 -0
  128. package/dist/schemas/errors.js +22 -0
  129. package/dist/schemas/index.d.ts +1 -0
  130. package/dist/schemas/index.js +1 -0
  131. package/dist/schemas/llamacpp-config.d.ts +4 -8
  132. package/dist/schemas/llamacpp-config.js +4 -4
  133. package/dist/schemas/load-model.d.ts +2604 -180
  134. package/dist/schemas/model-config-schemas.d.ts +228 -5
  135. package/dist/schemas/model-ops.d.ts +3 -0
  136. package/dist/schemas/model-ops.js +6 -0
  137. package/dist/schemas/plugin.d.ts +21 -11
  138. package/dist/schemas/sdcpp-config.d.ts +221 -4
  139. package/dist/schemas/sdcpp-config.js +128 -22
  140. package/dist/schemas/text-to-speech.d.ts +780 -19
  141. package/dist/schemas/text-to-speech.js +298 -44
  142. package/dist/schemas/transcription-config.js +7 -4
  143. package/dist/schemas/vector-index.d.ts +194 -0
  144. package/dist/schemas/vector-index.js +185 -0
  145. package/dist/utils/addon-responses.d.ts +30 -3
  146. package/dist/utils/aggregate-events.d.ts +3 -1
  147. package/dist/utils/aggregate-events.js +7 -1
  148. package/dist/utils/tts-stats.d.ts +15 -11
  149. package/dist/utils/tts-stats.js +33 -11
  150. package/package.json +29 -26
@@ -1,4 +1,3 @@
1
- import bciAddonLogging from '@qvac/bci-whispercpp/addonLogging';
2
1
  import { type CreateModelParams, type PluginModelResult } from '../../../schemas/index.js';
3
2
  import { resolveBciConfig } from './resolve-config.js';
4
3
  export declare const bciPlugin: {
@@ -167,7 +166,11 @@ export declare const bciPlugin: {
167
166
  }, import("zod/v4/core").$strip>>;
168
167
  };
169
168
  logging: {
170
- module: typeof bciAddonLogging;
169
+ module: () => Promise<{
170
+ default: typeof import("@qvac/bci-whispercpp/addonLogging");
171
+ setLogger: (callback: (priority: number, message: string) => void) => void;
172
+ releaseLogger: () => void;
173
+ }>;
171
174
  namespace: "bci-whispercpp-transcription";
172
175
  };
173
176
  };
@@ -1,4 +1,3 @@
1
- import bciAddonLogging from '@qvac/bci-whispercpp/addonLogging';
2
1
  import BCIWhispercpp from '@qvac/bci-whispercpp';
3
2
  import { definePlugin, defineHandler, defineDuplexHandler, bciTranscribeRequestSchema, bciTranscribeResponseSchema, bciTranscribeStreamRequestSchema, bciTranscribeStreamResponseSchema, ModelType, bciConfigSchema, ADDON_BCI } from '../../../schemas/index.js';
4
3
  import { createStreamLogger, registerAddonLogger } from '../../../logging/index.js';
@@ -105,7 +104,7 @@ export const bciPlugin = definePlugin({
105
104
  })
106
105
  },
107
106
  logging: {
108
- module: bciAddonLogging,
107
+ module: () => import('@qvac/bci-whispercpp/addonLogging'),
109
108
  namespace: ModelType.bciWhispercppTranscription
110
109
  }
111
110
  });
@@ -96,7 +96,7 @@ export declare const ocrPlugin: {
96
96
  }, import("zod/v4/core").$strip>>;
97
97
  };
98
98
  logging: {
99
- module: import("@qvac/ocr-ggml/addonLogging").AddonLogging;
99
+ module: () => Promise<typeof import("@qvac/ocr-ggml/addonLogging")>;
100
100
  namespace: "ggml-ocr";
101
101
  };
102
102
  };
@@ -1,7 +1,6 @@
1
1
  import { definePlugin, defineHandler, ocrStreamRequestSchema, ocrStreamResponseSchema, ModelType, ocrConfigSchema, ADDON_OCR } from '../../../schemas/index.js';
2
2
  import { ModelLoadFailedError } from '../../../errors/index.js';
3
3
  import { createStreamLogger, registerAddonLogger } from '../../../logging/index.js';
4
- import ocrAddonLogging from '@qvac/ocr-ggml/addonLogging';
5
4
  import { OcrGgml } from '@qvac/ocr-ggml';
6
5
  import { ocr } from './ops/ocr-stream.js';
7
6
  import { attachModelExecutionMs } from '../../../profiling/model-execution.js';
@@ -106,7 +105,7 @@ export const ocrPlugin = definePlugin({
106
105
  })
107
106
  },
108
107
  logging: {
109
- module: ocrAddonLogging,
108
+ module: () => import('@qvac/ocr-ggml/addonLogging'),
110
109
  namespace: ModelType.ggmlOcr
111
110
  }
112
111
  });
@@ -3,7 +3,7 @@ import { getEngineLogger } from '../../../../logging/index.js';
3
3
  import { nowMs } from '../../../../profiling/index.js';
4
4
  import { buildStreamResult } from '../../../../profiling/model-execution.js';
5
5
  import { getResponseFormatJsonSchema } from '../../../../utils/response-format.js';
6
- import { transformMessages } from './completion-stream.js';
6
+ import { seedConfiguredSystemPrompt, transformMessages } from './completion-stream.js';
7
7
  import { normalizeCompletionStats } from './completion-stats.js';
8
8
  import { prependToolsToHistory } from '../../../../utils/tool-integration.js';
9
9
  const logger = getEngineLogger();
@@ -24,9 +24,10 @@ function mergeGenerationParams(generationParams, responseFormat) {
24
24
  }
25
25
  function renderPromptHistory(prompt, options) {
26
26
  const tools = options.toolsEnabled && prompt.tools && prompt.tools.length > 0 ? prompt.tools : undefined;
27
- let historyWithTools = prompt.history;
27
+ const history = seedConfiguredSystemPrompt(prompt.history, options.modelConfig);
28
+ let historyWithTools = history;
28
29
  if (tools) {
29
- historyWithTools = prependToolsToHistory(prompt.history, tools);
30
+ historyWithTools = prependToolsToHistory(history, tools);
30
31
  }
31
32
  // Uses the same attachment expansion as single completion: each
32
33
  // attachment becomes an addon `type: "media"` message before the text turn.
@@ -57,7 +58,8 @@ export async function* batchCompletion(params, opts) {
57
58
  const model = getModel(modelId);
58
59
  const modelConfig = getModelConfig(modelId);
59
60
  const renderOptions = {
60
- toolsEnabled: modelConfig.tools === true
61
+ toolsEnabled: modelConfig.tools === true,
62
+ modelConfig
61
63
  };
62
64
  // Cancel via the batch response, mirroring completion(): the addon routes
63
65
  // response.cancel() to cancelJob for this batch group only, so a concurrent
@@ -1,6 +1,5 @@
1
1
  export declare function logCacheStatus(cacheKey: string, isReusing: boolean): void;
2
- export declare function logCacheInit(cacheKey: string, systemPrompt: string): void;
3
- export declare function logMessagesToAddon(messages: unknown[], phase?: 'CACHE_INIT' | 'PROMPT_SEND' | 'NO_CACHE'): void;
2
+ export declare function logMessagesToAddon(messages: unknown[], phase?: 'PROMPT_SEND' | 'NO_CACHE'): void;
4
3
  export declare function logCacheDisabled(): void;
5
4
  export declare function logCacheSave(sessionPath: string): void;
6
5
  export declare function logCacheSaveError(sessionPath: string, err: unknown): void;
@@ -17,10 +17,6 @@ export function logCacheStatus(cacheKey, isReusing) {
17
17
  const status = isReusing ? 'REUSING' : 'CREATING';
18
18
  logger.debug(`[kv-cache] [${cacheKey}] ${status} cache`);
19
19
  }
20
- export function logCacheInit(cacheKey, systemPrompt) {
21
- const promptLen = systemPrompt.length;
22
- logger.debug(`[kv-cache] [${cacheKey}] Initializing cache (prompt: ${promptLen} chars)`);
23
- }
24
20
  export function logMessagesToAddon(messages, phase = 'PROMPT_SEND') {
25
21
  const typedMessages = messages;
26
22
  logger.debug(`[kv-cache] [${phase}] Sending ${typedMessages.length} msg(s): [${formatMessages(typedMessages)}]`);
@@ -3,11 +3,13 @@ import type { LlmStats } from '../../../../utils/addon-responses.js';
3
3
  export declare function normalizeCompletionStats(stats: LlmStats | undefined): {
4
4
  timeToFirstToken?: number | undefined;
5
5
  tokensPerSecond?: number | undefined;
6
+ promptTokensPerSecond?: number | undefined;
6
7
  cacheTokens?: number | undefined;
7
8
  promptTokens?: number | undefined;
8
9
  generatedTokens?: number | undefined;
9
10
  emittedTokens?: number | undefined;
10
11
  avgConcurrentSeq?: number | undefined;
12
+ toolDefinitionsDropped?: number | undefined;
11
13
  backendDevice?: "gpu" | "cpu" | undefined;
12
14
  } | undefined;
13
15
  /**
@@ -6,25 +6,31 @@ export function normalizeCompletionStats(stats) {
6
6
  return undefined;
7
7
  const timeToFirstToken = finiteNumber(stats.TTFT);
8
8
  const tokensPerSecond = finiteNumber(stats.TPS);
9
+ const promptTokensPerSecond = finiteNumber(stats.ppTPS);
9
10
  const cacheTokens = finiteNumber(stats.CacheTokens);
10
11
  const promptTokens = finiteNumber(stats.promptTokens);
11
12
  const generatedTokens = finiteNumber(stats.generatedTokens);
12
13
  const avgConcurrentSeq = finiteNumber(stats.avgConcurrentSeq);
14
+ const toolDefinitionsDropped = finiteNumber(stats.toolDefinitionsDropped);
13
15
  const normalized = {
14
16
  ...(timeToFirstToken !== undefined && { timeToFirstToken }),
15
17
  ...(tokensPerSecond !== undefined && { tokensPerSecond }),
18
+ ...(promptTokensPerSecond !== undefined && { promptTokensPerSecond }),
16
19
  ...(cacheTokens !== undefined && { cacheTokens }),
17
20
  ...(promptTokens !== undefined && { promptTokens }),
18
21
  ...(generatedTokens !== undefined && { generatedTokens }),
19
22
  ...(avgConcurrentSeq !== undefined && { avgConcurrentSeq }),
23
+ ...(toolDefinitionsDropped !== undefined && { toolDefinitionsDropped }),
20
24
  ...(stats.backendDevice !== undefined && { backendDevice: stats.backendDevice })
21
25
  };
22
26
  if (timeToFirstToken === undefined &&
23
27
  tokensPerSecond === undefined &&
28
+ promptTokensPerSecond === undefined &&
24
29
  cacheTokens === undefined &&
25
30
  promptTokens === undefined &&
26
31
  generatedTokens === undefined &&
27
32
  avgConcurrentSeq === undefined &&
33
+ toolDefinitionsDropped === undefined &&
28
34
  stats.backendDevice === undefined) {
29
35
  return undefined;
30
36
  }
@@ -26,6 +26,15 @@ export declare function transformMessages(messages: Array<{
26
26
  path: string;
27
27
  }[] | undefined;
28
28
  } | Tool>): ChatHistory[];
29
+ /**
30
+ * Put the model's configured system prompt in front of a history that carries
31
+ * none. `transform.ts` strips `system_prompt` from the config handed to the
32
+ * addon, so the conversation is the only way it reaches the model.
33
+ */
34
+ export declare function seedConfiguredSystemPrompt<T extends {
35
+ role: string;
36
+ content: string;
37
+ }>(history: T[], modelConfig: unknown): T[];
29
38
  export declare function completion(params: CompletionParams & {
30
39
  tools?: Tool[];
31
40
  generationParams?: GenerationParams;
@@ -1,12 +1,12 @@
1
- import { logCacheDisabled, logCacheInit, logCacheSave, logMessagesToAddon } from './cache-logger.js';
1
+ import { logCacheDisabled, logCacheSave, logMessagesToAddon } from './cache-logger.js';
2
2
  import { extractSystemPrompt, getCurrentCacheInfo } from '../../../ops/kv-cache-utils.js';
3
- import { isAddonPreMutationRefusal, isAddonContextOverflowError } from './context-overflow.js';
4
3
  import { getModel, getModelConfig } from '../../../../runtime/model-registry.js';
5
4
  import { decideCachedHistorySlice, shouldCommitCachedTurn } from './kv-cache-state.js';
6
5
  import { createKvCacheSession, generateConfigHash } from './kv-cache-session.js';
7
6
  import { detectToolDialect, prependToolsToHistory } from '../../../../utils/tool-integration.js';
8
7
  import { parseToolCalls } from '../../../../utils/tools/index.js';
9
8
  import { getResponseFormatJsonSchema } from '../../../../utils/response-format.js';
9
+ import { toolChoiceDemandsCall } from '../../../../schemas/completion-stream.js';
10
10
  import { buildAutoCacheSaveHistory } from '../../../../utils/index.js';
11
11
  import { getEngineLogger } from '../../../../logging/index.js';
12
12
  import { AttachmentNotFoundError } from '../../../../errors/index.js';
@@ -57,90 +57,77 @@ export function transformMessages(messages) {
57
57
  return transformed;
58
58
  }
59
59
  /**
60
- * Prime the cache prefix with the system prompt only.
61
- *
62
- * Tools deliberately stay out of the prefix: a message list with no user turn
63
- * is not a renderable conversation for every chat template (Qwen3.5 raises
64
- * `No user query found in messages.` because it anchors its tool block on the
65
- * last user query), and a template failure degrades the whole render rather
66
- * than just the tool block. Tools travel with the turn instead.
60
+ * Put the model's configured system prompt in front of a history that carries
61
+ * none. `transform.ts` strips `system_prompt` from the config handed to the
62
+ * addon, so the conversation is the only way it reaches the model.
67
63
  */
68
- async function initSystemPromptCache(model, cachePathToUse, systemPromptToUse, cacheKey, onResponse) {
69
- const primeMessages = [{ role: 'system', content: systemPromptToUse }];
70
- logCacheInit(cacheKey, systemPromptToUse);
71
- logMessagesToAddon(primeMessages, 'CACHE_INIT');
72
- const primeResponse = await runModel(model, primeMessages, {
73
- cacheKey: cachePathToUse,
74
- saveCacheToDisk: true,
75
- prefill: true
76
- });
77
- // Register the prime so an abort during a cold-cache prefill cancels it.
78
- onResponse?.(primeResponse);
79
- await primeResponse.await();
64
+ export function seedConfiguredSystemPrompt(history, modelConfig) {
65
+ const configured = modelConfig.system_prompt;
66
+ if (!configured || extractSystemPrompt(history) !== null)
67
+ return history;
68
+ // Every caller's message type makes everything past role and content optional.
69
+ return [{ role: 'system', content: configured }, ...history];
80
70
  }
81
71
  /**
82
- * Attach the tool block ahead of a turn payload, mirroring the no-kv-cache
83
- * path (`prependToolsToHistory`).
72
+ * Attach the tool block to a turn payload, mirroring the no-kv-cache path
73
+ * (`prependToolsToHistory`): after a system message when the payload carries
74
+ * one, ahead of everything otherwise.
84
75
  */
85
76
  function withToolBlock(messages, toolBlock) {
86
77
  if (toolBlock.length === 0)
87
78
  return messages;
88
- return [...toolBlock, ...messages];
79
+ const systemIndex = messages.findIndex((msg) => msg.role === 'system');
80
+ if (systemIndex === -1)
81
+ return [...toolBlock, ...messages];
82
+ return [...messages.slice(0, systemIndex + 1), ...toolBlock, ...messages.slice(systemIndex + 1)];
89
83
  }
90
84
  /**
91
- * Whether a payload carrying this tool block actually gets it in front of the
92
- * model. Qwen-family templates anchor their tool section on the last user
93
- * query and raise without one, and the addon answers that by re-rendering the
94
- * turn with tools stripped — a usable prompt with no tools in it. Recording
95
- * such a turn as "the block is cached now" would suppress the block for the
96
- * rest of the session, so require a user message before believing it landed.
85
+ * Fallback guess for whether a payload carrying this tool block gets it in
86
+ * front of the model, used only when the addon does not report
87
+ * `toolDefinitionsDropped`. Qwen-family templates anchor their tool section on
88
+ * the last user query and raise without one, and the addon answers that by
89
+ * re-rendering with tools stripped, so require a user message before believing
90
+ * the block landed.
97
91
  */
98
92
  function rendersToolBlock(messages, toolBlock) {
99
93
  if (toolBlock.length === 0)
100
94
  return false;
101
95
  return messages.some((msg) => msg.role === 'user');
102
96
  }
97
+ /**
98
+ * Settle whether the committed prefix holds the full tool block. The addon's
99
+ * `toolDefinitionsDropped` is the template's own word on whether the block it
100
+ * was handed reached the model; without it (a payload with no block, or a
101
+ * stand-in model) the payload's estimate stands.
102
+ */
103
+ function resolveToolBlockCached(payload, stats) {
104
+ const dropped = stats?.toolDefinitionsDropped;
105
+ if (!payload.toolBlockSent || dropped === undefined)
106
+ return payload.toolBlockCached;
107
+ return payload.prefixHoldsBlock || (payload.toolBlockFull && dropped === 0);
108
+ }
103
109
  /**
104
110
  * Pick the messages that need to reach the model for the next turn.
105
111
  *
106
- * Tools are never baked into the primed prefix — a prefix with no user turn is
107
- * not a renderable conversation for every template — so they travel with a
108
- * turn instead.
109
- *
110
- * - Empty history: nothing to slice; send whatever non-system messages
111
- * exist. (The call site always reports the cache as existing, so this
112
- * is the only way into this branch.)
113
- * - Cache hit with a recorded `savedCount`: send only the unsaved tail
114
- * (`history.slice(savedCount)`), so a multi-message turn (e.g. a
115
- * consumer pushing both an assistant transcript and a follow-up user
116
- * message between completions) all reaches the model.
117
- * - Cache hit with a stale/missing `savedCount`: fall back to the full
118
- * non-system history. The session is told (`dropStaleSavedCount`) so
119
- * the bad boundary doesn't propagate into the next turn.
120
- * - The tool block travels only with the turn that writes it into the
121
- * cache; see `skipToolBlock` below.
112
+ * The cache holds whatever a committed turn sent, the tool block included, so
113
+ * the block travels with a turn rather than being written into the prefix on
114
+ * its own, and only with the turn that writes it into the cache — see
115
+ * `skipToolBlock` below.
122
116
  */
123
- function prepareMessagesForCache(session, turn, cacheExists, history, tools) {
117
+ async function prepareMessagesForCache(session, turn, history, tools, toolChoice) {
124
118
  const toolBlock = tools?.length ? transformMessages(tools) : [];
125
- if (!(cacheExists && history.length > 0)) {
126
- const historyWithoutSystem = history.filter((msg) => msg.role !== 'system');
127
- return {
128
- messages: withToolBlock(transformMessages(historyWithoutSystem), toolBlock),
129
- toolBlockCached: rendersToolBlock(historyWithoutSystem, toolBlock)
130
- };
131
- }
132
119
  // Slice from the turn's `savedCount` so callers can
133
120
  // stage multiple messages between completions. `decideCachedHistorySlice`
134
121
  // also guards against the QVAC-17780 stale-count regression: if the
135
122
  // saved boundary would slice the history down to an empty payload
136
123
  // (e.g. after a cancelled mid-decode), it falls back to the full
137
- // non-system history and signals the caller to drop the bad entry.
138
- // The session owns the entry; `dropStaleSavedCount` clears it
139
- // without touching the on-disk file (the file is still trustworthy
140
- // — only the boundary count is wrong).
141
- const { messages, clearStaleCount } = decideCachedHistorySlice(turn.savedCount, cacheExists, history);
124
+ // history and signals the caller to drop the bad entry.
125
+ // The session owns the entry; `dropStaleSavedCount` clears it in memory
126
+ // and on disk without touching the cache file (the file is still
127
+ // trustworthy — only the boundary count is wrong).
128
+ const { messages, clearStaleCount } = decideCachedHistorySlice(turn.savedCount, history);
142
129
  if (clearStaleCount) {
143
- session.dropStaleSavedCount(turn);
130
+ await session.dropStaleSavedCount(turn);
144
131
  }
145
132
  // The block is never trimmed back out of the cache, so re-sending it every
146
133
  // turn would leave one copy per turn and grow the prefix with the
@@ -148,14 +135,24 @@ function prepareMessagesForCache(session, turn, cacheExists, history, tools) {
148
135
  // one: `toolBlockCached` records that a previous turn actually got it into
149
136
  // the cache, which a committed message count does not prove. A stale
150
137
  // boundary means we are resending the whole conversation anyway.
151
- const skipToolBlock = turn.toolBlockCached && !clearStaleCount;
138
+ const prefixHoldsBlock = turn.toolBlockCached && !clearStaleCount;
139
+ // The addon arms the tool-call grammar only for a payload that carries
140
+ // tools, and `required` / a named tool cannot be honoured without it. Such a
141
+ // turn resends the block even into a prefix that holds one; the second copy
142
+ // in the cache is the price of the guarantee.
143
+ const demandsCall = toolChoiceDemandsCall(toolChoice) && toolBlock.length > 0;
144
+ const skipToolBlock = prefixHoldsBlock && !demandsCall;
152
145
  const blockToSend = skipToolBlock ? [] : toolBlock;
146
+ const toolBlockFull = blockToSend.length > 0 && (!demandsCall || toolChoice === 'required');
153
147
  return {
154
148
  messages: withToolBlock(transformMessages(messages), blockToSend),
155
- toolBlockCached: skipToolBlock || rendersToolBlock(messages, blockToSend)
149
+ toolBlockSent: blockToSend.length > 0,
150
+ toolBlockFull,
151
+ prefixHoldsBlock,
152
+ toolBlockCached: prefixHoldsBlock || (toolBlockFull && rendersToolBlock(messages, blockToSend))
156
153
  };
157
154
  }
158
- async function* processModelResponse(model, messagesToSend, tools, generationParams, cacheOptions, dialect, onResponse) {
155
+ async function* processModelResponse(model, messagesToSend, tools, generationParams, cacheOptions, dialect, onResponse, onRunSettled) {
159
156
  const runOptions = {
160
157
  ...(generationParams && { generationParams }),
161
158
  ...(cacheOptions?.cacheKey !== undefined && {
@@ -184,6 +181,8 @@ async function* processModelResponse(model, messagesToSend, tools, generationPar
184
181
  yield { token: tokenStr };
185
182
  }
186
183
  const modelExecutionMs = nowMs() - modelStart;
184
+ // The addon has finished, and saved the cache file if it was going to.
185
+ onRunSettled?.();
187
186
  if (cacheOptions?.saveCacheToDisk && cacheOptions.cacheKey) {
188
187
  logCacheSave(cacheOptions.cacheKey);
189
188
  }
@@ -193,19 +192,22 @@ async function* processModelResponse(model, messagesToSend, tools, generationPar
193
192
  }
194
193
  const responseWithStats = response;
195
194
  const stats = withEmittedTokens(normalizeCompletionStats(responseWithStats.stats), emittedPieces);
195
+ const stopReason = responseWithStats.stats?.stopReason;
196
196
  return {
197
197
  ...buildStreamResult(modelExecutionMs, stats),
198
198
  toolCalls: toolCallsResult,
199
199
  responseText: accumulatedText,
200
200
  producedTokens,
201
- stoppedAtContextBoundary: responseWithStats.stats?.stopReason === 'contextOverflow'
201
+ stoppedAtContextBoundary: stopReason === 'contextOverflow',
202
+ generationFinished: stopReason !== 'none'
202
203
  };
203
204
  }
204
205
  export async function* completion(params, opts) {
205
- const { history, modelId, kvCache, tools, generationParams, responseFormat } = params;
206
+ const { history: requestHistory, modelId, kvCache, tools, generationParams, responseFormat } = params;
206
207
  const { signal, scope } = opts;
207
208
  const requestLogger = opts.logger ?? logger;
208
209
  const modelConfig = getModelConfig(modelId);
210
+ const history = seedConfiguredSystemPrompt(requestHistory, modelConfig);
209
211
  const toolsEnabled = modelConfig.tools === true;
210
212
  const toolsActive = !!tools?.length && toolsEnabled;
211
213
  const dialect = tools && tools.length > 0 ? (params.toolDialect ?? detectToolDialect(modelId)) : undefined;
@@ -267,30 +269,25 @@ export async function* completion(params, opts) {
267
269
  logMessagesToAddon(transformedHistory, 'NO_CACHE');
268
270
  return yield* processModelResponse(model, transformedHistory, tools, mergedGenerationParams, undefined, dialect, setActiveResponse);
269
271
  }
270
- // ---- KV-cache path. The session owns every bookkeeping layer; the
271
- // handler registers one deferred unwind (`rollback`, or the non-destructive
272
- // `releaseTurn` on a recognised pre-mutation refusal) that `commitTurn`
273
- // short-circuits on the happy path. Cancellations / zero-token replies /
274
- // rename failures all still unwind destructively through the same hook. ----
272
+ // ---- KV-cache path. The session owns every bookkeeping layer; the handler
273
+ // registers one deferred unwind that `commitTurn` short-circuits on the happy
274
+ // path. It is the non-destructive `releaseTurn` when the committed file is
275
+ // known to be intact — a throw before the addon run settled, or a cancel the
276
+ // addon rewound — and the destructive `rollback` for everything else,
277
+ // including zero-token replies, budget and context stops, and rename
278
+ // failures. ----
275
279
  const session = createKvCacheSession(modelId, { logger: requestLogger });
276
280
  const systemPromptFromHistory = extractSystemPrompt(history);
277
281
  // The tool block is baked into the cache on the turn that first sends it and
278
282
  // never trimmed, so a late or changed tool set has to land on a fresh cache
279
283
  // rather than a warm prefix holding the old block.
280
284
  const configHash = generateConfigHash(systemPromptFromHistory, toolsActive ? tools : undefined);
281
- const systemPromptToUse = systemPromptFromHistory ||
282
- modelConfig.system_prompt ||
283
- 'You are a helpful assistant.';
284
- const primeIfMissing = async (cachePath) => {
285
- await initSystemPromptCache(model, cachePath, systemPromptToUse, typeof kvCache === 'string' ? kvCache : 'auto', setActiveResponse);
286
- };
287
285
  let turn;
288
286
  if (typeof kvCache === 'string') {
289
287
  turn = await session.beginTurn({
290
288
  kind: 'custom',
291
289
  customKey: kvCache,
292
290
  configHash,
293
- primeIfMissing,
294
291
  signal
295
292
  });
296
293
  }
@@ -304,7 +301,6 @@ export async function* completion(params, opts) {
304
301
  kind: 'auto',
305
302
  configHash,
306
303
  history: cacheMessages,
307
- primeIfMissing,
308
304
  signal
309
305
  });
310
306
  }
@@ -312,17 +308,14 @@ export async function* completion(params, opts) {
312
308
  // flips the turn's internal `committed` flag so this becomes a no-op
313
309
  // on the happy path. Scope unwinding is LIFO — registered after the
314
310
  // `removeEventListener` defer above so rollback runs before the
315
- // listener detach. A thrown overflow, pre-mutation refusal, or pre-addon
316
- // attachment failure never persists the turn; the committed cache stays valid.
311
+ // listener detach. `preserveCacheOnUnwind` selects the non-destructive
312
+ // `releaseTurn` when the file on disk is known to still hold the last
313
+ // committed turn; `releaseTurn` still drops a cache this turn created.
317
314
  let preserveCacheOnUnwind = false;
318
315
  scope.defer(() => (preserveCacheOnUnwind ? session.releaseTurn(turn) : session.rollback(turn)));
319
- // `cacheExists` is implied by `beginTurn` — the session either found
320
- // an existing cache or just primed one. Pass `true` to the message
321
- // selector so the slicing branches engage.
322
316
  let payload;
323
317
  try {
324
- payload = prepareMessagesForCache(session, turn,
325
- /* cacheExists */ true, history, toolsActive ? tools : undefined);
318
+ payload = await prepareMessagesForCache(session, turn, history, toolsActive ? tools : undefined, mergedGenerationParams?.tool_choice);
326
319
  }
327
320
  catch (error) {
328
321
  // A missing attachment is caller input rejected before the addon runs,
@@ -333,11 +326,18 @@ export async function* completion(params, opts) {
333
326
  const messagesToSend = payload.messages;
334
327
  logMessagesToAddon(messagesToSend, 'PROMPT_SEND');
335
328
  let result;
329
+ let addonRunSettled = false;
336
330
  try {
337
- result = yield* processModelResponse(model, messagesToSend, tools, mergedGenerationParams, { cacheKey: turn.cachePath, saveCacheToDisk: true }, dialect, setActiveResponse);
331
+ result = yield* processModelResponse(model, messagesToSend, tools, mergedGenerationParams, { cacheKey: turn.cachePath, saveCacheToDisk: true }, dialect, setActiveResponse, () => {
332
+ addonRunSettled = true;
333
+ });
338
334
  }
339
335
  catch (error) {
340
- preserveCacheOnUnwind = isAddonContextOverflowError(error) || isAddonPreMutationRefusal(error);
336
+ // The addon writes the cache file only after a run completes and skips the
337
+ // save on every error path, so a run that threw left the committed file as
338
+ // it was. An engine-side throw after the run settled is the opposite: the
339
+ // file already holds this turn while no boundary was recorded for it.
340
+ preserveCacheOnUnwind = !addonRunSettled;
341
341
  throw error;
342
342
  }
343
343
  const shouldCommitTurn = shouldCommitCachedTurn({
@@ -347,6 +347,13 @@ export async function* completion(params, opts) {
347
347
  predict: mergedGenerationParams?.predict ?? modelConfig.predict,
348
348
  stoppedAtContextBoundary: result.stoppedAtContextBoundary
349
349
  });
350
+ // A cancelled run is rewound by the addon to the pre-request state before the
351
+ // file is re-saved, so the committed cache is intact. Every other non-commit
352
+ // finish (zero tokens, budget or context stop) was saved as-is and must go.
353
+ // An abort that landed after the addon reported a stop reason is the latter.
354
+ if (!shouldCommitTurn) {
355
+ preserveCacheOnUnwind = signal.aborted && !result.generationFinished;
356
+ }
350
357
  if (typeof kvCache === 'string') {
351
358
  // Custom-key path: the addon wrote the new cache state inline at
352
359
  // the same path. Either commit (records the boundary, suppresses
@@ -355,7 +362,7 @@ export async function* completion(params, opts) {
355
362
  await session.commitTurn(turn, {
356
363
  kind: 'static',
357
364
  messageCount: history.length + 1,
358
- toolBlockCached: payload.toolBlockCached
365
+ toolBlockCached: resolveToolBlockCached(payload, result.stats)
359
366
  });
360
367
  }
361
368
  return result;
@@ -387,7 +394,7 @@ export async function* completion(params, opts) {
387
394
  kind: 'autoRename',
388
395
  targetCachePath: postResponseCacheInfo.cachePath,
389
396
  messageCount: savedHistory.length,
390
- toolBlockCached: payload.toolBlockCached
397
+ toolBlockCached: resolveToolBlockCached(payload, result.stats)
391
398
  });
392
399
  return result;
393
400
  }
@@ -15,7 +15,6 @@
15
15
  * + extraction logic without a real model load.
16
16
  */
17
17
  export declare function isAddonContextOverflowError(err: unknown): boolean;
18
- export declare function isAddonPreMutationRefusal(err: unknown): boolean;
19
18
  export type ContextOverflowSizes = {
20
19
  /** The prompt alone, in tokens; unset when the guard reports KV cells. */
21
20
  promptTokens?: number;
@@ -48,37 +48,6 @@ export function isAddonContextOverflowError(err) {
48
48
  const trimmed = message.trim();
49
49
  return CONTEXT_OVERFLOW_FORMS.some((form) => form.test(trimmed));
50
50
  }
51
- // Refusals thrown before any decode or disk save; the async transport delivers
52
- // exception.what() alone, so each form is matched complete and end-anchored.
53
- const PRE_MUTATION_REFUSAL_FORMS = [
54
- /^ContinuousBatchScheduler::submit: prompt of \d+ KV cells exceeds per-sequence cap \d+ \(ctxTotalTokens \/ n_parallel\)$/,
55
- /^ContinuousBatchScheduler::submit: prompt of \d+ tokens leaves no room under per-sequence cap \d+ \(ctxTotalTokens \/ n_parallel\)$/,
56
- /^ContinuousBatchScheduler::submit: prefill prompt of \d+ tokens exceeds per-sequence cap \d+ \(ctxTotalTokens \/ n_parallel\)$/,
57
- /^ContinuousBatchScheduler::submit: n_predict [1-9]\d* \+ prompt \d+ KV cells exceeds per-sequence cap \d+ \(ctxTotalTokens \/ n_parallel\)$/,
58
- /^ContinuousBatchScheduler::submit: failed to add to batch \(MultiRequestBatcher::AddStatus=[1-4]\)$/,
59
- /^invalid generationParams\.json_schema: [^\r\n]*$/,
60
- /^failed to initialise sampler with per-request generationParams \(invalid grammar or json_schema\?\)$/,
61
- /^\[MtmdLlm\] Media buffer is empty$/,
62
- /^\[MtmdLlm\] Filename is empty$/,
63
- /^\[MtmdLlm\] Failed to load media from file: [^\r\n]*$/,
64
- /^\[MtmdLlm\] preparePrefill: prompt must end with text after the last media item$/
65
- ];
66
- export function isAddonPreMutationRefusal(err) {
67
- if (typeof err !== 'object' || err === null)
68
- return false;
69
- // A status code only exists on the synchronous throw path; when present it
70
- // must be InvalidArgument, and its absence (the async transport) is fine.
71
- const code = err.code;
72
- if (code !== undefined &&
73
- (typeof code !== 'string' || !/^\[\s*[\w.-]+\s*::\s*InvalidArgument\s*\]$/.test(code))) {
74
- return false;
75
- }
76
- const message = err.message;
77
- if (typeof message !== 'string')
78
- return false;
79
- const trimmed = message.trim();
80
- return PRE_MUTATION_REFUSAL_FORMS.some((form) => form.test(trimmed));
81
- }
82
51
  // One entry per number-formatting guard, most specific first; keep in step
83
52
  // with TextLlmContext.cpp / MtmdLlmContext.cpp or fields silently vanish.
84
53
  const MESSAGE_PATTERNS = [