doppler-gpu 0.4.5 → 0.4.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (155) hide show
  1. package/README.md +45 -29
  2. package/assets/doppler-runtime-map.svg +82 -0
  3. package/assets/doppler.svg +1 -3
  4. package/package.json +25 -17
  5. package/src/cli/doppler-quickstart.js +1 -1
  6. package/src/cli/doppler-serve.js +2 -2
  7. package/src/client/doppler-registry.d.ts +19 -0
  8. package/src/client/doppler-registry.js +12 -0
  9. package/src/client/doppler-registry.json +50 -1
  10. package/src/client/runtime/model-session.d.ts +1 -0
  11. package/src/client/runtime/model-session.js +9 -0
  12. package/src/config/conversion/README.md +4 -0
  13. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +28 -4
  14. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +29 -4
  15. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +29 -4
  16. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +29 -4
  17. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +29 -4
  18. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +29 -4
  19. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +29 -4
  20. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +29 -4
  21. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +29 -4
  22. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +31 -5
  23. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +30 -4
  24. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +30 -4
  25. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +30 -4
  26. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +30 -5
  27. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +29 -4
  28. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +30 -5
  29. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +29 -4
  30. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +29 -4
  31. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +29 -4
  32. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +29 -4
  33. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +29 -4
  34. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +29 -4
  35. package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +481 -0
  36. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +29 -4
  37. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +29 -4
  38. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +30 -5
  39. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +29 -4
  40. package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +395 -0
  41. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +429 -0
  42. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +540 -0
  43. package/src/config/execution-contract-check.js +14 -10
  44. package/src/config/merge-contract-check.js +1 -0
  45. package/src/config/merge.js +30 -0
  46. package/src/config/param-validator.d.ts +5 -0
  47. package/src/config/param-validator.js +42 -8
  48. package/src/config/required-inference-fields-contract-check.js +14 -0
  49. package/src/config/schema/debug.schema.d.ts +2 -0
  50. package/src/config/schema/debug.schema.js +1 -0
  51. package/src/config/schema/doppler.schema.d.ts +5 -1
  52. package/src/config/schema/doppler.schema.js +82 -292
  53. package/src/config/schema/inference.schema.d.ts +17 -3
  54. package/src/config/schema/kvcache.schema.d.ts +19 -3
  55. package/src/config/schema/kvcache.schema.js +2 -1
  56. package/src/config/schema/manifest.schema.d.ts +38 -1
  57. package/src/config/schema/manifest.schema.js +18 -0
  58. package/src/config/schema/policy-schema-registry.json +5 -0
  59. package/src/config/source-packages/litert/gemma-4-12b-it.json +20 -1
  60. package/src/config/source-packages/litert/gemma-4-e2b-it.json +20 -1
  61. package/src/config/source-packages/litert/gemma-4-e4b-it.json +21 -1
  62. package/src/config/transforms/execution-graph-transforms.d.ts +6 -0
  63. package/src/config/transforms/execution-graph-transforms.js +8 -0
  64. package/src/converter/conversion-plan.js +2 -0
  65. package/src/converter/parsers/transformer.js +27 -1
  66. package/src/converter/quantization-info.js +16 -4
  67. package/src/converter/rope-config.js +55 -3
  68. package/src/experimental/training/distillation/student-fixture.js +1 -1
  69. package/src/experimental/training/lora-pipeline.js +1 -1
  70. package/src/experimental/training/lora.js +2 -2
  71. package/src/experimental/training/suite.js +1 -1
  72. package/src/gpu/device.js +54 -11
  73. package/src/gpu/kernels/README.md +8 -1
  74. package/src/gpu/kernels/backward/attention_backward.js +8 -0
  75. package/src/gpu/kernels/dequant.d.ts +6 -6
  76. package/src/gpu/kernels/dequant.js +17 -6
  77. package/src/gpu/kernels/linear-attention-core.js +1 -6
  78. package/src/gpu/kernels/matmul-selection.d.ts +2 -2
  79. package/src/gpu/kernels/matmul-selection.js +9 -2
  80. package/src/gpu/kernels/matmul.d.ts +1 -1
  81. package/src/gpu/kernels/matmul.js +63 -9
  82. package/src/gpu/kernels/moe.d.ts +5 -1
  83. package/src/gpu/kernels/moe.js +5 -2
  84. package/src/gpu/kernels/softmax.d.ts +3 -3
  85. package/src/gpu/kernels/softmax.js +9 -2
  86. package/src/gpu/kernels/turboquant-codebook.js +1 -1
  87. package/src/gpu/kernels/types.d.ts +1 -1
  88. package/src/gpu/profiler.js +11 -1
  89. package/src/inference/browser-harness-model-helpers.js +4 -1
  90. package/src/inference/browser-harness-text-helpers.d.ts +28 -0
  91. package/src/inference/browser-harness-text-helpers.js +357 -116
  92. package/src/inference/browser-harness.d.ts +3 -3
  93. package/src/inference/browser-harness.js +266 -11
  94. package/src/inference/fixtures/rerank-semantic-fixtures.json +38 -0
  95. package/src/inference/kv-cache/quantized.js +16 -2
  96. package/src/inference/kv-cache/tiered.js +50 -15
  97. package/src/inference/moe-router.js +38 -7
  98. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +1 -1
  99. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +8 -6
  100. package/src/inference/pipelines/text/config.d.ts +5 -1
  101. package/src/inference/pipelines/text/config.js +135 -112
  102. package/src/inference/pipelines/text/embed.d.ts +1 -0
  103. package/src/inference/pipelines/text/embed.js +39 -12
  104. package/src/inference/pipelines/text/execution-plan.js +8 -2
  105. package/src/inference/pipelines/text/ffn/dense.js +15 -4
  106. package/src/inference/pipelines/text/ffn/moe.js +4 -3
  107. package/src/inference/pipelines/text/ffn/standard.d.ts +2 -1
  108. package/src/inference/pipelines/text/ffn/standard.js +25 -2
  109. package/src/inference/pipelines/text/generator-helpers.js +2 -1
  110. package/src/inference/pipelines/text/generator-runtime.d.ts +6 -2
  111. package/src/inference/pipelines/text/generator-runtime.js +53 -20
  112. package/src/inference/pipelines/text/generator-steps.js +3 -0
  113. package/src/inference/pipelines/text/generator.d.ts +1 -0
  114. package/src/inference/pipelines/text/generator.js +14 -1
  115. package/src/inference/pipelines/text/init-rope.js +54 -6
  116. package/src/inference/pipelines/text/init.js +63 -15
  117. package/src/inference/pipelines/text/layer.js +32 -5
  118. package/src/inference/pipelines/text/linear-attention.js +12 -8
  119. package/src/inference/pipelines/text/logits/gpu.d.ts +1 -1
  120. package/src/inference/pipelines/text/logits/gpu.js +76 -113
  121. package/src/inference/pipelines/text/logits/index.js +44 -87
  122. package/src/inference/pipelines/text/logits/precision-policy.d.ts +11 -0
  123. package/src/inference/pipelines/text/logits/precision-policy.js +103 -0
  124. package/src/inference/pipelines/text/logits/types.d.ts +1 -0
  125. package/src/inference/pipelines/text/logits/utils.d.ts +2 -0
  126. package/src/inference/pipelines/text/logits/utils.js +11 -0
  127. package/src/inference/pipelines/text/lora-apply.js +6 -4
  128. package/src/inference/pipelines/text/moe-cpu.js +25 -0
  129. package/src/inference/pipelines/text/moe-gpu.js +148 -97
  130. package/src/inference/pipelines/text/moe-shape-validator.d.ts +33 -3
  131. package/src/inference/pipelines/text/moe-shape-validator.js +149 -31
  132. package/src/inference/pipelines/text/ops.d.ts +3 -0
  133. package/src/inference/pipelines/text/ops.js +26 -1
  134. package/src/inference/pipelines/text/per-layer-inputs.js +27 -12
  135. package/src/inference/pipelines/text/sampling.js +52 -10
  136. package/src/inference/pipelines/text/weights.js +7 -2
  137. package/src/inference/pipelines/text.js +24 -4
  138. package/src/inference/tokenizers/behavior-flags.js +23 -2
  139. package/src/loader/final-weights-loader.js +12 -1
  140. package/src/loader/tensors/tensor-loader.js +37 -3
  141. package/src/rules/execution-rules-contract-check.js +12 -3
  142. package/src/rules/inference/capability-transforms.rules.json +14 -2
  143. package/src/rules/inference/config.rules.json +26 -2
  144. package/src/rules/inference/dtype.rules.json +12 -5
  145. package/src/rules/inference/execution.rules.json +5 -19
  146. package/src/rules/inference/moe.rules.json +97 -0
  147. package/src/rules/loader/weights.rules.json +1 -0
  148. package/src/tooling/command-api-constants.js +2 -2
  149. package/src/tooling/command-api-family-normalizers.js +2 -1
  150. package/src/tooling/command-api.d.ts +1 -1
  151. package/src/tooling/command-runner-shared.d.ts +1 -1
  152. package/src/tooling/command-runner-shared.js +3 -1
  153. package/src/tooling/node-converter.js +20 -0
  154. package/src/tooling/node-source-runtime.js +5 -1
  155. package/src/version.js +1 -1
@@ -4,6 +4,11 @@ import { TOOLING_INTENTS, TOOLING_DIAGNOSTICS } from './schema/tooling.schema.js
4
4
  import { validateEcosystemConfig } from './schema/ecosystem.schema.js';
5
5
  import { isPlainObject } from '../utils/plain-object.js';
6
6
 
7
+ const MODEL_OVERRIDE_ALLOWED_PREFIXES = Object.freeze([
8
+ 'vision_config',
9
+ 'audio_config',
10
+ ]);
11
+
7
12
  export function validateCallTimeOptions(options) {
8
13
  if (!options) return;
9
14
 
@@ -23,7 +28,7 @@ export function validateCallTimeOptions(options) {
23
28
 
24
29
  const violation = violations[0];
25
30
  const guidance = violation.category === 'model'
26
- ? 'Set via runtime.inference.modelOverrides (experimental) or manifest.'
31
+ ? 'Set in the conversion config/manifest. Runtime modelOverrides are limited to an explicit allowlist.'
27
32
  : 'Set via setRuntimeConfig() before generation.';
28
33
 
29
34
  throw new Error(
@@ -39,25 +44,39 @@ export function validateRuntimeOverrides(overrides) {
39
44
  }
40
45
 
41
46
  for (const key of ['shared', 'loading', 'inference', 'emulation']) {
42
- assertRequiredRuntimeOverrideNotNull(overrides, key);
47
+ assertRuntimeOverrideObject(overrides, key);
43
48
  }
44
49
  for (const key of ['batching', 'compute', 'generation', 'kernelPathPolicy']) {
45
- assertRequiredRuntimeOverrideNotNull(overrides?.inference, key, 'runtime.inference');
50
+ assertRuntimeOverrideObject(overrides?.inference, key, 'runtime.inference');
46
51
  }
47
52
  validateRuntimeKernelPath('runtime.inference.kernelPath', overrides?.inference?.kernelPath);
48
53
 
49
54
  const modelOverrides = overrides?.inference?.modelOverrides;
55
+ validateModelOverrides(modelOverrides, 'runtime.inference.modelOverrides');
56
+ }
57
+
58
+ export function validateModelOverrides(modelOverrides, label = 'runtime.inference.modelOverrides') {
50
59
  if (modelOverrides !== undefined && modelOverrides !== null && !isPlainObject(modelOverrides)) {
51
- throw new Error('DopplerConfigError: runtime.inference.modelOverrides must be an object when provided.');
60
+ throw new Error(`DopplerConfigError: ${label} must be an object when provided.`);
52
61
  }
53
62
  if (!modelOverrides) return;
54
63
 
55
64
  const params = flattenObject(modelOverrides);
56
65
  if (params.length === 0) return;
57
66
 
67
+ const disallowed = params.filter((param) => !isAllowedModelOverridePath(param));
68
+ if (disallowed.length > 0) {
69
+ throw new Error(
70
+ `DopplerConfigError: ${label} may only override ` +
71
+ `${MODEL_OVERRIDE_ALLOWED_PREFIXES.join(', ')}. ` +
72
+ `Disallowed model param(s): ${disallowed.join(', ')}. ` +
73
+ 'Move model inference params to the conversion config/manifest.'
74
+ );
75
+ }
76
+
58
77
  log.warn(
59
78
  'Config',
60
- `Experimental: Overriding ${params.length} model param(s) via runtime: ${params.join(', ')}. ` +
79
+ `Experimental: Overriding ${params.length} allowlisted model param(s) via runtime: ${params.join(', ')}. ` +
61
80
  'Manifest values are recommended.'
62
81
  );
63
82
  }
@@ -89,6 +108,12 @@ export function validateRuntimeConfig(runtimeConfig) {
89
108
  if (compute?.deferredRoundingWindowTokens !== undefined) {
90
109
  assertPositiveInt('runtime.inference.compute.deferredRoundingWindowTokens', compute.deferredRoundingWindowTokens);
91
110
  }
111
+ if (generation?.maxTokens !== undefined) {
112
+ assertPositiveInt('runtime.inference.generation.maxTokens', generation.maxTokens);
113
+ }
114
+ if (generation?.multimodalMaxTokens !== undefined) {
115
+ assertPositiveInt('runtime.inference.generation.multimodalMaxTokens', generation.multimodalMaxTokens);
116
+ }
92
117
  if (session?.prefillTokenChunkSize !== undefined) {
93
118
  assertNullablePositiveInt('runtime.inference.session.prefillTokenChunkSize', session.prefillTokenChunkSize);
94
119
  }
@@ -194,9 +219,9 @@ function validateToolingIntent(runtimeConfig) {
194
219
  function flattenObject(obj, prefix = '') {
195
220
  const result = [];
196
221
  for (const [key, value] of Object.entries(obj)) {
197
- if (value === undefined || value === null) continue;
222
+ if (value === undefined) continue;
198
223
  const path = prefix ? `${prefix}.${key}` : key;
199
- if (typeof value === 'object' && !Array.isArray(value)) {
224
+ if (isPlainObject(value)) {
200
225
  result.push(...flattenObject(value, path));
201
226
  } else {
202
227
  result.push(path);
@@ -205,6 +230,12 @@ function flattenObject(obj, prefix = '') {
205
230
  return result;
206
231
  }
207
232
 
233
+ function isAllowedModelOverridePath(path) {
234
+ return MODEL_OVERRIDE_ALLOWED_PREFIXES.some((prefix) => {
235
+ return path === prefix || path.startsWith(`${prefix}.`);
236
+ });
237
+ }
238
+
208
239
  function isDebugMode(debug) {
209
240
  if (!debug) return false;
210
241
  if (debug.pipeline?.enabled) return true;
@@ -290,13 +321,16 @@ function validateKernelPathPolicy(label, value) {
290
321
  }
291
322
  }
292
323
 
293
- function assertRequiredRuntimeOverrideNotNull(container, key, prefix = 'runtime') {
324
+ function assertRuntimeOverrideObject(container, key, prefix = 'runtime') {
294
325
  if (!isPlainObject(container) || !Object.prototype.hasOwnProperty.call(container, key)) {
295
326
  return;
296
327
  }
297
328
  if (container[key] === null) {
298
329
  throw new Error(`DopplerConfigError: ${prefix}.${key} must not be null.`);
299
330
  }
331
+ if (!isPlainObject(container[key])) {
332
+ throw new Error(`DopplerConfigError: ${prefix}.${key} must be an object when provided.`);
333
+ }
300
334
  }
301
335
 
302
336
  function arraysEqual(left, right) {
@@ -38,6 +38,7 @@ function createValidInferenceFixture() {
38
38
  ffn: {
39
39
  activation: 'gelu',
40
40
  gatedActivation: true,
41
+ branchMode: 'auto',
41
42
  useDoubleWideMlp: false,
42
43
  swigluLimit: null,
43
44
  },
@@ -55,6 +56,9 @@ function createValidInferenceFixture() {
55
56
  yarnBetaFast: null,
56
57
  yarnBetaSlow: null,
57
58
  yarnOriginalMaxPos: null,
59
+ longropeShortFactor: null,
60
+ longropeLongFactor: null,
61
+ longropeOriginalMaxPos: null,
58
62
  ropeLocalYarnBetaFast: null,
59
63
  ropeLocalYarnBetaSlow: null,
60
64
  ropeLocalYarnOriginalMaxPos: null,
@@ -62,6 +66,8 @@ function createValidInferenceFixture() {
62
66
  output: {
63
67
  tieWordEmbeddings: true,
64
68
  scaleEmbeddings: true,
69
+ embeddingScale: null,
70
+ logitInputScale: 1,
65
71
  embeddingTranspose: false,
66
72
  finalLogitSoftcapping: null,
67
73
  embeddingVocabSize: null,
@@ -72,6 +78,7 @@ function createValidInferenceFixture() {
72
78
  globalPattern: null,
73
79
  period: 6,
74
80
  offset: 0,
81
+ residualBranchScale: 1,
75
82
  },
76
83
  chatTemplate: {
77
84
  type: null,
@@ -95,6 +102,7 @@ const FIELD_CASES = Object.freeze([
95
102
  { kind: 'nonNullable', path: ['normalization', 'postFeedforwardNorm'], message: 'normalization.postFeedforwardNorm is required' },
96
103
  { kind: 'nonNullable', path: ['ffn', 'activation'], message: 'ffn.activation is required' },
97
104
  { kind: 'nonNullable', path: ['ffn', 'gatedActivation'], message: 'ffn.gatedActivation is required' },
105
+ { kind: 'nonNullable', path: ['ffn', 'branchMode'], message: 'ffn.branchMode is required' },
98
106
  { kind: 'nonNullable', path: ['ffn', 'useDoubleWideMlp'], message: 'ffn.useDoubleWideMlp is required' },
99
107
  { kind: 'nullable', path: ['ffn', 'swigluLimit'], message: 'ffn.swigluLimit must be explicitly set' },
100
108
  { kind: 'nonNullable', path: ['rope', 'ropeTheta'], message: 'rope.ropeTheta is required' },
@@ -110,11 +118,16 @@ const FIELD_CASES = Object.freeze([
110
118
  { kind: 'nullable', path: ['rope', 'yarnBetaFast'], message: 'rope.yarnBetaFast must be explicitly set' },
111
119
  { kind: 'nullable', path: ['rope', 'yarnBetaSlow'], message: 'rope.yarnBetaSlow must be explicitly set' },
112
120
  { kind: 'nullable', path: ['rope', 'yarnOriginalMaxPos'], message: 'rope.yarnOriginalMaxPos must be explicitly set' },
121
+ { kind: 'nullable', path: ['rope', 'longropeShortFactor'], message: 'rope.longropeShortFactor must be explicitly set' },
122
+ { kind: 'nullable', path: ['rope', 'longropeLongFactor'], message: 'rope.longropeLongFactor must be explicitly set' },
123
+ { kind: 'nullable', path: ['rope', 'longropeOriginalMaxPos'], message: 'rope.longropeOriginalMaxPos must be explicitly set' },
113
124
  { kind: 'nullable', path: ['rope', 'ropeLocalYarnBetaFast'], message: 'rope.ropeLocalYarnBetaFast must be explicitly set' },
114
125
  { kind: 'nullable', path: ['rope', 'ropeLocalYarnBetaSlow'], message: 'rope.ropeLocalYarnBetaSlow must be explicitly set' },
115
126
  { kind: 'nullable', path: ['rope', 'ropeLocalYarnOriginalMaxPos'], message: 'rope.ropeLocalYarnOriginalMaxPos must be explicitly set' },
116
127
  { kind: 'nonNullable', path: ['output', 'tieWordEmbeddings'], message: 'output.tieWordEmbeddings is required' },
117
128
  { kind: 'nonNullable', path: ['output', 'scaleEmbeddings'], message: 'output.scaleEmbeddings is required' },
129
+ { kind: 'nullable', path: ['output', 'embeddingScale'], message: 'output.embeddingScale must be explicitly set' },
130
+ { kind: 'nonNullable', path: ['output', 'logitInputScale'], message: 'output.logitInputScale is required' },
118
131
  { kind: 'nonNullable', path: ['output', 'embeddingTranspose'], message: 'output.embeddingTranspose is required' },
119
132
  { kind: 'nullable', path: ['output', 'finalLogitSoftcapping'], message: 'output.finalLogitSoftcapping must be explicitly set' },
120
133
  { kind: 'nullable', path: ['output', 'embeddingVocabSize'], message: 'output.embeddingVocabSize must be explicitly set' },
@@ -123,6 +136,7 @@ const FIELD_CASES = Object.freeze([
123
136
  { kind: 'nullable', path: ['layerPattern', 'globalPattern'], message: 'layerPattern.globalPattern must be explicitly set' },
124
137
  { kind: 'nullable', path: ['layerPattern', 'period'], message: 'layerPattern.period must be explicitly set' },
125
138
  { kind: 'nullable', path: ['layerPattern', 'offset'], message: 'layerPattern.offset must be explicitly set' },
139
+ { kind: 'nonNullable', path: ['layerPattern', 'residualBranchScale'], message: 'layerPattern.residualBranchScale is required' },
126
140
  { kind: 'nullable', path: ['chatTemplate', 'type'], message: 'chatTemplate.type must be explicitly set' },
127
141
  { kind: 'nonNullable', path: ['chatTemplate', 'enabled'], message: 'chatTemplate.enabled is required' },
128
142
  ]);
@@ -206,6 +206,8 @@ export interface ProfilerConfigSchema {
206
206
  maxDurationMs: number;
207
207
  /** Log every N decode profile steps (<=1 logs all steps) */
208
208
  logEveryDecodeSteps: number;
209
+ /** Maximum number of distinct labels retained in history */
210
+ maxHistoryLabels: number;
209
211
  }
210
212
 
211
213
  /** Default profiler configuration */
@@ -102,6 +102,7 @@ export const DEFAULT_PROFILER_CONFIG = {
102
102
  maxSamples: 100,
103
103
  maxDurationMs: 60000,
104
104
  logEveryDecodeSteps: 64,
105
+ maxHistoryLabels: 1024,
105
106
  };
106
107
 
107
108
  // =============================================================================
@@ -30,6 +30,10 @@ export interface LargeWeightConfigSchema {
30
30
 
31
31
  export interface RuntimeInferenceConfigSchema {
32
32
  prompt?: string | null;
33
+ rerank?: {
34
+ query?: string | null;
35
+ documents?: string[] | null;
36
+ } | null;
33
37
  debugTokens?: boolean;
34
38
  batching: Record<string, unknown>;
35
39
  sampling: Record<string, unknown>;
@@ -49,7 +53,7 @@ export interface RuntimeInferenceConfigSchema {
49
53
  kernelPath?: KernelPathSchema | null;
50
54
  kernelPathSource?: 'config' | 'model' | 'manifest' | 'execution-v1' | 'execution-v1-transform' | 'none';
51
55
  kernelPathPolicy?: Record<string, unknown> | null;
52
- modelOverrides?: Record<string, unknown>;
56
+ modelOverrides?: Record<string, unknown> | null;
53
57
  pipeline?: Record<string, unknown> | null;
54
58
  }
55
59
 
@@ -5,9 +5,11 @@ import { DEFAULT_EMULATION_CONFIG, createEmulationConfig } from './emulation.sch
5
5
  import { DEFAULT_KVCACHE_CONFIG } from './kvcache.schema.js';
6
6
  import { DEFAULT_MOE_RUNTIME_CONFIG } from './moe.schema.js';
7
7
  import { DEFAULT_SPECULATIVE_CONFIG } from './speculative.schema.js';
8
- import { mergeEcosystemConfig } from './ecosystem.schema.js';
8
+ import { DEFAULT_SELF_SPECULATION_CONFIG } from './speculation-self.schema.js';
9
+ import { mergeRuntimeValues } from '../runtime-merge.js';
10
+ import { validateRuntimeOverrides } from '../param-validator.js';
11
+ import { isPlainObject } from '../../utils/plain-object.js';
9
12
  import {
10
- chooseNullish,
11
13
  chooseDefined,
12
14
  mergeExecutionPatchLists,
13
15
  mergeKernelPathPolicy,
@@ -59,6 +61,8 @@ export const DEFAULT_RUNTIME_CONFIG = {
59
61
  moe: DEFAULT_MOE_RUNTIME_CONFIG,
60
62
  speculative: DEFAULT_SPECULATIVE_CONFIG,
61
63
  generation: {
64
+ maxTokens: 256,
65
+ multimodalMaxTokens: 512,
62
66
  disableMultiTokenDecode: false,
63
67
  },
64
68
  kernelPathPolicy: mergeKernelPathPolicy(undefined, {
@@ -69,6 +73,7 @@ export const DEFAULT_RUNTIME_CONFIG = {
69
73
  chatTemplate: DEFAULT_CHAT_TEMPLATE_CONFIG,
70
74
  session: {
71
75
  kvcache: DEFAULT_KVCACHE_CONFIG,
76
+ speculation: { ...DEFAULT_SELF_SPECULATION_CONFIG },
72
77
  // "sync" (default): wait for each prefill chunk to finish on GPU before
73
78
  // recording the next. Required when profile timings are being
74
79
  // resolved and when low-memory constraints demand deterministic
@@ -186,17 +191,18 @@ export const DEFAULT_DOPPLER_CONFIG = {
186
191
  export function createDopplerConfig(
187
192
  overrides
188
193
  ) {
194
+ const runtimeBase = cloneConfigTree(DEFAULT_RUNTIME_CONFIG);
195
+
189
196
  if (!overrides) {
190
197
  return {
191
198
  model: DEFAULT_DOPPLER_CONFIG.model,
192
- runtime: cloneConfigTree(DEFAULT_RUNTIME_CONFIG),
199
+ runtime: runtimeBase,
193
200
  };
194
201
  }
195
202
 
196
203
  const runtimeOverrides = overrides.runtime ?? {};
197
- const runtimeBase = cloneConfigTree(DEFAULT_RUNTIME_CONFIG);
198
204
  const runtime = overrides.runtime
199
- ? mergeRuntimeConfig(runtimeBase, runtimeOverrides)
205
+ ? createRuntimeConfig(runtimeBase, runtimeOverrides)
200
206
  : runtimeBase;
201
207
  const config = {
202
208
  model: overrides.model ?? DEFAULT_DOPPLER_CONFIG.model,
@@ -205,234 +211,85 @@ export function createDopplerConfig(
205
211
  return config;
206
212
  }
207
213
 
208
- function mergeRuntimeConfig(
209
- base,
210
- overrides
211
- ) {
212
- return {
213
- shared: overrides.shared
214
- ? mergeSharedRuntimeConfig(base.shared, overrides.shared)
215
- : { ...base.shared },
216
- loading: overrides.loading
217
- ? mergeLoadingConfig(base.loading, overrides.loading)
218
- : { ...base.loading },
219
- inference: overrides.inference
220
- ? mergeInferenceConfig(base.inference, overrides.inference)
221
- : { ...base.inference },
222
- emulation: overrides.emulation
223
- ? mergeEmulationConfig(base.emulation, overrides.emulation)
224
- : { ...base.emulation },
225
- };
214
+ function createRuntimeConfig(base, overrides) {
215
+ validateRuntimeOverrides(overrides);
216
+ const runtime = mergeRuntimeValues(base, overrides);
217
+ normalizeRuntimeContracts(runtime, base, overrides);
218
+ return runtime;
226
219
  }
227
220
 
228
- function mergeSharedRuntimeConfig(
229
- base,
230
- overrides
231
- ) {
232
- return {
233
- debug: overrides.debug
234
- ? mergeDebugConfig(base.debug, overrides.debug)
235
- : { ...base.debug },
236
- benchmark: overrides.benchmark
237
- ? mergeBenchmarkConfig(base.benchmark, overrides.benchmark)
238
- : { ...base.benchmark },
239
- harness: overrides.harness
240
- ? { ...base.harness, ...overrides.harness }
241
- : { ...base.harness },
242
- tooling: overrides.tooling
243
- ? { ...base.tooling, ...overrides.tooling }
244
- : { ...base.tooling },
245
- ecosystem: overrides.ecosystem
246
- ? mergeEcosystemConfig(base.ecosystem, overrides.ecosystem)
247
- : mergeEcosystemConfig(base.ecosystem, {}),
248
- platform: overrides.platform ?? base.platform,
249
- kernelRegistry: { ...base.kernelRegistry, ...overrides.kernelRegistry },
250
- kernelThresholds: overrides.kernelThresholds
251
- ? mergeKernelThresholds(base.kernelThresholds, overrides.kernelThresholds)
252
- : { ...base.kernelThresholds },
253
- kernelWarmup: overrides.kernelWarmup
254
- ? { ...base.kernelWarmup, ...overrides.kernelWarmup }
255
- : { ...base.kernelWarmup },
256
- bufferPool: overrides.bufferPool
257
- ? {
258
- bucket: { ...base.bufferPool.bucket, ...overrides.bufferPool.bucket },
259
- limits: { ...base.bufferPool.limits, ...overrides.bufferPool.limits },
260
- alignment: { ...base.bufferPool.alignment, ...overrides.bufferPool.alignment },
261
- budget: { ...base.bufferPool.budget, ...overrides.bufferPool.budget },
262
- }
263
- : { ...base.bufferPool },
264
- gpuCache: { ...base.gpuCache, ...overrides.gpuCache },
265
- tuner: { ...base.tuner, ...overrides.tuner },
266
- memory: overrides.memory
267
- ? {
268
- heapTesting: { ...base.memory.heapTesting, ...overrides.memory.heapTesting },
269
- segmentTesting: { ...base.memory.segmentTesting, ...overrides.memory.segmentTesting },
270
- addressSpace: { ...base.memory.addressSpace, ...overrides.memory.addressSpace },
271
- segmentAllocation: { ...base.memory.segmentAllocation, ...overrides.memory.segmentAllocation },
272
- }
273
- : { ...base.memory },
274
- hotSwap: overrides.hotSwap
275
- ? {
276
- ...base.hotSwap,
277
- ...overrides.hotSwap,
278
- trustedSigners: overrides.hotSwap.trustedSigners ?? base.hotSwap.trustedSigners,
279
- }
280
- : { ...base.hotSwap },
281
- intentBundle: overrides.intentBundle
282
- ? { ...base.intentBundle, ...overrides.intentBundle }
283
- : { ...base.intentBundle },
284
- bridge: { ...base.bridge, ...overrides.bridge },
285
- };
221
+ function normalizeRuntimeContracts(runtime, base, overrides) {
222
+ if (isPlainObject(overrides.inference)) {
223
+ normalizeInferenceContracts(runtime.inference, base.inference, overrides.inference);
224
+ }
225
+ if (isPlainObject(overrides.emulation)) {
226
+ runtime.emulation = createEmulationConfig(overrides.emulation);
227
+ }
286
228
  }
287
229
 
288
- function mergeLoadingConfig(
289
- base,
290
- overrides
291
- ) {
292
- return {
293
- storage: overrides.storage
294
- ? {
295
- quota: { ...base.storage.quota, ...overrides.storage.quota },
296
- vramEstimation: { ...base.storage.vramEstimation, ...overrides.storage.vramEstimation },
297
- alignment: { ...base.storage.alignment, ...overrides.storage.alignment },
298
- backend: overrides.storage.backend
299
- ? {
300
- backend: overrides.storage.backend.backend ?? base.storage.backend.backend,
301
- opfs: { ...base.storage.backend.opfs, ...overrides.storage.backend.opfs },
302
- indexeddb: { ...base.storage.backend.indexeddb, ...overrides.storage.backend.indexeddb },
303
- memory: { ...base.storage.backend.memory, ...overrides.storage.backend.memory },
304
- streaming: { ...base.storage.backend.streaming, ...overrides.storage.backend.streaming },
305
- }
306
- : { ...base.storage.backend },
307
- }
308
- : { ...base.storage },
309
- distribution: { ...base.distribution, ...overrides.distribution },
310
- shardCache: { ...base.shardCache, ...overrides.shardCache },
311
- memoryManagement: overrides.memoryManagement
312
- ? {
313
- ...base.memoryManagement,
314
- ...overrides.memoryManagement,
315
- budget: {
316
- ...base.memoryManagement.budget,
317
- ...overrides.memoryManagement.budget,
318
- },
319
- }
320
- : { ...base.memoryManagement },
321
- prefetch: { ...base.prefetch, ...overrides.prefetch },
322
- opfsPath: { ...base.opfsPath, ...overrides.opfsPath },
323
- expertCache: { ...base.expertCache, ...overrides.expertCache },
324
- allowF32UpcastNonMatmul: overrides.allowF32UpcastNonMatmul ?? base.allowF32UpcastNonMatmul,
325
- };
230
+ function normalizeInferenceContracts(inference, base, overrides) {
231
+ if (hasOwn(overrides, 'kernelPath')) {
232
+ inference.kernelPath = mergeRuntimeKernelPath(base.kernelPath, overrides.kernelPath);
233
+ }
234
+ if (hasOwn(overrides, 'kernelPathPolicy')) {
235
+ inference.kernelPathPolicy = mergeKernelPathPolicy(
236
+ base.kernelPathPolicy ?? {},
237
+ overrides.kernelPathPolicy ?? {}
238
+ );
239
+ }
240
+ if (hasOwn(overrides, 'chatTemplate')) {
241
+ inference.chatTemplate = mergeShallowObject(base.chatTemplate, overrides.chatTemplate);
242
+ }
243
+ if (hasOwn(overrides, 'executionPatch')) {
244
+ inference.executionPatch = mergeExecutionPatchLists(
245
+ base.executionPatch ?? {},
246
+ overrides.executionPatch ?? {}
247
+ );
248
+ }
249
+ if (hasOwn(overrides, 'modelOverrides')) {
250
+ inference.modelOverrides = chooseDefined(overrides.modelOverrides, base.modelOverrides);
251
+ }
252
+ normalizeSessionContracts(inference, base, overrides);
326
253
  }
327
254
 
328
- function mergeInferenceConfig(
329
- base,
330
- overrides
331
- ) {
255
+ function normalizeSessionContracts(inference, base, overrides) {
256
+ if (!isPlainObject(overrides.session) || !isPlainObject(inference.session)) {
257
+ return;
258
+ }
259
+
332
260
  const baseSession = base.session ?? {};
333
- const overrideSession = overrides.session ?? {};
334
- const baseSessionCompute = baseSession.compute ?? {};
335
- const overrideSessionCompute = overrideSession.compute ?? {};
336
- const baseSessionComputeDefaults = baseSessionCompute.defaults ?? {};
337
- const overrideSessionComputeDefaults = overrideSessionCompute.defaults ?? {};
338
- const baseExecutionPatch = base.executionPatch ?? {};
339
- const overrideExecutionPatch = overrides.executionPatch ?? {};
340
- const baseKernelPathPolicy = base.kernelPathPolicy ?? {};
341
- const overrideKernelPathPolicy = overrides.kernelPathPolicy ?? {};
342
- const baseDiffusion = base.diffusion ?? {};
343
- const baseDiffusionDecode = baseDiffusion.decode ?? {};
344
- const baseDiffusionGemma = base.diffusionGemma ?? {};
345
- const baseEnergy = base.energy ?? {};
346
- const baseEnergyQuintel = baseEnergy.quintel ?? {};
347
- const baseMoe = base.moe ?? {};
348
- const hasRuntimeKernelProfiles = Object.prototype.hasOwnProperty.call(
349
- overrideSessionCompute,
350
- 'kernelProfiles'
351
- );
261
+ const overrideSession = overrides.session;
262
+ if (hasOwn(overrideSession, 'kvcache')) {
263
+ inference.session.kvcache = replaceSubtree(overrideSession.kvcache, baseSession.kvcache);
264
+ }
265
+ if (hasOwn(overrideSession, 'decodeLoop')) {
266
+ inference.session.decodeLoop = replaceSubtree(overrideSession.decodeLoop, baseSession.decodeLoop);
267
+ }
268
+ if (hasOwn(overrideSession, 'perLayerInputs')) {
269
+ inference.session.perLayerInputs = replaceSubtree(
270
+ overrideSession.perLayerInputs,
271
+ baseSession.perLayerInputs
272
+ );
273
+ }
274
+ if (hasOwn(overrideSession, 'speculation')) {
275
+ inference.session.speculation = replaceSubtree(
276
+ overrideSession.speculation,
277
+ baseSession.speculation
278
+ );
279
+ }
352
280
 
353
- return {
354
- prompt: overrides.prompt ?? base.prompt,
355
- debugTokens: overrides.debugTokens ?? base.debugTokens,
356
- batching: { ...base.batching, ...overrides.batching },
357
- sampling: { ...base.sampling, ...overrides.sampling },
358
- compute: { ...base.compute, ...overrides.compute },
359
- tokenizer: { ...base.tokenizer, ...overrides.tokenizer },
360
- largeWeights: { ...base.largeWeights, ...overrides.largeWeights },
361
- kvcache: { ...base.kvcache, ...overrides.kvcache },
362
- diffusion: overrides.diffusion
363
- ? {
364
- ...baseDiffusion,
365
- ...overrides.diffusion,
366
- scheduler: { ...baseDiffusion.scheduler, ...overrides.diffusion.scheduler },
367
- latent: { ...baseDiffusion.latent, ...overrides.diffusion.latent },
368
- textEncoder: { ...baseDiffusion.textEncoder, ...overrides.diffusion.textEncoder },
369
- decode: {
370
- ...baseDiffusionDecode,
371
- ...overrides.diffusion.decode,
372
- tiling: { ...baseDiffusionDecode.tiling, ...overrides.diffusion.decode?.tiling },
373
- },
374
- swapper: { ...baseDiffusion.swapper, ...overrides.diffusion.swapper },
375
- quantization: { ...baseDiffusion.quantization, ...overrides.diffusion.quantization },
376
- }
377
- : { ...baseDiffusion },
378
- diffusionGemma: { ...baseDiffusionGemma, ...overrides.diffusionGemma },
379
- energy: overrides.energy
380
- ? {
381
- ...baseEnergy,
382
- ...overrides.energy,
383
- problem: overrides.energy.problem ?? baseEnergy.problem,
384
- state: { ...baseEnergy.state, ...overrides.energy.state },
385
- init: { ...baseEnergy.init, ...overrides.energy.init },
386
- target: { ...baseEnergy.target, ...overrides.energy.target },
387
- loop: { ...baseEnergy.loop, ...overrides.energy.loop },
388
- diagnostics: { ...baseEnergy.diagnostics, ...overrides.energy.diagnostics },
389
- quintel: overrides.energy.quintel
390
- ? {
391
- ...baseEnergyQuintel,
392
- ...overrides.energy.quintel,
393
- rules: { ...baseEnergyQuintel.rules, ...overrides.energy.quintel.rules },
394
- weights: { ...baseEnergyQuintel.weights, ...overrides.energy.quintel.weights },
395
- clamp: { ...baseEnergyQuintel.clamp, ...overrides.energy.quintel.clamp },
396
- }
397
- : { ...baseEnergyQuintel },
398
- }
399
- : { ...baseEnergy },
400
- moe: overrides.moe
401
- ? {
402
- routing: { ...baseMoe.routing, ...overrides.moe.routing },
403
- cache: { ...baseMoe.cache, ...overrides.moe.cache },
404
- }
405
- : { ...baseMoe },
406
- speculative: { ...base.speculative, ...overrides.speculative },
407
- generation: { ...base.generation, ...overrides.generation },
408
- pipeline: overrides.pipeline ?? base.pipeline,
409
- kernelPath: mergeRuntimeKernelPath(base.kernelPath, overrides.kernelPath),
410
- kernelPathSource: overrides.kernelPathSource ?? base.kernelPathSource,
411
- kernelPathPolicy: mergeKernelPathPolicy(baseKernelPathPolicy, overrideKernelPathPolicy),
412
- chatTemplate: mergeShallowObject(base.chatTemplate, overrides.chatTemplate),
413
- session: {
414
- ...baseSession,
415
- ...overrideSession,
416
- compute: {
417
- ...baseSessionCompute,
418
- ...overrideSessionCompute,
419
- defaults: {
420
- ...baseSessionComputeDefaults,
421
- ...overrideSessionComputeDefaults,
422
- },
423
- ...(hasRuntimeKernelProfiles
424
- ? { kernelProfiles: overrideSessionCompute.kernelProfiles }
425
- : { kernelProfiles: baseSessionCompute.kernelProfiles }),
426
- },
427
- kvcache: replaceSubtree(overrideSession.kvcache, baseSession.kvcache),
428
- decodeLoop: replaceSubtree(overrideSession.decodeLoop, baseSession.decodeLoop),
429
- perLayerInputs: replaceSubtree(overrideSession.perLayerInputs, baseSession.perLayerInputs),
430
- prefillTokenChunkSize: overrideSession.prefillTokenChunkSize ?? baseSession.prefillTokenChunkSize,
431
- },
432
- executionPatch: mergeExecutionPatchLists(baseExecutionPatch, overrideExecutionPatch),
433
- // Model-specific inference overrides (merged with manifest.inference at load time)
434
- modelOverrides: overrides.modelOverrides ?? base.modelOverrides,
435
- };
281
+ const overrideSessionCompute = overrideSession.compute;
282
+ if (isPlainObject(overrideSessionCompute) && hasOwn(overrideSessionCompute, 'kernelProfiles')) {
283
+ const compute = isPlainObject(inference.session.compute) ? inference.session.compute : {};
284
+ inference.session.compute = {
285
+ ...compute,
286
+ kernelProfiles: overrideSessionCompute.kernelProfiles,
287
+ };
288
+ }
289
+ }
290
+
291
+ function hasOwn(container, key) {
292
+ return Object.prototype.hasOwnProperty.call(container, key);
436
293
  }
437
294
 
438
295
  function mergeRuntimeKernelPath(
@@ -460,70 +317,3 @@ function assertRuntimeKernelPath(kernelPath) {
460
317
  );
461
318
  }
462
319
  }
463
-
464
- function mergeKernelThresholds(
465
- base,
466
- overrides
467
- ) {
468
- return {
469
- ...base,
470
- ...overrides,
471
- matmul: { ...base.matmul, ...overrides.matmul },
472
- rmsnorm: { ...base.rmsnorm, ...overrides.rmsnorm },
473
- rope: { ...base.rope, ...overrides.rope },
474
- attention: { ...base.attention, ...overrides.attention },
475
- fusedMatmul: { ...base.fusedMatmul, ...overrides.fusedMatmul },
476
- cast: { ...base.cast, ...overrides.cast },
477
- };
478
- }
479
-
480
- function mergeDebugConfig(
481
- base,
482
- overrides
483
- ) {
484
- if (!overrides) {
485
- return { ...base };
486
- }
487
-
488
- return {
489
- logOutput: { ...base.logOutput, ...overrides.logOutput },
490
- logHistory: { ...base.logHistory, ...overrides.logHistory },
491
- logLevel: { ...base.logLevel, ...overrides.logLevel },
492
- trace: { ...base.trace, ...overrides.trace },
493
- pipeline: { ...base.pipeline, ...overrides.pipeline },
494
- loader: { ...base.loader, ...overrides.loader },
495
- matmul: { ...base.matmul, ...overrides.matmul },
496
- kernelTrace: { ...base.kernelTrace, ...overrides.kernelTrace },
497
- probes: overrides.probes ?? base.probes,
498
- profiler: { ...base.profiler, ...overrides.profiler },
499
- perfGuards: { ...base.perfGuards, ...overrides.perfGuards },
500
- };
501
- }
502
-
503
- function mergeBenchmarkConfig(
504
- base,
505
- overrides
506
- ) {
507
- if (!overrides) {
508
- return { ...base };
509
- }
510
-
511
- return {
512
- output: { ...base.output, ...overrides.output },
513
- run: { ...base.run, ...overrides.run },
514
- stats: { ...base.stats, ...overrides.stats },
515
- comparison: { ...base.comparison, ...overrides.comparison },
516
- baselines: { ...base.baselines, ...overrides.baselines },
517
- };
518
- }
519
-
520
- function mergeEmulationConfig(
521
- base,
522
- overrides
523
- ) {
524
- if (!overrides) {
525
- return { ...base };
526
- }
527
-
528
- return createEmulationConfig(overrides);
529
- }