doppler-gpu 0.4.5 → 0.4.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (155) hide show
  1. package/README.md +45 -29
  2. package/assets/doppler-runtime-map.svg +82 -0
  3. package/assets/doppler.svg +1 -3
  4. package/package.json +25 -17
  5. package/src/cli/doppler-quickstart.js +1 -1
  6. package/src/cli/doppler-serve.js +2 -2
  7. package/src/client/doppler-registry.d.ts +19 -0
  8. package/src/client/doppler-registry.js +12 -0
  9. package/src/client/doppler-registry.json +50 -1
  10. package/src/client/runtime/model-session.d.ts +1 -0
  11. package/src/client/runtime/model-session.js +9 -0
  12. package/src/config/conversion/README.md +4 -0
  13. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +28 -4
  14. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +29 -4
  15. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +29 -4
  16. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +29 -4
  17. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +29 -4
  18. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +29 -4
  19. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +29 -4
  20. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +29 -4
  21. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +29 -4
  22. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +31 -5
  23. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +30 -4
  24. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +30 -4
  25. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +30 -4
  26. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +30 -5
  27. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +29 -4
  28. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +30 -5
  29. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +29 -4
  30. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +29 -4
  31. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +29 -4
  32. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +29 -4
  33. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +29 -4
  34. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +29 -4
  35. package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +481 -0
  36. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +29 -4
  37. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +29 -4
  38. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +30 -5
  39. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +29 -4
  40. package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +395 -0
  41. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +429 -0
  42. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +540 -0
  43. package/src/config/execution-contract-check.js +14 -10
  44. package/src/config/merge-contract-check.js +1 -0
  45. package/src/config/merge.js +30 -0
  46. package/src/config/param-validator.d.ts +5 -0
  47. package/src/config/param-validator.js +42 -8
  48. package/src/config/required-inference-fields-contract-check.js +14 -0
  49. package/src/config/schema/debug.schema.d.ts +2 -0
  50. package/src/config/schema/debug.schema.js +1 -0
  51. package/src/config/schema/doppler.schema.d.ts +5 -1
  52. package/src/config/schema/doppler.schema.js +82 -292
  53. package/src/config/schema/inference.schema.d.ts +17 -3
  54. package/src/config/schema/kvcache.schema.d.ts +19 -3
  55. package/src/config/schema/kvcache.schema.js +2 -1
  56. package/src/config/schema/manifest.schema.d.ts +38 -1
  57. package/src/config/schema/manifest.schema.js +18 -0
  58. package/src/config/schema/policy-schema-registry.json +5 -0
  59. package/src/config/source-packages/litert/gemma-4-12b-it.json +20 -1
  60. package/src/config/source-packages/litert/gemma-4-e2b-it.json +20 -1
  61. package/src/config/source-packages/litert/gemma-4-e4b-it.json +21 -1
  62. package/src/config/transforms/execution-graph-transforms.d.ts +6 -0
  63. package/src/config/transforms/execution-graph-transforms.js +8 -0
  64. package/src/converter/conversion-plan.js +2 -0
  65. package/src/converter/parsers/transformer.js +27 -1
  66. package/src/converter/quantization-info.js +16 -4
  67. package/src/converter/rope-config.js +55 -3
  68. package/src/experimental/training/distillation/student-fixture.js +1 -1
  69. package/src/experimental/training/lora-pipeline.js +1 -1
  70. package/src/experimental/training/lora.js +2 -2
  71. package/src/experimental/training/suite.js +1 -1
  72. package/src/gpu/device.js +54 -11
  73. package/src/gpu/kernels/README.md +8 -1
  74. package/src/gpu/kernels/backward/attention_backward.js +8 -0
  75. package/src/gpu/kernels/dequant.d.ts +6 -6
  76. package/src/gpu/kernels/dequant.js +17 -6
  77. package/src/gpu/kernels/linear-attention-core.js +1 -6
  78. package/src/gpu/kernels/matmul-selection.d.ts +2 -2
  79. package/src/gpu/kernels/matmul-selection.js +9 -2
  80. package/src/gpu/kernels/matmul.d.ts +1 -1
  81. package/src/gpu/kernels/matmul.js +63 -9
  82. package/src/gpu/kernels/moe.d.ts +5 -1
  83. package/src/gpu/kernels/moe.js +5 -2
  84. package/src/gpu/kernels/softmax.d.ts +3 -3
  85. package/src/gpu/kernels/softmax.js +9 -2
  86. package/src/gpu/kernels/turboquant-codebook.js +1 -1
  87. package/src/gpu/kernels/types.d.ts +1 -1
  88. package/src/gpu/profiler.js +11 -1
  89. package/src/inference/browser-harness-model-helpers.js +4 -1
  90. package/src/inference/browser-harness-text-helpers.d.ts +28 -0
  91. package/src/inference/browser-harness-text-helpers.js +357 -116
  92. package/src/inference/browser-harness.d.ts +3 -3
  93. package/src/inference/browser-harness.js +266 -11
  94. package/src/inference/fixtures/rerank-semantic-fixtures.json +38 -0
  95. package/src/inference/kv-cache/quantized.js +16 -2
  96. package/src/inference/kv-cache/tiered.js +50 -15
  97. package/src/inference/moe-router.js +38 -7
  98. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +1 -1
  99. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +8 -6
  100. package/src/inference/pipelines/text/config.d.ts +5 -1
  101. package/src/inference/pipelines/text/config.js +135 -112
  102. package/src/inference/pipelines/text/embed.d.ts +1 -0
  103. package/src/inference/pipelines/text/embed.js +39 -12
  104. package/src/inference/pipelines/text/execution-plan.js +8 -2
  105. package/src/inference/pipelines/text/ffn/dense.js +15 -4
  106. package/src/inference/pipelines/text/ffn/moe.js +4 -3
  107. package/src/inference/pipelines/text/ffn/standard.d.ts +2 -1
  108. package/src/inference/pipelines/text/ffn/standard.js +25 -2
  109. package/src/inference/pipelines/text/generator-helpers.js +2 -1
  110. package/src/inference/pipelines/text/generator-runtime.d.ts +6 -2
  111. package/src/inference/pipelines/text/generator-runtime.js +53 -20
  112. package/src/inference/pipelines/text/generator-steps.js +3 -0
  113. package/src/inference/pipelines/text/generator.d.ts +1 -0
  114. package/src/inference/pipelines/text/generator.js +14 -1
  115. package/src/inference/pipelines/text/init-rope.js +54 -6
  116. package/src/inference/pipelines/text/init.js +63 -15
  117. package/src/inference/pipelines/text/layer.js +32 -5
  118. package/src/inference/pipelines/text/linear-attention.js +12 -8
  119. package/src/inference/pipelines/text/logits/gpu.d.ts +1 -1
  120. package/src/inference/pipelines/text/logits/gpu.js +76 -113
  121. package/src/inference/pipelines/text/logits/index.js +44 -87
  122. package/src/inference/pipelines/text/logits/precision-policy.d.ts +11 -0
  123. package/src/inference/pipelines/text/logits/precision-policy.js +103 -0
  124. package/src/inference/pipelines/text/logits/types.d.ts +1 -0
  125. package/src/inference/pipelines/text/logits/utils.d.ts +2 -0
  126. package/src/inference/pipelines/text/logits/utils.js +11 -0
  127. package/src/inference/pipelines/text/lora-apply.js +6 -4
  128. package/src/inference/pipelines/text/moe-cpu.js +25 -0
  129. package/src/inference/pipelines/text/moe-gpu.js +148 -97
  130. package/src/inference/pipelines/text/moe-shape-validator.d.ts +33 -3
  131. package/src/inference/pipelines/text/moe-shape-validator.js +149 -31
  132. package/src/inference/pipelines/text/ops.d.ts +3 -0
  133. package/src/inference/pipelines/text/ops.js +26 -1
  134. package/src/inference/pipelines/text/per-layer-inputs.js +27 -12
  135. package/src/inference/pipelines/text/sampling.js +52 -10
  136. package/src/inference/pipelines/text/weights.js +7 -2
  137. package/src/inference/pipelines/text.js +24 -4
  138. package/src/inference/tokenizers/behavior-flags.js +23 -2
  139. package/src/loader/final-weights-loader.js +12 -1
  140. package/src/loader/tensors/tensor-loader.js +37 -3
  141. package/src/rules/execution-rules-contract-check.js +12 -3
  142. package/src/rules/inference/capability-transforms.rules.json +14 -2
  143. package/src/rules/inference/config.rules.json +26 -2
  144. package/src/rules/inference/dtype.rules.json +12 -5
  145. package/src/rules/inference/execution.rules.json +5 -19
  146. package/src/rules/inference/moe.rules.json +97 -0
  147. package/src/rules/loader/weights.rules.json +1 -0
  148. package/src/tooling/command-api-constants.js +2 -2
  149. package/src/tooling/command-api-family-normalizers.js +2 -1
  150. package/src/tooling/command-api.d.ts +1 -1
  151. package/src/tooling/command-runner-shared.d.ts +1 -1
  152. package/src/tooling/command-runner-shared.js +3 -1
  153. package/src/tooling/node-converter.js +20 -0
  154. package/src/tooling/node-source-runtime.js +5 -1
  155. package/src/version.js +1 -1
@@ -7,6 +7,7 @@ import { isMoELayerLocal } from './types.js';
7
7
  import { runDenseFFNGPU } from './dense.js';
8
8
  import { runMoEFFNGPU } from './moe.js';
9
9
  import { acquireBuffer, readBuffer } from '../../../../memory/buffer-pool.js';
10
+ import { runScale, recordScale } from '../../../../gpu/kernel-selector.js';
10
11
  import { isGpuBufferInstance, isWeightBuffer } from '../../../../gpu/weight-buffer.js';
11
12
  import { shouldDebugLayerOutput, decodeReadback, getLogitsHealth } from '../debug-utils/index.js';
12
13
  import { trace, isTraceEnabled } from '../../../../debug/index.js';
@@ -44,7 +45,8 @@ export async function processFFNStandard(
44
45
  context,
45
46
  layerWeights,
46
47
  fusedResidualInput,
47
- finalOutputScale = 1
48
+ finalOutputScale = 1,
49
+ residualBranchScale = 1
48
50
  ) {
49
51
  const { config, weightConfig, debugFlags, recorder, decodeBuffers } = context;
50
52
  const { hiddenSize, rmsNormEps } = config;
@@ -52,6 +54,13 @@ export async function processFFNStandard(
52
54
  if (!Number.isFinite(requestedFinalOutputScale)) {
53
55
  throw new Error(`Layer ${layerIdx} finalOutputScale must be finite; got "${String(finalOutputScale)}".`);
54
56
  }
57
+ const requestedResidualBranchScale = Number(residualBranchScale);
58
+ if (!Number.isFinite(requestedResidualBranchScale) || requestedResidualBranchScale <= 0) {
59
+ throw new Error(
60
+ `Layer ${layerIdx} residualBranchScale must be a positive finite number; ` +
61
+ `got "${String(residualBranchScale)}".`
62
+ );
63
+ }
55
64
  context.__layerScalarFusedFired = false;
56
65
 
57
66
  const decodeOutputBuffer = numTokens === 1 && decodeBuffers
@@ -112,7 +121,9 @@ export async function processFFNStandard(
112
121
  const pendingResidualForFfn = prenormSumBuffer
113
122
  ? { buffer: prenormSumBuffer, dtype: postAttn.dtype, shape: postAttn.shape }
114
123
  : postAttn;
115
- context.__pendingFfnResidualTensor = pendingResidualForFfn;
124
+ context.__pendingFfnResidualTensor = requestedResidualBranchScale === 1
125
+ ? pendingResidualForFfn
126
+ : null;
116
127
  context.__ffnResidualFusedFired = false;
117
128
 
118
129
  let ffnOutput;
@@ -156,6 +167,18 @@ export async function processFFNStandard(
156
167
  if (residualInputOwned) {
157
168
  residualInput = await doCast(ffnOutput, residualTensor.dtype, recorder);
158
169
  }
170
+ if (requestedResidualBranchScale !== 1) {
171
+ const unscaledResidualInput = residualInput;
172
+ residualInput = recorder
173
+ ? await recordScale(recorder, unscaledResidualInput, requestedResidualBranchScale, { count: size })
174
+ : await runScale(unscaledResidualInput, requestedResidualBranchScale, { count: size });
175
+ if (residualInputOwned) {
176
+ releaseOrTrack(recorder, unscaledResidualInput.buffer, decodeBuffers);
177
+ } else if (unscaledResidualInput.buffer !== ffnOutput.buffer) {
178
+ releaseOrTrack(recorder, unscaledResidualInput.buffer, decodeBuffers);
179
+ }
180
+ residualInputOwned = true;
181
+ }
159
182
  const residualOutputScale = requestedFinalOutputScale !== 1
160
183
  && residualTensor.dtype === context.activationDtype
161
184
  && !context.debugProbes?.length
@@ -202,7 +202,7 @@ function getWeightBufferConfig(state) {
202
202
  export function getLogitsWeights(state) {
203
203
  const finalNorm = state.weights.get('final_norm');
204
204
  const lmHead = state.weights.get('lm_head');
205
- if (!finalNorm || !(isGpuBufferInstance(finalNorm) || finalNorm instanceof Float32Array)) {
205
+ if (!finalNorm || !(isGpuBufferInstance(finalNorm) || finalNorm instanceof Float32Array || isWeightBuffer(finalNorm))) {
206
206
  throw new Error('Final norm not found or invalid type');
207
207
  }
208
208
  if (!lmHead || !(isGpuBufferInstance(lmHead) || lmHead instanceof Float32Array || isWeightBuffer(lmHead) || isCpuWeightBuffer(lmHead) || isSplitWeightBuffer(lmHead))) {
@@ -225,6 +225,7 @@ export function getLogitsConfig(state) {
225
225
  useTiedEmbeddings: state.useTiedEmbeddings,
226
226
  embeddingVocabSize: state.embeddingVocabSize,
227
227
  finalLogitSoftcapping: config.finalLogitSoftcapping,
228
+ logitInputScale: config.logitInputScale,
228
229
  largeWeights: state.runtimeConfig.inference.largeWeights,
229
230
  activationDtype: effectiveActivationDtype,
230
231
  kernelPath: activeKernelPath,
@@ -72,8 +72,12 @@ export function resolveAdvanceEmbeddingMode(
72
72
 
73
73
  export function resolveFloatDtypeFromByteSize(
74
74
  totalBytes: number,
75
- expectedLength: number,
76
- fallback?: 'f16' | 'f32'
75
+ expectedLength: number
76
+ ): 'f16' | 'f32';
77
+
78
+ export function resolveFloatDtypeFromBufferMetadata(
79
+ buffer: GPUBuffer,
80
+ expectedLength: number
77
81
  ): 'f16' | 'f32';
78
82
 
79
83
  export function getFinalNormWeights(state: PipelineState): Promise<Float32Array>;
@@ -1,12 +1,13 @@
1
1
  import { readBuffer } from '../../../memory/buffer-pool.js';
2
2
  import { matmulCPU, rmsNormCPU } from './logits/index.js';
3
- import { isGpuBufferInstance, isWeightBuffer, isCpuWeightBuffer } from '../../../gpu/weight-buffer.js';
3
+ import { isGpuBufferInstance, isWeightBuffer, isCpuWeightBuffer, getBufferDtype } from '../../../gpu/weight-buffer.js';
4
4
  import { decodeReadback } from './debug-utils/index.js';
5
5
  import { resolveExecutionSessionPlan } from './execution-plan.js';
6
6
  import { selectRuleValue } from '../../../rules/rule-registry.js';
7
+ import { validateSelfSpeculationConfig } from '../../../config/schema/speculation-self.schema.js';
7
8
 
8
9
  const UNKNOWN_TOKENIZER_VOCAB_SIZE = 'unknown';
9
- const DEFAULT_DTYPE = 'f32';
10
+ const INVALID_DTYPE_SENTINEL = '__invalid_dtype__';
10
11
 
11
12
  function resolveConfiguredValue(value, defaultValue, context, validate) {
12
13
  if (value === undefined) {
@@ -242,16 +243,28 @@ export function resolveGenerateOptions(state, options = {}) {
242
243
  }
243
244
 
244
245
  function resolveSpeculationConfig(state, options) {
245
- const sessionSpeculation = state.runtimeConfig?.inference?.session?.speculation ?? null;
246
+ const sessionSpeculation = state.runtimeConfig?.inference?.session?.speculation;
246
247
  const callSpeculation = options.speculation ?? null;
247
- if (!sessionSpeculation && !callSpeculation) return null;
248
- return {
249
- mode: callSpeculation?.mode ?? sessionSpeculation?.mode ?? 'none',
250
- tokens: callSpeculation?.tokens ?? sessionSpeculation?.tokens ?? 1,
251
- verify: callSpeculation?.verify ?? sessionSpeculation?.verify ?? 'greedy',
252
- threshold: callSpeculation?.threshold ?? sessionSpeculation?.threshold ?? null,
253
- rollbackOnReject: callSpeculation?.rollbackOnReject ?? sessionSpeculation?.rollbackOnReject ?? true,
248
+ if (sessionSpeculation === undefined) {
249
+ throw new Error('[Pipeline] runtime.inference.session.speculation is required.');
250
+ }
251
+ if (callSpeculation === null) {
252
+ if (sessionSpeculation === null) return null;
253
+ validateSelfSpeculationConfig(sessionSpeculation);
254
+ return sessionSpeculation.mode === 'none' ? null : sessionSpeculation;
255
+ }
256
+ if (!sessionSpeculation || typeof sessionSpeculation !== 'object') {
257
+ throw new Error('[Pipeline] options.speculation requires runtime.inference.session.speculation defaults.');
258
+ }
259
+ if (!callSpeculation || typeof callSpeculation !== 'object') {
260
+ throw new Error('[Pipeline] options.speculation must be an object.');
261
+ }
262
+ const merged = {
263
+ ...sessionSpeculation,
264
+ ...callSpeculation,
254
265
  };
266
+ validateSelfSpeculationConfig(merged);
267
+ return merged.mode === 'none' ? null : merged;
255
268
  }
256
269
 
257
270
  export function resolvePrefillOptions(state, options = {}) {
@@ -333,22 +346,39 @@ export function resolveAdvanceEmbeddingMode(state, options = {}) {
333
346
 
334
347
  function resolveFloatDtypeFromAlias(dtype) {
335
348
  const normalized = typeof dtype === 'string' ? dtype.trim().toLowerCase() : '';
336
- if (!normalized) return DEFAULT_DTYPE;
337
- return selectRuleValue('inference', 'dtype', 'dtypeFromAlias', {
349
+ if (!normalized) {
350
+ throw new Error('[Pipeline] float weight dtype is required.');
351
+ }
352
+ const resolved = selectRuleValue('inference', 'dtype', 'dtypeFromAlias', {
338
353
  dtype: normalized,
339
- fallback: DEFAULT_DTYPE,
354
+ fallback: INVALID_DTYPE_SENTINEL,
340
355
  });
356
+ if (resolved === INVALID_DTYPE_SENTINEL) {
357
+ throw new Error(`[Pipeline] Unsupported float weight dtype "${dtype}".`);
358
+ }
359
+ return resolved;
341
360
  }
342
361
 
343
362
  export function resolveFloatDtypeFromByteSize(totalBytes, expectedLength) {
344
363
  if (!Number.isFinite(totalBytes) || totalBytes <= 0 || !Number.isFinite(expectedLength) || expectedLength <= 0) {
345
- return DEFAULT_DTYPE;
364
+ throw new Error(
365
+ `[Pipeline] Cannot infer float dtype from invalid size metadata (totalBytes=${totalBytes}, expectedLength=${expectedLength}).`
366
+ );
346
367
  }
347
368
  const bytesPerElement = totalBytes / expectedLength;
348
- return selectRuleValue('inference', 'dtype', 'f16OrF32FromBytesOrFallback', {
349
- bytesPerElement,
350
- fallback: DEFAULT_DTYPE,
351
- });
369
+ if (bytesPerElement !== 2 && bytesPerElement !== 4) {
370
+ throw new Error(
371
+ `[Pipeline] Cannot infer float dtype from bytesPerElement=${bytesPerElement}; expected 2 or 4.`
372
+ );
373
+ }
374
+ return selectRuleValue('inference', 'dtype', 'f16OrF32FromBytes', { bytesPerElement });
375
+ }
376
+
377
+ export function resolveFloatDtypeFromBufferMetadata(buffer, expectedLength) {
378
+ const taggedDtype = getBufferDtype(buffer);
379
+ return taggedDtype
380
+ ? resolveFloatDtypeFromAlias(taggedDtype)
381
+ : resolveFloatDtypeFromByteSize(buffer?.size, expectedLength);
352
382
  }
353
383
 
354
384
  function decodeFloatWeights(data, dtype, expectedLength, label) {
@@ -385,8 +415,11 @@ export async function getFinalNormWeights(state) {
385
415
  const dtypeValue = typeof finalNorm.dtype === 'string' ? finalNorm.dtype.trim().toLowerCase() : '';
386
416
  const dtype = selectRuleValue('inference', 'dtype', 'f16OrF32FromDtypeAlias', {
387
417
  dtype: dtypeValue === '' ? undefined : dtypeValue,
388
- fallback: DEFAULT_DTYPE,
418
+ fallback: INVALID_DTYPE_SENTINEL,
389
419
  });
420
+ if (dtype === INVALID_DTYPE_SENTINEL) {
421
+ throw new Error(`[Pipeline] Unsupported final_norm dtype "${finalNorm.dtype}".`);
422
+ }
390
423
  const bytesPerElement = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype });
391
424
  const readSize = hiddenSize * bytesPerElement;
392
425
  const data = await readBuffer(finalNorm.buffer, readSize);
@@ -395,7 +428,7 @@ export async function getFinalNormWeights(state) {
395
428
  }
396
429
  weights = decodeFloatWeights(data, dtype, hiddenSize, 'final_norm');
397
430
  } else if (isGpuBufferInstance(finalNorm)) {
398
- const dtype = resolveFloatDtypeFromByteSize(finalNorm.size, hiddenSize);
431
+ const dtype = resolveFloatDtypeFromBufferMetadata(finalNorm, hiddenSize);
399
432
  const bytesPerElement = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype });
400
433
  const readSize = hiddenSize * bytesPerElement;
401
434
  const data = await readBuffer(finalNorm, readSize);
@@ -467,6 +467,7 @@ async function runDecodeLayers(state, tokenId, opts, helpers) {
467
467
  hiddenSize: config.hiddenSize,
468
468
  vocabSize: config.vocabSize,
469
469
  scaleEmbeddings: config.scaleEmbeddings,
470
+ embeddingScale: config.embeddingScale,
470
471
  outputBuffer: decodeHiddenBuffer ?? undefined,
471
472
  transpose: state.embeddingTranspose,
472
473
  debugProbes: state.runtimeConfig.shared.debug.probes,
@@ -619,6 +620,7 @@ export async function decodeStep(state, currentIds, opts, helpers) {
619
620
  hiddenSize: config.hiddenSize,
620
621
  vocabSize: config.vocabSize,
621
622
  scaleEmbeddings: config.scaleEmbeddings,
623
+ embeddingScale: config.embeddingScale,
622
624
  recorder,
623
625
  outputBuffer: decodeHiddenBuffer ?? undefined,
624
626
  transpose: state.embeddingTranspose,
@@ -1527,6 +1529,7 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
1527
1529
  hiddenSize: config.hiddenSize,
1528
1530
  vocabSize: config.vocabSize,
1529
1531
  scaleEmbeddings: config.scaleEmbeddings,
1532
+ embeddingScale: config.embeddingScale,
1530
1533
  recorder,
1531
1534
  outputBuffer: context.decodeBuffers?.getHiddenBuffer() ?? undefined,
1532
1535
  transpose: state.embeddingTranspose,
@@ -62,6 +62,7 @@ export declare class PipelineGenerator {
62
62
  prompt: PromptInput,
63
63
  options?: GenerateOptions
64
64
  ): Promise<{ tokenIds: number[]; stats: import('./types.js').PipelineStats }>;
65
+ resetGenerationState(): void;
65
66
  resetToSeqLen(seqLen: number): void;
66
67
  prefillKVOnly(prompt: PromptInput, options?: GenerateOptions): Promise<KVCacheSnapshot>;
67
68
  computeDiffusionGemmaCanvasLogits(
@@ -430,6 +430,14 @@ export class PipelineGenerator {
430
430
  this.#state.currentSeqLen = 0;
431
431
  }
432
432
 
433
+ resetGenerationState() {
434
+ if (this.#state.isGenerating) {
435
+ throw new Error('InferencePipeline.resetGenerationState: cannot reset while generation is in progress');
436
+ }
437
+ this._resetReplayPrefillRuntimeState();
438
+ this._resetDecodeRuntimeState();
439
+ }
440
+
433
441
  async _replayPrefillDecodeLogits(currentIds, opts) {
434
442
  // Guard: cap replay-prefill sequence length to the config-owned maxSeqLen.
435
443
  // Without KV cache creation, this bound is not enforced elsewhere.
@@ -1290,6 +1298,7 @@ export class PipelineGenerator {
1290
1298
  hiddenSize,
1291
1299
  vocabSize,
1292
1300
  scaleEmbeddings: config.scaleEmbeddings,
1301
+ embeddingScale: config.embeddingScale,
1293
1302
  debug: opts.debug,
1294
1303
  recorder: null,
1295
1304
  transpose: this.#state.embeddingTranspose,
@@ -2529,7 +2538,10 @@ export class PipelineGenerator {
2529
2538
  // and speculative use the same weights and state. The benefit is
2530
2539
  // amortizing per-iteration overhead for models where batch decode is
2531
2540
  // disabled (e.g., linear attention).
2532
- const speculativeBurstTokens = Math.max(1, Math.trunc(opts.speculation?.tokens ?? 1));
2541
+ const speculativeBurstTokens = opts.speculation.tokens;
2542
+ if (!Number.isInteger(speculativeBurstTokens) || speculativeBurstTokens < 1) {
2543
+ throw new Error('[Pipeline] resolved self-speculation tokens must be a positive integer.');
2544
+ }
2533
2545
  const doSpecDecode = hasLinearLayers
2534
2546
  ? () => this._decodeStep(generatedIds, opts)
2535
2547
  : decodeSingleTokenViaLogits;
@@ -2855,6 +2867,7 @@ export class PipelineGenerator {
2855
2867
  hiddenSize: config.hiddenSize,
2856
2868
  vocabSize: config.vocabSize,
2857
2869
  scaleEmbeddings: config.scaleEmbeddings,
2870
+ embeddingScale: config.embeddingScale,
2858
2871
  debug: opts.debug,
2859
2872
  recorder,
2860
2873
  transpose: this.#state.embeddingTranspose,
@@ -28,6 +28,8 @@ function computeRoPEFreqsForTheta(
28
28
  // Compute per-dimension scaling factors
29
29
  const scales = new Float32Array(halfDim);
30
30
  const isYarn = ropeScalingType === 'yarn';
31
+ const isLongRoPE = ropeScalingType === 'longrope';
32
+ let magnitudeScale = 1;
31
33
  if (isYarn) {
32
34
  // YARN scaling - validate ALL required params (fail fast on incomplete manifest)
33
35
  if (ropeScaling?.factor == null ||
@@ -61,13 +63,53 @@ function computeRoPEFreqsForTheta(
61
63
  scales[i] = 1.0 + (yarnFactor - 1.0) * t;
62
64
  }
63
65
  }
66
+ } else if (isLongRoPE) {
67
+ if (
68
+ !Array.isArray(ropeScaling?.short_factor)
69
+ || !Array.isArray(ropeScaling?.long_factor)
70
+ || ropeScaling?.original_max_position_embeddings == null
71
+ ) {
72
+ throw new Error(
73
+ `RoPE scaling type is 'longrope' but LongRoPE params are missing. ` +
74
+ `Manifest must provide short_factor, long_factor, and original_max_position_embeddings. ` +
75
+ `Got: short_factor=${Array.isArray(ropeScaling?.short_factor) ? ropeScaling.short_factor.length : ropeScaling?.short_factor}, ` +
76
+ `long_factor=${Array.isArray(ropeScaling?.long_factor) ? ropeScaling.long_factor.length : ropeScaling?.long_factor}, ` +
77
+ `original_max_position_embeddings=${ropeScaling?.original_max_position_embeddings}`
78
+ );
79
+ }
80
+ if (ropeScaling.short_factor.length !== halfDim || ropeScaling.long_factor.length !== halfDim) {
81
+ throw new Error(
82
+ `LongRoPE factor length mismatch: expected ${halfDim}, ` +
83
+ `got short_factor=${ropeScaling.short_factor.length}, long_factor=${ropeScaling.long_factor.length}.`
84
+ );
85
+ }
86
+ const originalMaxPos = Number(ropeScaling.original_max_position_embeddings);
87
+ if (!Number.isFinite(originalMaxPos) || originalMaxPos <= 0) {
88
+ throw new Error(`LongRoPE original_max_position_embeddings must be positive; got "${String(originalMaxPos)}".`);
89
+ }
90
+ const selectedFactors = maxSeqLen > originalMaxPos
91
+ ? ropeScaling.long_factor
92
+ : ropeScaling.short_factor;
93
+ for (let i = 0; i < halfDim; i++) {
94
+ const factor = Number(selectedFactors[i]);
95
+ if (!Number.isFinite(factor) || factor <= 0) {
96
+ throw new Error(`LongRoPE factor[${i}] must be a positive finite number; got "${String(selectedFactors[i])}".`);
97
+ }
98
+ scales[i] = factor;
99
+ }
100
+ magnitudeScale = Math.sqrt(1 + Math.log(maxSeqLen / originalMaxPos) / Math.log(originalMaxPos));
101
+ if (!Number.isFinite(magnitudeScale) || magnitudeScale <= 0) {
102
+ throw new Error(
103
+ `LongRoPE magnitude scale is invalid for maxSeqLen=${maxSeqLen}, ` +
104
+ `original_max_position_embeddings=${originalMaxPos}.`
105
+ );
106
+ }
64
107
  } else {
65
108
  // Linear scaling: uniform across all dimensions
66
109
  if (ropeScalingType != null && ropeScalingType !== 'linear') {
67
- log.warn(
68
- 'Pipeline',
69
- `Unrecognized RoPE scaling type "${ropeScalingType}"; falling back to linear scaling. ` +
70
- 'Known types: "linear", "yarn".'
110
+ throw new Error(
111
+ `Unsupported RoPE scaling type "${ropeScalingType}". ` +
112
+ 'Supported types: null, "linear", "yarn", "longrope".'
71
113
  );
72
114
  }
73
115
  for (let i = 0; i < halfDim; i++) {
@@ -83,8 +125,8 @@ function computeRoPEFreqsForTheta(
83
125
  for (let i = 0; i < halfDim; i++) {
84
126
  const scaledPos = pos / scales[i];
85
127
  const angle = scaledPos * freqs[i];
86
- cosValues[pos * halfDim + i] = Math.cos(angle);
87
- sinValues[pos * halfDim + i] = Math.sin(angle);
128
+ cosValues[pos * halfDim + i] = Math.cos(angle) * magnitudeScale;
129
+ sinValues[pos * halfDim + i] = Math.sin(angle) * magnitudeScale;
88
130
  }
89
131
  }
90
132
 
@@ -101,6 +143,12 @@ function isSameRoPEScalingConfig(
101
143
  ) {
102
144
  if (leftType !== rightType) return false;
103
145
  if (leftScale !== rightScale) return false;
146
+ if (leftType === 'longrope') {
147
+ return JSON.stringify(leftScaling?.short_factor ?? null) === JSON.stringify(rightScaling?.short_factor ?? null)
148
+ && JSON.stringify(leftScaling?.long_factor ?? null) === JSON.stringify(rightScaling?.long_factor ?? null)
149
+ && (leftScaling?.original_max_position_embeddings ?? null)
150
+ === (rightScaling?.original_max_position_embeddings ?? null);
151
+ }
104
152
  if (leftType !== 'yarn') return true;
105
153
  return (leftScaling?.beta_fast ?? null) === (rightScaling?.beta_fast ?? null)
106
154
  && (leftScaling?.beta_slow ?? null) === (rightScaling?.beta_slow ?? null)
@@ -442,22 +442,35 @@ function getRequiredKernelMaxKVLen(operation, variant, label) {
442
442
  }
443
443
 
444
444
  function resolveTieredRequestedQuantMode(runtimeKV) {
445
- const tieringMode = String(runtimeKV?.tiering?.mode ?? 'off').trim().toLowerCase();
445
+ const tiering = requirePlainObject(
446
+ runtimeKV?.tiering,
447
+ 'runtime.inference.session.kvcache.tiering'
448
+ );
449
+ const tieringMode = requireNonEmptyString(
450
+ tiering.mode,
451
+ 'runtime.inference.session.kvcache.tiering.mode'
452
+ ).toLowerCase();
446
453
  assertSupportedTurboQuantMode(tieringMode, 'runtime.inference.session.kvcache.tiering.mode');
447
- const compressionMode = String(
448
- runtimeKV?.tiering?.compression?.mode
449
- ?? (tieringMode === 'int8'
450
- || tieringMode === 'int4'
451
- || tieringMode === 'turboquant'
452
- || tieringMode === 'turboquant_prod'
453
- ? tieringMode
454
- : 'none')
455
- ).trim().toLowerCase();
454
+ const compression = requirePlainObject(
455
+ tiering.compression,
456
+ 'runtime.inference.session.kvcache.tiering.compression'
457
+ );
458
+ const compressionMode = requireNonEmptyString(
459
+ compression.mode,
460
+ 'runtime.inference.session.kvcache.tiering.compression.mode'
461
+ ).toLowerCase();
456
462
  assertSupportedTurboQuantMode(
457
463
  compressionMode,
458
464
  'runtime.inference.session.kvcache.tiering.compression.mode'
459
465
  );
460
- const gatingMode = String(runtimeKV?.tiering?.gating?.mode ?? 'auto').trim().toLowerCase();
466
+ const gating = requirePlainObject(
467
+ tiering.gating,
468
+ 'runtime.inference.session.kvcache.tiering.gating'
469
+ );
470
+ const gatingMode = requireNonEmptyString(
471
+ gating.mode,
472
+ 'runtime.inference.session.kvcache.tiering.gating.mode'
473
+ ).toLowerCase();
461
474
  if (gatingMode === 'force_off') {
462
475
  return 'none';
463
476
  }
@@ -465,7 +478,14 @@ function resolveTieredRequestedQuantMode(runtimeKV) {
465
478
  }
466
479
 
467
480
  function resolveContiguousRequestedQuantMode(runtimeKV) {
468
- const quantMode = String(runtimeKV?.quantization?.mode ?? 'none').trim().toLowerCase();
481
+ const quantization = requirePlainObject(
482
+ runtimeKV?.quantization,
483
+ 'runtime.inference.session.kvcache.quantization'
484
+ );
485
+ const quantMode = requireNonEmptyString(
486
+ quantization.mode,
487
+ 'runtime.inference.session.kvcache.quantization.mode'
488
+ ).toLowerCase();
469
489
  assertSupportedTurboQuantMode(
470
490
  quantMode,
471
491
  'runtime.inference.session.kvcache.quantization.mode'
@@ -725,17 +745,24 @@ export function createKVCache(modelConfig, useGPU, debug = false, runtimeConfig)
725
745
  tiering: runtimeKV.tiering,
726
746
  });
727
747
  } else if (cacheLayout === 'contiguous_quantized') {
728
- const quantCfg = runtimeKV.quantization ?? {};
748
+ const quantCfg = requirePlainObject(
749
+ runtimeKV.quantization,
750
+ 'runtime.inference.kvcache.quantization'
751
+ );
752
+ const bitWidth = requirePositiveInteger(
753
+ quantCfg.bitWidth,
754
+ 'runtime.inference.kvcache.quantization.bitWidth'
755
+ );
729
756
  const qCache = new QuantizedKVCache({
730
757
  ...cacheConfig,
731
758
  quantMode,
732
- bitWidth: quantCfg.bitWidth ?? 4,
759
+ bitWidth,
733
760
  prodMode: quantCfg.prodMode === true,
734
761
  });
735
762
  const device = getDevice();
736
763
  qCache.setSharedBuffers(retainTurboQuantSharedBuffers(device, {
737
764
  headDim: modelConfig.headDim,
738
- bitWidth: quantCfg.bitWidth ?? 4,
765
+ bitWidth,
739
766
  prodMode: quantCfg.prodMode === true,
740
767
  }));
741
768
  kvCache = qCache;
@@ -783,6 +810,27 @@ export function createKVCache(modelConfig, useGPU, debug = false, runtimeConfig)
783
810
  return kvCache;
784
811
  }
785
812
 
813
+ function requirePlainObject(value, label) {
814
+ if (!isPlainObject(value)) {
815
+ throw new Error(`${label} is required.`);
816
+ }
817
+ return value;
818
+ }
819
+
820
+ function requireNonEmptyString(value, label) {
821
+ if (typeof value !== 'string' || value.trim().length === 0) {
822
+ throw new Error(`${label} is required.`);
823
+ }
824
+ return value.trim();
825
+ }
826
+
827
+ function requirePositiveInteger(value, label) {
828
+ if (!Number.isInteger(value) || value <= 0) {
829
+ throw new Error(`${label} must be a positive integer.`);
830
+ }
831
+ return value;
832
+ }
833
+
786
834
  // ============================================================================
787
835
  // Tokenizer Setup
788
836
  // ============================================================================
@@ -628,6 +628,13 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
628
628
 
629
629
  assertSupportedLayerRuntime(layerIdx, config);
630
630
  const { hiddenSize, numHeads, numKVHeads, headDim, rmsNormEps } = config;
631
+ const residualBranchScale = Number(config.residualBranchScale);
632
+ if (!Number.isFinite(residualBranchScale) || residualBranchScale <= 0) {
633
+ throw new Error(
634
+ `Layer ${layerIdx} residualBranchScale must be a positive finite number; ` +
635
+ `got "${String(config.residualBranchScale)}".`
636
+ );
637
+ }
631
638
 
632
639
  // Determine activation dtype from context (defaults to f32)
633
640
 
@@ -638,6 +645,12 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
638
645
 
639
646
  const layerWeights = (weights.get(`layer_${layerIdx}`));
640
647
  const sandwichNorm = detectSandwichNorm(config);
648
+ if (sandwichNorm.useSandwichNorm && residualBranchScale !== 1) {
649
+ throw new Error(
650
+ `Layer ${layerIdx} uses sandwich norms with residualBranchScale=${residualBranchScale}. ` +
651
+ 'Scaled residual branches for sandwich-norm layers are not implemented.'
652
+ );
653
+ }
641
654
  const lastTokenIdx = Math.max(0, numTokens - 1);
642
655
 
643
656
  await runProbes('layer_in', inputBuffer, {
@@ -651,6 +664,12 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
651
664
  });
652
665
 
653
666
  if (context.pipelinePlan) {
667
+ if (residualBranchScale !== 1) {
668
+ throw new Error(
669
+ `Layer ${layerIdx} has residualBranchScale=${residualBranchScale}, but pipelinePlan execution ` +
670
+ 'does not implement scaled residual branches.'
671
+ );
672
+ }
654
673
  return processLayerPlanGPU(layerIdx, inputBuffer, numTokens, isPrefill, size, context, layerWeights, sandwichNorm);
655
674
  }
656
675
 
@@ -763,7 +782,7 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
763
782
  activationDtype,
764
783
  slidingWindow: config.slidingWindow,
765
784
  layerType,
766
- residualTensor: (numTokens === 1 && !(sandwichNorm.useSandwichNorm && sandwichNorm.hasPostAttentionNorm))
785
+ residualTensor: (numTokens === 1 && !(sandwichNorm.useSandwichNorm && sandwichNorm.hasPostAttentionNorm) && residualBranchScale === 1)
767
786
  ? inputTensor
768
787
  : null,
769
788
  attnSoftcap: config.attnLogitSoftcapping === null ? 0 : config.attnLogitSoftcapping,
@@ -870,6 +889,13 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
870
889
  operatorDiagnostics: context.operatorDiagnostics,
871
890
  dtype: attnOutput.dtype,
872
891
  });
892
+ if (!residualFused && residualBranchScale !== 1) {
893
+ const rawAttnOutput = attnOutput;
894
+ attnOutput = recorder
895
+ ? await recordScale(recorder, rawAttnOutput, residualBranchScale, { count: size })
896
+ : await runScale(rawAttnOutput, residualBranchScale, { count: size });
897
+ releaseOrTrack(recorder, rawAttnOutput.buffer, context.decodeBuffers);
898
+ }
873
899
 
874
900
  // 2. Handle residual connection based on architecture
875
901
 
@@ -1061,10 +1087,11 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
1061
1087
  numTokens,
1062
1088
  size,
1063
1089
  context,
1064
- layerWeights,
1065
- fusedResidualForFFN,
1066
- requestFfnLayerScalarFusion ? layerScalar : 1
1067
- );
1090
+ layerWeights,
1091
+ fusedResidualForFFN,
1092
+ requestFfnLayerScalarFusion ? layerScalar : 1,
1093
+ residualBranchScale
1094
+ );
1068
1095
  layerScalarFused = context.__layerScalarFusedFired === true;
1069
1096
  context.__layerScalarFusedFired = false;
1070
1097
  }
@@ -34,9 +34,7 @@ function normalizeLinearNormMode(value) {
34
34
  }
35
35
 
36
36
  function bytesFromDtype(dtype) {
37
- const normalized = String(dtype ?? '').toLowerCase();
38
- if (normalized === 'f16' || normalized === 'bf16') return 2;
39
- return 4;
37
+ return selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype });
40
38
  }
41
39
 
42
40
  function resolveMatmulStepDtype(role, phase, layerIdx, kernelPath, fallback, field) {
@@ -199,13 +197,19 @@ function inferLinearNormModeFromWeight(weight, projectionLayout) {
199
197
  if (weight instanceof ArrayBuffer) {
200
198
  return classify(Math.trunc(weight.byteLength / Float32Array.BYTES_PER_ELEMENT));
201
199
  }
200
+ if (!weight || typeof weight !== 'object') {
201
+ return null;
202
+ }
202
203
  const explicitDtype = typeof weight?.dtype === 'string' ? weight.dtype.toLowerCase() : null;
203
204
  const trackedDtype = isGpuBuffer(weight) ? String(getBufferDtype(weight) ?? '').toLowerCase() : '';
204
- const bytesPerElement = bytesFromDtype(explicitDtype || trackedDtype || null);
205
- const sizedElements = Number.isFinite(weight?.size)
206
- ? Math.trunc(Number(weight.size) / bytesPerElement)
207
- : null;
208
- if (sizedElements && Number(weight.size) % bytesPerElement === 0) {
205
+ const sourceDtype = explicitDtype || trackedDtype;
206
+ if (!sourceDtype || !Number.isFinite(weight?.size)) {
207
+ return null;
208
+ }
209
+ const bytesPerElement = bytesFromDtype(sourceDtype);
210
+ const size = Number(weight.size);
211
+ const sizedElements = Math.trunc(size / bytesPerElement);
212
+ if (sizedElements && size % bytesPerElement === 0) {
209
213
  return classify(sizedElements);
210
214
  }
211
215
  return null;
@@ -38,7 +38,7 @@ export function extractLmHeadChunk(
38
38
  vocabSize: number,
39
39
  rowOffset: number,
40
40
  rowCount: number,
41
- sourceDtype?: 'f16' | 'f32' | 'bf16'
41
+ sourceDtype: 'f16' | 'f32' | 'bf16'
42
42
  ): Promise<Float32Array>;
43
43
 
44
44
  /**