doppler-gpu 0.4.5 → 0.4.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (155) hide show
  1. package/README.md +45 -29
  2. package/assets/doppler-runtime-map.svg +82 -0
  3. package/assets/doppler.svg +1 -3
  4. package/package.json +25 -17
  5. package/src/cli/doppler-quickstart.js +1 -1
  6. package/src/cli/doppler-serve.js +2 -2
  7. package/src/client/doppler-registry.d.ts +19 -0
  8. package/src/client/doppler-registry.js +12 -0
  9. package/src/client/doppler-registry.json +50 -1
  10. package/src/client/runtime/model-session.d.ts +1 -0
  11. package/src/client/runtime/model-session.js +9 -0
  12. package/src/config/conversion/README.md +4 -0
  13. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +28 -4
  14. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +29 -4
  15. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +29 -4
  16. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +29 -4
  17. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +29 -4
  18. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +29 -4
  19. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +29 -4
  20. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +29 -4
  21. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +29 -4
  22. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +31 -5
  23. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +30 -4
  24. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +30 -4
  25. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +30 -4
  26. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +30 -5
  27. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +29 -4
  28. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +30 -5
  29. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +29 -4
  30. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +29 -4
  31. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +29 -4
  32. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +29 -4
  33. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +29 -4
  34. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +29 -4
  35. package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +481 -0
  36. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +29 -4
  37. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +29 -4
  38. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +30 -5
  39. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +29 -4
  40. package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +395 -0
  41. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +429 -0
  42. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +540 -0
  43. package/src/config/execution-contract-check.js +14 -10
  44. package/src/config/merge-contract-check.js +1 -0
  45. package/src/config/merge.js +30 -0
  46. package/src/config/param-validator.d.ts +5 -0
  47. package/src/config/param-validator.js +42 -8
  48. package/src/config/required-inference-fields-contract-check.js +14 -0
  49. package/src/config/schema/debug.schema.d.ts +2 -0
  50. package/src/config/schema/debug.schema.js +1 -0
  51. package/src/config/schema/doppler.schema.d.ts +5 -1
  52. package/src/config/schema/doppler.schema.js +82 -292
  53. package/src/config/schema/inference.schema.d.ts +17 -3
  54. package/src/config/schema/kvcache.schema.d.ts +19 -3
  55. package/src/config/schema/kvcache.schema.js +2 -1
  56. package/src/config/schema/manifest.schema.d.ts +38 -1
  57. package/src/config/schema/manifest.schema.js +18 -0
  58. package/src/config/schema/policy-schema-registry.json +5 -0
  59. package/src/config/source-packages/litert/gemma-4-12b-it.json +20 -1
  60. package/src/config/source-packages/litert/gemma-4-e2b-it.json +20 -1
  61. package/src/config/source-packages/litert/gemma-4-e4b-it.json +21 -1
  62. package/src/config/transforms/execution-graph-transforms.d.ts +6 -0
  63. package/src/config/transforms/execution-graph-transforms.js +8 -0
  64. package/src/converter/conversion-plan.js +2 -0
  65. package/src/converter/parsers/transformer.js +27 -1
  66. package/src/converter/quantization-info.js +16 -4
  67. package/src/converter/rope-config.js +55 -3
  68. package/src/experimental/training/distillation/student-fixture.js +1 -1
  69. package/src/experimental/training/lora-pipeline.js +1 -1
  70. package/src/experimental/training/lora.js +2 -2
  71. package/src/experimental/training/suite.js +1 -1
  72. package/src/gpu/device.js +54 -11
  73. package/src/gpu/kernels/README.md +8 -1
  74. package/src/gpu/kernels/backward/attention_backward.js +8 -0
  75. package/src/gpu/kernels/dequant.d.ts +6 -6
  76. package/src/gpu/kernels/dequant.js +17 -6
  77. package/src/gpu/kernels/linear-attention-core.js +1 -6
  78. package/src/gpu/kernels/matmul-selection.d.ts +2 -2
  79. package/src/gpu/kernels/matmul-selection.js +9 -2
  80. package/src/gpu/kernels/matmul.d.ts +1 -1
  81. package/src/gpu/kernels/matmul.js +63 -9
  82. package/src/gpu/kernels/moe.d.ts +5 -1
  83. package/src/gpu/kernels/moe.js +5 -2
  84. package/src/gpu/kernels/softmax.d.ts +3 -3
  85. package/src/gpu/kernels/softmax.js +9 -2
  86. package/src/gpu/kernels/turboquant-codebook.js +1 -1
  87. package/src/gpu/kernels/types.d.ts +1 -1
  88. package/src/gpu/profiler.js +11 -1
  89. package/src/inference/browser-harness-model-helpers.js +4 -1
  90. package/src/inference/browser-harness-text-helpers.d.ts +28 -0
  91. package/src/inference/browser-harness-text-helpers.js +357 -116
  92. package/src/inference/browser-harness.d.ts +3 -3
  93. package/src/inference/browser-harness.js +266 -11
  94. package/src/inference/fixtures/rerank-semantic-fixtures.json +38 -0
  95. package/src/inference/kv-cache/quantized.js +16 -2
  96. package/src/inference/kv-cache/tiered.js +50 -15
  97. package/src/inference/moe-router.js +38 -7
  98. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +1 -1
  99. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +8 -6
  100. package/src/inference/pipelines/text/config.d.ts +5 -1
  101. package/src/inference/pipelines/text/config.js +135 -112
  102. package/src/inference/pipelines/text/embed.d.ts +1 -0
  103. package/src/inference/pipelines/text/embed.js +39 -12
  104. package/src/inference/pipelines/text/execution-plan.js +8 -2
  105. package/src/inference/pipelines/text/ffn/dense.js +15 -4
  106. package/src/inference/pipelines/text/ffn/moe.js +4 -3
  107. package/src/inference/pipelines/text/ffn/standard.d.ts +2 -1
  108. package/src/inference/pipelines/text/ffn/standard.js +25 -2
  109. package/src/inference/pipelines/text/generator-helpers.js +2 -1
  110. package/src/inference/pipelines/text/generator-runtime.d.ts +6 -2
  111. package/src/inference/pipelines/text/generator-runtime.js +53 -20
  112. package/src/inference/pipelines/text/generator-steps.js +3 -0
  113. package/src/inference/pipelines/text/generator.d.ts +1 -0
  114. package/src/inference/pipelines/text/generator.js +14 -1
  115. package/src/inference/pipelines/text/init-rope.js +54 -6
  116. package/src/inference/pipelines/text/init.js +63 -15
  117. package/src/inference/pipelines/text/layer.js +32 -5
  118. package/src/inference/pipelines/text/linear-attention.js +12 -8
  119. package/src/inference/pipelines/text/logits/gpu.d.ts +1 -1
  120. package/src/inference/pipelines/text/logits/gpu.js +76 -113
  121. package/src/inference/pipelines/text/logits/index.js +44 -87
  122. package/src/inference/pipelines/text/logits/precision-policy.d.ts +11 -0
  123. package/src/inference/pipelines/text/logits/precision-policy.js +103 -0
  124. package/src/inference/pipelines/text/logits/types.d.ts +1 -0
  125. package/src/inference/pipelines/text/logits/utils.d.ts +2 -0
  126. package/src/inference/pipelines/text/logits/utils.js +11 -0
  127. package/src/inference/pipelines/text/lora-apply.js +6 -4
  128. package/src/inference/pipelines/text/moe-cpu.js +25 -0
  129. package/src/inference/pipelines/text/moe-gpu.js +148 -97
  130. package/src/inference/pipelines/text/moe-shape-validator.d.ts +33 -3
  131. package/src/inference/pipelines/text/moe-shape-validator.js +149 -31
  132. package/src/inference/pipelines/text/ops.d.ts +3 -0
  133. package/src/inference/pipelines/text/ops.js +26 -1
  134. package/src/inference/pipelines/text/per-layer-inputs.js +27 -12
  135. package/src/inference/pipelines/text/sampling.js +52 -10
  136. package/src/inference/pipelines/text/weights.js +7 -2
  137. package/src/inference/pipelines/text.js +24 -4
  138. package/src/inference/tokenizers/behavior-flags.js +23 -2
  139. package/src/loader/final-weights-loader.js +12 -1
  140. package/src/loader/tensors/tensor-loader.js +37 -3
  141. package/src/rules/execution-rules-contract-check.js +12 -3
  142. package/src/rules/inference/capability-transforms.rules.json +14 -2
  143. package/src/rules/inference/config.rules.json +26 -2
  144. package/src/rules/inference/dtype.rules.json +12 -5
  145. package/src/rules/inference/execution.rules.json +5 -19
  146. package/src/rules/inference/moe.rules.json +97 -0
  147. package/src/rules/loader/weights.rules.json +1 -0
  148. package/src/tooling/command-api-constants.js +2 -2
  149. package/src/tooling/command-api-family-normalizers.js +2 -1
  150. package/src/tooling/command-api.d.ts +1 -1
  151. package/src/tooling/command-runner-shared.d.ts +1 -1
  152. package/src/tooling/command-runner-shared.js +3 -1
  153. package/src/tooling/node-converter.js +20 -0
  154. package/src/tooling/node-source-runtime.js +5 -1
  155. package/src/version.js +1 -1
@@ -1,39 +1,131 @@
1
1
  import { getKernelCapabilities } from '../../../gpu/device.js';
2
2
  import { selectRuleValue } from '../../../rules/rule-registry.js';
3
3
 
4
+ const KERNEL_PATH_PROFILE_RESOLVERS = Object.freeze({
5
+ 'gpt-oss': resolveGptOssKernelPathProfile,
6
+ mixtral: resolveMixtralKernelPathProfile,
7
+ });
8
+
4
9
  function asVendorString(caps) {
5
10
  const raw = caps?.adapterInfo?.vendor;
6
11
  return typeof raw === 'string' && raw.trim() !== '' ? raw.toLowerCase() : 'unknown';
7
12
  }
8
13
 
9
- export function resolveMoeVendorProfile(modelType) {
10
- const caps = getKernelCapabilities();
11
- const vendor = asVendorString(caps);
12
- if (modelType === 'gpt-oss') {
13
- return selectRuleValue('kernels', 'moeGptoss', 'vendorQuirkProfile', { vendor });
14
+ function requireRuleContextValue(value, label) {
15
+ if (value == null || value === '') {
16
+ throw new Error(`[MoE] ${label} is required for rule selection.`);
17
+ }
18
+ return value;
19
+ }
20
+
21
+ function requireMoeProfileField(profile, field) {
22
+ const value = profile?.[field];
23
+ if (typeof value !== 'string' || value.length === 0) {
24
+ throw new Error(`[MoE] execution profile is missing ${field}.`);
25
+ }
26
+ return value;
27
+ }
28
+
29
+ function requireManifestString(value, label) {
30
+ if (typeof value !== 'string' || value.length === 0) {
31
+ throw new Error(`[MoE] ${label} is required for MoE execution profile resolution.`);
32
+ }
33
+ return value;
34
+ }
35
+
36
+ function validateVendorProfile(profile) {
37
+ if (!profile || typeof profile !== 'object' || Array.isArray(profile)) {
38
+ throw new Error('[MoE] vendor profile must resolve to an object.');
39
+ }
40
+ if (typeof profile.preferVec4Dequant !== 'boolean') {
41
+ throw new Error('[MoE] vendor profile missing preferVec4Dequant boolean.');
42
+ }
43
+ if (profile.dequantTileShape !== null && profile.dequantTileShape !== 'vec4' && profile.dequantTileShape !== 'scalar') {
44
+ throw new Error('[MoE] vendor profile dequantTileShape must be null, "vec4", or "scalar".');
45
+ }
46
+ if (!Number.isFinite(profile.routerWorkgroupSize) || profile.routerWorkgroupSize <= 0) {
47
+ throw new Error('[MoE] vendor profile missing positive routerWorkgroupSize.');
48
+ }
49
+ if (!Number.isFinite(profile.maxTokensPerExpertScale) || profile.maxTokensPerExpertScale <= 0) {
50
+ throw new Error('[MoE] vendor profile missing positive maxTokensPerExpertScale.');
51
+ }
52
+ return profile;
53
+ }
54
+
55
+ function validateMoeExecutionProfile(profile) {
56
+ if (!profile || typeof profile !== 'object' || Array.isArray(profile)) {
57
+ throw new Error('[MoE] executionProfile rule must resolve to an object.');
58
+ }
59
+ requireMoeProfileField(profile, 'id');
60
+ requireMoeProfileField(profile, 'label');
61
+ requireMoeProfileField(profile, 'expertExecutor');
62
+ requireMoeProfileField(profile, 'intermediateSizeSource');
63
+ if (typeof profile.requiresShaderF16 !== 'boolean') {
64
+ throw new Error('[MoE] execution profile is missing requiresShaderF16 boolean.');
65
+ }
66
+ if (profile.routerScaleMode !== 'none' && profile.routerScaleMode !== 'optional' && profile.routerScaleMode !== 'required') {
67
+ throw new Error('[MoE] execution profile routerScaleMode must be none, optional, or required.');
68
+ }
69
+ if (!profile.vendorProfile || typeof profile.vendorProfile !== 'object' || Array.isArray(profile.vendorProfile)) {
70
+ throw new Error('[MoE] execution profile is missing vendorProfile selector.');
71
+ }
72
+ return profile;
73
+ }
74
+
75
+ export function resolveMoeExecutionProfile(config, options = {}) {
76
+ const expertFormat = requireManifestString(config?.expertFormat, 'manifest moeConfig.expertFormat');
77
+ const modelType = requireManifestString(
78
+ typeof options?.modelType === 'string' && options.modelType.length > 0
79
+ ? options.modelType
80
+ : config?.modelType,
81
+ 'manifest modelType'
82
+ );
83
+ const profile = selectRuleValue('inference', 'moe', 'executionProfile', {
84
+ modelType,
85
+ expertFormat,
86
+ });
87
+ return validateMoeExecutionProfile(profile);
88
+ }
89
+
90
+ export function resolveMoeIntermediateSize(config, moeProfile) {
91
+ const source = requireMoeProfileField(moeProfile, 'intermediateSizeSource');
92
+ const resolvers = {
93
+ architecture: () => config?.intermediateSize,
94
+ expert: () => config?.expertIntermediateSize,
95
+ };
96
+ const resolver = resolvers[source];
97
+ if (typeof resolver !== 'function') {
98
+ throw new Error(`[MoE] Unknown intermediateSizeSource "${source}" in execution profile "${moeProfile.id}".`);
14
99
  }
15
- if (modelType === 'mixtral') {
16
- return selectRuleValue('kernels', 'moeMixtral', 'vendorQuirkProfile', { vendor });
100
+ const value = Number(resolver());
101
+ if (!Number.isFinite(value) || value <= 0) {
102
+ throw new Error(`[MoE] Invalid ${source} intermediate size for profile "${moeProfile.id}": ${String(value)}.`);
103
+ }
104
+ return value;
105
+ }
106
+
107
+ export function resolveMoeVendorProfile(moeProfile) {
108
+ const selector = moeProfile?.vendorProfile;
109
+ if (selector?.kind === 'rule') {
110
+ const caps = getKernelCapabilities();
111
+ const vendor = asVendorString(caps);
112
+ const profile = selectRuleValue(selector.domain, selector.group, selector.name, { vendor });
113
+ return validateVendorProfile(profile);
17
114
  }
18
- if (modelType === 'gemma4' || modelType === 'diffusion_gemma') {
19
- return {
20
- preferVec4Dequant: false,
21
- dequantTileShape: null,
22
- routerWorkgroupSize: 256,
23
- maxTokensPerExpertScale: 1.0,
24
- };
115
+ if (selector?.kind === 'static') {
116
+ return validateVendorProfile(selector.value);
25
117
  }
26
- throw new Error(`[MoE] Unknown modelType "${modelType}" for vendor profile resolution.`);
118
+ throw new Error(`[MoE] execution profile "${moeProfile?.id ?? 'unknown'}" has invalid vendorProfile selector.`);
27
119
  }
28
120
 
29
121
  function resolveGptOssRuleContext(context) {
30
122
  return {
31
- modelType: 'gpt-oss',
123
+ modelType: requireRuleContextValue(context?.modelType, 'GPT-OSS kernel rule modelType'),
32
124
  hasF16: context?.hasF16,
33
125
  hasSubgroups: context?.hasSubgroups,
34
- inputDtype: context?.routerDtype ?? context?.inputDtype,
35
- weightsDtype: context?.weightsDtype,
36
- outputDtype: context?.outputDtype ?? context?.weightsDtype,
126
+ inputDtype: requireRuleContextValue(context?.routerDtype ?? context?.inputDtype, 'GPT-OSS router/input dtype'),
127
+ weightsDtype: requireRuleContextValue(context?.weightsDtype, 'GPT-OSS weights dtype'),
128
+ outputDtype: requireRuleContextValue(context?.outputDtype, 'GPT-OSS output dtype'),
37
129
  groupSize: context?.groupSize,
38
130
  dequantTileShape: context?.tileShape ?? context?.dequantTileShape,
39
131
  };
@@ -49,12 +141,12 @@ export async function resolveGptOssKernelPathProfile(context) {
49
141
 
50
142
  function resolveMixtralRuleContext(context) {
51
143
  return {
52
- modelType: 'mixtral',
144
+ modelType: requireRuleContextValue(context?.modelType, 'Mixtral kernel rule modelType'),
53
145
  hasF16: context?.hasF16,
54
146
  hasSubgroups: context?.hasSubgroups,
55
- routerDtype: context?.routerDtype ?? 'f32',
56
- weightsDtype: context?.weightsDtype,
57
- outputDtype: context?.outputDtype ?? context?.weightsDtype,
147
+ routerDtype: requireRuleContextValue(context?.routerDtype, 'Mixtral router dtype'),
148
+ weightsDtype: requireRuleContextValue(context?.weightsDtype, 'Mixtral weights dtype'),
149
+ outputDtype: requireRuleContextValue(context?.outputDtype, 'Mixtral output dtype'),
58
150
  };
59
151
  }
60
152
 
@@ -66,6 +158,30 @@ export async function resolveMixtralKernelPathProfile(context) {
66
158
  };
67
159
  }
68
160
 
161
+ export async function resolveMoeKernelPathProfile(moeProfile, context) {
162
+ const resolverId = moeProfile?.kernelPathProfileResolver ?? null;
163
+ if (resolverId === null) {
164
+ return null;
165
+ }
166
+ const resolver = KERNEL_PATH_PROFILE_RESOLVERS[resolverId];
167
+ if (typeof resolver !== 'function') {
168
+ throw new Error(`[MoE] Unknown kernelPathProfileResolver "${resolverId}" in execution profile "${moeProfile.id}".`);
169
+ }
170
+ const modelType = requireRuleContextValue(moeProfile.kernelRuleModelType, `${moeProfile.id} kernel rule modelType`);
171
+ return resolver({ ...context, modelType });
172
+ }
173
+
174
+ function resolveShapePolicy(moeProfile, context) {
175
+ const selector = moeProfile?.shapePolicy ?? null;
176
+ if (selector === null) {
177
+ return null;
178
+ }
179
+ if (selector.kind !== 'rule') {
180
+ throw new Error(`[MoE] execution profile "${moeProfile.id}" has invalid shapePolicy selector.`);
181
+ }
182
+ return selectRuleValue(selector.domain, selector.group, selector.name, context);
183
+ }
184
+
69
185
  export function validateMoeShape(config, options = {}) {
70
186
  const {
71
187
  hiddenSize,
@@ -74,10 +190,12 @@ export function validateMoeShape(config, options = {}) {
74
190
  numExperts,
75
191
  expertFormat,
76
192
  } = config;
77
- const modelType = options.modelType ?? (
78
- expertFormat === 'gpt-oss'
79
- ? 'gpt-oss'
80
- : (expertFormat === 'gemma4' ? 'gemma4' : 'mixtral')
193
+ if (typeof options.modelType !== 'string' || options.modelType.length === 0) {
194
+ throw new Error('[MoE] validateMoeShape requires options.modelType from the manifest.');
195
+ }
196
+ const moeProfile = options.moeProfile ?? resolveMoeExecutionProfile(
197
+ { expertFormat, modelType: options.modelType },
198
+ { modelType: options.modelType }
81
199
  );
82
200
 
83
201
  if (!Number.isFinite(hiddenSize) || hiddenSize <= 0) {
@@ -93,16 +211,16 @@ export function validateMoeShape(config, options = {}) {
93
211
  throw new Error(`[MoE] topK must be in range [1, ${numExperts}], got ${moeTopK}.`);
94
212
  }
95
213
 
96
- if (modelType === 'gpt-oss') {
97
- const policy = selectRuleValue('kernels', 'moeGptoss', 'shapePolicy', { modelType });
214
+ const policy = resolveShapePolicy(moeProfile, { modelType: options.modelType });
215
+ if (policy != null) {
98
216
  if (policy.hiddenSizeDivisor == null || policy.intermediateSizeDivisor == null) {
99
- throw new Error('[MoE] GPT-OSS shapePolicy is missing hiddenSizeDivisor or intermediateSizeDivisor.');
217
+ throw new Error(`[MoE] ${moeProfile.label} shapePolicy is missing hiddenSizeDivisor or intermediateSizeDivisor.`);
100
218
  }
101
219
  const hiddenDivisor = policy.hiddenSizeDivisor;
102
220
  const intermediateDivisor = policy.intermediateSizeDivisor;
103
221
  if (hiddenSize % hiddenDivisor !== 0 || intermediateSize % intermediateDivisor !== 0) {
104
222
  throw new Error(
105
- `[MoE] GPT-OSS shape policy violation: hiddenSize (${hiddenSize}) % ${hiddenDivisor} = ${hiddenSize % hiddenDivisor}, ` +
223
+ `[MoE] ${moeProfile.label} shape policy violation: hiddenSize (${hiddenSize}) % ${hiddenDivisor} = ${hiddenSize % hiddenDivisor}, ` +
106
224
  `intermediateSize (${intermediateSize}) % ${intermediateDivisor} = ${intermediateSize % intermediateDivisor}.`
107
225
  );
108
226
  }
@@ -162,6 +162,9 @@ export function doConv(
162
162
  layerIdx?: number;
163
163
  label?: string;
164
164
  kernelPath?: unknown;
165
+ weightDtype?: 'f16' | 'f32' | 'q4k';
166
+ convInProjDtype?: 'f16' | 'f32' | 'q4k';
167
+ convOutProjDtype?: 'f16' | 'f32' | 'q4k';
165
168
  conv2d?: {
166
169
  enabled: boolean;
167
170
  inChannels: number;
@@ -16,7 +16,7 @@ import {
16
16
  } from '../../../gpu/kernels/cast.js';
17
17
  import { createTensor } from '../../../gpu/tensor.js';
18
18
  import { releaseBuffer, readBuffer, acquireBuffer, uploadData } from '../../../memory/buffer-pool.js';
19
- import { isGpuBufferInstance } from '../../../gpu/weight-buffer.js';
19
+ import { getWeightDtype, isGpuBufferInstance, isWeightBuffer } from '../../../gpu/weight-buffer.js';
20
20
  import { kernelTrace, traceStep } from './kernel-trace.js';
21
21
  import {
22
22
  runLayerAttentionGPU,
@@ -180,6 +180,17 @@ export async function doMatmulRMSNormFused(input, weight, normWeight, options, r
180
180
  return resultTensor;
181
181
  }
182
182
 
183
+ function requireProjectionWeightDtype(weight, declared, label) {
184
+ if (isWeightBuffer(weight)) {
185
+ return getWeightDtype(weight);
186
+ }
187
+ const dtype = declared ?? weight?.dtype;
188
+ if (dtype === 'f16' || dtype === 'f32' || dtype === 'q4k') {
189
+ return dtype;
190
+ }
191
+ throw new Error(`${label} requires explicit weight dtype.`);
192
+ }
193
+
183
194
  export async function doConv(
184
195
  inputTensor,
185
196
  convInProj,
@@ -200,6 +211,16 @@ export async function doConv(
200
211
  if (!Number.isFinite(hiddenSize) || hiddenSize <= 0) {
201
212
  throw new Error('doConv requires hiddenSize > 0.');
202
213
  }
214
+ const convInProjDtype = requireProjectionWeightDtype(
215
+ convInProj,
216
+ options.convInProjDtype ?? options.weightDtype,
217
+ `${label}.in_proj`
218
+ );
219
+ const convOutProjDtype = requireProjectionWeightDtype(
220
+ convOutProj,
221
+ options.convOutProjDtype ?? options.weightDtype,
222
+ `${label}.out_proj`
223
+ );
203
224
 
204
225
  // LFM2 gated short convolution (GPU-native):
205
226
  // in_proj → 3×hidden → GPU kernel: split(B,C,x) + B*x + causal conv1d + C*conv_out → out_proj
@@ -225,6 +246,8 @@ export async function doConv(
225
246
  kernelPath,
226
247
  role: 'conv_in_proj',
227
248
  executionPolicies: options.executionPolicies ?? null,
249
+ bDtype: convInProjDtype,
250
+ outputDtype: inputTensor.dtype,
228
251
  },
229
252
  recorder
230
253
  );
@@ -265,6 +288,8 @@ export async function doConv(
265
288
  kernelPath,
266
289
  role: 'conv_out_proj',
267
290
  executionPolicies: options.executionPolicies ?? null,
291
+ bDtype: convOutProjDtype,
292
+ outputDtype: convOut.dtype,
268
293
  },
269
294
  recorder
270
295
  );
@@ -431,7 +431,7 @@ export async function ensurePleScaledProjectionNormWeight(context, combineScale
431
431
  }
432
432
  scaledProjectionNormWeight = createPleProjectionNormTensor(
433
433
  scaledBuffer,
434
- getPleProjectionNormDtype(projectionNormWeight) ?? 'f32',
434
+ inferPleProjectionNormDtype(projectionNormWeight, hiddenSizePerLayerInput),
435
435
  hiddenSizePerLayerInput,
436
436
  'per_layer_projection_norm_scaled'
437
437
  );
@@ -474,6 +474,17 @@ function getEmbeddingDtype(weight) {
474
474
  return getWeightDtype(weight);
475
475
  }
476
476
 
477
+ function requirePleSourceDtype(rawDtype, label, allowed = ['f16', 'f32']) {
478
+ if (rawDtype == null) {
479
+ throw new Error(`${label} requires source dtype metadata.`);
480
+ }
481
+ const sourceDtype = String(rawDtype).toLowerCase();
482
+ if (!allowed.includes(sourceDtype)) {
483
+ throw new Error(`${label} requires ${allowed.join('/')} source rows; got "${sourceDtype}".`);
484
+ }
485
+ return sourceDtype;
486
+ }
487
+
477
488
  function getEmbeddingTranspose(weight) {
478
489
  if (isWeightBuffer(weight) || isCpuWeightBuffer(weight)) {
479
490
  return weight.layout === 'column';
@@ -721,11 +732,12 @@ function storePreparedTokenEntry(cache, tokenId, buffers, sessionConfig, activat
721
732
  }
722
733
 
723
734
  function getPleRangeRowLoadConfig(embedTokensPerLayer, totalPerLayerHiddenSize) {
724
- const sourceDtype = String(
735
+ const sourceDtype = requirePleSourceDtype(
725
736
  (isCpuWeightBuffer(embedTokensPerLayer) ? embedTokensPerLayer.data?.sourceDtype : null)
726
- ?? embedTokensPerLayer?.dtype
727
- ?? 'f32'
728
- ).toLowerCase();
737
+ ?? embedTokensPerLayer?.dtype,
738
+ 'Gemma 4 range-backed per-layer input rows',
739
+ ['f16', 'bf16', 'f32']
740
+ );
729
741
  const bytesPerElement = (sourceDtype === 'f16' || sourceDtype === 'bf16') ? 2 : 4;
730
742
  return {
731
743
  sourceDtype,
@@ -916,12 +928,10 @@ export async function ensurePleGpuSplitTablesRuntime(context) {
916
928
  throw new Error('Gemma 4 gpu_split_tables materialization requires a range-backed CPU embedTokensPerLayer source.');
917
929
  }
918
930
 
919
- const sourceDtype = String(embedTokensPerLayer.data?.sourceDtype ?? embedTokensPerLayer.dtype ?? 'f32').toLowerCase();
920
- if (sourceDtype !== 'f16' && sourceDtype !== 'f32') {
921
- throw new Error(
922
- `Gemma 4 gpu_split_tables materialization requires f16/f32 source rows; got "${sourceDtype}".`
923
- );
924
- }
931
+ const sourceDtype = requirePleSourceDtype(
932
+ embedTokensPerLayer.data?.sourceDtype ?? embedTokensPerLayer.dtype,
933
+ 'Gemma 4 gpu_split_tables materialization'
934
+ );
925
935
 
926
936
  const device = getDevice();
927
937
  if (!device) {
@@ -1124,7 +1134,11 @@ export async function ensurePleGpuHotVocabularyRuntime(context) {
1124
1134
  return null;
1125
1135
  }
1126
1136
 
1127
- const sourceDtype = String(embedTokensPerLayer.data?.sourceDtype ?? embedTokensPerLayer.dtype ?? 'f32').toLowerCase();
1137
+ const sourceDtype = requirePleSourceDtype(
1138
+ embedTokensPerLayer.data?.sourceDtype ?? embedTokensPerLayer.dtype,
1139
+ 'Gemma 4 hot vocabulary cache',
1140
+ [policy.outputDtype]
1141
+ );
1128
1142
  if (sourceDtype !== policy.outputDtype) {
1129
1143
  throw new Error(
1130
1144
  `Gemma 4 hot vocabulary cache requires source dtype "${policy.outputDtype}" for zero-copy row packing; ` +
@@ -1551,6 +1565,7 @@ export async function preparePerLayerInputs(tokenIds, inputEmbedsTensor, context
1551
1565
  hiddenSize: hiddenSizePerLayerInput,
1552
1566
  vocabSize: useHotVocabularyTables ? (hotVocabularyRuntime.sentinelIndex + 1) : vocabSizePerLayerInput,
1553
1567
  scaleEmbeddings: true,
1568
+ embeddingScale: null,
1554
1569
  probeStage: 'per_layer_embed_out',
1555
1570
  recorder,
1556
1571
  numTokens,
@@ -58,24 +58,65 @@ export function softmax(logits) {
58
58
  return exps;
59
59
  }
60
60
 
61
- function countFiniteCandidates(logits) {
62
- let finiteCandidateCount = 0;
61
+ function summarizeLogitHealth(logits) {
62
+ const health = {
63
+ length: logits.length,
64
+ finite: 0,
65
+ nan: 0,
66
+ posInf: 0,
67
+ negInf: 0,
68
+ min: Infinity,
69
+ max: -Infinity,
70
+ };
63
71
  for (let i = 0; i < logits.length; i++) {
64
- if (Number.isFinite(logits[i])) {
65
- finiteCandidateCount += 1;
72
+ const value = logits[i];
73
+ if (Number.isNaN(value)) {
74
+ health.nan += 1;
75
+ continue;
76
+ }
77
+ if (value === Infinity) {
78
+ health.posInf += 1;
79
+ continue;
80
+ }
81
+ if (value === -Infinity) {
82
+ health.negInf += 1;
83
+ continue;
66
84
  }
85
+ if (Number.isFinite(value)) {
86
+ health.finite += 1;
87
+ if (value < health.min) health.min = value;
88
+ if (value > health.max) health.max = value;
89
+ }
90
+ }
91
+ if (health.finite === 0) {
92
+ health.min = null;
93
+ health.max = null;
67
94
  }
68
- return finiteCandidateCount;
95
+ return health;
96
+ }
97
+
98
+ function formatLogitHealth(health) {
99
+ return [
100
+ `len=${health.length}`,
101
+ `finite=${health.finite}`,
102
+ `nan=${health.nan}`,
103
+ `posInf=${health.posInf}`,
104
+ `negInf=${health.negInf}`,
105
+ `min=${health.min === null ? 'n/a' : Number(health.min).toPrecision(6)}`,
106
+ `max=${health.max === null ? 'n/a' : Number(health.max).toPrecision(6)}`,
107
+ ].join(',');
69
108
  }
70
109
 
71
- function assertFiniteSamplingCandidates(logits, label) {
72
- const finiteCandidateCount = countFiniteCandidates(logits);
73
- if (finiteCandidateCount > 0) {
110
+ function assertFiniteSamplingCandidates(logits, label, beforeMaskHealth) {
111
+ const afterMaskHealth = summarizeLogitHealth(logits);
112
+ if (afterMaskHealth.finite > 0) {
74
113
  return;
75
114
  }
76
115
  throw new Error(
77
116
  `[Sampling] ${label} has no finite candidate logits after masking suppressed tokens. ` +
78
- 'Upstream decode likely produced NaN/Inf or an all-masked distribution.'
117
+ 'Upstream decode likely produced NaN/Inf or an all-masked distribution. ' +
118
+ `beforeMask={${formatLogitHealth(beforeMaskHealth || afterMaskHealth)}} ` +
119
+ `afterMask={${formatLogitHealth(afterMaskHealth)}}`
79
120
  );
80
121
  }
81
122
 
@@ -215,6 +256,7 @@ function sampleFromLogitCandidates(candidates, temperature, seed, decode, debug,
215
256
 
216
257
  export function sample(logits, opts) {
217
258
  const { temperature, topP, topK, decode, debug = false, padTokenId, seed, suppressTokenIds } = opts;
259
+ const beforeMaskHealth = summarizeLogitHealth(logits);
218
260
 
219
261
  if (padTokenId !== undefined && padTokenId >= 0 && padTokenId < logits.length) {
220
262
  logits[padTokenId] = -Infinity;
@@ -227,7 +269,7 @@ export function sample(logits, opts) {
227
269
  }
228
270
  }
229
271
 
230
- assertFiniteSamplingCandidates(logits, 'Logits');
272
+ assertFiniteSamplingCandidates(logits, 'Logits', beforeMaskHealth);
231
273
 
232
274
  // Greedy (argmax) when temperature = 0
233
275
  if (temperature === 0) {
@@ -42,14 +42,19 @@ export function getWeightBuffer(weight, label, deviceOverride = null) {
42
42
 
43
43
 
44
44
  let data;
45
- let bufferDtype = 'f32';
45
+ let bufferDtype;
46
46
  if (isCpuWeightBuffer(weight)) {
47
47
  data = weight.data;
48
- bufferDtype = weight.dtype ?? 'f32';
48
+ if (!weight.dtype) {
49
+ throw new Error(`Weight buffer "${label}" is missing dtype metadata.`);
50
+ }
51
+ bufferDtype = weight.dtype;
49
52
  } else if (weight instanceof Float32Array) {
50
53
  data = weight;
54
+ bufferDtype = 'f32';
51
55
  } else {
52
56
  data = new Float32Array( (weight));
57
+ bufferDtype = 'f32';
53
58
  }
54
59
 
55
60
  const buf = acquireBuffer(data.byteLength, undefined, label);
@@ -245,6 +245,23 @@ export function buildConservativeMultimodalGenerationOptions(options = {}) {
245
245
  };
246
246
  }
247
247
 
248
+ function resolveMultimodalMaxTokens(runtimeConfig, requestedMaxTokens) {
249
+ if (requestedMaxTokens !== undefined) {
250
+ return requirePositiveInteger(requestedMaxTokens, 'maxTokens');
251
+ }
252
+ return requirePositiveInteger(
253
+ runtimeConfig?.inference?.generation?.multimodalMaxTokens,
254
+ 'runtime.inference.generation.multimodalMaxTokens'
255
+ );
256
+ }
257
+
258
+ function requirePositiveInteger(value, label) {
259
+ if (!Number.isInteger(value) || value <= 0) {
260
+ throw new Error(`${label} must be a positive integer.`);
261
+ }
262
+ return value;
263
+ }
264
+
248
265
  // ============================================================================
249
266
  // Main Inference Pipeline Class
250
267
  // ============================================================================
@@ -1145,7 +1162,7 @@ export class InferencePipeline extends PipelineState {
1145
1162
 
1146
1163
  // Step 3: Generate with embedding override at the image token offset.
1147
1164
  const tokens = [];
1148
- const maxGen = maxTokens ?? 512;
1165
+ const maxGen = resolveMultimodalMaxTokens(this.runtimeConfig, maxTokens);
1149
1166
  const stopTokenIds = this.modelConfig.stopTokenIds;
1150
1167
 
1151
1168
  try {
@@ -1274,7 +1291,7 @@ export class InferencePipeline extends PipelineState {
1274
1291
 
1275
1292
  // Step 3: Generate with embedding override at the video token offset
1276
1293
  const tokens = [];
1277
- const maxGen = maxTokens ?? 512;
1294
+ const maxGen = resolveMultimodalMaxTokens(this.runtimeConfig, maxTokens);
1278
1295
  const stopTokenIds = this.modelConfig.stopTokenIds;
1279
1296
 
1280
1297
  try {
@@ -1404,7 +1421,7 @@ export class InferencePipeline extends PipelineState {
1404
1421
 
1405
1422
  // Step 4: Generate with embedding override at the audio token offset
1406
1423
  const tokens = [];
1407
- const maxGen = maxTokens ?? 512;
1424
+ const maxGen = resolveMultimodalMaxTokens(this.runtimeConfig, maxTokens);
1408
1425
  const stopTokenIds = this.modelConfig.stopTokenIds;
1409
1426
 
1410
1427
  try {
@@ -1560,7 +1577,10 @@ export class InferencePipeline extends PipelineState {
1560
1577
 
1561
1578
  async embed(prompt, options = {}) {
1562
1579
  assertNotAborted(options?.signal);
1563
- const result = await this.prefillWithEmbedding(prompt, options);
1580
+ const result = await this.prefillWithEmbedding(prompt, {
1581
+ ...options,
1582
+ __skipStateSnapshot: true,
1583
+ });
1564
1584
  assertNotAborted(options?.signal);
1565
1585
  return {
1566
1586
  embedding: result.embedding,
@@ -12,7 +12,10 @@ function isBosLikeLabel(value) {
12
12
 
13
13
  function isEosLikeLabel(value) {
14
14
  const normalized = normalizeSpecialTokenLabel(value);
15
- return normalized === '<eos>' || normalized === '</s>' || normalized.includes('eos');
15
+ return normalized === '<eos>'
16
+ || normalized === '</s>'
17
+ || normalized.includes('eos')
18
+ || normalized.includes('endoftext');
16
19
  }
17
20
 
18
21
  function classifySpecialTokenReference(ref, specialTokens) {
@@ -84,7 +87,25 @@ export function inferBundledTokenizerBehaviorFlags(tokenizerJson, specialTokens
84
87
  return { addBosToken: null, addEosToken: null };
85
88
  }
86
89
 
87
- const postProcessor = tokenizerJson.post_processor;
90
+ const findTemplateProcessing = (postProcessor) => {
91
+ if (!postProcessor || typeof postProcessor !== 'object' || Array.isArray(postProcessor)) {
92
+ return null;
93
+ }
94
+ if (postProcessor.type === 'TemplateProcessing') {
95
+ return postProcessor;
96
+ }
97
+ if (postProcessor.type === 'Sequence' && Array.isArray(postProcessor.processors)) {
98
+ for (const child of postProcessor.processors) {
99
+ const template = findTemplateProcessing(child);
100
+ if (template) {
101
+ return template;
102
+ }
103
+ }
104
+ }
105
+ return null;
106
+ };
107
+
108
+ const postProcessor = findTemplateProcessing(tokenizerJson.post_processor);
88
109
  if (!postProcessor || postProcessor.type !== 'TemplateProcessing') {
89
110
  return { addBosToken: null, addEosToken: null };
90
111
  }
@@ -167,6 +167,16 @@ async function loadFirstExistingTensor(ctx, prefixes, suffixes, options = {}) {
167
167
  return null;
168
168
  }
169
169
 
170
+ function wrapFinalNormTensor(ctx, tensor, name, location) {
171
+ if (!isGpuBufferInstance(tensor) || !location?.shape) {
172
+ return tensor;
173
+ }
174
+ const dtype = selectRuleValue('loader', 'weights', 'floatLocationDtype', {
175
+ locationDtype: location.dtype,
176
+ });
177
+ return createWeightBuffer(tensor, dtype, 'row', location.shape, name);
178
+ }
179
+
170
180
  async function loadDiffusionGemmaSelfConditioning(ctx) {
171
181
  const preNorm = await loadFirstExistingTensor(
172
182
  ctx,
@@ -284,7 +294,8 @@ async function loadFinalNorm(ctx) {
284
294
  for (const name of finalNormNames) {
285
295
  const location = ctx.tensorLocations.get(name);
286
296
  if (location) {
287
- finalNorm = (await ctx.loadTensor(name, true, true));
297
+ const tensor = await ctx.loadTensor(name, true, true);
298
+ finalNorm = wrapFinalNormTensor(ctx, tensor, name, location);
288
299
  break;
289
300
  }
290
301
  }