doppler-gpu 0.4.5 → 0.4.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +45 -29
- package/assets/doppler-runtime-map.svg +82 -0
- package/assets/doppler.svg +1 -3
- package/package.json +25 -17
- package/src/cli/doppler-quickstart.js +1 -1
- package/src/cli/doppler-serve.js +2 -2
- package/src/client/doppler-registry.d.ts +19 -0
- package/src/client/doppler-registry.js +12 -0
- package/src/client/doppler-registry.json +50 -1
- package/src/client/runtime/model-session.d.ts +1 -0
- package/src/client/runtime/model-session.js +9 -0
- package/src/config/conversion/README.md +4 -0
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +28 -4
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +31 -5
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +30 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +30 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +30 -4
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +30 -5
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +30 -5
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +29 -4
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +29 -4
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +481 -0
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +29 -4
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +29 -4
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +30 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +29 -4
- package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +395 -0
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +429 -0
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +540 -0
- package/src/config/execution-contract-check.js +14 -10
- package/src/config/merge-contract-check.js +1 -0
- package/src/config/merge.js +30 -0
- package/src/config/param-validator.d.ts +5 -0
- package/src/config/param-validator.js +42 -8
- package/src/config/required-inference-fields-contract-check.js +14 -0
- package/src/config/schema/debug.schema.d.ts +2 -0
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/doppler.schema.d.ts +5 -1
- package/src/config/schema/doppler.schema.js +82 -292
- package/src/config/schema/inference.schema.d.ts +17 -3
- package/src/config/schema/kvcache.schema.d.ts +19 -3
- package/src/config/schema/kvcache.schema.js +2 -1
- package/src/config/schema/manifest.schema.d.ts +38 -1
- package/src/config/schema/manifest.schema.js +18 -0
- package/src/config/schema/policy-schema-registry.json +5 -0
- package/src/config/source-packages/litert/gemma-4-12b-it.json +20 -1
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +20 -1
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +21 -1
- package/src/config/transforms/execution-graph-transforms.d.ts +6 -0
- package/src/config/transforms/execution-graph-transforms.js +8 -0
- package/src/converter/conversion-plan.js +2 -0
- package/src/converter/parsers/transformer.js +27 -1
- package/src/converter/quantization-info.js +16 -4
- package/src/converter/rope-config.js +55 -3
- package/src/experimental/training/distillation/student-fixture.js +1 -1
- package/src/experimental/training/lora-pipeline.js +1 -1
- package/src/experimental/training/lora.js +2 -2
- package/src/experimental/training/suite.js +1 -1
- package/src/gpu/device.js +54 -11
- package/src/gpu/kernels/README.md +8 -1
- package/src/gpu/kernels/backward/attention_backward.js +8 -0
- package/src/gpu/kernels/dequant.d.ts +6 -6
- package/src/gpu/kernels/dequant.js +17 -6
- package/src/gpu/kernels/linear-attention-core.js +1 -6
- package/src/gpu/kernels/matmul-selection.d.ts +2 -2
- package/src/gpu/kernels/matmul-selection.js +9 -2
- package/src/gpu/kernels/matmul.d.ts +1 -1
- package/src/gpu/kernels/matmul.js +63 -9
- package/src/gpu/kernels/moe.d.ts +5 -1
- package/src/gpu/kernels/moe.js +5 -2
- package/src/gpu/kernels/softmax.d.ts +3 -3
- package/src/gpu/kernels/softmax.js +9 -2
- package/src/gpu/kernels/turboquant-codebook.js +1 -1
- package/src/gpu/kernels/types.d.ts +1 -1
- package/src/gpu/profiler.js +11 -1
- package/src/inference/browser-harness-model-helpers.js +4 -1
- package/src/inference/browser-harness-text-helpers.d.ts +28 -0
- package/src/inference/browser-harness-text-helpers.js +357 -116
- package/src/inference/browser-harness.d.ts +3 -3
- package/src/inference/browser-harness.js +266 -11
- package/src/inference/fixtures/rerank-semantic-fixtures.json +38 -0
- package/src/inference/kv-cache/quantized.js +16 -2
- package/src/inference/kv-cache/tiered.js +50 -15
- package/src/inference/moe-router.js +38 -7
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +1 -1
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +8 -6
- package/src/inference/pipelines/text/config.d.ts +5 -1
- package/src/inference/pipelines/text/config.js +135 -112
- package/src/inference/pipelines/text/embed.d.ts +1 -0
- package/src/inference/pipelines/text/embed.js +39 -12
- package/src/inference/pipelines/text/execution-plan.js +8 -2
- package/src/inference/pipelines/text/ffn/dense.js +15 -4
- package/src/inference/pipelines/text/ffn/moe.js +4 -3
- package/src/inference/pipelines/text/ffn/standard.d.ts +2 -1
- package/src/inference/pipelines/text/ffn/standard.js +25 -2
- package/src/inference/pipelines/text/generator-helpers.js +2 -1
- package/src/inference/pipelines/text/generator-runtime.d.ts +6 -2
- package/src/inference/pipelines/text/generator-runtime.js +53 -20
- package/src/inference/pipelines/text/generator-steps.js +3 -0
- package/src/inference/pipelines/text/generator.d.ts +1 -0
- package/src/inference/pipelines/text/generator.js +14 -1
- package/src/inference/pipelines/text/init-rope.js +54 -6
- package/src/inference/pipelines/text/init.js +63 -15
- package/src/inference/pipelines/text/layer.js +32 -5
- package/src/inference/pipelines/text/linear-attention.js +12 -8
- package/src/inference/pipelines/text/logits/gpu.d.ts +1 -1
- package/src/inference/pipelines/text/logits/gpu.js +76 -113
- package/src/inference/pipelines/text/logits/index.js +44 -87
- package/src/inference/pipelines/text/logits/precision-policy.d.ts +11 -0
- package/src/inference/pipelines/text/logits/precision-policy.js +103 -0
- package/src/inference/pipelines/text/logits/types.d.ts +1 -0
- package/src/inference/pipelines/text/logits/utils.d.ts +2 -0
- package/src/inference/pipelines/text/logits/utils.js +11 -0
- package/src/inference/pipelines/text/lora-apply.js +6 -4
- package/src/inference/pipelines/text/moe-cpu.js +25 -0
- package/src/inference/pipelines/text/moe-gpu.js +148 -97
- package/src/inference/pipelines/text/moe-shape-validator.d.ts +33 -3
- package/src/inference/pipelines/text/moe-shape-validator.js +149 -31
- package/src/inference/pipelines/text/ops.d.ts +3 -0
- package/src/inference/pipelines/text/ops.js +26 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +27 -12
- package/src/inference/pipelines/text/sampling.js +52 -10
- package/src/inference/pipelines/text/weights.js +7 -2
- package/src/inference/pipelines/text.js +24 -4
- package/src/inference/tokenizers/behavior-flags.js +23 -2
- package/src/loader/final-weights-loader.js +12 -1
- package/src/loader/tensors/tensor-loader.js +37 -3
- package/src/rules/execution-rules-contract-check.js +12 -3
- package/src/rules/inference/capability-transforms.rules.json +14 -2
- package/src/rules/inference/config.rules.json +26 -2
- package/src/rules/inference/dtype.rules.json +12 -5
- package/src/rules/inference/execution.rules.json +5 -19
- package/src/rules/inference/moe.rules.json +97 -0
- package/src/rules/loader/weights.rules.json +1 -0
- package/src/tooling/command-api-constants.js +2 -2
- package/src/tooling/command-api-family-normalizers.js +2 -1
- package/src/tooling/command-api.d.ts +1 -1
- package/src/tooling/command-runner-shared.d.ts +1 -1
- package/src/tooling/command-runner-shared.js +3 -1
- package/src/tooling/node-converter.js +20 -0
- package/src/tooling/node-source-runtime.js +5 -1
- package/src/version.js +1 -1
|
@@ -82,6 +82,7 @@
|
|
|
82
82
|
"rms_norm_eps": 0.000001,
|
|
83
83
|
"audio_architecture": "gemma4",
|
|
84
84
|
"num_hidden_layers": 0,
|
|
85
|
+
"num_attention_heads": 1,
|
|
85
86
|
"conv_kernel_size": 1,
|
|
86
87
|
"subsampling_conv_channels": [
|
|
87
88
|
1,
|
|
@@ -126,6 +127,7 @@
|
|
|
126
127
|
"ffn": {
|
|
127
128
|
"activation": "gelu",
|
|
128
129
|
"gatedActivation": true,
|
|
130
|
+
"branchMode": "auto",
|
|
129
131
|
"useDoubleWideMlp": false,
|
|
130
132
|
"swigluLimit": null
|
|
131
133
|
},
|
|
@@ -148,7 +150,10 @@
|
|
|
148
150
|
"yarnOriginalMaxPos": null,
|
|
149
151
|
"ropeLocalYarnBetaFast": null,
|
|
150
152
|
"ropeLocalYarnBetaSlow": null,
|
|
151
|
-
"ropeLocalYarnOriginalMaxPos": null
|
|
153
|
+
"ropeLocalYarnOriginalMaxPos": null,
|
|
154
|
+
"longropeShortFactor": null,
|
|
155
|
+
"longropeLongFactor": null,
|
|
156
|
+
"longropeOriginalMaxPos": null
|
|
152
157
|
},
|
|
153
158
|
"output": {
|
|
154
159
|
"finalLogitSoftcapping": 30,
|
|
@@ -156,7 +161,9 @@
|
|
|
156
161
|
"scaleEmbeddings": true,
|
|
157
162
|
"embeddingTranspose": false,
|
|
158
163
|
"embeddingVocabSize": 262144,
|
|
159
|
-
"embeddingPostprocessor": null
|
|
164
|
+
"embeddingPostprocessor": null,
|
|
165
|
+
"embeddingScale": null,
|
|
166
|
+
"logitInputScale": 1
|
|
160
167
|
},
|
|
161
168
|
"layerPattern": {
|
|
162
169
|
"type": "custom",
|
|
@@ -212,7 +219,8 @@
|
|
|
212
219
|
"sliding_attention",
|
|
213
220
|
"sliding_attention",
|
|
214
221
|
"full_attention"
|
|
215
|
-
]
|
|
222
|
+
],
|
|
223
|
+
"residualBranchScale": 1
|
|
216
224
|
},
|
|
217
225
|
"chatTemplate": {
|
|
218
226
|
"type": "gemma4",
|
|
@@ -233,7 +241,25 @@
|
|
|
233
241
|
"layout": "contiguous",
|
|
234
242
|
"pageSize": 256,
|
|
235
243
|
"tiering": {
|
|
236
|
-
"mode": "off"
|
|
244
|
+
"mode": "off",
|
|
245
|
+
"hotWindow": 1024,
|
|
246
|
+
"coldPageSize": 256,
|
|
247
|
+
"coldDtype": "f16",
|
|
248
|
+
"compression": {
|
|
249
|
+
"mode": "none",
|
|
250
|
+
"blockSize": 1,
|
|
251
|
+
"bitWidth": 4,
|
|
252
|
+
"prodMode": false
|
|
253
|
+
},
|
|
254
|
+
"gating": {
|
|
255
|
+
"mode": "auto",
|
|
256
|
+
"minAluBwRatio": 0
|
|
257
|
+
}
|
|
258
|
+
},
|
|
259
|
+
"quantization": {
|
|
260
|
+
"mode": "none",
|
|
261
|
+
"bitWidth": 4,
|
|
262
|
+
"prodMode": false
|
|
237
263
|
}
|
|
238
264
|
},
|
|
239
265
|
"decodeLoop": {
|
|
@@ -84,6 +84,7 @@
|
|
|
84
84
|
"rms_norm_eps": 0.000001,
|
|
85
85
|
"audio_architecture": "gemma4",
|
|
86
86
|
"num_hidden_layers": 0,
|
|
87
|
+
"num_attention_heads": 1,
|
|
87
88
|
"conv_kernel_size": 1,
|
|
88
89
|
"subsampling_conv_channels": [
|
|
89
90
|
1,
|
|
@@ -128,6 +129,7 @@
|
|
|
128
129
|
"ffn": {
|
|
129
130
|
"activation": "gelu",
|
|
130
131
|
"gatedActivation": true,
|
|
132
|
+
"branchMode": "auto",
|
|
131
133
|
"useDoubleWideMlp": false,
|
|
132
134
|
"swigluLimit": null
|
|
133
135
|
},
|
|
@@ -150,7 +152,10 @@
|
|
|
150
152
|
"yarnOriginalMaxPos": null,
|
|
151
153
|
"ropeLocalYarnBetaFast": null,
|
|
152
154
|
"ropeLocalYarnBetaSlow": null,
|
|
153
|
-
"ropeLocalYarnOriginalMaxPos": null
|
|
155
|
+
"ropeLocalYarnOriginalMaxPos": null,
|
|
156
|
+
"longropeShortFactor": null,
|
|
157
|
+
"longropeLongFactor": null,
|
|
158
|
+
"longropeOriginalMaxPos": null
|
|
154
159
|
},
|
|
155
160
|
"output": {
|
|
156
161
|
"finalLogitSoftcapping": 30,
|
|
@@ -158,7 +163,9 @@
|
|
|
158
163
|
"scaleEmbeddings": true,
|
|
159
164
|
"embeddingTranspose": false,
|
|
160
165
|
"embeddingVocabSize": 262144,
|
|
161
|
-
"embeddingPostprocessor": null
|
|
166
|
+
"embeddingPostprocessor": null,
|
|
167
|
+
"embeddingScale": null,
|
|
168
|
+
"logitInputScale": 1
|
|
162
169
|
},
|
|
163
170
|
"layerPattern": {
|
|
164
171
|
"type": "custom",
|
|
@@ -214,7 +221,8 @@
|
|
|
214
221
|
"sliding_attention",
|
|
215
222
|
"sliding_attention",
|
|
216
223
|
"full_attention"
|
|
217
|
-
]
|
|
224
|
+
],
|
|
225
|
+
"residualBranchScale": 1
|
|
218
226
|
},
|
|
219
227
|
"chatTemplate": {
|
|
220
228
|
"type": "gemma4",
|
|
@@ -235,7 +243,25 @@
|
|
|
235
243
|
"layout": "contiguous",
|
|
236
244
|
"pageSize": 256,
|
|
237
245
|
"tiering": {
|
|
238
|
-
"mode": "off"
|
|
246
|
+
"mode": "off",
|
|
247
|
+
"hotWindow": 1024,
|
|
248
|
+
"coldPageSize": 256,
|
|
249
|
+
"coldDtype": "f16",
|
|
250
|
+
"compression": {
|
|
251
|
+
"mode": "none",
|
|
252
|
+
"blockSize": 1,
|
|
253
|
+
"bitWidth": 4,
|
|
254
|
+
"prodMode": false
|
|
255
|
+
},
|
|
256
|
+
"gating": {
|
|
257
|
+
"mode": "auto",
|
|
258
|
+
"minAluBwRatio": 0
|
|
259
|
+
}
|
|
260
|
+
},
|
|
261
|
+
"quantization": {
|
|
262
|
+
"mode": "none",
|
|
263
|
+
"bitWidth": 4,
|
|
264
|
+
"prodMode": false
|
|
239
265
|
}
|
|
240
266
|
},
|
|
241
267
|
"decodeLoop": {
|
|
@@ -69,7 +69,7 @@
|
|
|
69
69
|
"weightsRef": {
|
|
70
70
|
"weightPackId": "gemma-4-31b-it-text-q4k-ehf16-af32-wp-catalog-v1",
|
|
71
71
|
"artifactRoot": "../gemma-4-31b-it-text-q4k-ehf16-af32",
|
|
72
|
-
"manifestDigest": "sha256:
|
|
72
|
+
"manifestDigest": "sha256:dafb0b812dd714c2b63d7a3796d463c60a0ba3b16f9e0933744e24b291bf202c",
|
|
73
73
|
"shardSetHash": "sha256:54933058e14fa0be4480912e641b897730db3e8ee7e871bdb2a92fd404ade146"
|
|
74
74
|
}
|
|
75
75
|
},
|
|
@@ -94,6 +94,7 @@
|
|
|
94
94
|
"ffn": {
|
|
95
95
|
"activation": "gelu",
|
|
96
96
|
"gatedActivation": true,
|
|
97
|
+
"branchMode": "auto",
|
|
97
98
|
"useDoubleWideMlp": false,
|
|
98
99
|
"swigluLimit": null
|
|
99
100
|
},
|
|
@@ -116,7 +117,10 @@
|
|
|
116
117
|
"yarnOriginalMaxPos": null,
|
|
117
118
|
"ropeLocalYarnBetaFast": null,
|
|
118
119
|
"ropeLocalYarnBetaSlow": null,
|
|
119
|
-
"ropeLocalYarnOriginalMaxPos": null
|
|
120
|
+
"ropeLocalYarnOriginalMaxPos": null,
|
|
121
|
+
"longropeShortFactor": null,
|
|
122
|
+
"longropeLongFactor": null,
|
|
123
|
+
"longropeOriginalMaxPos": null
|
|
120
124
|
},
|
|
121
125
|
"output": {
|
|
122
126
|
"finalLogitSoftcapping": 30,
|
|
@@ -124,7 +128,9 @@
|
|
|
124
128
|
"scaleEmbeddings": true,
|
|
125
129
|
"embeddingTranspose": false,
|
|
126
130
|
"embeddingVocabSize": 262144,
|
|
127
|
-
"embeddingPostprocessor": null
|
|
131
|
+
"embeddingPostprocessor": null,
|
|
132
|
+
"embeddingScale": null,
|
|
133
|
+
"logitInputScale": 1
|
|
128
134
|
},
|
|
129
135
|
"layerPattern": {
|
|
130
136
|
"type": "custom",
|
|
@@ -192,7 +198,8 @@
|
|
|
192
198
|
"sliding_attention",
|
|
193
199
|
"sliding_attention",
|
|
194
200
|
"full_attention"
|
|
195
|
-
]
|
|
201
|
+
],
|
|
202
|
+
"residualBranchScale": 1
|
|
196
203
|
},
|
|
197
204
|
"chatTemplate": {
|
|
198
205
|
"type": "gemma4",
|
|
@@ -213,7 +220,25 @@
|
|
|
213
220
|
"layout": "contiguous",
|
|
214
221
|
"pageSize": 256,
|
|
215
222
|
"tiering": {
|
|
216
|
-
"mode": "off"
|
|
223
|
+
"mode": "off",
|
|
224
|
+
"hotWindow": 1024,
|
|
225
|
+
"coldPageSize": 256,
|
|
226
|
+
"coldDtype": "f16",
|
|
227
|
+
"compression": {
|
|
228
|
+
"mode": "none",
|
|
229
|
+
"blockSize": 1,
|
|
230
|
+
"bitWidth": 4,
|
|
231
|
+
"prodMode": false
|
|
232
|
+
},
|
|
233
|
+
"gating": {
|
|
234
|
+
"mode": "auto",
|
|
235
|
+
"minAluBwRatio": 0
|
|
236
|
+
}
|
|
237
|
+
},
|
|
238
|
+
"quantization": {
|
|
239
|
+
"mode": "none",
|
|
240
|
+
"bitWidth": 4,
|
|
241
|
+
"prodMode": false
|
|
217
242
|
}
|
|
218
243
|
},
|
|
219
244
|
"decodeLoop": {
|
|
@@ -88,6 +88,7 @@
|
|
|
88
88
|
"ffn": {
|
|
89
89
|
"activation": "gelu",
|
|
90
90
|
"gatedActivation": true,
|
|
91
|
+
"branchMode": "auto",
|
|
91
92
|
"useDoubleWideMlp": false,
|
|
92
93
|
"swigluLimit": null
|
|
93
94
|
},
|
|
@@ -110,7 +111,10 @@
|
|
|
110
111
|
"yarnOriginalMaxPos": null,
|
|
111
112
|
"ropeLocalYarnBetaFast": null,
|
|
112
113
|
"ropeLocalYarnBetaSlow": null,
|
|
113
|
-
"ropeLocalYarnOriginalMaxPos": null
|
|
114
|
+
"ropeLocalYarnOriginalMaxPos": null,
|
|
115
|
+
"longropeShortFactor": null,
|
|
116
|
+
"longropeLongFactor": null,
|
|
117
|
+
"longropeOriginalMaxPos": null
|
|
114
118
|
},
|
|
115
119
|
"output": {
|
|
116
120
|
"finalLogitSoftcapping": 30,
|
|
@@ -118,7 +122,9 @@
|
|
|
118
122
|
"scaleEmbeddings": true,
|
|
119
123
|
"embeddingTranspose": false,
|
|
120
124
|
"embeddingVocabSize": 262144,
|
|
121
|
-
"embeddingPostprocessor": null
|
|
125
|
+
"embeddingPostprocessor": null,
|
|
126
|
+
"embeddingScale": null,
|
|
127
|
+
"logitInputScale": 1
|
|
122
128
|
},
|
|
123
129
|
"layerPattern": {
|
|
124
130
|
"type": "custom",
|
|
@@ -186,7 +192,8 @@
|
|
|
186
192
|
"sliding_attention",
|
|
187
193
|
"sliding_attention",
|
|
188
194
|
"full_attention"
|
|
189
|
-
]
|
|
195
|
+
],
|
|
196
|
+
"residualBranchScale": 1
|
|
190
197
|
},
|
|
191
198
|
"chatTemplate": {
|
|
192
199
|
"type": "gemma4",
|
|
@@ -207,7 +214,25 @@
|
|
|
207
214
|
"layout": "contiguous",
|
|
208
215
|
"pageSize": 256,
|
|
209
216
|
"tiering": {
|
|
210
|
-
"mode": "off"
|
|
217
|
+
"mode": "off",
|
|
218
|
+
"hotWindow": 1024,
|
|
219
|
+
"coldPageSize": 256,
|
|
220
|
+
"coldDtype": "f16",
|
|
221
|
+
"compression": {
|
|
222
|
+
"mode": "none",
|
|
223
|
+
"blockSize": 1,
|
|
224
|
+
"bitWidth": 4,
|
|
225
|
+
"prodMode": false
|
|
226
|
+
},
|
|
227
|
+
"gating": {
|
|
228
|
+
"mode": "auto",
|
|
229
|
+
"minAluBwRatio": 0
|
|
230
|
+
}
|
|
231
|
+
},
|
|
232
|
+
"quantization": {
|
|
233
|
+
"mode": "none",
|
|
234
|
+
"bitWidth": 4,
|
|
235
|
+
"prodMode": false
|
|
211
236
|
}
|
|
212
237
|
},
|
|
213
238
|
"decodeLoop": {
|
|
@@ -67,7 +67,7 @@
|
|
|
67
67
|
"weightsRef": {
|
|
68
68
|
"weightPackId": "gemma-4-e2b-it-q4k-ehf16-af32-int4ple-wp-catalog-v1",
|
|
69
69
|
"artifactRoot": "../gemma-4-e2b-it-q4k-ehf16-af32-int4ple",
|
|
70
|
-
"manifestDigest": "sha256:
|
|
70
|
+
"manifestDigest": "sha256:9d6d08271f126f7a64fddb5ff0a2a30261e1676d9369baa1039b215cf8fd32d9",
|
|
71
71
|
"shardSetHash": "sha256:7575f4e89b3938a3ca4f8d7fc50fb46d4cfbd1d4dcc3b6ed62b6fb53424678ea"
|
|
72
72
|
}
|
|
73
73
|
},
|
|
@@ -95,6 +95,7 @@
|
|
|
95
95
|
"ffn": {
|
|
96
96
|
"activation": "gelu",
|
|
97
97
|
"gatedActivation": true,
|
|
98
|
+
"branchMode": "auto",
|
|
98
99
|
"useDoubleWideMlp": true,
|
|
99
100
|
"swigluLimit": null
|
|
100
101
|
},
|
|
@@ -117,7 +118,10 @@
|
|
|
117
118
|
"yarnOriginalMaxPos": null,
|
|
118
119
|
"ropeLocalYarnBetaFast": null,
|
|
119
120
|
"ropeLocalYarnBetaSlow": null,
|
|
120
|
-
"ropeLocalYarnOriginalMaxPos": null
|
|
121
|
+
"ropeLocalYarnOriginalMaxPos": null,
|
|
122
|
+
"longropeShortFactor": null,
|
|
123
|
+
"longropeLongFactor": null,
|
|
124
|
+
"longropeOriginalMaxPos": null
|
|
121
125
|
},
|
|
122
126
|
"output": {
|
|
123
127
|
"finalLogitSoftcapping": 30,
|
|
@@ -125,14 +129,17 @@
|
|
|
125
129
|
"scaleEmbeddings": true,
|
|
126
130
|
"embeddingTranspose": false,
|
|
127
131
|
"embeddingVocabSize": null,
|
|
128
|
-
"embeddingPostprocessor": null
|
|
132
|
+
"embeddingPostprocessor": null,
|
|
133
|
+
"embeddingScale": null,
|
|
134
|
+
"logitInputScale": 1
|
|
129
135
|
},
|
|
130
136
|
"layerPattern": {
|
|
131
137
|
"type": "every_n",
|
|
132
138
|
"globalPattern": null,
|
|
133
139
|
"period": 5,
|
|
134
140
|
"offset": 4,
|
|
135
|
-
"layerTypes": null
|
|
141
|
+
"layerTypes": null,
|
|
142
|
+
"residualBranchScale": 1
|
|
136
143
|
},
|
|
137
144
|
"chatTemplate": {
|
|
138
145
|
"type": "gemma4",
|
|
@@ -153,7 +160,25 @@
|
|
|
153
160
|
"layout": "contiguous",
|
|
154
161
|
"pageSize": 256,
|
|
155
162
|
"tiering": {
|
|
156
|
-
"mode": "off"
|
|
163
|
+
"mode": "off",
|
|
164
|
+
"hotWindow": 1024,
|
|
165
|
+
"coldPageSize": 256,
|
|
166
|
+
"coldDtype": "f16",
|
|
167
|
+
"compression": {
|
|
168
|
+
"mode": "none",
|
|
169
|
+
"blockSize": 1,
|
|
170
|
+
"bitWidth": 4,
|
|
171
|
+
"prodMode": false
|
|
172
|
+
},
|
|
173
|
+
"gating": {
|
|
174
|
+
"mode": "auto",
|
|
175
|
+
"minAluBwRatio": 0
|
|
176
|
+
}
|
|
177
|
+
},
|
|
178
|
+
"quantization": {
|
|
179
|
+
"mode": "none",
|
|
180
|
+
"bitWidth": 4,
|
|
181
|
+
"prodMode": false
|
|
157
182
|
}
|
|
158
183
|
},
|
|
159
184
|
"decodeLoop": {
|
|
@@ -88,6 +88,7 @@
|
|
|
88
88
|
"ffn": {
|
|
89
89
|
"activation": "gelu",
|
|
90
90
|
"gatedActivation": true,
|
|
91
|
+
"branchMode": "auto",
|
|
91
92
|
"useDoubleWideMlp": true,
|
|
92
93
|
"swigluLimit": null
|
|
93
94
|
},
|
|
@@ -110,7 +111,10 @@
|
|
|
110
111
|
"yarnOriginalMaxPos": null,
|
|
111
112
|
"ropeLocalYarnBetaFast": null,
|
|
112
113
|
"ropeLocalYarnBetaSlow": null,
|
|
113
|
-
"ropeLocalYarnOriginalMaxPos": null
|
|
114
|
+
"ropeLocalYarnOriginalMaxPos": null,
|
|
115
|
+
"longropeShortFactor": null,
|
|
116
|
+
"longropeLongFactor": null,
|
|
117
|
+
"longropeOriginalMaxPos": null
|
|
114
118
|
},
|
|
115
119
|
"output": {
|
|
116
120
|
"finalLogitSoftcapping": 30,
|
|
@@ -118,14 +122,17 @@
|
|
|
118
122
|
"scaleEmbeddings": true,
|
|
119
123
|
"embeddingTranspose": false,
|
|
120
124
|
"embeddingVocabSize": null,
|
|
121
|
-
"embeddingPostprocessor": null
|
|
125
|
+
"embeddingPostprocessor": null,
|
|
126
|
+
"embeddingScale": null,
|
|
127
|
+
"logitInputScale": 1
|
|
122
128
|
},
|
|
123
129
|
"layerPattern": {
|
|
124
130
|
"type": "every_n",
|
|
125
131
|
"globalPattern": null,
|
|
126
132
|
"period": 5,
|
|
127
133
|
"offset": 4,
|
|
128
|
-
"layerTypes": null
|
|
134
|
+
"layerTypes": null,
|
|
135
|
+
"residualBranchScale": 1
|
|
129
136
|
},
|
|
130
137
|
"chatTemplate": {
|
|
131
138
|
"type": "gemma4",
|
|
@@ -146,7 +153,25 @@
|
|
|
146
153
|
"layout": "contiguous",
|
|
147
154
|
"pageSize": 256,
|
|
148
155
|
"tiering": {
|
|
149
|
-
"mode": "off"
|
|
156
|
+
"mode": "off",
|
|
157
|
+
"hotWindow": 1024,
|
|
158
|
+
"coldPageSize": 256,
|
|
159
|
+
"coldDtype": "f16",
|
|
160
|
+
"compression": {
|
|
161
|
+
"mode": "none",
|
|
162
|
+
"blockSize": 1,
|
|
163
|
+
"bitWidth": 4,
|
|
164
|
+
"prodMode": false
|
|
165
|
+
},
|
|
166
|
+
"gating": {
|
|
167
|
+
"mode": "auto",
|
|
168
|
+
"minAluBwRatio": 0
|
|
169
|
+
}
|
|
170
|
+
},
|
|
171
|
+
"quantization": {
|
|
172
|
+
"mode": "none",
|
|
173
|
+
"bitWidth": 4,
|
|
174
|
+
"prodMode": false
|
|
150
175
|
}
|
|
151
176
|
},
|
|
152
177
|
"decodeLoop": {
|
|
@@ -84,6 +84,7 @@
|
|
|
84
84
|
"ffn": {
|
|
85
85
|
"activation": "gelu",
|
|
86
86
|
"gatedActivation": true,
|
|
87
|
+
"branchMode": "auto",
|
|
87
88
|
"useDoubleWideMlp": true,
|
|
88
89
|
"swigluLimit": null
|
|
89
90
|
},
|
|
@@ -106,7 +107,10 @@
|
|
|
106
107
|
"yarnOriginalMaxPos": null,
|
|
107
108
|
"ropeLocalYarnBetaFast": null,
|
|
108
109
|
"ropeLocalYarnBetaSlow": null,
|
|
109
|
-
"ropeLocalYarnOriginalMaxPos": null
|
|
110
|
+
"ropeLocalYarnOriginalMaxPos": null,
|
|
111
|
+
"longropeShortFactor": null,
|
|
112
|
+
"longropeLongFactor": null,
|
|
113
|
+
"longropeOriginalMaxPos": null
|
|
110
114
|
},
|
|
111
115
|
"output": {
|
|
112
116
|
"finalLogitSoftcapping": 30,
|
|
@@ -114,14 +118,17 @@
|
|
|
114
118
|
"scaleEmbeddings": true,
|
|
115
119
|
"embeddingTranspose": false,
|
|
116
120
|
"embeddingVocabSize": null,
|
|
117
|
-
"embeddingPostprocessor": null
|
|
121
|
+
"embeddingPostprocessor": null,
|
|
122
|
+
"embeddingScale": null,
|
|
123
|
+
"logitInputScale": 1
|
|
118
124
|
},
|
|
119
125
|
"layerPattern": {
|
|
120
126
|
"type": "every_n",
|
|
121
127
|
"globalPattern": null,
|
|
122
128
|
"period": 5,
|
|
123
129
|
"offset": 4,
|
|
124
|
-
"layerTypes": null
|
|
130
|
+
"layerTypes": null,
|
|
131
|
+
"residualBranchScale": 1
|
|
125
132
|
},
|
|
126
133
|
"chatTemplate": {
|
|
127
134
|
"type": "gemma4",
|
|
@@ -142,7 +149,25 @@
|
|
|
142
149
|
"layout": "contiguous",
|
|
143
150
|
"pageSize": 256,
|
|
144
151
|
"tiering": {
|
|
145
|
-
"mode": "off"
|
|
152
|
+
"mode": "off",
|
|
153
|
+
"hotWindow": 1024,
|
|
154
|
+
"coldPageSize": 256,
|
|
155
|
+
"coldDtype": "f16",
|
|
156
|
+
"compression": {
|
|
157
|
+
"mode": "none",
|
|
158
|
+
"blockSize": 1,
|
|
159
|
+
"bitWidth": 4,
|
|
160
|
+
"prodMode": false
|
|
161
|
+
},
|
|
162
|
+
"gating": {
|
|
163
|
+
"mode": "auto",
|
|
164
|
+
"minAluBwRatio": 0
|
|
165
|
+
}
|
|
166
|
+
},
|
|
167
|
+
"quantization": {
|
|
168
|
+
"mode": "none",
|
|
169
|
+
"bitWidth": 4,
|
|
170
|
+
"prodMode": false
|
|
146
171
|
}
|
|
147
172
|
},
|
|
148
173
|
"decodeLoop": {
|
|
@@ -31,6 +31,7 @@
|
|
|
31
31
|
"ffn": {
|
|
32
32
|
"activation": "gelu",
|
|
33
33
|
"gatedActivation": true,
|
|
34
|
+
"branchMode": "auto",
|
|
34
35
|
"useDoubleWideMlp": false,
|
|
35
36
|
"swigluLimit": null
|
|
36
37
|
},
|
|
@@ -49,21 +50,27 @@
|
|
|
49
50
|
"yarnOriginalMaxPos": 32768,
|
|
50
51
|
"ropeLocalYarnBetaFast": null,
|
|
51
52
|
"ropeLocalYarnBetaSlow": null,
|
|
52
|
-
"ropeLocalYarnOriginalMaxPos": null
|
|
53
|
+
"ropeLocalYarnOriginalMaxPos": null,
|
|
54
|
+
"longropeShortFactor": null,
|
|
55
|
+
"longropeLongFactor": null,
|
|
56
|
+
"longropeOriginalMaxPos": null
|
|
53
57
|
},
|
|
54
58
|
"output": {
|
|
55
59
|
"finalLogitSoftcapping": null,
|
|
56
60
|
"tieWordEmbeddings": true,
|
|
57
61
|
"scaleEmbeddings": true,
|
|
58
62
|
"embeddingTranspose": false,
|
|
59
|
-
"embeddingVocabSize": null
|
|
63
|
+
"embeddingVocabSize": null,
|
|
64
|
+
"embeddingScale": null,
|
|
65
|
+
"logitInputScale": 1
|
|
60
66
|
},
|
|
61
67
|
"layerPattern": {
|
|
62
68
|
"type": "uniform",
|
|
63
69
|
"globalPattern": null,
|
|
64
70
|
"period": null,
|
|
65
71
|
"offset": null,
|
|
66
|
-
"layerTypes": null
|
|
72
|
+
"layerTypes": null,
|
|
73
|
+
"residualBranchScale": 1
|
|
67
74
|
},
|
|
68
75
|
"moe": {
|
|
69
76
|
"kernelProfileId": "mixtral-moe-v1",
|
|
@@ -97,7 +104,25 @@
|
|
|
97
104
|
"layout": "contiguous",
|
|
98
105
|
"pageSize": 256,
|
|
99
106
|
"tiering": {
|
|
100
|
-
"mode": "off"
|
|
107
|
+
"mode": "off",
|
|
108
|
+
"hotWindow": 1024,
|
|
109
|
+
"coldPageSize": 256,
|
|
110
|
+
"coldDtype": "f16",
|
|
111
|
+
"compression": {
|
|
112
|
+
"mode": "none",
|
|
113
|
+
"blockSize": 1,
|
|
114
|
+
"bitWidth": 4,
|
|
115
|
+
"prodMode": false
|
|
116
|
+
},
|
|
117
|
+
"gating": {
|
|
118
|
+
"mode": "auto",
|
|
119
|
+
"minAluBwRatio": 0
|
|
120
|
+
}
|
|
121
|
+
},
|
|
122
|
+
"quantization": {
|
|
123
|
+
"mode": "none",
|
|
124
|
+
"bitWidth": 4,
|
|
125
|
+
"prodMode": false
|
|
101
126
|
}
|
|
102
127
|
},
|
|
103
128
|
"decodeLoop": {
|
|
@@ -34,6 +34,7 @@
|
|
|
34
34
|
"ffn": {
|
|
35
35
|
"activation": "silu",
|
|
36
36
|
"gatedActivation": true,
|
|
37
|
+
"branchMode": "auto",
|
|
37
38
|
"swigluLimit": null
|
|
38
39
|
},
|
|
39
40
|
"rope": {
|
|
@@ -51,14 +52,19 @@
|
|
|
51
52
|
"yarnOriginalMaxPos": 4096,
|
|
52
53
|
"ropeLocalYarnBetaFast": null,
|
|
53
54
|
"ropeLocalYarnBetaSlow": null,
|
|
54
|
-
"ropeLocalYarnOriginalMaxPos": null
|
|
55
|
+
"ropeLocalYarnOriginalMaxPos": null,
|
|
56
|
+
"longropeShortFactor": null,
|
|
57
|
+
"longropeLongFactor": null,
|
|
58
|
+
"longropeOriginalMaxPos": null
|
|
55
59
|
},
|
|
56
60
|
"output": {
|
|
57
61
|
"finalLogitSoftcapping": null,
|
|
58
62
|
"tieWordEmbeddings": false,
|
|
59
63
|
"scaleEmbeddings": false,
|
|
60
64
|
"embeddingTranspose": false,
|
|
61
|
-
"embeddingVocabSize": null
|
|
65
|
+
"embeddingVocabSize": null,
|
|
66
|
+
"embeddingScale": null,
|
|
67
|
+
"logitInputScale": 1
|
|
62
68
|
},
|
|
63
69
|
"moe": {
|
|
64
70
|
"kernelProfileId": "gpt-oss-moe-v1",
|
|
@@ -78,7 +84,8 @@
|
|
|
78
84
|
"globalPattern": "odd",
|
|
79
85
|
"period": null,
|
|
80
86
|
"offset": null,
|
|
81
|
-
"layerTypes": null
|
|
87
|
+
"layerTypes": null,
|
|
88
|
+
"residualBranchScale": 1
|
|
82
89
|
},
|
|
83
90
|
"chatTemplate": {
|
|
84
91
|
"type": "gpt-oss",
|
|
@@ -99,7 +106,25 @@
|
|
|
99
106
|
"layout": "contiguous",
|
|
100
107
|
"pageSize": 256,
|
|
101
108
|
"tiering": {
|
|
102
|
-
"mode": "off"
|
|
109
|
+
"mode": "off",
|
|
110
|
+
"hotWindow": 1024,
|
|
111
|
+
"coldPageSize": 256,
|
|
112
|
+
"coldDtype": "f16",
|
|
113
|
+
"compression": {
|
|
114
|
+
"mode": "none",
|
|
115
|
+
"blockSize": 1,
|
|
116
|
+
"bitWidth": 4,
|
|
117
|
+
"prodMode": false
|
|
118
|
+
},
|
|
119
|
+
"gating": {
|
|
120
|
+
"mode": "auto",
|
|
121
|
+
"minAluBwRatio": 0
|
|
122
|
+
}
|
|
123
|
+
},
|
|
124
|
+
"quantization": {
|
|
125
|
+
"mode": "none",
|
|
126
|
+
"bitWidth": 4,
|
|
127
|
+
"prodMode": false
|
|
103
128
|
}
|
|
104
129
|
},
|
|
105
130
|
"decodeLoop": {
|