doppler-gpu 0.4.5 → 0.4.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (155) hide show
  1. package/README.md +45 -29
  2. package/assets/doppler-runtime-map.svg +82 -0
  3. package/assets/doppler.svg +1 -3
  4. package/package.json +25 -17
  5. package/src/cli/doppler-quickstart.js +1 -1
  6. package/src/cli/doppler-serve.js +2 -2
  7. package/src/client/doppler-registry.d.ts +19 -0
  8. package/src/client/doppler-registry.js +12 -0
  9. package/src/client/doppler-registry.json +50 -1
  10. package/src/client/runtime/model-session.d.ts +1 -0
  11. package/src/client/runtime/model-session.js +9 -0
  12. package/src/config/conversion/README.md +4 -0
  13. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +28 -4
  14. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +29 -4
  15. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +29 -4
  16. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +29 -4
  17. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +29 -4
  18. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +29 -4
  19. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +29 -4
  20. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +29 -4
  21. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +29 -4
  22. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +31 -5
  23. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +30 -4
  24. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +30 -4
  25. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +30 -4
  26. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +30 -5
  27. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +29 -4
  28. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +30 -5
  29. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +29 -4
  30. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +29 -4
  31. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +29 -4
  32. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +29 -4
  33. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +29 -4
  34. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +29 -4
  35. package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +481 -0
  36. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +29 -4
  37. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +29 -4
  38. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +30 -5
  39. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +29 -4
  40. package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +395 -0
  41. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +429 -0
  42. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +540 -0
  43. package/src/config/execution-contract-check.js +14 -10
  44. package/src/config/merge-contract-check.js +1 -0
  45. package/src/config/merge.js +30 -0
  46. package/src/config/param-validator.d.ts +5 -0
  47. package/src/config/param-validator.js +42 -8
  48. package/src/config/required-inference-fields-contract-check.js +14 -0
  49. package/src/config/schema/debug.schema.d.ts +2 -0
  50. package/src/config/schema/debug.schema.js +1 -0
  51. package/src/config/schema/doppler.schema.d.ts +5 -1
  52. package/src/config/schema/doppler.schema.js +82 -292
  53. package/src/config/schema/inference.schema.d.ts +17 -3
  54. package/src/config/schema/kvcache.schema.d.ts +19 -3
  55. package/src/config/schema/kvcache.schema.js +2 -1
  56. package/src/config/schema/manifest.schema.d.ts +38 -1
  57. package/src/config/schema/manifest.schema.js +18 -0
  58. package/src/config/schema/policy-schema-registry.json +5 -0
  59. package/src/config/source-packages/litert/gemma-4-12b-it.json +20 -1
  60. package/src/config/source-packages/litert/gemma-4-e2b-it.json +20 -1
  61. package/src/config/source-packages/litert/gemma-4-e4b-it.json +21 -1
  62. package/src/config/transforms/execution-graph-transforms.d.ts +6 -0
  63. package/src/config/transforms/execution-graph-transforms.js +8 -0
  64. package/src/converter/conversion-plan.js +2 -0
  65. package/src/converter/parsers/transformer.js +27 -1
  66. package/src/converter/quantization-info.js +16 -4
  67. package/src/converter/rope-config.js +55 -3
  68. package/src/experimental/training/distillation/student-fixture.js +1 -1
  69. package/src/experimental/training/lora-pipeline.js +1 -1
  70. package/src/experimental/training/lora.js +2 -2
  71. package/src/experimental/training/suite.js +1 -1
  72. package/src/gpu/device.js +54 -11
  73. package/src/gpu/kernels/README.md +8 -1
  74. package/src/gpu/kernels/backward/attention_backward.js +8 -0
  75. package/src/gpu/kernels/dequant.d.ts +6 -6
  76. package/src/gpu/kernels/dequant.js +17 -6
  77. package/src/gpu/kernels/linear-attention-core.js +1 -6
  78. package/src/gpu/kernels/matmul-selection.d.ts +2 -2
  79. package/src/gpu/kernels/matmul-selection.js +9 -2
  80. package/src/gpu/kernels/matmul.d.ts +1 -1
  81. package/src/gpu/kernels/matmul.js +63 -9
  82. package/src/gpu/kernels/moe.d.ts +5 -1
  83. package/src/gpu/kernels/moe.js +5 -2
  84. package/src/gpu/kernels/softmax.d.ts +3 -3
  85. package/src/gpu/kernels/softmax.js +9 -2
  86. package/src/gpu/kernels/turboquant-codebook.js +1 -1
  87. package/src/gpu/kernels/types.d.ts +1 -1
  88. package/src/gpu/profiler.js +11 -1
  89. package/src/inference/browser-harness-model-helpers.js +4 -1
  90. package/src/inference/browser-harness-text-helpers.d.ts +28 -0
  91. package/src/inference/browser-harness-text-helpers.js +357 -116
  92. package/src/inference/browser-harness.d.ts +3 -3
  93. package/src/inference/browser-harness.js +266 -11
  94. package/src/inference/fixtures/rerank-semantic-fixtures.json +38 -0
  95. package/src/inference/kv-cache/quantized.js +16 -2
  96. package/src/inference/kv-cache/tiered.js +50 -15
  97. package/src/inference/moe-router.js +38 -7
  98. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +1 -1
  99. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +8 -6
  100. package/src/inference/pipelines/text/config.d.ts +5 -1
  101. package/src/inference/pipelines/text/config.js +135 -112
  102. package/src/inference/pipelines/text/embed.d.ts +1 -0
  103. package/src/inference/pipelines/text/embed.js +39 -12
  104. package/src/inference/pipelines/text/execution-plan.js +8 -2
  105. package/src/inference/pipelines/text/ffn/dense.js +15 -4
  106. package/src/inference/pipelines/text/ffn/moe.js +4 -3
  107. package/src/inference/pipelines/text/ffn/standard.d.ts +2 -1
  108. package/src/inference/pipelines/text/ffn/standard.js +25 -2
  109. package/src/inference/pipelines/text/generator-helpers.js +2 -1
  110. package/src/inference/pipelines/text/generator-runtime.d.ts +6 -2
  111. package/src/inference/pipelines/text/generator-runtime.js +53 -20
  112. package/src/inference/pipelines/text/generator-steps.js +3 -0
  113. package/src/inference/pipelines/text/generator.d.ts +1 -0
  114. package/src/inference/pipelines/text/generator.js +14 -1
  115. package/src/inference/pipelines/text/init-rope.js +54 -6
  116. package/src/inference/pipelines/text/init.js +63 -15
  117. package/src/inference/pipelines/text/layer.js +32 -5
  118. package/src/inference/pipelines/text/linear-attention.js +12 -8
  119. package/src/inference/pipelines/text/logits/gpu.d.ts +1 -1
  120. package/src/inference/pipelines/text/logits/gpu.js +76 -113
  121. package/src/inference/pipelines/text/logits/index.js +44 -87
  122. package/src/inference/pipelines/text/logits/precision-policy.d.ts +11 -0
  123. package/src/inference/pipelines/text/logits/precision-policy.js +103 -0
  124. package/src/inference/pipelines/text/logits/types.d.ts +1 -0
  125. package/src/inference/pipelines/text/logits/utils.d.ts +2 -0
  126. package/src/inference/pipelines/text/logits/utils.js +11 -0
  127. package/src/inference/pipelines/text/lora-apply.js +6 -4
  128. package/src/inference/pipelines/text/moe-cpu.js +25 -0
  129. package/src/inference/pipelines/text/moe-gpu.js +148 -97
  130. package/src/inference/pipelines/text/moe-shape-validator.d.ts +33 -3
  131. package/src/inference/pipelines/text/moe-shape-validator.js +149 -31
  132. package/src/inference/pipelines/text/ops.d.ts +3 -0
  133. package/src/inference/pipelines/text/ops.js +26 -1
  134. package/src/inference/pipelines/text/per-layer-inputs.js +27 -12
  135. package/src/inference/pipelines/text/sampling.js +52 -10
  136. package/src/inference/pipelines/text/weights.js +7 -2
  137. package/src/inference/pipelines/text.js +24 -4
  138. package/src/inference/tokenizers/behavior-flags.js +23 -2
  139. package/src/loader/final-weights-loader.js +12 -1
  140. package/src/loader/tensors/tensor-loader.js +37 -3
  141. package/src/rules/execution-rules-contract-check.js +12 -3
  142. package/src/rules/inference/capability-transforms.rules.json +14 -2
  143. package/src/rules/inference/config.rules.json +26 -2
  144. package/src/rules/inference/dtype.rules.json +12 -5
  145. package/src/rules/inference/execution.rules.json +5 -19
  146. package/src/rules/inference/moe.rules.json +97 -0
  147. package/src/rules/loader/weights.rules.json +1 -0
  148. package/src/tooling/command-api-constants.js +2 -2
  149. package/src/tooling/command-api-family-normalizers.js +2 -1
  150. package/src/tooling/command-api.d.ts +1 -1
  151. package/src/tooling/command-runner-shared.d.ts +1 -1
  152. package/src/tooling/command-runner-shared.js +3 -1
  153. package/src/tooling/node-converter.js +20 -0
  154. package/src/tooling/node-source-runtime.js +5 -1
  155. package/src/version.js +1 -1
@@ -34,6 +34,7 @@
34
34
  "ffn": {
35
35
  "activation": "silu",
36
36
  "gatedActivation": true,
37
+ "branchMode": "auto",
37
38
  "swigluLimit": null
38
39
  },
39
40
  "rope": {
@@ -51,21 +52,27 @@
51
52
  "yarnOriginalMaxPos": null,
52
53
  "ropeLocalYarnBetaFast": null,
53
54
  "ropeLocalYarnBetaSlow": null,
54
- "ropeLocalYarnOriginalMaxPos": null
55
+ "ropeLocalYarnOriginalMaxPos": null,
56
+ "longropeShortFactor": null,
57
+ "longropeLongFactor": null,
58
+ "longropeOriginalMaxPos": null
55
59
  },
56
60
  "output": {
57
61
  "finalLogitSoftcapping": null,
58
62
  "tieWordEmbeddings": false,
59
63
  "scaleEmbeddings": false,
60
64
  "embeddingTranspose": false,
61
- "embeddingVocabSize": null
65
+ "embeddingVocabSize": null,
66
+ "embeddingScale": null,
67
+ "logitInputScale": 1
62
68
  },
63
69
  "layerPattern": {
64
70
  "type": "all_attention",
65
71
  "globalPattern": null,
66
72
  "period": null,
67
73
  "offset": null,
68
- "layerTypes": null
74
+ "layerTypes": null,
75
+ "residualBranchScale": 1
69
76
  },
70
77
  "chatTemplate": {
71
78
  "type": null,
@@ -86,7 +93,25 @@
86
93
  "layout": "contiguous",
87
94
  "pageSize": 256,
88
95
  "tiering": {
89
- "mode": "off"
96
+ "mode": "off",
97
+ "hotWindow": 1024,
98
+ "coldPageSize": 256,
99
+ "coldDtype": "f16",
100
+ "compression": {
101
+ "mode": "none",
102
+ "blockSize": 1,
103
+ "bitWidth": 4,
104
+ "prodMode": false
105
+ },
106
+ "gating": {
107
+ "mode": "auto",
108
+ "minAluBwRatio": 0
109
+ }
110
+ },
111
+ "quantization": {
112
+ "mode": "none",
113
+ "bitWidth": 4,
114
+ "prodMode": false
90
115
  }
91
116
  },
92
117
  "decodeLoop": {
@@ -31,6 +31,7 @@
31
31
  "ffn": {
32
32
  "activation": "silu",
33
33
  "gatedActivation": true,
34
+ "branchMode": "auto",
34
35
  "swigluLimit": null,
35
36
  "useDoubleWideMlp": false
36
37
  },
@@ -52,7 +53,10 @@
52
53
  "ropeLocalYarnOriginalMaxPos": null,
53
54
  "ropeLocalPartialRotaryFactor": null,
54
55
  "ropeFrequencyBaseDim": null,
55
- "ropeLocalFrequencyBaseDim": null
56
+ "ropeLocalFrequencyBaseDim": null,
57
+ "longropeShortFactor": null,
58
+ "longropeLongFactor": null,
59
+ "longropeOriginalMaxPos": null
56
60
  },
57
61
  "output": {
58
62
  "finalLogitSoftcapping": null,
@@ -60,7 +64,9 @@
60
64
  "scaleEmbeddings": false,
61
65
  "embeddingTranspose": false,
62
66
  "embeddingVocabSize": null,
63
- "embeddingPostprocessor": null
67
+ "embeddingPostprocessor": null,
68
+ "embeddingScale": null,
69
+ "logitInputScale": 1
64
70
  },
65
71
  "layerPattern": {
66
72
  "type": "custom",
@@ -84,7 +90,8 @@
84
90
  "conv",
85
91
  "full_attention",
86
92
  "conv"
87
- ]
93
+ ],
94
+ "residualBranchScale": 1
88
95
  },
89
96
  "chatTemplate": {
90
97
  "type": "chatml",
@@ -105,7 +112,25 @@
105
112
  "layout": "contiguous",
106
113
  "pageSize": 256,
107
114
  "tiering": {
108
- "mode": "off"
115
+ "mode": "off",
116
+ "hotWindow": 1024,
117
+ "coldPageSize": 256,
118
+ "coldDtype": "f16",
119
+ "compression": {
120
+ "mode": "none",
121
+ "blockSize": 1,
122
+ "bitWidth": 4,
123
+ "prodMode": false
124
+ },
125
+ "gating": {
126
+ "mode": "auto",
127
+ "minAluBwRatio": 0
128
+ }
129
+ },
130
+ "quantization": {
131
+ "mode": "none",
132
+ "bitWidth": 4,
133
+ "prodMode": false
109
134
  }
110
135
  },
111
136
  "decodeLoop": {
@@ -0,0 +1,481 @@
1
+ {
2
+ "modelType": "transformer",
3
+ "output": {
4
+ "baseDir": "models/local",
5
+ "modelBaseId": "minicpm4-0-5b-f16-af32",
6
+ "textOnly": true,
7
+ "fast": false
8
+ },
9
+ "quantization": {
10
+ "weights": "f16",
11
+ "embeddings": "f16",
12
+ "lmHead": "f16",
13
+ "computePrecision": "f32"
14
+ },
15
+ "manifest": {
16
+ "hashAlgorithm": "blake3",
17
+ "artifactIdentity": {
18
+ "sourceCheckpointId": "openbmb/MiniCPM4-0.5B",
19
+ "weightPackId": "minicpm4-0-5b-f16-af32-wp-catalog-v1",
20
+ "manifestVariantId": "minicpm4-0-5b-f16-af32-mv-exec-v1",
21
+ "artifactCompleteness": "complete"
22
+ },
23
+ "eosTokenId": 73440
24
+ },
25
+ "inference": {
26
+ "attention": {
27
+ "queryPreAttnScalar": 64,
28
+ "attnLogitSoftcapping": null,
29
+ "slidingWindow": null,
30
+ "queryKeyNorm": false,
31
+ "valueNorm": false,
32
+ "causal": true,
33
+ "attentionBias": false,
34
+ "attentionOutputGate": false
35
+ },
36
+ "normalization": {
37
+ "rmsNormEps": 0.00001,
38
+ "rmsNormWeightOffset": false,
39
+ "postAttentionNorm": true,
40
+ "preFeedforwardNorm": false,
41
+ "postFeedforwardNorm": false
42
+ },
43
+ "ffn": {
44
+ "activation": "silu",
45
+ "gatedActivation": true,
46
+ "branchMode": "dense",
47
+ "useDoubleWideMlp": false,
48
+ "swigluLimit": null
49
+ },
50
+ "rope": {
51
+ "ropeTheta": 10000,
52
+ "ropeLocalTheta": null,
53
+ "mropeInterleaved": false,
54
+ "mropeSection": null,
55
+ "partialRotaryFactor": null,
56
+ "ropeLocalPartialRotaryFactor": null,
57
+ "ropeFrequencyBaseDim": null,
58
+ "ropeLocalFrequencyBaseDim": null,
59
+ "ropeScalingType": "longrope",
60
+ "ropeScalingFactor": 1,
61
+ "ropeLocalScalingType": null,
62
+ "ropeLocalScalingFactor": 1,
63
+ "yarnBetaFast": null,
64
+ "yarnBetaSlow": null,
65
+ "yarnOriginalMaxPos": null,
66
+ "longropeShortFactor": [
67
+ 1.0004360675811768,
68
+ 1.0668443441390991,
69
+ 1.1631425619125366,
70
+ 1.3025742769241333,
71
+ 1.5040205717086792,
72
+ 1.7941505908966064,
73
+ 2.2101221084594727,
74
+ 2.802666664123535,
75
+ 3.6389970779418945,
76
+ 4.804192543029785,
77
+ 6.39855432510376,
78
+ 8.527148246765137,
79
+ 11.277542114257812,
80
+ 14.684998512268066,
81
+ 18.69317054748535,
82
+ 23.13019371032715,
83
+ 27.72362518310547,
84
+ 32.1606559753418,
85
+ 36.168827056884766,
86
+ 39.57627868652344,
87
+ 42.32667541503906,
88
+ 44.45526885986328,
89
+ 46.04962921142578,
90
+ 47.21482849121094,
91
+ 48.05115509033203,
92
+ 48.64370346069336,
93
+ 49.05967712402344,
94
+ 49.34980392456055,
95
+ 49.551246643066406,
96
+ 49.69068145751953,
97
+ 49.78697967529297,
98
+ 49.85338592529297
99
+ ],
100
+ "longropeLongFactor": [
101
+ 1.0004360675811768,
102
+ 1.0668443441390991,
103
+ 1.1631425619125366,
104
+ 1.3025742769241333,
105
+ 1.5040205717086792,
106
+ 1.7941505908966064,
107
+ 2.2101221084594727,
108
+ 2.802666664123535,
109
+ 3.6389970779418945,
110
+ 4.804192543029785,
111
+ 6.39855432510376,
112
+ 8.527148246765137,
113
+ 11.277542114257812,
114
+ 14.684998512268066,
115
+ 18.69317054748535,
116
+ 23.13019371032715,
117
+ 27.72362518310547,
118
+ 32.1606559753418,
119
+ 36.168827056884766,
120
+ 39.57627868652344,
121
+ 42.32667541503906,
122
+ 44.45526885986328,
123
+ 46.04962921142578,
124
+ 47.21482849121094,
125
+ 48.05115509033203,
126
+ 48.64370346069336,
127
+ 49.05967712402344,
128
+ 49.34980392456055,
129
+ 49.551246643066406,
130
+ 49.69068145751953,
131
+ 49.78697967529297,
132
+ 49.85338592529297
133
+ ],
134
+ "longropeOriginalMaxPos": 32768,
135
+ "ropeLocalYarnBetaFast": null,
136
+ "ropeLocalYarnBetaSlow": null,
137
+ "ropeLocalYarnOriginalMaxPos": null
138
+ },
139
+ "output": {
140
+ "finalLogitSoftcapping": null,
141
+ "tieWordEmbeddings": true,
142
+ "scaleEmbeddings": false,
143
+ "embeddingTranspose": false,
144
+ "embeddingVocabSize": null,
145
+ "embeddingPostprocessor": null,
146
+ "embeddingScale": 12,
147
+ "logitInputScale": 0.25
148
+ },
149
+ "layerPattern": {
150
+ "type": "uniform",
151
+ "globalPattern": null,
152
+ "period": null,
153
+ "offset": null,
154
+ "layerTypes": null,
155
+ "residualBranchScale": 0.28577380332470415
156
+ },
157
+ "chatTemplate": {
158
+ "type": "chatml",
159
+ "enabled": true
160
+ }
161
+ },
162
+ "session": {
163
+ "compute": {
164
+ "defaults": {
165
+ "activationDtype": "f32",
166
+ "mathDtype": "f32",
167
+ "accumDtype": "f32",
168
+ "outputDtype": "f32"
169
+ }
170
+ },
171
+ "kvcache": {
172
+ "kvDtype": "f16",
173
+ "layout": "contiguous",
174
+ "maxSeqLen": 8192,
175
+ "pageSize": 256,
176
+ "tiering": {
177
+ "mode": "off",
178
+ "hotWindow": 1024,
179
+ "coldPageSize": 256,
180
+ "coldDtype": "f16",
181
+ "compression": {
182
+ "mode": "none",
183
+ "blockSize": 1,
184
+ "bitWidth": 4,
185
+ "prodMode": false
186
+ },
187
+ "gating": {
188
+ "mode": "auto",
189
+ "minAluBwRatio": 0
190
+ }
191
+ },
192
+ "quantization": {
193
+ "mode": "none",
194
+ "bitWidth": 4,
195
+ "prodMode": false
196
+ }
197
+ },
198
+ "decodeLoop": {
199
+ "batchSize": 4,
200
+ "stopCheckMode": "batch",
201
+ "readbackInterval": 1,
202
+ "readbackMode": "sequential",
203
+ "submitLatencyThresholdMs": null,
204
+ "ringTokens": 1,
205
+ "ringStop": 1,
206
+ "ringStaging": 1,
207
+ "disableCommandBatching": false
208
+ },
209
+ "perLayerInputs": {
210
+ "materialization": "auto",
211
+ "rowCache": {
212
+ "mode": "lru",
213
+ "maxRows": 256,
214
+ "maxBytes": 134217728,
215
+ "decodedDtype": "f32"
216
+ },
217
+ "prefetch": {
218
+ "mode": "next_token",
219
+ "rowsAhead": 1
220
+ },
221
+ "gpuUpload": {
222
+ "mode": "per_step_slices",
223
+ "stagingRows": 2
224
+ },
225
+ "hotCache": {
226
+ "mode": "prepared_tokens",
227
+ "maxTokens": 1024,
228
+ "maxBytes": 134217728,
229
+ "outputDtype": "f32"
230
+ }
231
+ },
232
+ "prefillChunkSubmitMode": "sync",
233
+ "useFlashPrefillAttention": false,
234
+ "useWideTileQ4KPrefill": false,
235
+ "retainQ4KMaterialization": false
236
+ },
237
+ "largeWeights": {
238
+ "gpuResidentOverrides": null
239
+ },
240
+ "execution": {
241
+ "inlineKernelPath": true,
242
+ "policies": {
243
+ "unsupportedPrecision": "error",
244
+ "dtypeTransition": "require_cast_step",
245
+ "unresolvedKernel": "error"
246
+ },
247
+ "kernels": {
248
+ "embed": {
249
+ "kernel": "gather_f16.wgsl",
250
+ "entry": "main",
251
+ "digest": "sha256:8995b4a790a1f3b89fbb801f89d595985eee46c9bfca4133f74396259083eda9"
252
+ },
253
+ "rmsnorm": {
254
+ "kernel": "rmsnorm.wgsl",
255
+ "entry": "main",
256
+ "digest": "sha256:284d3efb0ad0991fc57ece5f634cbc8a931fcab36f193b2b6832561d4cc79ef1"
257
+ },
258
+ "tiled": {
259
+ "kernel": "matmul_f16w_f32a.wgsl",
260
+ "entry": "main",
261
+ "digest": "sha256:027a8f1cd9713cbe0b0ada160bd175e0542bb90896ad85441b023522d9a1befc"
262
+ },
263
+ "lm_head_gemv": {
264
+ "kernel": "matmul_gemv_subgroup.wgsl",
265
+ "entry": "main_multicol",
266
+ "digest": "sha256:e41f94574d5ac54dd2710036da5d5acc643a483b79b2d74b86825efdaaa7438f",
267
+ "constants": {
268
+ "MULTICOL_COLS_PER_WG": 8,
269
+ "MULTICOL_THREADS_PER_COL": 32
270
+ }
271
+ },
272
+ "rope": {
273
+ "kernel": "rope.wgsl",
274
+ "entry": "main",
275
+ "digest": "sha256:b2da9d396668981dab9794c2973b668279f768994466b083d2105730555e1a5b"
276
+ },
277
+ "attn_decode": {
278
+ "kernel": "attention_decode_online_f16kv.wgsl",
279
+ "entry": "main",
280
+ "digest": "sha256:91808be35895e921c0fa43fc47bfe8cab69c437c388ab561f595d9dc983f66e1",
281
+ "precision": {
282
+ "kvDtype": "f16"
283
+ }
284
+ },
285
+ "attn_stream": {
286
+ "kernel": "attention_streaming_f16kv.wgsl",
287
+ "entry": "main",
288
+ "digest": "sha256:c3ec920699a81ee6df04a64eb0ea2477bec168ebb809af29f2d4b0924a4bdf8f",
289
+ "precision": {
290
+ "kvDtype": "f16"
291
+ }
292
+ },
293
+ "residual": {
294
+ "kernel": "residual.wgsl",
295
+ "entry": "main",
296
+ "digest": "sha256:abd19bc08ad668a7cad562e2bc0a4be1aa8827ad4300d8d474b7fc2af4c27117"
297
+ },
298
+ "silu": {
299
+ "kernel": "silu.wgsl",
300
+ "entry": "main",
301
+ "digest": "sha256:7b52d30fb741beef2dbf728e0c4ecffe5b08d9661d63c306caecb4cb3ced85e5"
302
+ },
303
+ "sample": {
304
+ "kernel": "sample.wgsl",
305
+ "entry": "sample_single_pass",
306
+ "digest": "sha256:7ff3dd675b4737d920e0b3889e15c4fedde8410a353bbc6d4651db76ce69eb9b"
307
+ },
308
+ "gemv": {
309
+ "kernel": "matmul_gemv_subgroup.wgsl",
310
+ "entry": "main_vec4",
311
+ "digest": "sha256:2ea9aea659367210fbf61d922c0d4ee17cdac68ffb506b3bc5516e498071e011"
312
+ }
313
+ },
314
+ "preLayer": [
315
+ [
316
+ "embed",
317
+ "embed",
318
+ "embed_tokens"
319
+ ]
320
+ ],
321
+ "decode": [
322
+ [
323
+ "input_norm",
324
+ "rmsnorm"
325
+ ],
326
+ [
327
+ "q_proj",
328
+ "gemv",
329
+ "layer.{L}.self_attn.q_proj"
330
+ ],
331
+ [
332
+ "k_proj",
333
+ "gemv",
334
+ "layer.{L}.self_attn.k_proj"
335
+ ],
336
+ [
337
+ "v_proj",
338
+ "gemv",
339
+ "layer.{L}.self_attn.v_proj"
340
+ ],
341
+ [
342
+ "rope_q",
343
+ "rope"
344
+ ],
345
+ [
346
+ "rope_k",
347
+ "rope"
348
+ ],
349
+ [
350
+ "attention",
351
+ "attn_decode"
352
+ ],
353
+ [
354
+ "o_proj",
355
+ "gemv",
356
+ "layer.{L}.self_attn.o_proj"
357
+ ],
358
+ [
359
+ "attn_residual",
360
+ "residual"
361
+ ],
362
+ [
363
+ "post_attn_norm",
364
+ "rmsnorm"
365
+ ],
366
+ [
367
+ "gate_proj",
368
+ "gemv",
369
+ "layer.{L}.mlp.gate_proj"
370
+ ],
371
+ [
372
+ "up_proj",
373
+ "gemv",
374
+ "layer.{L}.mlp.up_proj"
375
+ ],
376
+ [
377
+ "activation",
378
+ "silu"
379
+ ],
380
+ [
381
+ "down_proj",
382
+ "gemv",
383
+ "layer.{L}.mlp.down_proj"
384
+ ],
385
+ [
386
+ "ffn_residual",
387
+ "residual"
388
+ ]
389
+ ],
390
+ "prefill": [
391
+ [
392
+ "input_norm",
393
+ "rmsnorm"
394
+ ],
395
+ [
396
+ "q_proj",
397
+ "tiled",
398
+ "layer.{L}.self_attn.q_proj"
399
+ ],
400
+ [
401
+ "k_proj",
402
+ "tiled",
403
+ "layer.{L}.self_attn.k_proj"
404
+ ],
405
+ [
406
+ "v_proj",
407
+ "tiled",
408
+ "layer.{L}.self_attn.v_proj"
409
+ ],
410
+ [
411
+ "rope_q",
412
+ "rope"
413
+ ],
414
+ [
415
+ "rope_k",
416
+ "rope"
417
+ ],
418
+ [
419
+ "attention",
420
+ "attn_stream"
421
+ ],
422
+ [
423
+ "o_proj",
424
+ "tiled",
425
+ "layer.{L}.self_attn.o_proj"
426
+ ],
427
+ [
428
+ "attn_residual",
429
+ "residual"
430
+ ],
431
+ [
432
+ "post_attn_norm",
433
+ "rmsnorm"
434
+ ],
435
+ [
436
+ "gate_proj",
437
+ "tiled",
438
+ "layer.{L}.mlp.gate_proj"
439
+ ],
440
+ [
441
+ "up_proj",
442
+ "tiled",
443
+ "layer.{L}.mlp.up_proj"
444
+ ],
445
+ [
446
+ "activation",
447
+ "silu"
448
+ ],
449
+ [
450
+ "down_proj",
451
+ "tiled",
452
+ "layer.{L}.mlp.down_proj"
453
+ ],
454
+ [
455
+ "ffn_residual",
456
+ "residual"
457
+ ]
458
+ ],
459
+ "postLayer": [
460
+ [
461
+ "final_norm",
462
+ "rmsnorm"
463
+ ],
464
+ [
465
+ "lm_head",
466
+ "lm_head_gemv",
467
+ "lm_head"
468
+ ],
469
+ [
470
+ "lm_head_prefill",
471
+ "tiled",
472
+ "lm_head"
473
+ ],
474
+ [
475
+ "sample",
476
+ "sample"
477
+ ]
478
+ ],
479
+ "sampling": null
480
+ }
481
+ }
@@ -85,6 +85,7 @@
85
85
  "ffn": {
86
86
  "activation": "silu",
87
87
  "gatedActivation": true,
88
+ "branchMode": "auto",
88
89
  "useDoubleWideMlp": false,
89
90
  "swigluLimit": null
90
91
  },
@@ -110,7 +111,10 @@
110
111
  "yarnOriginalMaxPos": null,
111
112
  "ropeLocalYarnBetaFast": null,
112
113
  "ropeLocalYarnBetaSlow": null,
113
- "ropeLocalYarnOriginalMaxPos": null
114
+ "ropeLocalYarnOriginalMaxPos": null,
115
+ "longropeShortFactor": null,
116
+ "longropeLongFactor": null,
117
+ "longropeOriginalMaxPos": null
114
118
  },
115
119
  "output": {
116
120
  "finalLogitSoftcapping": null,
@@ -118,7 +122,9 @@
118
122
  "scaleEmbeddings": false,
119
123
  "embeddingTranspose": false,
120
124
  "embeddingVocabSize": null,
121
- "embeddingPostprocessor": null
125
+ "embeddingPostprocessor": null,
126
+ "embeddingScale": null,
127
+ "logitInputScale": 1
122
128
  },
123
129
  "layerPattern": {
124
130
  "type": "custom",
@@ -150,7 +156,8 @@
150
156
  "linear_attention",
151
157
  "linear_attention",
152
158
  "full_attention"
153
- ]
159
+ ],
160
+ "residualBranchScale": 1
154
161
  },
155
162
  "chatTemplate": {
156
163
  "type": "chatml",
@@ -172,7 +179,25 @@
172
179
  "maxSeqLen": 8192,
173
180
  "pageSize": 256,
174
181
  "tiering": {
175
- "mode": "off"
182
+ "mode": "off",
183
+ "hotWindow": 1024,
184
+ "coldPageSize": 256,
185
+ "coldDtype": "f16",
186
+ "compression": {
187
+ "mode": "none",
188
+ "blockSize": 1,
189
+ "bitWidth": 4,
190
+ "prodMode": false
191
+ },
192
+ "gating": {
193
+ "mode": "auto",
194
+ "minAluBwRatio": 0
195
+ }
196
+ },
197
+ "quantization": {
198
+ "mode": "none",
199
+ "bitWidth": 4,
200
+ "prodMode": false
176
201
  }
177
202
  },
178
203
  "decodeLoop": {