doppler-gpu 0.4.6 → 0.4.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (149) hide show
  1. package/README.md +43 -31
  2. package/assets/doppler-runtime-map.svg +1 -3
  3. package/assets/doppler.svg +1 -3
  4. package/package.json +25 -17
  5. package/src/cli/doppler-quickstart.js +1 -1
  6. package/src/cli/doppler-serve.js +2 -2
  7. package/src/client/doppler-registry.d.ts +19 -0
  8. package/src/client/doppler-registry.js +12 -0
  9. package/src/client/doppler-registry.json +50 -1
  10. package/src/config/conversion/README.md +4 -0
  11. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +28 -4
  12. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +29 -4
  13. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +29 -4
  14. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +29 -4
  15. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +29 -4
  16. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +29 -4
  17. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +29 -4
  18. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +29 -4
  19. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +29 -4
  20. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +31 -5
  21. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +30 -4
  22. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +30 -4
  23. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +30 -4
  24. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +30 -5
  25. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +29 -4
  26. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +30 -5
  27. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +29 -4
  28. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +29 -4
  29. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +29 -4
  30. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +29 -4
  31. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +29 -4
  32. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +29 -4
  33. package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +481 -0
  34. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +29 -4
  35. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +29 -4
  36. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +30 -5
  37. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +29 -4
  38. package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +395 -0
  39. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +429 -0
  40. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +540 -0
  41. package/src/config/execution-contract-check.js +14 -10
  42. package/src/config/merge-contract-check.js +1 -0
  43. package/src/config/merge.js +30 -0
  44. package/src/config/param-validator.d.ts +5 -0
  45. package/src/config/param-validator.js +42 -8
  46. package/src/config/required-inference-fields-contract-check.js +14 -0
  47. package/src/config/schema/debug.schema.d.ts +2 -0
  48. package/src/config/schema/debug.schema.js +1 -0
  49. package/src/config/schema/doppler.schema.d.ts +5 -1
  50. package/src/config/schema/doppler.schema.js +82 -292
  51. package/src/config/schema/inference.schema.d.ts +17 -3
  52. package/src/config/schema/kvcache.schema.d.ts +19 -3
  53. package/src/config/schema/kvcache.schema.js +2 -1
  54. package/src/config/schema/manifest.schema.d.ts +38 -1
  55. package/src/config/schema/manifest.schema.js +18 -0
  56. package/src/config/schema/policy-schema-registry.json +5 -0
  57. package/src/config/source-packages/litert/gemma-4-12b-it.json +20 -1
  58. package/src/config/source-packages/litert/gemma-4-e2b-it.json +20 -1
  59. package/src/config/source-packages/litert/gemma-4-e4b-it.json +21 -1
  60. package/src/config/transforms/execution-graph-transforms.d.ts +6 -0
  61. package/src/config/transforms/execution-graph-transforms.js +8 -0
  62. package/src/converter/conversion-plan.js +2 -0
  63. package/src/converter/parsers/transformer.js +27 -1
  64. package/src/converter/quantization-info.js +16 -4
  65. package/src/converter/rope-config.js +55 -3
  66. package/src/experimental/training/distillation/student-fixture.js +1 -1
  67. package/src/experimental/training/lora-pipeline.js +1 -1
  68. package/src/experimental/training/lora.js +2 -2
  69. package/src/experimental/training/suite.js +1 -1
  70. package/src/gpu/kernels/README.md +8 -1
  71. package/src/gpu/kernels/backward/attention_backward.js +8 -0
  72. package/src/gpu/kernels/dequant.d.ts +6 -6
  73. package/src/gpu/kernels/dequant.js +17 -6
  74. package/src/gpu/kernels/linear-attention-core.js +1 -6
  75. package/src/gpu/kernels/matmul-selection.d.ts +2 -2
  76. package/src/gpu/kernels/matmul-selection.js +4 -1
  77. package/src/gpu/kernels/matmul.d.ts +1 -1
  78. package/src/gpu/kernels/matmul.js +24 -6
  79. package/src/gpu/kernels/moe.d.ts +5 -1
  80. package/src/gpu/kernels/moe.js +5 -2
  81. package/src/gpu/kernels/softmax.d.ts +3 -3
  82. package/src/gpu/kernels/softmax.js +9 -2
  83. package/src/gpu/kernels/turboquant-codebook.js +1 -1
  84. package/src/gpu/kernels/types.d.ts +1 -1
  85. package/src/gpu/profiler.js +11 -1
  86. package/src/inference/browser-harness-model-helpers.js +4 -1
  87. package/src/inference/browser-harness-text-helpers.d.ts +28 -0
  88. package/src/inference/browser-harness-text-helpers.js +357 -116
  89. package/src/inference/browser-harness.d.ts +3 -3
  90. package/src/inference/browser-harness.js +266 -11
  91. package/src/inference/fixtures/rerank-semantic-fixtures.json +38 -0
  92. package/src/inference/kv-cache/quantized.js +16 -2
  93. package/src/inference/kv-cache/tiered.js +50 -15
  94. package/src/inference/moe-router.js +38 -7
  95. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +1 -1
  96. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +8 -6
  97. package/src/inference/pipelines/text/config.d.ts +5 -1
  98. package/src/inference/pipelines/text/config.js +135 -112
  99. package/src/inference/pipelines/text/embed.d.ts +1 -0
  100. package/src/inference/pipelines/text/embed.js +39 -12
  101. package/src/inference/pipelines/text/execution-plan.js +8 -2
  102. package/src/inference/pipelines/text/ffn/dense.js +15 -4
  103. package/src/inference/pipelines/text/ffn/moe.js +4 -3
  104. package/src/inference/pipelines/text/ffn/standard.d.ts +2 -1
  105. package/src/inference/pipelines/text/ffn/standard.js +25 -2
  106. package/src/inference/pipelines/text/generator-helpers.js +2 -1
  107. package/src/inference/pipelines/text/generator-runtime.d.ts +6 -2
  108. package/src/inference/pipelines/text/generator-runtime.js +53 -20
  109. package/src/inference/pipelines/text/generator-steps.js +3 -0
  110. package/src/inference/pipelines/text/generator.js +6 -1
  111. package/src/inference/pipelines/text/init-rope.js +54 -6
  112. package/src/inference/pipelines/text/init.js +63 -15
  113. package/src/inference/pipelines/text/layer.js +32 -5
  114. package/src/inference/pipelines/text/linear-attention.js +12 -8
  115. package/src/inference/pipelines/text/logits/gpu.d.ts +1 -1
  116. package/src/inference/pipelines/text/logits/gpu.js +76 -113
  117. package/src/inference/pipelines/text/logits/index.js +44 -87
  118. package/src/inference/pipelines/text/logits/precision-policy.d.ts +11 -0
  119. package/src/inference/pipelines/text/logits/precision-policy.js +103 -0
  120. package/src/inference/pipelines/text/logits/types.d.ts +1 -0
  121. package/src/inference/pipelines/text/logits/utils.d.ts +2 -0
  122. package/src/inference/pipelines/text/logits/utils.js +11 -0
  123. package/src/inference/pipelines/text/lora-apply.js +6 -4
  124. package/src/inference/pipelines/text/moe-cpu.js +25 -0
  125. package/src/inference/pipelines/text/moe-gpu.js +148 -97
  126. package/src/inference/pipelines/text/moe-shape-validator.d.ts +33 -3
  127. package/src/inference/pipelines/text/moe-shape-validator.js +149 -31
  128. package/src/inference/pipelines/text/ops.d.ts +3 -0
  129. package/src/inference/pipelines/text/ops.js +26 -1
  130. package/src/inference/pipelines/text/per-layer-inputs.js +27 -12
  131. package/src/inference/pipelines/text/weights.js +7 -2
  132. package/src/inference/pipelines/text.js +24 -4
  133. package/src/inference/tokenizers/behavior-flags.js +23 -2
  134. package/src/loader/final-weights-loader.js +12 -1
  135. package/src/rules/execution-rules-contract-check.js +12 -3
  136. package/src/rules/inference/capability-transforms.rules.json +14 -2
  137. package/src/rules/inference/config.rules.json +26 -2
  138. package/src/rules/inference/dtype.rules.json +12 -5
  139. package/src/rules/inference/execution.rules.json +5 -19
  140. package/src/rules/inference/moe.rules.json +97 -0
  141. package/src/tooling/command-api-constants.js +2 -2
  142. package/src/tooling/command-api-family-normalizers.js +2 -1
  143. package/src/tooling/command-api.d.ts +1 -1
  144. package/src/tooling/command-runner-shared.d.ts +1 -1
  145. package/src/tooling/command-runner-shared.js +3 -1
  146. package/src/tooling/node-converter.js +20 -0
  147. package/src/tooling/node-source-runtime.js +5 -1
  148. package/src/version.js +1 -1
  149. package/assets/doppler-readme-hero.svg +0 -84
@@ -58,6 +58,7 @@
58
58
  "ffn": {
59
59
  "activation": "silu",
60
60
  "gatedActivation": true,
61
+ "branchMode": "auto",
61
62
  "useDoubleWideMlp": false,
62
63
  "swigluLimit": null
63
64
  },
@@ -83,7 +84,10 @@
83
84
  "yarnOriginalMaxPos": null,
84
85
  "ropeLocalYarnBetaFast": null,
85
86
  "ropeLocalYarnBetaSlow": null,
86
- "ropeLocalYarnOriginalMaxPos": null
87
+ "ropeLocalYarnOriginalMaxPos": null,
88
+ "longropeShortFactor": null,
89
+ "longropeLongFactor": null,
90
+ "longropeOriginalMaxPos": null
87
91
  },
88
92
  "output": {
89
93
  "finalLogitSoftcapping": null,
@@ -91,7 +95,9 @@
91
95
  "scaleEmbeddings": false,
92
96
  "embeddingTranspose": false,
93
97
  "embeddingVocabSize": null,
94
- "embeddingPostprocessor": null
98
+ "embeddingPostprocessor": null,
99
+ "embeddingScale": null,
100
+ "logitInputScale": 1
95
101
  },
96
102
  "layerPattern": {
97
103
  "type": "custom",
@@ -163,7 +169,8 @@
163
169
  "linear_attention",
164
170
  "linear_attention",
165
171
  "full_attention"
166
- ]
172
+ ],
173
+ "residualBranchScale": 1
167
174
  },
168
175
  "chatTemplate": {
169
176
  "type": "qwen",
@@ -185,7 +192,25 @@
185
192
  "maxSeqLen": 8192,
186
193
  "pageSize": 256,
187
194
  "tiering": {
188
- "mode": "off"
195
+ "mode": "off",
196
+ "hotWindow": 1024,
197
+ "coldPageSize": 256,
198
+ "coldDtype": "f16",
199
+ "compression": {
200
+ "mode": "none",
201
+ "blockSize": 1,
202
+ "bitWidth": 4,
203
+ "prodMode": false
204
+ },
205
+ "gating": {
206
+ "mode": "auto",
207
+ "minAluBwRatio": 0
208
+ }
209
+ },
210
+ "quantization": {
211
+ "mode": "none",
212
+ "bitWidth": 4,
213
+ "prodMode": false
189
214
  }
190
215
  },
191
216
  "decodeLoop": {
@@ -0,0 +1,395 @@
1
+ {
2
+ "modelType": "embedding",
3
+ "output": {
4
+ "baseDir": "models/local",
5
+ "modelBaseId": "qwen-3-embedding-0-6b-q4k-ehf16-af32",
6
+ "textOnly": true,
7
+ "fast": false
8
+ },
9
+ "quantization": {
10
+ "weights": "q4k",
11
+ "embeddings": "f16",
12
+ "lmHead": "f16",
13
+ "computePrecision": "f32",
14
+ "q4kLayout": "row"
15
+ },
16
+ "manifest": {
17
+ "hashAlgorithm": "blake3",
18
+ "artifactIdentity": {
19
+ "sourceCheckpointId": "Qwen/Qwen3-Embedding-0.6B",
20
+ "weightPackId": "qwen-3-embedding-0-6b-q4k-ehf16-af32-wp-catalog-v1",
21
+ "manifestVariantId": "qwen-3-embedding-0-6b-q4k-ehf16-af32-mv-exec-v1",
22
+ "artifactCompleteness": "complete"
23
+ },
24
+ "eosTokenId": 151643
25
+ },
26
+ "inference": {
27
+ "attention": {
28
+ "queryPreAttnScalar": 128,
29
+ "attnLogitSoftcapping": null,
30
+ "slidingWindow": null,
31
+ "queryKeyNorm": true,
32
+ "valueNorm": false,
33
+ "causal": true,
34
+ "attentionBias": false,
35
+ "attentionOutputGate": false
36
+ },
37
+ "normalization": {
38
+ "rmsNormEps": 0.000001,
39
+ "rmsNormWeightOffset": false,
40
+ "postAttentionNorm": true,
41
+ "preFeedforwardNorm": false,
42
+ "postFeedforwardNorm": false
43
+ },
44
+ "ffn": {
45
+ "activation": "silu",
46
+ "gatedActivation": true,
47
+ "branchMode": "dense",
48
+ "useDoubleWideMlp": false,
49
+ "swigluLimit": null
50
+ },
51
+ "rope": {
52
+ "ropeTheta": 1000000,
53
+ "ropeLocalTheta": null,
54
+ "mropeInterleaved": false,
55
+ "mropeSection": null,
56
+ "partialRotaryFactor": null,
57
+ "ropeLocalPartialRotaryFactor": null,
58
+ "ropeFrequencyBaseDim": null,
59
+ "ropeLocalFrequencyBaseDim": null,
60
+ "ropeScalingType": null,
61
+ "ropeScalingFactor": 1,
62
+ "ropeLocalScalingType": null,
63
+ "ropeLocalScalingFactor": 1,
64
+ "yarnBetaFast": null,
65
+ "yarnBetaSlow": null,
66
+ "yarnOriginalMaxPos": null,
67
+ "ropeLocalYarnBetaFast": null,
68
+ "ropeLocalYarnBetaSlow": null,
69
+ "ropeLocalYarnOriginalMaxPos": null,
70
+ "longropeShortFactor": null,
71
+ "longropeLongFactor": null,
72
+ "longropeOriginalMaxPos": null
73
+ },
74
+ "output": {
75
+ "finalLogitSoftcapping": null,
76
+ "tieWordEmbeddings": true,
77
+ "scaleEmbeddings": false,
78
+ "embeddingTranspose": false,
79
+ "embeddingVocabSize": null,
80
+ "embeddingPostprocessor": {
81
+ "poolingMode": "last",
82
+ "includePrompt": true,
83
+ "projections": [],
84
+ "normalize": "l2"
85
+ },
86
+ "embeddingScale": null,
87
+ "logitInputScale": 1
88
+ },
89
+ "layerPattern": {
90
+ "type": "uniform",
91
+ "globalPattern": null,
92
+ "period": null,
93
+ "offset": null,
94
+ "layerTypes": null,
95
+ "residualBranchScale": 1
96
+ },
97
+ "chatTemplate": {
98
+ "type": null,
99
+ "enabled": false
100
+ }
101
+ },
102
+ "session": {
103
+ "compute": {
104
+ "defaults": {
105
+ "activationDtype": "f32",
106
+ "mathDtype": "f32",
107
+ "accumDtype": "f32",
108
+ "outputDtype": "f32"
109
+ }
110
+ },
111
+ "kvcache": {
112
+ "kvDtype": "f16",
113
+ "layout": "contiguous",
114
+ "maxSeqLen": 8192,
115
+ "pageSize": 256,
116
+ "tiering": {
117
+ "mode": "off",
118
+ "hotWindow": 1024,
119
+ "coldPageSize": 256,
120
+ "coldDtype": "f16",
121
+ "compression": {
122
+ "mode": "none",
123
+ "blockSize": 1,
124
+ "bitWidth": 4,
125
+ "prodMode": false
126
+ },
127
+ "gating": {
128
+ "mode": "auto",
129
+ "minAluBwRatio": 0
130
+ }
131
+ },
132
+ "quantization": {
133
+ "mode": "none",
134
+ "bitWidth": 4,
135
+ "prodMode": false
136
+ }
137
+ },
138
+ "decodeLoop": {
139
+ "batchSize": 4,
140
+ "stopCheckMode": "batch",
141
+ "readbackInterval": 1,
142
+ "readbackMode": "sequential",
143
+ "submitLatencyThresholdMs": null,
144
+ "ringTokens": 1,
145
+ "ringStop": 1,
146
+ "ringStaging": 1,
147
+ "disableCommandBatching": false
148
+ },
149
+ "perLayerInputs": {
150
+ "materialization": "auto",
151
+ "rowCache": {
152
+ "mode": "lru",
153
+ "maxRows": 256,
154
+ "maxBytes": 134217728,
155
+ "decodedDtype": "f32"
156
+ },
157
+ "prefetch": {
158
+ "mode": "next_token",
159
+ "rowsAhead": 1
160
+ },
161
+ "gpuUpload": {
162
+ "mode": "per_step_slices",
163
+ "stagingRows": 2
164
+ },
165
+ "hotCache": {
166
+ "mode": "prepared_tokens",
167
+ "maxTokens": 1024,
168
+ "maxBytes": 134217728,
169
+ "outputDtype": "f32"
170
+ }
171
+ },
172
+ "prefillChunkSubmitMode": "sync",
173
+ "useFlashPrefillAttention": false,
174
+ "useWideTileQ4KPrefill": false,
175
+ "retainQ4KMaterialization": false
176
+ },
177
+ "largeWeights": {
178
+ "gpuResidentOverrides": null
179
+ },
180
+ "execution": {
181
+ "inlineKernelPath": true,
182
+ "kernels": {
183
+ "embed": {
184
+ "kernel": "gather_f16.wgsl",
185
+ "entry": "main",
186
+ "digest": "sha256:8995b4a790a1f3b89fbb801f89d595985eee46c9bfca4133f74396259083eda9"
187
+ },
188
+ "rmsnorm": {
189
+ "kernel": "rmsnorm.wgsl",
190
+ "entry": "main",
191
+ "digest": "sha256:284d3efb0ad0991fc57ece5f634cbc8a931fcab36f193b2b6832561d4cc79ef1"
192
+ },
193
+ "q4_decode_gemv": {
194
+ "kernel": "fused_matmul_q4.wgsl",
195
+ "entry": "main_gemv",
196
+ "digest": "sha256:c547743b9806fec47142c6a9f9ac2bd047981fd8cded46ce40dc01725e8aa71d"
197
+ },
198
+ "q4_widetile": {
199
+ "kernel": "fused_matmul_q4_widetile.wgsl",
200
+ "entry": "main",
201
+ "digest": "sha256:5631ce6f19f805c7ac85722bac17365048bc7b9e2c422f3ef9477b63cb8dad62"
202
+ },
203
+ "rope": {
204
+ "kernel": "rope.wgsl",
205
+ "entry": "main",
206
+ "digest": "sha256:b2da9d396668981dab9794c2973b668279f768994466b083d2105730555e1a5b"
207
+ },
208
+ "attn_decode": {
209
+ "kernel": "attention_decode_online_f16kv.wgsl",
210
+ "entry": "main",
211
+ "digest": "sha256:91808be35895e921c0fa43fc47bfe8cab69c437c388ab561f595d9dc983f66e1",
212
+ "precision": {
213
+ "kvDtype": "f16"
214
+ }
215
+ },
216
+ "attn_stream": {
217
+ "kernel": "attention_streaming_f16kv.wgsl",
218
+ "entry": "main",
219
+ "digest": "sha256:c3ec920699a81ee6df04a64eb0ea2477bec168ebb809af29f2d4b0924a4bdf8f",
220
+ "precision": {
221
+ "kvDtype": "f16"
222
+ }
223
+ },
224
+ "residual": {
225
+ "kernel": "residual.wgsl",
226
+ "entry": "main",
227
+ "digest": "sha256:abd19bc08ad668a7cad562e2bc0a4be1aa8827ad4300d8d474b7fc2af4c27117"
228
+ },
229
+ "silu": {
230
+ "kernel": "silu.wgsl",
231
+ "entry": "main",
232
+ "digest": "sha256:7b52d30fb741beef2dbf728e0c4ecffe5b08d9661d63c306caecb4cb3ced85e5",
233
+ "constants": {
234
+ "HAS_GATE": true
235
+ }
236
+ }
237
+ },
238
+ "preLayer": [
239
+ [
240
+ "embed",
241
+ "embed",
242
+ "embed_tokens"
243
+ ]
244
+ ],
245
+ "decode": [
246
+ [
247
+ "input_norm",
248
+ "rmsnorm"
249
+ ],
250
+ [
251
+ "q_proj",
252
+ "q4_decode_gemv",
253
+ "layer.{L}.self_attn.q_proj"
254
+ ],
255
+ [
256
+ "k_proj",
257
+ "q4_decode_gemv",
258
+ "layer.{L}.self_attn.k_proj"
259
+ ],
260
+ [
261
+ "v_proj",
262
+ "q4_decode_gemv",
263
+ "layer.{L}.self_attn.v_proj"
264
+ ],
265
+ [
266
+ "rope_q",
267
+ "rope"
268
+ ],
269
+ [
270
+ "rope_k",
271
+ "rope"
272
+ ],
273
+ [
274
+ "attention",
275
+ "attn_decode"
276
+ ],
277
+ [
278
+ "o_proj",
279
+ "q4_decode_gemv",
280
+ "layer.{L}.self_attn.o_proj"
281
+ ],
282
+ [
283
+ "attn_residual",
284
+ "residual"
285
+ ],
286
+ [
287
+ "post_attn_norm",
288
+ "rmsnorm"
289
+ ],
290
+ [
291
+ "gate_proj",
292
+ "q4_decode_gemv",
293
+ "layer.{L}.mlp.gate_proj"
294
+ ],
295
+ [
296
+ "up_proj",
297
+ "q4_decode_gemv",
298
+ "layer.{L}.mlp.up_proj"
299
+ ],
300
+ [
301
+ "activation",
302
+ "silu"
303
+ ],
304
+ [
305
+ "down_proj",
306
+ "q4_decode_gemv",
307
+ "layer.{L}.mlp.down_proj"
308
+ ],
309
+ [
310
+ "ffn_residual",
311
+ "residual"
312
+ ]
313
+ ],
314
+ "prefill": [
315
+ [
316
+ "input_norm",
317
+ "rmsnorm"
318
+ ],
319
+ [
320
+ "q_proj",
321
+ "q4_widetile",
322
+ "layer.{L}.self_attn.q_proj"
323
+ ],
324
+ [
325
+ "k_proj",
326
+ "q4_widetile",
327
+ "layer.{L}.self_attn.k_proj"
328
+ ],
329
+ [
330
+ "v_proj",
331
+ "q4_widetile",
332
+ "layer.{L}.self_attn.v_proj"
333
+ ],
334
+ [
335
+ "rope_q",
336
+ "rope"
337
+ ],
338
+ [
339
+ "rope_k",
340
+ "rope"
341
+ ],
342
+ [
343
+ "attention",
344
+ "attn_stream"
345
+ ],
346
+ [
347
+ "o_proj",
348
+ "q4_widetile",
349
+ "layer.{L}.self_attn.o_proj"
350
+ ],
351
+ [
352
+ "attn_residual",
353
+ "residual"
354
+ ],
355
+ [
356
+ "post_attn_norm",
357
+ "rmsnorm"
358
+ ],
359
+ [
360
+ "gate_proj",
361
+ "q4_widetile",
362
+ "layer.{L}.mlp.gate_proj"
363
+ ],
364
+ [
365
+ "up_proj",
366
+ "q4_widetile",
367
+ "layer.{L}.mlp.up_proj"
368
+ ],
369
+ [
370
+ "activation",
371
+ "silu"
372
+ ],
373
+ [
374
+ "down_proj",
375
+ "q4_widetile",
376
+ "layer.{L}.mlp.down_proj"
377
+ ],
378
+ [
379
+ "ffn_residual",
380
+ "residual"
381
+ ]
382
+ ],
383
+ "postLayer": [
384
+ [
385
+ "final_norm",
386
+ "rmsnorm"
387
+ ]
388
+ ],
389
+ "policies": {
390
+ "unsupportedPrecision": "error",
391
+ "dtypeTransition": "require_cast_step",
392
+ "unresolvedKernel": "error"
393
+ }
394
+ }
395
+ }