doppler-gpu 0.4.8 → 0.4.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/CHANGELOG.md +29 -0
  2. package/README.md +62 -12
  3. package/models/catalog.json +21 -7
  4. package/package.json +25 -1
  5. package/src/config/conversion/qwen3/qwen-3-5-9b-f16-af32.json +474 -0
  6. package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +509 -0
  7. package/src/config/kernels/backward-registry.json +13 -2
  8. package/src/config/kernels/kernel-ref-digests.js +8 -4
  9. package/src/config/kernels/registry.json +1035 -2677
  10. package/src/config/runtime/profiles/qwen-3-5-9b-metal-correctness.json +46 -0
  11. package/src/config/schema/training-metrics.schema.js +4 -0
  12. package/src/experimental/bridge/trainer-artifact-bridge.d.ts +161 -0
  13. package/src/experimental/bridge/trainer-artifact-bridge.js +573 -0
  14. package/src/experimental/training/attention-backward.js +123 -55
  15. package/src/experimental/training/autograd.js +101 -5
  16. package/src/experimental/training/clip.js +6 -1
  17. package/src/experimental/training/datasets/text-pairs.js +80 -8
  18. package/src/experimental/training/distillation/student-fixture.js +470 -114
  19. package/src/experimental/training/lora-pipeline.js +202 -27
  20. package/src/experimental/training/lora.js +10 -4
  21. package/src/experimental/training/optimizer.js +9 -1
  22. package/src/experimental/training/runner.js +18 -4
  23. package/src/experimental/training/workloads.js +11 -0
  24. package/src/gpu/device.js +100 -32
  25. package/src/gpu/kernels/backward/adam.js +6 -4
  26. package/src/gpu/kernels/backward/causal_conv1d_silu_backward.d.ts +24 -0
  27. package/src/gpu/kernels/backward/causal_conv1d_silu_backward.js +74 -0
  28. package/src/gpu/kernels/backward/causal_conv1d_silu_backward.wgsl +59 -0
  29. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +4 -0
  30. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.d.ts +39 -0
  31. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.js +124 -0
  32. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.wgsl +177 -0
  33. package/src/gpu/kernels/backward/gated_rmsnorm_backward.d.ts +32 -0
  34. package/src/gpu/kernels/backward/gated_rmsnorm_backward.js +90 -0
  35. package/src/gpu/kernels/backward/gated_rmsnorm_backward.wgsl +86 -0
  36. package/src/gpu/kernels/backward/index.d.ts +3 -0
  37. package/src/gpu/kernels/backward/index.js +3 -0
  38. package/src/gpu/kernels/backward/matmul_backward.wgsl +6 -6
  39. package/src/gpu/kernels/backward/matmul_backward_f16w.wgsl +82 -0
  40. package/src/gpu/kernels/backward/rmsnorm_backward.js +26 -4
  41. package/src/gpu/kernels/backward/rmsnorm_backward.wgsl +6 -3
  42. package/src/gpu/kernels/backward/utils.d.ts +12 -2
  43. package/src/gpu/kernels/backward/utils.js +9 -2
  44. package/src/gpu/kernels/cast_f16_to_f32.wgsl +6 -2
  45. package/src/gpu/kernels/index.d.ts +5 -0
  46. package/src/gpu/kernels/index.js +5 -0
  47. package/src/gpu/kernels/matmul-selection.js +5 -0
  48. package/src/gpu/kernels/silu.d.ts +1 -0
  49. package/src/gpu/kernels/silu.js +19 -4
  50. package/src/inference/pipelines/text/embed.js +1 -1
  51. package/src/inference/pipelines/text/execution-v1.js +8 -2
  52. package/src/inference/pipelines/text/ffn/dense.js +2 -0
  53. package/src/inference/pipelines/text/generator.js +29 -0
  54. package/src/inference/pipelines/text/lora-apply.js +4 -4
  55. package/src/inference/pipelines/text/moe-cpu.js +1 -0
  56. package/src/inference/pipelines/text/moe-gpu.js +1 -0
  57. package/src/inference/tokenizers/bundled.js +105 -32
  58. package/src/rules/kernels/matmul.rules.json +1 -0
  59. package/src/tooling/node-converter.js +20 -5
  60. package/src/tooling/runtime-optimization.d.ts +109 -0
  61. package/src/tooling/runtime-optimization.js +872 -0
  62. package/src/tooling-experimental-exports.shared.d.ts +25 -0
  63. package/src/tooling-experimental-exports.shared.js +18 -0
  64. package/src/tooling-exports.shared.d.ts +19 -0
  65. package/src/tooling-exports.shared.js +11 -0
  66. package/src/version.js +1 -1
@@ -0,0 +1,474 @@
1
+ {
2
+ "modelType": "transformer",
3
+ "output": {
4
+ "baseDir": "models/local",
5
+ "modelBaseId": "qwen-3-5-9b-f16-af32",
6
+ "textOnly": true,
7
+ "fast": false
8
+ },
9
+ "quantization": {
10
+ "weights": "f16",
11
+ "embeddings": "f16",
12
+ "lmHead": "f16",
13
+ "computePrecision": "f32"
14
+ },
15
+ "manifest": {
16
+ "hashAlgorithm": "blake3",
17
+ "artifactIdentity": {
18
+ "sourceCheckpointId": "Qwen/Qwen3.5-9B",
19
+ "sourceRepo": "Qwen/Qwen3.5-9B",
20
+ "sourceRevision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a",
21
+ "weightPackId": "qwen-3-5-9b-f16-af32-wp-catalog-v1",
22
+ "manifestVariantId": "qwen-3-5-9b-f16-af32-mv-exec-v1",
23
+ "artifactCompleteness": "complete"
24
+ },
25
+ "eosTokenId": [
26
+ 248046,
27
+ 248044
28
+ ]
29
+ },
30
+ "inference": {
31
+ "attention": {
32
+ "queryPreAttnScalar": 256,
33
+ "attnLogitSoftcapping": null,
34
+ "slidingWindow": null,
35
+ "queryKeyNorm": true,
36
+ "valueNorm": false,
37
+ "causal": true,
38
+ "attentionBias": false,
39
+ "attentionOutputGate": true
40
+ },
41
+ "normalization": {
42
+ "rmsNormEps": 0.000001,
43
+ "rmsNormWeightOffset": true,
44
+ "postAttentionNorm": true,
45
+ "preFeedforwardNorm": false,
46
+ "postFeedforwardNorm": false
47
+ },
48
+ "ffn": {
49
+ "activation": "silu",
50
+ "gatedActivation": true,
51
+ "branchMode": "auto",
52
+ "useDoubleWideMlp": false,
53
+ "swigluLimit": null
54
+ },
55
+ "rope": {
56
+ "ropeTheta": 10000000,
57
+ "ropeLocalTheta": null,
58
+ "mropeInterleaved": true,
59
+ "mropeSection": [
60
+ 11,
61
+ 11,
62
+ 10
63
+ ],
64
+ "partialRotaryFactor": 0.25,
65
+ "ropeLocalPartialRotaryFactor": null,
66
+ "ropeFrequencyBaseDim": null,
67
+ "ropeLocalFrequencyBaseDim": null,
68
+ "ropeScalingType": null,
69
+ "ropeScalingFactor": 1,
70
+ "ropeLocalScalingType": null,
71
+ "ropeLocalScalingFactor": 1,
72
+ "yarnBetaFast": null,
73
+ "yarnBetaSlow": null,
74
+ "yarnOriginalMaxPos": null,
75
+ "ropeLocalYarnBetaFast": null,
76
+ "ropeLocalYarnBetaSlow": null,
77
+ "ropeLocalYarnOriginalMaxPos": null,
78
+ "longropeShortFactor": null,
79
+ "longropeLongFactor": null,
80
+ "longropeOriginalMaxPos": null
81
+ },
82
+ "output": {
83
+ "finalLogitSoftcapping": null,
84
+ "tieWordEmbeddings": false,
85
+ "scaleEmbeddings": false,
86
+ "embeddingTranspose": false,
87
+ "embeddingVocabSize": null,
88
+ "embeddingPostprocessor": null,
89
+ "embeddingScale": null,
90
+ "logitInputScale": 1
91
+ },
92
+ "layerPattern": {
93
+ "type": "custom",
94
+ "globalPattern": null,
95
+ "period": null,
96
+ "offset": null,
97
+ "layerTypes": [
98
+ "linear_attention",
99
+ "linear_attention",
100
+ "linear_attention",
101
+ "full_attention",
102
+ "linear_attention",
103
+ "linear_attention",
104
+ "linear_attention",
105
+ "full_attention",
106
+ "linear_attention",
107
+ "linear_attention",
108
+ "linear_attention",
109
+ "full_attention",
110
+ "linear_attention",
111
+ "linear_attention",
112
+ "linear_attention",
113
+ "full_attention",
114
+ "linear_attention",
115
+ "linear_attention",
116
+ "linear_attention",
117
+ "full_attention",
118
+ "linear_attention",
119
+ "linear_attention",
120
+ "linear_attention",
121
+ "full_attention",
122
+ "linear_attention",
123
+ "linear_attention",
124
+ "linear_attention",
125
+ "full_attention",
126
+ "linear_attention",
127
+ "linear_attention",
128
+ "linear_attention",
129
+ "full_attention"
130
+ ],
131
+ "residualBranchScale": 1
132
+ },
133
+ "chatTemplate": {
134
+ "type": "qwen",
135
+ "enabled": true
136
+ }
137
+ },
138
+ "session": {
139
+ "compute": {
140
+ "defaults": {
141
+ "activationDtype": "f32",
142
+ "mathDtype": "f32",
143
+ "accumDtype": "f32",
144
+ "outputDtype": "f32"
145
+ }
146
+ },
147
+ "kvcache": {
148
+ "kvDtype": "f16",
149
+ "layout": "contiguous",
150
+ "maxSeqLen": 8192,
151
+ "pageSize": 256,
152
+ "tiering": {
153
+ "mode": "off",
154
+ "hotWindow": 1024,
155
+ "coldPageSize": 256,
156
+ "coldDtype": "f16",
157
+ "compression": {
158
+ "mode": "none",
159
+ "blockSize": 1,
160
+ "bitWidth": 4,
161
+ "prodMode": false
162
+ },
163
+ "gating": {
164
+ "mode": "auto",
165
+ "minAluBwRatio": 0
166
+ }
167
+ },
168
+ "quantization": {
169
+ "mode": "none",
170
+ "bitWidth": 4,
171
+ "prodMode": false
172
+ }
173
+ },
174
+ "decodeLoop": {
175
+ "batchSize": 12,
176
+ "stopCheckMode": "batch",
177
+ "readbackInterval": 32,
178
+ "readbackMode": "sequential",
179
+ "submitLatencyThresholdMs": null,
180
+ "ringTokens": 1,
181
+ "ringStop": 1,
182
+ "ringStaging": 1,
183
+ "disableCommandBatching": true
184
+ },
185
+ "perLayerInputs": {
186
+ "materialization": "auto",
187
+ "rowCache": {
188
+ "mode": "lru",
189
+ "maxRows": 256,
190
+ "maxBytes": 134217728,
191
+ "decodedDtype": "f32"
192
+ },
193
+ "prefetch": {
194
+ "mode": "next_token",
195
+ "rowsAhead": 1
196
+ },
197
+ "gpuUpload": {
198
+ "mode": "per_step_slices",
199
+ "stagingRows": 2
200
+ },
201
+ "hotCache": {
202
+ "mode": "prepared_tokens",
203
+ "maxTokens": 1024,
204
+ "maxBytes": 134217728,
205
+ "outputDtype": "f32"
206
+ }
207
+ },
208
+ "speculation": {
209
+ "mode": "self",
210
+ "tokens": 1,
211
+ "verify": "greedy",
212
+ "threshold": null,
213
+ "rollbackOnReject": true
214
+ },
215
+ "prefillChunkSubmitMode": "async",
216
+ "useFlashPrefillAttention": false,
217
+ "useWideTileQ4KPrefill": false,
218
+ "retainQ4KMaterialization": false
219
+ },
220
+ "largeWeights": {
221
+ "gpuResidentOverrides": null
222
+ },
223
+ "execution": {
224
+ "inlineKernelPath": true,
225
+ "kernels": {
226
+ "embed": {
227
+ "kernel": "gather_f16.wgsl",
228
+ "entry": "main",
229
+ "digest": "sha256:8995b4a790a1f3b89fbb801f89d595985eee46c9bfca4133f74396259083eda9"
230
+ },
231
+ "rmsnorm": {
232
+ "kernel": "rmsnorm.wgsl",
233
+ "entry": "main",
234
+ "digest": "sha256:284d3efb0ad0991fc57ece5f634cbc8a931fcab36f193b2b6832561d4cc79ef1"
235
+ },
236
+ "gemv": {
237
+ "kernel": "matmul_gemv_subgroup.wgsl",
238
+ "entry": "main_vec4",
239
+ "digest": "sha256:2ea9aea659367210fbf61d922c0d4ee17cdac68ffb506b3bc5516e498071e011"
240
+ },
241
+ "rope": {
242
+ "kernel": "rope.wgsl",
243
+ "entry": "main",
244
+ "digest": "sha256:b2da9d396668981dab9794c2973b668279f768994466b083d2105730555e1a5b"
245
+ },
246
+ "attn_decode": {
247
+ "kernel": "attention_decode_online_f16kv.wgsl",
248
+ "entry": "main",
249
+ "digest": "sha256:91808be35895e921c0fa43fc47bfe8cab69c437c388ab561f595d9dc983f66e1",
250
+ "precision": {
251
+ "kvDtype": "f16"
252
+ }
253
+ },
254
+ "residual": {
255
+ "kernel": "residual.wgsl",
256
+ "entry": "main",
257
+ "digest": "sha256:abd19bc08ad668a7cad562e2bc0a4be1aa8827ad4300d8d474b7fc2af4c27117"
258
+ },
259
+ "silu": {
260
+ "kernel": "silu.wgsl",
261
+ "entry": "main",
262
+ "digest": "sha256:7b52d30fb741beef2dbf728e0c4ecffe5b08d9661d63c306caecb4cb3ced85e5",
263
+ "constants": {
264
+ "HAS_GATE": true
265
+ }
266
+ },
267
+ "tiled": {
268
+ "kernel": "matmul_f16w_f32a.wgsl",
269
+ "entry": "main",
270
+ "digest": "sha256:027a8f1cd9713cbe0b0ada160bd175e0542bb90896ad85441b023522d9a1befc"
271
+ },
272
+ "attn_stream": {
273
+ "kernel": "attention_streaming_f16kv.wgsl",
274
+ "entry": "main",
275
+ "digest": "sha256:c3ec920699a81ee6df04a64eb0ea2477bec168ebb809af29f2d4b0924a4bdf8f",
276
+ "precision": {
277
+ "kvDtype": "f16"
278
+ }
279
+ },
280
+ "attn_head256": {
281
+ "kernel": "attention_head256_f16kv.wgsl",
282
+ "entry": "main",
283
+ "digest": "sha256:4ecdc079e322a770350414244cb383d72ae7ffa47ed620c64ff425c93413e97a",
284
+ "precision": {
285
+ "kvDtype": "f16"
286
+ }
287
+ },
288
+ "lm_head_gemv": {
289
+ "kernel": "matmul_gemv_subgroup.wgsl",
290
+ "entry": "main_multicol",
291
+ "digest": "sha256:e41f94574d5ac54dd2710036da5d5acc643a483b79b2d74b86825efdaaa7438f",
292
+ "constants": {
293
+ "MULTICOL_COLS_PER_WG": 64,
294
+ "MULTICOL_THREADS_PER_COL": 4
295
+ }
296
+ },
297
+ "sample": {
298
+ "kernel": "sample.wgsl",
299
+ "entry": "sample_single_pass",
300
+ "digest": "sha256:7ff3dd675b4737d920e0b3889e15c4fedde8410a353bbc6d4651db76ce69eb9b"
301
+ }
302
+ },
303
+ "preLayer": [
304
+ [
305
+ "embed",
306
+ "embed",
307
+ "embed_tokens"
308
+ ]
309
+ ],
310
+ "decode": [
311
+ [
312
+ "input_norm",
313
+ "rmsnorm"
314
+ ],
315
+ [
316
+ "q_proj",
317
+ "gemv",
318
+ "layer.{L}.self_attn.q_proj"
319
+ ],
320
+ [
321
+ "k_proj",
322
+ "gemv",
323
+ "layer.{L}.self_attn.k_proj"
324
+ ],
325
+ [
326
+ "v_proj",
327
+ "gemv",
328
+ "layer.{L}.self_attn.v_proj"
329
+ ],
330
+ [
331
+ "rope_q",
332
+ "rope"
333
+ ],
334
+ [
335
+ "rope_k",
336
+ "rope"
337
+ ],
338
+ [
339
+ "attention",
340
+ "attn_decode"
341
+ ],
342
+ [
343
+ "o_proj",
344
+ "gemv",
345
+ "layer.{L}.self_attn.o_proj"
346
+ ],
347
+ [
348
+ "attn_residual",
349
+ "residual"
350
+ ],
351
+ [
352
+ "post_attn_norm",
353
+ "rmsnorm"
354
+ ],
355
+ [
356
+ "gate_proj",
357
+ "gemv",
358
+ "layer.{L}.mlp.gate_proj"
359
+ ],
360
+ [
361
+ "up_proj",
362
+ "gemv",
363
+ "layer.{L}.mlp.up_proj"
364
+ ],
365
+ [
366
+ "activation",
367
+ "silu"
368
+ ],
369
+ [
370
+ "down_proj",
371
+ "gemv",
372
+ "layer.{L}.mlp.down_proj"
373
+ ],
374
+ [
375
+ "ffn_residual",
376
+ "residual"
377
+ ]
378
+ ],
379
+ "prefill": [
380
+ [
381
+ "input_norm",
382
+ "rmsnorm"
383
+ ],
384
+ [
385
+ "q_proj",
386
+ "tiled",
387
+ "layer.{L}.self_attn.q_proj"
388
+ ],
389
+ [
390
+ "k_proj",
391
+ "tiled",
392
+ "layer.{L}.self_attn.k_proj"
393
+ ],
394
+ [
395
+ "v_proj",
396
+ "tiled",
397
+ "layer.{L}.self_attn.v_proj"
398
+ ],
399
+ [
400
+ "rope_q",
401
+ "rope"
402
+ ],
403
+ [
404
+ "rope_k",
405
+ "rope"
406
+ ],
407
+ [
408
+ "attention",
409
+ "attn_head256"
410
+ ],
411
+ [
412
+ "o_proj",
413
+ "tiled",
414
+ "layer.{L}.self_attn.o_proj"
415
+ ],
416
+ [
417
+ "attn_residual",
418
+ "residual"
419
+ ],
420
+ [
421
+ "post_attn_norm",
422
+ "rmsnorm"
423
+ ],
424
+ [
425
+ "gate_proj",
426
+ "tiled",
427
+ "layer.{L}.mlp.gate_proj"
428
+ ],
429
+ [
430
+ "up_proj",
431
+ "tiled",
432
+ "layer.{L}.mlp.up_proj"
433
+ ],
434
+ [
435
+ "activation",
436
+ "silu"
437
+ ],
438
+ [
439
+ "down_proj",
440
+ "tiled",
441
+ "layer.{L}.mlp.down_proj"
442
+ ],
443
+ [
444
+ "ffn_residual",
445
+ "residual"
446
+ ]
447
+ ],
448
+ "postLayer": [
449
+ [
450
+ "final_norm",
451
+ "rmsnorm"
452
+ ],
453
+ [
454
+ "lm_head",
455
+ "lm_head_gemv",
456
+ "lm_head"
457
+ ],
458
+ [
459
+ "lm_head_prefill",
460
+ "tiled",
461
+ "lm_head"
462
+ ],
463
+ [
464
+ "sample",
465
+ "sample"
466
+ ]
467
+ ],
468
+ "policies": {
469
+ "unsupportedPrecision": "error",
470
+ "dtypeTransition": "require_cast_step",
471
+ "unresolvedKernel": "error"
472
+ }
473
+ }
474
+ }