doppler-gpu 0.4.9 → 0.4.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. package/CHANGELOG.md +53 -0
  2. package/README.md +89 -12
  3. package/models/catalog.json +612 -7
  4. package/package.json +30 -1
  5. package/src/client/doppler-registry.js +6 -0
  6. package/src/client/doppler-registry.json +126 -4
  7. package/src/client/runtime/lora.d.ts +10 -0
  8. package/src/client/runtime/lora.js +64 -0
  9. package/src/client/runtime/model-manager.js +6 -167
  10. package/src/client/runtime/model-session.d.ts +3 -1
  11. package/src/client/runtime/model-session.js +6 -0
  12. package/src/client/wrap-pipeline-handle.js +7 -0
  13. package/src/config/conversion/amplify/amplify-120m-f16-af32.json +516 -0
  14. package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +571 -0
  15. package/src/config/conversion/esmc/esmc-300m-f32-af32.json +342 -0
  16. package/src/config/conversion/nucleotide-transformer/nucleotide-transformer-v2-50m-f32-af32.json +388 -0
  17. package/src/config/kernels/registry.json +52 -9
  18. package/src/config/merge.js +24 -0
  19. package/src/config/required-inference-fields-contract-check.js +2 -0
  20. package/src/config/schema/converter.schema.d.ts +7 -0
  21. package/src/config/schema/converter.schema.js +15 -0
  22. package/src/config/schema/inference.schema.d.ts +8 -0
  23. package/src/config/schema/manifest.schema.d.ts +32 -1
  24. package/src/config/schema/manifest.schema.js +26 -0
  25. package/src/converter/conversion-plan.js +15 -1
  26. package/src/converter/core.js +5 -4
  27. package/src/converter/source-tensor-rules.d.ts +50 -0
  28. package/src/converter/source-tensor-rules.js +180 -0
  29. package/src/converter/source-tokenizer.d.ts +45 -0
  30. package/src/converter/source-tokenizer.js +155 -0
  31. package/src/experimental/adapters/lora-loader.js +15 -0
  32. package/src/experimental/bridge/trainer-artifact-bridge.d.ts +2 -1
  33. package/src/experimental/bridge/trainer-artifact-bridge.js +35 -13
  34. package/src/gpu/device.js +100 -32
  35. package/src/inference/pipelines/text/attention/attn-config.js +2 -0
  36. package/src/inference/pipelines/text/attention/projections.js +161 -24
  37. package/src/inference/pipelines/text/attention/record.js +67 -9
  38. package/src/inference/pipelines/text/attention/run.js +60 -7
  39. package/src/inference/pipelines/text/attention/types.js +18 -9
  40. package/src/inference/pipelines/text/config.d.ts +5 -0
  41. package/src/inference/pipelines/text/config.js +91 -0
  42. package/src/inference/pipelines/text/ffn/dense.js +189 -2
  43. package/src/inference/pipelines/text/ffn/standard.js +50 -3
  44. package/src/inference/pipelines/text/generator-helpers.js +11 -1
  45. package/src/inference/pipelines/text/generator-runtime.js +63 -15
  46. package/src/inference/pipelines/text/generator-steps.js +3 -1
  47. package/src/inference/pipelines/text/generator.js +90 -3
  48. package/src/inference/pipelines/text/init-qkv-fusion.js +85 -1
  49. package/src/inference/pipelines/text/init.d.ts +2 -0
  50. package/src/inference/pipelines/text/init.js +2 -0
  51. package/src/inference/pipelines/text/layer.js +6 -0
  52. package/src/inference/pipelines/text/logits/cpu.d.ts +6 -0
  53. package/src/inference/pipelines/text/logits/cpu.js +22 -0
  54. package/src/inference/pipelines/text/logits/index.d.ts +1 -1
  55. package/src/inference/pipelines/text/logits/index.js +85 -11
  56. package/src/inference/pipelines/text/logits/types.d.ts +4 -0
  57. package/src/inference/pipelines/text/ops.js +32 -0
  58. package/src/inference/pipelines/text/types.d.ts +38 -0
  59. package/src/inference/pipelines/text/weights.d.ts +10 -0
  60. package/src/inference/pipelines/text/weights.js +33 -1
  61. package/src/inference/pipelines/text.d.ts +4 -2
  62. package/src/inference/pipelines/text.js +112 -0
  63. package/src/inference/tokenizers/behavior-flags.js +5 -1
  64. package/src/inference/tokenizers/bundled.js +99 -8
  65. package/src/loader/doppler-loader.d.ts +2 -0
  66. package/src/loader/doppler-loader.js +9 -0
  67. package/src/loader/final-weights-loader.d.ts +8 -0
  68. package/src/loader/final-weights-loader.js +42 -0
  69. package/src/loader/layer-loader.d.ts +6 -0
  70. package/src/loader/layer-loader.js +102 -3
  71. package/src/loader/loader-types.d.ts +15 -2
  72. package/src/tooling/node-converter.js +33 -1
  73. package/src/tooling/node-webgpu.js +85 -2
  74. package/src/tooling/runtime-optimization.d.ts +109 -0
  75. package/src/tooling/runtime-optimization.js +872 -0
  76. package/src/tooling-exports.shared.d.ts +19 -0
  77. package/src/tooling-exports.shared.js +11 -0
  78. package/src/version.js +1 -1
@@ -0,0 +1,342 @@
1
+ {
2
+ "modelType": "embedding",
3
+ "output": {
4
+ "baseDir": "models/local",
5
+ "modelBaseId": "esmc-300m-f32-af32",
6
+ "textOnly": true,
7
+ "fast": false
8
+ },
9
+ "architecture": {
10
+ "numLayers": 30,
11
+ "hiddenSize": 960,
12
+ "intermediateSize": 2560,
13
+ "numAttentionHeads": 15,
14
+ "numKeyValueHeads": 15,
15
+ "headDim": 64,
16
+ "vocabSize": 64,
17
+ "maxSeqLen": 2048,
18
+ "ropeTheta": 10000
19
+ },
20
+ "quantization": {
21
+ "weights": "f32",
22
+ "embeddings": "f32",
23
+ "lmHead": "f32",
24
+ "q4kLayout": "row",
25
+ "computePrecision": "f32"
26
+ },
27
+ "sourceTensors": {
28
+ "requireAll": true,
29
+ "rules": [
30
+ {
31
+ "kind": "rename",
32
+ "match": "^esmc\\.embed\\.weight$",
33
+ "replace": "model.embed_tokens.weight",
34
+ "expectedMatches": 1,
35
+ "role": "embedding",
36
+ "group": "embed"
37
+ },
38
+ {
39
+ "kind": "rename",
40
+ "match": "^esmc\\.transformer\\.norm\\.weight$",
41
+ "replace": "model.norm.weight",
42
+ "expectedMatches": 1,
43
+ "role": "norm",
44
+ "group": "head"
45
+ },
46
+ {
47
+ "kind": "rename",
48
+ "match": "^esmc\\.transformer\\.blocks\\.(\\d+)\\.attn\\.layernorm_qkv\\.layer_norm_weight$",
49
+ "replace": "model.layers.$1.input_layernorm.weight",
50
+ "expectedMatches": 30
51
+ },
52
+ {
53
+ "kind": "rename",
54
+ "match": "^esmc\\.transformer\\.blocks\\.(\\d+)\\.attn\\.layernorm_qkv\\.layer_norm_bias$",
55
+ "replace": "model.layers.$1.input_layernorm.bias",
56
+ "expectedMatches": 30
57
+ },
58
+ {
59
+ "kind": "split",
60
+ "match": "^esmc\\.transformer\\.blocks\\.(\\d+)\\.attn\\.layernorm_qkv\\.weight$",
61
+ "axis": 0,
62
+ "expectedMatches": 30,
63
+ "parts": [
64
+ { "replace": "model.layers.$1.self_attn.q_proj.weight", "size": 960 },
65
+ { "replace": "model.layers.$1.self_attn.k_proj.weight", "size": 960 },
66
+ { "replace": "model.layers.$1.self_attn.v_proj.weight", "size": 960 }
67
+ ]
68
+ },
69
+ {
70
+ "kind": "rename",
71
+ "match": "^esmc\\.transformer\\.blocks\\.(\\d+)\\.attn\\.q_ln\\.weight$",
72
+ "replace": "model.layers.$1.self_attn.q_norm.weight",
73
+ "expectedMatches": 30
74
+ },
75
+ {
76
+ "kind": "rename",
77
+ "match": "^esmc\\.transformer\\.blocks\\.(\\d+)\\.attn\\.k_ln\\.weight$",
78
+ "replace": "model.layers.$1.self_attn.k_norm.weight",
79
+ "expectedMatches": 30
80
+ },
81
+ {
82
+ "kind": "rename",
83
+ "match": "^esmc\\.transformer\\.blocks\\.(\\d+)\\.attn\\.out_proj\\.weight$",
84
+ "replace": "model.layers.$1.self_attn.o_proj.weight",
85
+ "expectedMatches": 30
86
+ },
87
+ {
88
+ "kind": "rename",
89
+ "match": "^esmc\\.transformer\\.blocks\\.(\\d+)\\.ffn\\.layer_norm_weight$",
90
+ "replace": "model.layers.$1.post_attention_layernorm.weight",
91
+ "expectedMatches": 30
92
+ },
93
+ {
94
+ "kind": "rename",
95
+ "match": "^esmc\\.transformer\\.blocks\\.(\\d+)\\.ffn\\.layer_norm_bias$",
96
+ "replace": "model.layers.$1.post_attention_layernorm.bias",
97
+ "expectedMatches": 30
98
+ },
99
+ {
100
+ "kind": "split",
101
+ "match": "^esmc\\.transformer\\.blocks\\.(\\d+)\\.ffn\\.fc1_weight$",
102
+ "axis": 0,
103
+ "expectedMatches": 30,
104
+ "parts": [
105
+ { "replace": "model.layers.$1.mlp.gate_proj.weight", "size": 2560 },
106
+ { "replace": "model.layers.$1.mlp.up_proj.weight", "size": 2560 }
107
+ ]
108
+ },
109
+ {
110
+ "kind": "rename",
111
+ "match": "^esmc\\.transformer\\.blocks\\.(\\d+)\\.ffn\\.fc2_weight$",
112
+ "replace": "model.layers.$1.mlp.down_proj.weight",
113
+ "expectedMatches": 30
114
+ },
115
+ {
116
+ "kind": "ignore",
117
+ "match": "^esmc\\.transformer\\.blocks\\.\\d+\\.(?:attn\\.(?:layernorm_qkv|out_proj)|ffn)\\._extra_state$",
118
+ "expectedMatches": 90,
119
+ "reason": "PyTorch module extra-state markers contain no inference parameters."
120
+ },
121
+ {
122
+ "kind": "ignore",
123
+ "match": "^lm_head\\.",
124
+ "expectedMatches": 6,
125
+ "reason": "The initial ESMC lane exposes embeddings. Its dense-GELU-LayerNorm masked-token head requires a separately qualified output pipeline."
126
+ }
127
+ ]
128
+ },
129
+ "manifest": {
130
+ "hashAlgorithm": "blake3",
131
+ "artifactIdentity": {
132
+ "sourceCheckpointId": "biohub/ESMC-300M@a59b831785f907e96e6a246b1d142bfb76df31ee",
133
+ "sourceRepo": "biohub/ESMC-300M",
134
+ "sourceRevision": "a59b831785f907e96e6a246b1d142bfb76df31ee",
135
+ "sourceFormat": "safetensors",
136
+ "conversionConfigPath": "src/config/conversion/esmc/esmc-300m-f32-af32.json",
137
+ "weightPackId": "esmc-300m-f32-af32-wp-v1",
138
+ "manifestVariantId": "esmc-300m-f32-af32-mv-exec-v1",
139
+ "modalitySet": ["protein_sequence"],
140
+ "materializationProfile": "f32-weights-f32-activations",
141
+ "artifactCompleteness": "complete"
142
+ },
143
+ "eosTokenId": 2
144
+ },
145
+ "inference": {
146
+ "supportsEmbedding": true,
147
+ "supportsSequence": true,
148
+ "sequence": {
149
+ "alphabet": "amino_acid",
150
+ "tokenEmbeddings": true,
151
+ "pooledEmbedding": {
152
+ "mode": "mean",
153
+ "excludeTokenIds": [0, 1, 2, 3, 31, 32]
154
+ },
155
+ "logits": false
156
+ },
157
+ "attention": {
158
+ "queryPreAttnScalar": 64,
159
+ "attnLogitSoftcapping": null,
160
+ "slidingWindow": null,
161
+ "queryKeyNorm": true,
162
+ "queryKeyNormType": "layernorm",
163
+ "queryKeyNormAxis": "projection",
164
+ "queryKeyNormLayers": null,
165
+ "queryKeyNormWeightLayers": null,
166
+ "valueNorm": false,
167
+ "causal": false,
168
+ "attentionBias": true,
169
+ "attentionOutputGate": false,
170
+ "outputGateType": null
171
+ },
172
+ "normalization": {
173
+ "type": "layernorm",
174
+ "rmsNormEps": 0.00001,
175
+ "rmsNormWeightOffset": false,
176
+ "postAttentionNorm": true,
177
+ "preFeedforwardNorm": false,
178
+ "postFeedforwardNorm": false,
179
+ "finalNormBiasTensor": null
180
+ },
181
+ "ffn": {
182
+ "activation": "silu",
183
+ "gatedActivation": true,
184
+ "branchMode": "dense",
185
+ "useDoubleWideMlp": false,
186
+ "swigluLimit": null
187
+ },
188
+ "rope": {
189
+ "ropeTheta": 10000,
190
+ "ropeLocalTheta": null,
191
+ "ropeInterleaved": false,
192
+ "mropeInterleaved": false,
193
+ "mropeSection": null,
194
+ "partialRotaryFactor": null,
195
+ "ropeLocalPartialRotaryFactor": null,
196
+ "ropeFrequencyBaseDim": null,
197
+ "ropeLocalFrequencyBaseDim": null,
198
+ "ropeScalingType": null,
199
+ "ropeScalingFactor": 1,
200
+ "ropeLocalScalingType": null,
201
+ "ropeLocalScalingFactor": 1,
202
+ "yarnBetaFast": null,
203
+ "yarnBetaSlow": null,
204
+ "yarnOriginalMaxPos": null,
205
+ "ropeLocalYarnBetaFast": null,
206
+ "ropeLocalYarnBetaSlow": null,
207
+ "ropeLocalYarnOriginalMaxPos": null,
208
+ "longropeShortFactor": null,
209
+ "longropeLongFactor": null,
210
+ "longropeOriginalMaxPos": null
211
+ },
212
+ "output": {
213
+ "finalLogitSoftcapping": null,
214
+ "tieWordEmbeddings": false,
215
+ "scaleEmbeddings": false,
216
+ "embeddingScale": null,
217
+ "logitInputScale": 1,
218
+ "embeddingTranspose": false,
219
+ "embeddingVocabSize": null,
220
+ "embeddingPostprocessor": null,
221
+ "lmHeadBiasTensor": null
222
+ },
223
+ "layerPattern": {
224
+ "type": "uniform",
225
+ "globalPattern": null,
226
+ "period": null,
227
+ "offset": null,
228
+ "layerTypes": null,
229
+ "residualBranchScale": 1.0954451150103321
230
+ },
231
+ "chatTemplate": { "type": null, "enabled": false }
232
+ },
233
+ "session": {
234
+ "compute": {
235
+ "defaults": {
236
+ "activationDtype": "f32",
237
+ "mathDtype": "f32",
238
+ "accumDtype": "f32",
239
+ "outputDtype": "f32"
240
+ }
241
+ },
242
+ "kvcache": {
243
+ "kvDtype": "f32",
244
+ "layout": "contiguous",
245
+ "maxSeqLen": 2048,
246
+ "pageSize": 256,
247
+ "tiering": {
248
+ "mode": "off",
249
+ "hotWindow": 1024,
250
+ "coldPageSize": 256,
251
+ "coldDtype": "f16",
252
+ "compression": {
253
+ "mode": "none",
254
+ "blockSize": 1,
255
+ "bitWidth": 4,
256
+ "prodMode": false
257
+ },
258
+ "gating": { "mode": "auto", "minAluBwRatio": 0 }
259
+ },
260
+ "quantization": { "mode": "none", "bitWidth": 4, "prodMode": false }
261
+ },
262
+ "decodeLoop": {
263
+ "batchSize": 1,
264
+ "stopCheckMode": "per-token",
265
+ "readbackInterval": 1,
266
+ "readbackMode": "sequential",
267
+ "submitLatencyThresholdMs": null,
268
+ "ringTokens": 1,
269
+ "ringStop": 1,
270
+ "ringStaging": 1,
271
+ "disableCommandBatching": false
272
+ },
273
+ "perLayerInputs": {
274
+ "materialization": "auto",
275
+ "rowCache": { "mode": "lru", "maxRows": 256, "maxBytes": 134217728, "decodedDtype": "f32" },
276
+ "prefetch": { "mode": "next_token", "rowsAhead": 1 },
277
+ "gpuUpload": { "mode": "per_step_slices", "stagingRows": 2 },
278
+ "hotCache": { "mode": "prepared_tokens", "maxTokens": 1024, "maxBytes": 134217728, "outputDtype": "f32" }
279
+ },
280
+ "skipEmbeddingKVCacheWrites": true,
281
+ "prefillChunkSubmitMode": "sync",
282
+ "useFlashPrefillAttention": false,
283
+ "useWideTileQ4KPrefill": false,
284
+ "retainQ4KMaterialization": false
285
+ },
286
+ "execution": {
287
+ "inlineKernelPath": true,
288
+ "policies": {
289
+ "unsupportedPrecision": "error",
290
+ "dtypeTransition": "require_cast_step",
291
+ "unresolvedKernel": "error"
292
+ },
293
+ "kernels": {
294
+ "embed": { "kernel": "gather.wgsl", "entry": "main", "digest": "sha256:4b12653c53247b32ebde7f6cf6a989d6248977e3816c761540b990b5f9818cb6" },
295
+ "layernorm": { "kernel": "layernorm.wgsl", "entry": "main", "digest": "sha256:8ff885017bfcfc8fdb8d16e09c358c000830bfd5fbf3d87187694edb0cb44f7c" },
296
+ "tiled": { "kernel": "matmul_f32.wgsl", "entry": "main", "digest": "sha256:b5bb8e3d8014136e33de7935dd2a1f074c988044fe05cf5b559718c6f061eaa8" },
297
+ "bias_add": { "kernel": "bias_add.wgsl", "entry": "main", "digest": "sha256:e7d3b4a5f9efc8b0569c56bcdeb63e921fa6d1d006901deabc734cab40346715" },
298
+ "rope": { "kernel": "rope.wgsl", "entry": "main", "digest": "sha256:b2da9d396668981dab9794c2973b668279f768994466b083d2105730555e1a5b" },
299
+ "attn_small": { "kernel": "attention_small.wgsl", "entry": "main", "digest": "sha256:6752ddd7ab53e6235c9b5b1a9515141c0d111df7fac9f4c0d7a38f9943490ed4", "precision": { "kvDtype": "f32" } },
300
+ "residual": { "kernel": "residual.wgsl", "entry": "main", "digest": "sha256:abd19bc08ad668a7cad562e2bc0a4be1aa8827ad4300d8d474b7fc2af4c27117" },
301
+ "silu": { "kernel": "silu.wgsl", "entry": "main", "digest": "sha256:d6b21e62031ac0d748617f7ebfb43834552a5b8f9590c18dc19cd0cd41a2fbca", "constants": { "HAS_GATE": true } }
302
+ },
303
+ "preLayer": [["embed", "embed", "embed_tokens"]],
304
+ "decode": [
305
+ ["input_norm", "layernorm"],
306
+ ["q_proj", "tiled", "layer.{L}.self_attn.q_proj"],
307
+ ["k_proj", "tiled", "layer.{L}.self_attn.k_proj"],
308
+ ["v_proj", "tiled", "layer.{L}.self_attn.v_proj"],
309
+ ["rope_q", "rope"],
310
+ ["rope_k", "rope"],
311
+ ["attention", "attn_small"],
312
+ ["o_proj", "tiled", "layer.{L}.self_attn.o_proj"],
313
+ ["attn_residual", "residual"],
314
+ ["post_attn_norm", "layernorm"],
315
+ ["gate_proj", "tiled", "layer.{L}.mlp.gate_proj"],
316
+ ["up_proj", "tiled", "layer.{L}.mlp.up_proj"],
317
+ ["activation", "silu"],
318
+ ["down_proj", "tiled", "layer.{L}.mlp.down_proj"],
319
+ ["ffn_residual", "residual"]
320
+ ],
321
+ "prefill": [
322
+ ["input_norm", "layernorm"],
323
+ ["q_proj", "tiled", "layer.{L}.self_attn.q_proj"],
324
+ ["k_proj", "tiled", "layer.{L}.self_attn.k_proj"],
325
+ ["v_proj", "tiled", "layer.{L}.self_attn.v_proj"],
326
+ ["rope_q", "rope"],
327
+ ["rope_k", "rope"],
328
+ ["attention", "attn_small"],
329
+ ["o_proj", "tiled", "layer.{L}.self_attn.o_proj"],
330
+ ["attn_residual", "residual"],
331
+ ["post_attn_norm", "layernorm"],
332
+ ["gate_proj", "tiled", "layer.{L}.mlp.gate_proj"],
333
+ ["up_proj", "tiled", "layer.{L}.mlp.up_proj"],
334
+ ["activation", "silu"],
335
+ ["down_proj", "tiled", "layer.{L}.mlp.down_proj"],
336
+ ["ffn_residual", "residual"]
337
+ ],
338
+ "postLayer": [["final_norm", "layernorm"]],
339
+ "sampling": null
340
+ },
341
+ "largeWeights": { "gpuResidentOverrides": null }
342
+ }