doppler-gpu 0.4.3 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/README.md +54 -33
- package/assets/doppler.svg +57 -1601
- package/package.json +36 -5
- package/src/cli/cli-output.js +34 -1
- package/src/cli/doppler-cli.d.ts +18 -0
- package/src/cli/doppler-cli.js +68 -35
- package/src/cli/doppler-serve.d.ts +210 -0
- package/src/cli/doppler-serve.js +468 -42
- package/src/client/doppler-registry.json +3 -5
- package/src/client/provider.d.ts +1 -1
- package/src/client/provider.js +1 -1
- package/src/client/runtime/index.js +13 -0
- package/src/client/runtime/lora.d.ts +11 -1
- package/src/client/runtime/lora.js +28 -7
- package/src/client/runtime/model-manager.js +35 -11
- package/src/client/runtime/model-session.d.ts +2 -1
- package/src/client/runtime/model-session.js +2 -2
- package/src/client/runtime/model-source.js +20 -1
- package/src/client/runtime/types.d.ts +8 -0
- package/src/client/wrap-pipeline-handle.d.ts +62 -0
- package/src/client/wrap-pipeline-handle.js +188 -13
- package/src/config/conversion/README.md +5 -0
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
- package/src/config/goal-completion-matrix.json +552 -0
- package/src/config/kernel-path-loader.js +192 -38
- package/src/config/kernels/kernel-ref-digests.js +52 -18
- package/src/config/kernels/registry.json +3017 -744
- package/src/config/merge-contract-check.js +7 -3
- package/src/config/merge-helpers.d.ts +2 -0
- package/src/config/merge-helpers.js +1 -0
- package/src/config/merge.d.ts +1 -0
- package/src/config/merge.js +30 -0
- package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
- package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
- package/src/config/param-validator.js +4 -0
- package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
- package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
- package/src/config/schema/buffer-pool.schema.js +3 -3
- package/src/config/schema/claim-evidence-contract.schema.json +93 -0
- package/src/config/schema/command-surface-contract.schema.json +81 -0
- package/src/config/schema/converter.schema.d.ts +11 -0
- package/src/config/schema/converter.schema.js +8 -0
- package/src/config/schema/doppler.schema.d.ts +1 -0
- package/src/config/schema/doppler.schema.js +38 -6
- package/src/config/schema/execution-v1.schema.d.ts +27 -0
- package/src/config/schema/execution-v1.schema.js +28 -19
- package/src/config/schema/goal-completion-matrix.schema.json +223 -0
- package/src/config/schema/inference.schema.d.ts +11 -0
- package/src/config/schema/manifest.schema.d.ts +107 -1
- package/src/config/schema/manifest.schema.js +10 -1
- package/src/config/schema/model-artifact-contract.schema.json +119 -0
- package/src/config/schema/moe.schema.d.ts +13 -0
- package/src/config/schema/moe.schema.js +1 -1
- package/src/config/schema/policy-schema-registry.json +26 -0
- package/src/config/schema/subsystem-support-contract.schema.json +48 -0
- package/src/config/schema/training-metrics.schema.d.ts +1 -1
- package/src/config/schema/training-metrics.schema.js +4 -3
- package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
- package/src/config/source-packages/registry.json +25 -0
- package/src/config/supported-operations.json +5 -0
- package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
- package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
- package/src/config/transforms/execution-graph-transforms.js +227 -230
- package/src/converter/conversion-plan.d.ts +2 -0
- package/src/converter/conversion-plan.js +18 -4
- package/src/converter/core.d.ts +8 -0
- package/src/converter/core.js +574 -12
- package/src/converter/quantization-info.js +179 -2
- package/src/converter/quantizer.d.ts +5 -4
- package/src/converter/quantizer.js +16 -2
- package/src/experimental/adapters/lora-loader.js +109 -24
- package/src/experimental/diffusion/index.d.ts +24 -0
- package/src/experimental/diffusion/index.js +10 -0
- package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
- package/src/experimental/distribution/descriptor-transport.js +342 -0
- package/src/experimental/training/README.md +11 -0
- package/src/experimental/training/autograd.js +33 -8
- package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
- package/src/experimental/training/datasets/gepa-frontier.js +220 -0
- package/src/experimental/training/datasets/index.d.ts +13 -1
- package/src/experimental/training/datasets/index.js +8 -1
- package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
- package/src/experimental/training/datasets/teacher-traces.js +279 -0
- package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
- package/src/experimental/training/datasets/text-pairs.js +82 -5
- package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
- package/src/experimental/training/distillation/student-fixture.js +290 -32
- package/src/experimental/training/export.d.ts +9 -0
- package/src/experimental/training/export.js +113 -7
- package/src/experimental/training/index.d.ts +13 -1
- package/src/experimental/training/index.js +13 -1
- package/src/experimental/training/lora-pipeline.d.ts +143 -0
- package/src/experimental/training/lora-pipeline.js +1737 -49
- package/src/experimental/training/lora.js +23 -1
- package/src/experimental/training/loss.js +1 -1
- package/src/experimental/training/operator-agent-eval.d.ts +65 -0
- package/src/experimental/training/operator-agent-eval.js +326 -0
- package/src/experimental/training/operator-command.js +310 -51
- package/src/experimental/training/runner.d.ts +1 -0
- package/src/experimental/training/runner.js +4 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
- package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/registry.json +186 -6
- package/src/experimental/training/workloads.d.ts +37 -0
- package/src/experimental/training/workloads.js +110 -1
- package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
- package/src/formats/rdrr/functional-descriptor.js +230 -0
- package/src/formats/rdrr/index.d.ts +1 -0
- package/src/formats/rdrr/index.js +1 -0
- package/src/formats/rdrr/parsing.js +22 -6
- package/src/formats/rdrr/source-transform-contract.js +10 -1
- package/src/formats/rdrr/storage-descriptor.js +9 -2
- package/src/formats/rdrr/types.d.ts +5 -3
- package/src/formats/rdrr/validation.js +15 -1
- package/src/gpu/command-recorder.d.ts +43 -3
- package/src/gpu/command-recorder.js +122 -25
- package/src/gpu/device.js +111 -11
- package/src/gpu/kernel-tuner/cache.js +1 -5
- package/src/gpu/kernels/attention.js +17 -9
- package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
- package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
- package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
- package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
- package/src/gpu/kernels/cast.js +74 -67
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
- package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
- package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
- package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
- package/src/gpu/kernels/dispatch.js +13 -3
- package/src/gpu/kernels/fused_ffn.js +3 -4
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
- package/src/gpu/kernels/gather.d.ts +84 -1
- package/src/gpu/kernels/gather.js +353 -5
- package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
- package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
- package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
- package/src/gpu/kernels/gelu.wgsl +11 -7
- package/src/gpu/kernels/gelu_f16.wgsl +11 -7
- package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
- package/src/gpu/kernels/gemma4-route-expert.js +126 -0
- package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
- package/src/gpu/kernels/index.d.ts +71 -0
- package/src/gpu/kernels/index.js +52 -0
- package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
- package/src/gpu/kernels/kv-cache-write.js +126 -0
- package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
- package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
- package/src/gpu/kernels/lm-head-argmax.js +218 -0
- package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
- package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
- package/src/gpu/kernels/matmul-dispatch.js +37 -1
- package/src/gpu/kernels/matmul-selection.d.ts +5 -5
- package/src/gpu/kernels/matmul-selection.js +104 -9
- package/src/gpu/kernels/matmul.d.ts +1 -1
- package/src/gpu/kernels/matmul.js +151 -43
- package/src/gpu/kernels/moe.d.ts +1 -0
- package/src/gpu/kernels/moe.js +46 -18
- package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
- package/src/gpu/kernels/pipeline-cache.js +24 -0
- package/src/gpu/kernels/rep-penalty.d.ts +21 -0
- package/src/gpu/kernels/rep-penalty.js +17 -9
- package/src/gpu/kernels/residual.d.ts +1 -0
- package/src/gpu/kernels/residual.js +13 -1
- package/src/gpu/kernels/residual.wgsl +3 -3
- package/src/gpu/kernels/residual_f16.wgsl +5 -3
- package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
- package/src/gpu/kernels/residual_vec4.wgsl +6 -6
- package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
- package/src/gpu/kernels/rmsnorm-pair.js +152 -0
- package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
- package/src/gpu/kernels/rmsnorm-qk.js +128 -0
- package/src/gpu/kernels/rmsnorm.d.ts +31 -0
- package/src/gpu/kernels/rmsnorm.js +32 -7
- package/src/gpu/kernels/rmsnorm.wgsl +10 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
- package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
- package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
- package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
- package/src/gpu/kernels/rope-qk.d.ts +41 -0
- package/src/gpu/kernels/rope-qk.js +143 -0
- package/src/gpu/kernels/rope_qk.wgsl +95 -0
- package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
- package/src/gpu/kernels/rule-registry.js +50 -10
- package/src/gpu/kernels/sample.js +212 -187
- package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
- package/src/gpu/kernels/shader-cache.js +1 -21
- package/src/gpu/kernels/soft-embedding.d.ts +28 -0
- package/src/gpu/kernels/soft-embedding.js +447 -0
- package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
- package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
- package/src/gpu/kernels/softmax.wgsl +5 -80
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
- package/src/gpu/kernels/uniform-utils.js +2 -5
- package/src/gpu/kernels/utils.d.ts +16 -1
- package/src/gpu/kernels/utils.js +42 -13
- package/src/gpu/readback-utils.js +10 -2
- package/src/gpu/uniform-cache.d.ts +10 -1
- package/src/gpu/uniform-cache.js +25 -7
- package/src/gpu/weight-buffer.d.ts +76 -7
- package/src/gpu/weight-buffer.js +85 -1
- package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
- package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
- package/src/inference/browser-harness-suite-helpers.js +438 -2
- package/src/inference/browser-harness-text-helpers.d.ts +77 -0
- package/src/inference/browser-harness-text-helpers.js +236 -11
- package/src/inference/browser-harness.d.ts +8 -0
- package/src/inference/browser-harness.js +292 -8
- package/src/inference/decode-ring.d.ts +1 -0
- package/src/inference/decode-ring.js +7 -5
- package/src/inference/kv-cache/base.d.ts +25 -0
- package/src/inference/kv-cache/base.js +109 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
- package/src/inference/kv-cache/mixed-geometry.js +109 -2
- package/src/inference/kv-cache/sliding-window.d.ts +14 -0
- package/src/inference/kv-cache/sliding-window.js +88 -0
- package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
- package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
- package/src/inference/moe-router.d.ts +14 -1
- package/src/inference/moe-router.js +24 -1
- package/src/inference/pipelines/audio/gemma4.js +52 -1
- package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
- package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
- package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
- package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
- package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
- package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
- package/src/inference/pipelines/text/attention/projections.js +329 -49
- package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
- package/src/inference/pipelines/text/attention/record.js +446 -73
- package/src/inference/pipelines/text/attention/run.js +251 -49
- package/src/inference/pipelines/text/attention/types.d.ts +5 -1
- package/src/inference/pipelines/text/config.d.ts +22 -1
- package/src/inference/pipelines/text/config.js +199 -22
- package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
- package/src/inference/pipelines/text/dtype-contract.js +4 -1
- package/src/inference/pipelines/text/embed.d.ts +4 -3
- package/src/inference/pipelines/text/embed.js +16 -6
- package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
- package/src/inference/pipelines/text/execution-plan.js +6 -0
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
- package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
- package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
- package/src/inference/pipelines/text/execution-v1.js +373 -139
- package/src/inference/pipelines/text/ffn/dense.js +44 -13
- package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
- package/src/inference/pipelines/text/ffn/moe.js +9 -2
- package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
- package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/standard.js +44 -4
- package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
- package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
- package/src/inference/pipelines/text/generator-helpers.js +5 -2
- package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
- package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
- package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
- package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
- package/src/inference/pipelines/text/generator-steps.js +268 -77
- package/src/inference/pipelines/text/generator.d.ts +39 -0
- package/src/inference/pipelines/text/generator.js +1105 -18
- package/src/inference/pipelines/text/init.d.ts +8 -1
- package/src/inference/pipelines/text/init.js +30 -1
- package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
- package/src/inference/pipelines/text/layer.js +272 -25
- package/src/inference/pipelines/text/linear-attention.js +6 -4
- package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
- package/src/inference/pipelines/text/logits/gpu.js +417 -66
- package/src/inference/pipelines/text/logits/index.d.ts +2 -2
- package/src/inference/pipelines/text/logits/index.js +96 -42
- package/src/inference/pipelines/text/logits/types.d.ts +2 -2
- package/src/inference/pipelines/text/lora-apply.js +17 -0
- package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
- package/src/inference/pipelines/text/moe-gpu.js +445 -16
- package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
- package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
- package/src/inference/pipelines/text/ops.d.ts +29 -2
- package/src/inference/pipelines/text/ops.js +19 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
- package/src/inference/pipelines/text/probes.js +4 -5
- package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
- package/src/inference/pipelines/text/sampling-config.js +103 -64
- package/src/inference/pipelines/text/sampling.js +13 -9
- package/src/inference/pipelines/text/state.js +17 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
- package/src/inference/pipelines/text/types.d.ts +84 -3
- package/src/inference/pipelines/text/weights.d.ts +4 -2
- package/src/inference/pipelines/text/weights.js +4 -5
- package/src/inference/pipelines/text.d.ts +24 -0
- package/src/inference/pipelines/text.js +162 -38
- package/src/inference/pipelines/vision/gemma4.js +20 -9
- package/src/inference/tokenizer.d.ts +30 -0
- package/src/inference/tokenizer.js +340 -164
- package/src/inference/tokenizers/bundled.js +71 -49
- package/src/loader/doppler-loader.d.ts +11 -0
- package/src/loader/doppler-loader.js +290 -15
- package/src/loader/embedding-limit-preflight.d.ts +31 -0
- package/src/loader/embedding-limit-preflight.js +216 -0
- package/src/loader/embedding-loader.d.ts +8 -2
- package/src/loader/embedding-loader.js +263 -8
- package/src/loader/experts/expert-loader.d.ts +4 -1
- package/src/loader/experts/expert-loader.js +174 -5
- package/src/loader/final-weights-loader.d.ts +5 -0
- package/src/loader/final-weights-loader.js +167 -4
- package/src/loader/layer-loader.d.ts +2 -0
- package/src/loader/layer-loader.js +74 -6
- package/src/loader/loader-types.d.ts +59 -2
- package/src/loader/manifest-config.d.ts +8 -0
- package/src/loader/shard-resolver.js +37 -4
- package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
- package/src/loader/tensors/functional-descriptor-loader.js +377 -0
- package/src/loader/tensors/tensor-loader.d.ts +36 -1
- package/src/loader/tensors/tensor-loader.js +437 -21
- package/src/loader/tensors/tensor-reader.d.ts +4 -1
- package/src/loader/tensors/tensor-reader.js +11 -4
- package/src/loader/weights.d.ts +3 -1
- package/src/memory/buffer-pool.js +12 -9
- package/src/models/diffusiongemma.d.ts +7 -0
- package/src/models/diffusiongemma.js +25 -0
- package/src/models/gemma4.js +24 -0
- package/src/models/qwen3.js +2 -2
- package/src/rules/execution-rules-contract-check.js +93 -10
- package/src/rules/inference/capability-transforms.rules.json +26 -2
- package/src/rules/inference/config.rules.json +1 -0
- package/src/rules/inference/dtype.rules.json +7 -0
- package/src/rules/inference/execution.rules.json +108 -17
- package/src/rules/inference/ffn.rules.json +18 -0
- package/src/rules/kernels/gather.rules.json +81 -0
- package/src/rules/kernels/matmul.rules.json +19 -0
- package/src/rules/kernels/moe.rules.json +3 -0
- package/src/rules/kernels/rep-penalty.rules.json +6 -0
- package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
- package/src/rules/kernels/rmsnorm.rules.json +42 -7
- package/src/rules/kernels/rope-qk.rules.json +6 -0
- package/src/rules/loader/tensor-loader.rules.json +8 -0
- package/src/rules/rule-registry.js +6 -0
- package/src/storage/artifact-storage-context.d.ts +1 -0
- package/src/storage/artifact-storage-context.js +10 -0
- package/src/tooling/browser-command-runner.d.ts +0 -3
- package/src/tooling/browser-command-runner.js +2 -14
- package/src/tooling/command-api-family-normalizers.js +4 -2
- package/src/tooling/command-api-helpers.js +2 -1
- package/src/tooling/command-api.d.ts +2 -0
- package/src/tooling/command-api.js +1 -0
- package/src/tooling/command-runner-shared.d.ts +4 -0
- package/src/tooling/command-runner-shared.js +2 -1
- package/src/tooling/diagnose-runner.js +2 -0
- package/src/tooling/litert-package-runtime.js +553 -16
- package/src/tooling/node-browser-command-runner.js +10 -4
- package/src/tooling/node-command-runner.js +7 -2
- package/src/tooling/node-convert-worker-pool.js +3 -0
- package/src/tooling/node-convert-worker.js +1 -0
- package/src/tooling/node-converter.js +12 -0
- package/src/tooling/node-webgpu.d.ts +1 -0
- package/src/tooling/node-webgpu.js +23 -11
- package/src/tooling/program-bundle.d.ts +52 -1
- package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
- package/src/tooling/source-artifact-adapter.js +7 -2
- package/src/tooling/source-package-profiles.d.ts +18 -1
- package/src/tooling/source-package-profiles.js +109 -1
- package/src/tooling/source-runtime-bundle.d.ts +1 -0
- package/src/tooling/source-runtime-bundle.js +27 -0
- package/src/tooling/source-runtime-converter-config.js +12 -3
- package/src/tooling-experimental-exports.shared.d.ts +10 -0
- package/src/tooling-experimental-exports.shared.js +10 -0
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
import { acquireBuffer, releaseBuffer } from '../../memory/buffer-pool.js';
|
|
2
|
+
import { padToQ4KBlock } from '../../config/schema/index.js';
|
|
3
|
+
import { createTensor, dtypeBytes } from '../tensor.js';
|
|
4
|
+
import { getBuffer } from '../weight-buffer.js';
|
|
5
|
+
import { unifiedKernelWrapper } from './utils.js';
|
|
6
|
+
import { selectRuleValue } from './rule-registry.js';
|
|
7
|
+
import {
|
|
8
|
+
assertRMSNormWeightBuffer,
|
|
9
|
+
planRMSNormDispatch,
|
|
10
|
+
resolveNormWeightDtype,
|
|
11
|
+
} from './rmsnorm.js';
|
|
12
|
+
|
|
13
|
+
function assertPositiveInteger(value, label) {
|
|
14
|
+
if (!Number.isInteger(value) || value <= 0) {
|
|
15
|
+
throw new Error(`RMSNorm QK requires ${label} to be a positive integer.`);
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
function assertBufferCapacity(buffer, requiredBytes, label) {
|
|
20
|
+
if (Number.isFinite(buffer?.size) && requiredBytes > buffer.size) {
|
|
21
|
+
throw new Error(`RMSNorm QK ${label} buffer is smaller than requested range (${requiredBytes} > ${buffer.size} bytes).`);
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function selectRMSNormQKVariant(dtype) {
|
|
26
|
+
return selectRuleValue('rmsnormQk', 'variant', { isF16: dtype === 'f16' });
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export function canUseRMSNormQK(qTensor, kTensor, options = {}) {
|
|
30
|
+
if (options.skipKNorm === true) {
|
|
31
|
+
return false;
|
|
32
|
+
}
|
|
33
|
+
if (!qTensor || !kTensor || qTensor.buffer === kTensor.buffer) {
|
|
34
|
+
return false;
|
|
35
|
+
}
|
|
36
|
+
return qTensor.dtype === kTensor.dtype && (qTensor.dtype === 'f32' || qTensor.dtype === 'f16');
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
async function rmsNormQK(target, qTensor, kTensor, qWeight, kWeight, eps, options = {}) {
|
|
40
|
+
const {
|
|
41
|
+
numTokens,
|
|
42
|
+
numHeads,
|
|
43
|
+
numKVHeads,
|
|
44
|
+
headDim,
|
|
45
|
+
rmsNormWeightOffset = false,
|
|
46
|
+
} = options;
|
|
47
|
+
|
|
48
|
+
assertPositiveInteger(numTokens, 'numTokens');
|
|
49
|
+
assertPositiveInteger(numHeads, 'numHeads');
|
|
50
|
+
assertPositiveInteger(numKVHeads, 'numKVHeads');
|
|
51
|
+
assertPositiveInteger(headDim, 'headDim');
|
|
52
|
+
if (qTensor.dtype !== kTensor.dtype) {
|
|
53
|
+
throw new Error(`RMSNorm QK requires matching Q/K dtypes, got ${qTensor.dtype} and ${kTensor.dtype}.`);
|
|
54
|
+
}
|
|
55
|
+
if (qTensor.dtype !== 'f32' && qTensor.dtype !== 'f16') {
|
|
56
|
+
throw new Error(`RMSNorm QK requires f32 or f16 tensors, got ${qTensor.dtype}.`);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
const qRows = numTokens * numHeads;
|
|
60
|
+
const kRows = numTokens * numKVHeads;
|
|
61
|
+
const bytesPerElement = dtypeBytes(qTensor.dtype);
|
|
62
|
+
assertBufferCapacity(qTensor.buffer, qRows * headDim * bytesPerElement, 'Q input');
|
|
63
|
+
assertBufferCapacity(kTensor.buffer, kRows * headDim * bytesPerElement, 'K input');
|
|
64
|
+
|
|
65
|
+
const qWeightBuffer = getBuffer(qWeight);
|
|
66
|
+
const kWeightBuffer = getBuffer(kWeight);
|
|
67
|
+
assertRMSNormWeightBuffer(qWeight, qWeightBuffer, headDim);
|
|
68
|
+
assertRMSNormWeightBuffer(kWeight, kWeightBuffer, headDim);
|
|
69
|
+
const qWeightDtype = resolveNormWeightDtype(qWeight, headDim);
|
|
70
|
+
const kWeightDtype = resolveNormWeightDtype(kWeight, headDim);
|
|
71
|
+
|
|
72
|
+
const paddedHeadDim = padToQ4KBlock(headDim);
|
|
73
|
+
const qOutputSize = qRows * paddedHeadDim * bytesPerElement;
|
|
74
|
+
const kOutputSize = kRows * paddedHeadDim * bytesPerElement;
|
|
75
|
+
let qOutputBuffer = null;
|
|
76
|
+
let kOutputBuffer = null;
|
|
77
|
+
|
|
78
|
+
try {
|
|
79
|
+
qOutputBuffer = acquireBuffer(qOutputSize, undefined, 'rmsnorm_qk_q_output');
|
|
80
|
+
kOutputBuffer = acquireBuffer(kOutputSize, undefined, 'rmsnorm_qk_k_output');
|
|
81
|
+
const dispatchPlan = planRMSNormDispatch(target, qRows + kRows);
|
|
82
|
+
await unifiedKernelWrapper(
|
|
83
|
+
'rmsnorm_qk',
|
|
84
|
+
target,
|
|
85
|
+
selectRMSNormQKVariant(qTensor.dtype),
|
|
86
|
+
[qTensor, qWeightBuffer, qOutputBuffer, kTensor, kWeightBuffer, kOutputBuffer],
|
|
87
|
+
{
|
|
88
|
+
q_rows: qRows,
|
|
89
|
+
k_rows: kRows,
|
|
90
|
+
head_dim: headDim,
|
|
91
|
+
row_stride: dispatchPlan.tokenStride,
|
|
92
|
+
eps,
|
|
93
|
+
_pad0: 0,
|
|
94
|
+
_pad1: 0,
|
|
95
|
+
_pad2: 0,
|
|
96
|
+
},
|
|
97
|
+
dispatchPlan.workgroups,
|
|
98
|
+
{
|
|
99
|
+
RMS_NORM_OFFSET: rmsNormWeightOffset,
|
|
100
|
+
Q_WEIGHT_IS_F16: qWeightDtype === 'f16',
|
|
101
|
+
K_WEIGHT_IS_F16: kWeightDtype === 'f16',
|
|
102
|
+
},
|
|
103
|
+
null,
|
|
104
|
+
'rmsnorm_qk'
|
|
105
|
+
);
|
|
106
|
+
|
|
107
|
+
return {
|
|
108
|
+
q: createTensor(qOutputBuffer, qTensor.dtype, [qRows, headDim], 'rmsnorm_qk_q_output'),
|
|
109
|
+
k: createTensor(kOutputBuffer, kTensor.dtype, [kRows, headDim], 'rmsnorm_qk_k_output'),
|
|
110
|
+
};
|
|
111
|
+
} catch (error) {
|
|
112
|
+
if (qOutputBuffer) {
|
|
113
|
+
releaseBuffer(qOutputBuffer);
|
|
114
|
+
}
|
|
115
|
+
if (kOutputBuffer) {
|
|
116
|
+
releaseBuffer(kOutputBuffer);
|
|
117
|
+
}
|
|
118
|
+
throw error;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
export async function runRMSNormQK(qTensor, kTensor, qWeight, kWeight, eps, options = {}) {
|
|
123
|
+
return rmsNormQK(null, qTensor, kTensor, qWeight, kWeight, eps, options);
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
export async function recordRMSNormQK(recorder, qTensor, kTensor, qWeight, kWeight, eps, options = {}) {
|
|
127
|
+
return rmsNormQK(recorder, qTensor, kTensor, qWeight, kWeight, eps, options);
|
|
128
|
+
}
|
|
@@ -9,15 +9,46 @@ import type { CommandRecorder } from '../command-recorder.js';
|
|
|
9
9
|
import type { OutputBufferOptions } from './types.js';
|
|
10
10
|
import type { WeightBuffer, TensorLike } from '../weight-buffer.js';
|
|
11
11
|
|
|
12
|
+
/** Maximum hidden size supported by the cached residual RMSNorm variant. */
|
|
13
|
+
export declare const RMSNORM_CACHE_LIMIT: number;
|
|
14
|
+
|
|
12
15
|
/** RMSNorm kernel options */
|
|
13
16
|
export interface RMSNormOptions extends OutputBufferOptions {
|
|
14
17
|
batchSize?: number;
|
|
15
18
|
hiddenSize?: number | null;
|
|
16
19
|
residual?: Tensor | null;
|
|
20
|
+
preResidual?: Tensor | null;
|
|
21
|
+
residualSumOutput?: GPUBuffer | Tensor | null;
|
|
22
|
+
outputScale?: number | null;
|
|
17
23
|
/** Use (1+w)*x normalization for Gemma 2/3 */
|
|
18
24
|
rmsNormWeightOffset?: boolean;
|
|
25
|
+
label?: string;
|
|
19
26
|
}
|
|
20
27
|
|
|
28
|
+
/**
|
|
29
|
+
* Return true when the residual RMSNorm variant must avoid the cached WGSL path.
|
|
30
|
+
*/
|
|
31
|
+
export declare function residualVariantBypassesCache(
|
|
32
|
+
residual: Tensor | GPUBuffer | WeightBuffer | TensorLike | null | undefined,
|
|
33
|
+
hiddenSize: number | null | undefined
|
|
34
|
+
): boolean;
|
|
35
|
+
|
|
36
|
+
export declare function resolveNormWeightDtype(
|
|
37
|
+
weight: GPUBuffer | WeightBuffer | TensorLike,
|
|
38
|
+
hiddenSize: number | null | undefined
|
|
39
|
+
): string;
|
|
40
|
+
|
|
41
|
+
export declare function assertRMSNormWeightBuffer(
|
|
42
|
+
weight: GPUBuffer | WeightBuffer | TensorLike,
|
|
43
|
+
weightBuffer: GPUBuffer | null | undefined,
|
|
44
|
+
hiddenSize: number | null | undefined
|
|
45
|
+
): void;
|
|
46
|
+
|
|
47
|
+
export declare function planRMSNormDispatch(
|
|
48
|
+
target: CommandRecorder | null | undefined,
|
|
49
|
+
numTokens: number
|
|
50
|
+
): { tokenStride: number; workgroups: [number, number, number] };
|
|
51
|
+
|
|
21
52
|
/**
|
|
22
53
|
* Select RMSNorm kernel variant based on options, tensor dtypes, and GPU capabilities.
|
|
23
54
|
*/
|
|
@@ -41,7 +41,7 @@ function normalizeNormWeightDtype(dtype) {
|
|
|
41
41
|
return null;
|
|
42
42
|
}
|
|
43
43
|
|
|
44
|
-
function resolveNormWeightDtype(weight, hiddenSize) {
|
|
44
|
+
export function resolveNormWeightDtype(weight, hiddenSize) {
|
|
45
45
|
const explicitDtype = normalizeNormWeightDtype(getWeightDtype(weight));
|
|
46
46
|
if (explicitDtype) {
|
|
47
47
|
return explicitDtype;
|
|
@@ -76,7 +76,7 @@ function resolveNormWeightDtype(weight, hiddenSize) {
|
|
|
76
76
|
return DEFAULT_DTYPE;
|
|
77
77
|
}
|
|
78
78
|
|
|
79
|
-
function assertRMSNormWeightBuffer(weight, weightBuffer, hiddenSize) {
|
|
79
|
+
export function assertRMSNormWeightBuffer(weight, weightBuffer, hiddenSize) {
|
|
80
80
|
const isGpuBuffer = weightBuffer && (
|
|
81
81
|
typeof GPUBuffer === 'undefined'
|
|
82
82
|
? true
|
|
@@ -94,7 +94,7 @@ function assertRMSNormWeightBuffer(weight, weightBuffer, hiddenSize) {
|
|
|
94
94
|
);
|
|
95
95
|
}
|
|
96
96
|
|
|
97
|
-
function planRMSNormDispatch(target, numTokens) {
|
|
97
|
+
export function planRMSNormDispatch(target, numTokens) {
|
|
98
98
|
const device = target?.device;
|
|
99
99
|
const maxPerDim = Number.isFinite(device?.limits?.maxComputeWorkgroupsPerDimension)
|
|
100
100
|
? device.limits.maxComputeWorkgroupsPerDimension
|
|
@@ -106,6 +106,25 @@ function planRMSNormDispatch(target, numTokens) {
|
|
|
106
106
|
};
|
|
107
107
|
}
|
|
108
108
|
|
|
109
|
+
function resolveRMSNormOutputScale(outputScale) {
|
|
110
|
+
if (outputScale == null) {
|
|
111
|
+
return 1;
|
|
112
|
+
}
|
|
113
|
+
const value = Number(outputScale);
|
|
114
|
+
if (!Number.isFinite(value)) {
|
|
115
|
+
throw new Error(`[rmsnorm] outputScale must be finite; got "${String(outputScale)}".`);
|
|
116
|
+
}
|
|
117
|
+
return value;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
function resolveRMSNormDispatchLabel(label) {
|
|
121
|
+
if (typeof label !== 'string' || label.length === 0) {
|
|
122
|
+
return 'rmsnorm';
|
|
123
|
+
}
|
|
124
|
+
const normalized = label.replace(/^L\d+\./, '').replace(/\s+/g, '_');
|
|
125
|
+
return `rmsnorm:${normalized}`;
|
|
126
|
+
}
|
|
127
|
+
|
|
109
128
|
export function selectRMSNormKernel(options = {}, isF16 = false) {
|
|
110
129
|
const { residual = null, hiddenSize = null } = options;
|
|
111
130
|
const { smallThreshold } = getKernelThresholds().rmsnorm;
|
|
@@ -133,7 +152,9 @@ export async function runRMSNorm(
|
|
|
133
152
|
const {
|
|
134
153
|
batchSize = 1, hiddenSize, residual = null, outputBuffer = null,
|
|
135
154
|
rmsNormWeightOffset = false, preResidual = null, residualSumOutput = null,
|
|
155
|
+
outputScale = null,
|
|
136
156
|
} = options;
|
|
157
|
+
const resolvedOutputScale = resolveRMSNormOutputScale(outputScale);
|
|
137
158
|
const isF16 = input.dtype === 'f16';
|
|
138
159
|
const variant = selectRMSNormKernel(options, isF16);
|
|
139
160
|
const inferredHiddenSize = inferHiddenSize(input, hiddenSize);
|
|
@@ -173,7 +194,7 @@ export async function runRMSNorm(
|
|
|
173
194
|
eps,
|
|
174
195
|
has_residual: residual ? 1 : 0,
|
|
175
196
|
token_stride: dispatchPlan.tokenStride,
|
|
176
|
-
|
|
197
|
+
output_scale: resolvedOutputScale,
|
|
177
198
|
_pad1: 0,
|
|
178
199
|
_pad2: 0,
|
|
179
200
|
},
|
|
@@ -184,7 +205,8 @@ export async function runRMSNorm(
|
|
|
184
205
|
PRE_RESIDUAL: !!preResidual,
|
|
185
206
|
OUTPUT_PRENORM: hasPrenormOutput,
|
|
186
207
|
},
|
|
187
|
-
extraBindings
|
|
208
|
+
extraBindings,
|
|
209
|
+
resolveRMSNormDispatchLabel(options.label)
|
|
188
210
|
);
|
|
189
211
|
|
|
190
212
|
if (ownedPrenormPlaceholder) releaseBuffer(ownedPrenormPlaceholder);
|
|
@@ -209,7 +231,9 @@ export async function recordRMSNorm(
|
|
|
209
231
|
const {
|
|
210
232
|
batchSize = 1, hiddenSize = null, residual = null, outputBuffer = null,
|
|
211
233
|
rmsNormWeightOffset = false, preResidual = null, residualSumOutput = null,
|
|
234
|
+
outputScale = null,
|
|
212
235
|
} = options;
|
|
236
|
+
const resolvedOutputScale = resolveRMSNormOutputScale(outputScale);
|
|
213
237
|
const isF16 = input.dtype === 'f16';
|
|
214
238
|
const variant = selectRMSNormKernel(options, isF16);
|
|
215
239
|
const inferredHiddenSize = inferHiddenSize(input, hiddenSize);
|
|
@@ -246,7 +270,7 @@ export async function recordRMSNorm(
|
|
|
246
270
|
eps,
|
|
247
271
|
has_residual: residual ? 1 : 0,
|
|
248
272
|
token_stride: dispatchPlan.tokenStride,
|
|
249
|
-
|
|
273
|
+
output_scale: resolvedOutputScale,
|
|
250
274
|
_pad1: 0,
|
|
251
275
|
_pad2: 0,
|
|
252
276
|
},
|
|
@@ -257,7 +281,8 @@ export async function recordRMSNorm(
|
|
|
257
281
|
PRE_RESIDUAL: !!preResidual,
|
|
258
282
|
OUTPUT_PRENORM: hasPrenormOutput,
|
|
259
283
|
},
|
|
260
|
-
extraBindings
|
|
284
|
+
extraBindings,
|
|
285
|
+
resolveRMSNormDispatchLabel(options.label)
|
|
261
286
|
);
|
|
262
287
|
|
|
263
288
|
if (ownedPrenormPlaceholder) releaseBuffer(ownedPrenormPlaceholder);
|
|
@@ -42,7 +42,7 @@ struct Uniforms {
|
|
|
42
42
|
eps: f32, // Epsilon for numerical stability (typically 1e-5 or 1e-6)
|
|
43
43
|
has_residual: u32, // Runtime flag: 1 = add residual after norm
|
|
44
44
|
token_stride: u32, // Workgroup rows per dispatch row
|
|
45
|
-
|
|
45
|
+
output_scale: f32, // Output epilogue scale
|
|
46
46
|
_pad1: u32,
|
|
47
47
|
_pad2: u32,
|
|
48
48
|
}
|
|
@@ -103,6 +103,10 @@ fn write_prenorm(base_offset: u32, idx: u32, val: f32) {
|
|
|
103
103
|
}
|
|
104
104
|
}
|
|
105
105
|
|
|
106
|
+
fn apply_output_scale(value: f32) -> f32 {
|
|
107
|
+
return value * u.output_scale;
|
|
108
|
+
}
|
|
109
|
+
|
|
106
110
|
fn token_index(wg_id: vec3<u32>) -> u32 {
|
|
107
111
|
return wg_id.y * max(u.token_stride, 1u) + wg_id.x;
|
|
108
112
|
}
|
|
@@ -177,7 +181,7 @@ fn main(
|
|
|
177
181
|
// Write pre-norm sum for downstream residual reuse
|
|
178
182
|
write_prenorm(base_offset, idx, x);
|
|
179
183
|
|
|
180
|
-
output[base_offset + idx] = result;
|
|
184
|
+
output[base_offset + idx] = apply_output_scale(result);
|
|
181
185
|
}
|
|
182
186
|
}
|
|
183
187
|
}
|
|
@@ -231,7 +235,7 @@ fn main_small(
|
|
|
231
235
|
result = result + residual[base_offset + thread_idx];
|
|
232
236
|
}
|
|
233
237
|
write_prenorm(base_offset, thread_idx, x);
|
|
234
|
-
output[base_offset + thread_idx] = result;
|
|
238
|
+
output[base_offset + thread_idx] = apply_output_scale(result);
|
|
235
239
|
}
|
|
236
240
|
}
|
|
237
241
|
|
|
@@ -295,7 +299,7 @@ fn main_cached(
|
|
|
295
299
|
result = result + residual[base_offset + idx];
|
|
296
300
|
}
|
|
297
301
|
write_prenorm(base_offset, idx, x);
|
|
298
|
-
output[base_offset + idx] = result;
|
|
302
|
+
output[base_offset + idx] = apply_output_scale(result);
|
|
299
303
|
}
|
|
300
304
|
}
|
|
301
305
|
}
|
|
@@ -378,7 +382,7 @@ fn main_subgroup(
|
|
|
378
382
|
}
|
|
379
383
|
|
|
380
384
|
write_prenorm(base_offset, idx, x);
|
|
381
|
-
output[base_offset + idx] = result;
|
|
385
|
+
output[base_offset + idx] = apply_output_scale(result);
|
|
382
386
|
}
|
|
383
387
|
}
|
|
384
388
|
}
|
|
@@ -443,6 +447,6 @@ fn main_small_subgroup(
|
|
|
443
447
|
result = result + residual[base_offset + thread_idx];
|
|
444
448
|
}
|
|
445
449
|
write_prenorm(base_offset, thread_idx, x);
|
|
446
|
-
output[base_offset + thread_idx] = result;
|
|
450
|
+
output[base_offset + thread_idx] = apply_output_scale(result);
|
|
447
451
|
}
|
|
448
452
|
}
|
|
@@ -1,5 +1,9 @@
|
|
|
1
1
|
// AUTO-GENERATED from src/gpu/kernels/rmsnorm.wgsl.
|
|
2
2
|
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
|
|
3
|
+
// AUTO-GENERATED from src/gpu/kernels/rmsnorm.wgsl.
|
|
4
|
+
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
|
|
5
|
+
// AUTO-GENERATED from src/gpu/kernels/rmsnorm.wgsl.
|
|
6
|
+
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
|
|
3
7
|
//
|
|
4
8
|
// F16 variant for reduced memory bandwidth when using F16 activations.
|
|
5
9
|
// The all-f16 lane keeps normalization arithmetic in f16.
|
|
@@ -22,7 +26,7 @@ struct Uniforms {
|
|
|
22
26
|
eps: f32, // Epsilon for numerical stability
|
|
23
27
|
has_residual: u32, // 1 if residual input provided, 0 otherwise
|
|
24
28
|
token_stride: u32, // Workgroup rows per dispatch row
|
|
25
|
-
|
|
29
|
+
output_scale: f32, // Output epilogue scale
|
|
26
30
|
_pad1: u32,
|
|
27
31
|
_pad2: u32,
|
|
28
32
|
}
|
|
@@ -156,7 +160,7 @@ fn main(
|
|
|
156
160
|
// Write pre-norm sum for downstream residual reuse
|
|
157
161
|
write_prenorm(base_offset, idx, x);
|
|
158
162
|
|
|
159
|
-
output[base_offset + idx] = result;
|
|
163
|
+
output[base_offset + idx] = f16(f32(result) * u.output_scale);
|
|
160
164
|
}
|
|
161
165
|
}
|
|
162
166
|
}
|
|
@@ -219,6 +223,6 @@ fn rmsnorm_small_f16(
|
|
|
219
223
|
result = result + residual[base_offset + thread_idx];
|
|
220
224
|
}
|
|
221
225
|
write_prenorm(base_offset, thread_idx, x);
|
|
222
|
-
output[base_offset + thread_idx] = result;
|
|
226
|
+
output[base_offset + thread_idx] = f16(f32(result) * u.output_scale);
|
|
223
227
|
}
|
|
224
228
|
}
|
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
// Sandwich RMSNorm pair kernel.
|
|
2
|
+
//
|
|
3
|
+
// Computes:
|
|
4
|
+
// post_output = residual + rmsnorm(input, post_weight)
|
|
5
|
+
// pre_output = rmsnorm(post_output, pre_weight)
|
|
6
|
+
//
|
|
7
|
+
// This is intended for decode sandwich-norm paths where post_attn_norm is
|
|
8
|
+
// immediately followed by pre_ffn_norm and both tensors are needed downstream.
|
|
9
|
+
|
|
10
|
+
enable subgroups;
|
|
11
|
+
|
|
12
|
+
override WORKGROUP_SIZE: u32 = 256u;
|
|
13
|
+
override RMS_NORM_OFFSET: bool = false;
|
|
14
|
+
override POST_WEIGHT_IS_F16: bool = false;
|
|
15
|
+
override PRE_WEIGHT_IS_F16: bool = false;
|
|
16
|
+
|
|
17
|
+
const MAX_WORKGROUP_SIZE: u32 = 256u;
|
|
18
|
+
const MAX_SUBGROUPS: u32 = 32u;
|
|
19
|
+
const MAX_PAIR_CACHE_SIZE: u32 = 4608u;
|
|
20
|
+
|
|
21
|
+
struct Uniforms {
|
|
22
|
+
size: u32,
|
|
23
|
+
num_tokens: u32,
|
|
24
|
+
eps: f32,
|
|
25
|
+
has_residual: u32,
|
|
26
|
+
token_stride: u32,
|
|
27
|
+
_pad0: u32,
|
|
28
|
+
_pad1: u32,
|
|
29
|
+
_pad2: u32,
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
@group(0) @binding(0) var<uniform> u: Uniforms;
|
|
33
|
+
@group(0) @binding(1) var<storage, read> input: array<f32>;
|
|
34
|
+
@group(0) @binding(2) var<storage, read> residual: array<f32>;
|
|
35
|
+
@group(0) @binding(3) var<storage, read> post_weight: array<u32>;
|
|
36
|
+
@group(0) @binding(4) var<storage, read> pre_weight: array<u32>;
|
|
37
|
+
@group(0) @binding(5) var<storage, read_write> post_output: array<f32>;
|
|
38
|
+
@group(0) @binding(6) var<storage, read_write> pre_output: array<f32>;
|
|
39
|
+
|
|
40
|
+
var<workgroup> shared_sum: array<f32, MAX_WORKGROUP_SIZE>;
|
|
41
|
+
var<workgroup> shared_post: array<f32, MAX_PAIR_CACHE_SIZE>;
|
|
42
|
+
var<workgroup> sg_partial_sums: array<f32, MAX_SUBGROUPS>;
|
|
43
|
+
|
|
44
|
+
fn apply_weight(w: f32) -> f32 {
|
|
45
|
+
if (RMS_NORM_OFFSET) {
|
|
46
|
+
return 1.0 + w;
|
|
47
|
+
}
|
|
48
|
+
return w;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
fn load_post_weight(idx: u32) -> f32 {
|
|
52
|
+
if (POST_WEIGHT_IS_F16) {
|
|
53
|
+
let packed = post_weight[idx >> 1u];
|
|
54
|
+
let pair = unpack2x16float(packed);
|
|
55
|
+
return select(pair.x, pair.y, (idx & 1u) == 1u);
|
|
56
|
+
}
|
|
57
|
+
return bitcast<f32>(post_weight[idx]);
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
fn load_pre_weight(idx: u32) -> f32 {
|
|
61
|
+
if (PRE_WEIGHT_IS_F16) {
|
|
62
|
+
let packed = pre_weight[idx >> 1u];
|
|
63
|
+
let pair = unpack2x16float(packed);
|
|
64
|
+
return select(pair.x, pair.y, (idx & 1u) == 1u);
|
|
65
|
+
}
|
|
66
|
+
return bitcast<f32>(pre_weight[idx]);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
fn token_index(wg_id: vec3<u32>) -> u32 {
|
|
70
|
+
return wg_id.y * max(u.token_stride, 1u) + wg_id.x;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
fn reduce_workgroup_sum(local_sum: f32, thread_idx: u32) -> f32 {
|
|
74
|
+
shared_sum[thread_idx] = local_sum;
|
|
75
|
+
workgroupBarrier();
|
|
76
|
+
|
|
77
|
+
for (var stride: u32 = WORKGROUP_SIZE / 2u; stride > 0u; stride = stride >> 1u) {
|
|
78
|
+
if (thread_idx < stride) {
|
|
79
|
+
shared_sum[thread_idx] = shared_sum[thread_idx] + shared_sum[thread_idx + stride];
|
|
80
|
+
}
|
|
81
|
+
workgroupBarrier();
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
return shared_sum[0];
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
fn reduce_subgroup_sum(local_sum: f32, thread_idx: u32, sg_lane: u32, sg_size: u32) -> f32 {
|
|
88
|
+
let subgroup_id = thread_idx / sg_size;
|
|
89
|
+
let num_subgroups = (WORKGROUP_SIZE + sg_size - 1u) / sg_size;
|
|
90
|
+
let sg_sum = subgroupAdd(local_sum);
|
|
91
|
+
|
|
92
|
+
if (sg_lane == 0u && subgroup_id < num_subgroups) {
|
|
93
|
+
sg_partial_sums[subgroup_id] = sg_sum;
|
|
94
|
+
}
|
|
95
|
+
workgroupBarrier();
|
|
96
|
+
|
|
97
|
+
if (thread_idx == 0u) {
|
|
98
|
+
var sum: f32 = 0.0;
|
|
99
|
+
for (var s = 0u; s < num_subgroups; s = s + 1u) {
|
|
100
|
+
sum = sum + sg_partial_sums[s];
|
|
101
|
+
}
|
|
102
|
+
sg_partial_sums[0] = sum;
|
|
103
|
+
}
|
|
104
|
+
workgroupBarrier();
|
|
105
|
+
|
|
106
|
+
return sg_partial_sums[0];
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
fn compute_post(base_offset: u32, idx: u32, input_inv_rms: f32) -> f32 {
|
|
110
|
+
let source = input[base_offset + idx];
|
|
111
|
+
var post = source * input_inv_rms * apply_weight(load_post_weight(idx));
|
|
112
|
+
if (u.has_residual != 0u) {
|
|
113
|
+
post = post + residual[base_offset + idx];
|
|
114
|
+
}
|
|
115
|
+
return post;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
@compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
|
|
119
|
+
fn main(
|
|
120
|
+
@builtin(local_invocation_id) local_id: vec3<u32>,
|
|
121
|
+
@builtin(workgroup_id) wg_id: vec3<u32>
|
|
122
|
+
) {
|
|
123
|
+
let token_idx = token_index(wg_id);
|
|
124
|
+
let thread_idx = local_id.x;
|
|
125
|
+
let size = u.size;
|
|
126
|
+
|
|
127
|
+
if (token_idx >= u.num_tokens) {
|
|
128
|
+
return;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
let base_offset = token_idx * size;
|
|
132
|
+
let elements_per_thread = (size + WORKGROUP_SIZE - 1u) / WORKGROUP_SIZE;
|
|
133
|
+
|
|
134
|
+
var input_sum_sq: f32 = 0.0;
|
|
135
|
+
for (var i: u32 = 0u; i < elements_per_thread; i = i + 1u) {
|
|
136
|
+
let idx = thread_idx * elements_per_thread + i;
|
|
137
|
+
if (idx < size) {
|
|
138
|
+
let source = input[base_offset + idx];
|
|
139
|
+
input_sum_sq = input_sum_sq + source * source;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
let input_total = reduce_workgroup_sum(input_sum_sq, thread_idx);
|
|
144
|
+
let input_inv_rms = 1.0 / sqrt(input_total / f32(size) + u.eps);
|
|
145
|
+
workgroupBarrier();
|
|
146
|
+
|
|
147
|
+
var post_sum_sq: f32 = 0.0;
|
|
148
|
+
for (var i: u32 = 0u; i < elements_per_thread; i = i + 1u) {
|
|
149
|
+
let idx = thread_idx * elements_per_thread + i;
|
|
150
|
+
if (idx < size) {
|
|
151
|
+
let post = compute_post(base_offset, idx, input_inv_rms);
|
|
152
|
+
shared_post[idx] = post;
|
|
153
|
+
post_output[base_offset + idx] = post;
|
|
154
|
+
post_sum_sq = post_sum_sq + post * post;
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
let post_total = reduce_workgroup_sum(post_sum_sq, thread_idx);
|
|
159
|
+
let post_inv_rms = 1.0 / sqrt(post_total / f32(size) + u.eps);
|
|
160
|
+
workgroupBarrier();
|
|
161
|
+
|
|
162
|
+
for (var i: u32 = 0u; i < elements_per_thread; i = i + 1u) {
|
|
163
|
+
let idx = thread_idx * elements_per_thread + i;
|
|
164
|
+
if (idx < size) {
|
|
165
|
+
let post = shared_post[idx];
|
|
166
|
+
pre_output[base_offset + idx] = post * post_inv_rms * apply_weight(load_pre_weight(idx));
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
@compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
|
|
172
|
+
fn main_subgroup(
|
|
173
|
+
@builtin(local_invocation_id) local_id: vec3<u32>,
|
|
174
|
+
@builtin(workgroup_id) wg_id: vec3<u32>,
|
|
175
|
+
@builtin(subgroup_invocation_id) sg_lane: u32,
|
|
176
|
+
@builtin(subgroup_size) sg_size: u32,
|
|
177
|
+
) {
|
|
178
|
+
let token_idx = token_index(wg_id);
|
|
179
|
+
let thread_idx = local_id.x;
|
|
180
|
+
let size = u.size;
|
|
181
|
+
|
|
182
|
+
if (token_idx >= u.num_tokens) {
|
|
183
|
+
return;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
let base_offset = token_idx * size;
|
|
187
|
+
let elements_per_thread = (size + WORKGROUP_SIZE - 1u) / WORKGROUP_SIZE;
|
|
188
|
+
|
|
189
|
+
var input_sum_sq: f32 = 0.0;
|
|
190
|
+
for (var i: u32 = 0u; i < elements_per_thread; i = i + 1u) {
|
|
191
|
+
let idx = thread_idx * elements_per_thread + i;
|
|
192
|
+
if (idx < size) {
|
|
193
|
+
let source = input[base_offset + idx];
|
|
194
|
+
input_sum_sq = input_sum_sq + source * source;
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
let input_total = reduce_subgroup_sum(input_sum_sq, thread_idx, sg_lane, sg_size);
|
|
199
|
+
let input_inv_rms = 1.0 / sqrt(input_total / f32(size) + u.eps);
|
|
200
|
+
workgroupBarrier();
|
|
201
|
+
|
|
202
|
+
var post_sum_sq: f32 = 0.0;
|
|
203
|
+
for (var i: u32 = 0u; i < elements_per_thread; i = i + 1u) {
|
|
204
|
+
let idx = thread_idx * elements_per_thread + i;
|
|
205
|
+
if (idx < size) {
|
|
206
|
+
let post = compute_post(base_offset, idx, input_inv_rms);
|
|
207
|
+
shared_post[idx] = post;
|
|
208
|
+
post_output[base_offset + idx] = post;
|
|
209
|
+
post_sum_sq = post_sum_sq + post * post;
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
let post_total = reduce_subgroup_sum(post_sum_sq, thread_idx, sg_lane, sg_size);
|
|
214
|
+
let post_inv_rms = 1.0 / sqrt(post_total / f32(size) + u.eps);
|
|
215
|
+
workgroupBarrier();
|
|
216
|
+
|
|
217
|
+
for (var i: u32 = 0u; i < elements_per_thread; i = i + 1u) {
|
|
218
|
+
let idx = thread_idx * elements_per_thread + i;
|
|
219
|
+
if (idx < size) {
|
|
220
|
+
let post = shared_post[idx];
|
|
221
|
+
pre_output[base_offset + idx] = post * post_inv_rms * apply_weight(load_pre_weight(idx));
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
}
|