doppler-gpu 0.4.3 → 0.4.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/README.md +67 -34
- package/assets/doppler.svg +57 -1601
- package/package.json +36 -5
- package/src/cli/cli-output.js +34 -1
- package/src/cli/doppler-cli.d.ts +18 -0
- package/src/cli/doppler-cli.js +68 -35
- package/src/cli/doppler-serve.d.ts +210 -0
- package/src/cli/doppler-serve.js +468 -42
- package/src/client/doppler-registry.json +3 -5
- package/src/client/provider.d.ts +1 -1
- package/src/client/provider.js +1 -1
- package/src/client/runtime/index.js +13 -0
- package/src/client/runtime/lora.d.ts +11 -1
- package/src/client/runtime/lora.js +28 -7
- package/src/client/runtime/model-manager.js +35 -11
- package/src/client/runtime/model-session.d.ts +2 -1
- package/src/client/runtime/model-session.js +2 -2
- package/src/client/runtime/model-source.js +20 -1
- package/src/client/runtime/types.d.ts +8 -0
- package/src/client/wrap-pipeline-handle.d.ts +62 -0
- package/src/client/wrap-pipeline-handle.js +188 -13
- package/src/config/conversion/README.md +5 -0
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
- package/src/config/goal-completion-matrix.json +552 -0
- package/src/config/kernel-path-loader.js +192 -38
- package/src/config/kernels/kernel-ref-digests.js +52 -18
- package/src/config/kernels/registry.json +3017 -744
- package/src/config/merge-contract-check.js +7 -3
- package/src/config/merge-helpers.d.ts +2 -0
- package/src/config/merge-helpers.js +1 -0
- package/src/config/merge.d.ts +1 -0
- package/src/config/merge.js +30 -0
- package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
- package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
- package/src/config/param-validator.js +4 -0
- package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
- package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
- package/src/config/schema/buffer-pool.schema.js +3 -3
- package/src/config/schema/claim-evidence-contract.schema.json +93 -0
- package/src/config/schema/command-surface-contract.schema.json +81 -0
- package/src/config/schema/converter.schema.d.ts +11 -0
- package/src/config/schema/converter.schema.js +8 -0
- package/src/config/schema/doppler.schema.d.ts +1 -0
- package/src/config/schema/doppler.schema.js +38 -6
- package/src/config/schema/execution-v1.schema.d.ts +27 -0
- package/src/config/schema/execution-v1.schema.js +28 -19
- package/src/config/schema/goal-completion-matrix.schema.json +223 -0
- package/src/config/schema/inference.schema.d.ts +11 -0
- package/src/config/schema/manifest.schema.d.ts +107 -1
- package/src/config/schema/manifest.schema.js +10 -1
- package/src/config/schema/model-artifact-contract.schema.json +119 -0
- package/src/config/schema/moe.schema.d.ts +13 -0
- package/src/config/schema/moe.schema.js +1 -1
- package/src/config/schema/policy-schema-registry.json +26 -0
- package/src/config/schema/subsystem-support-contract.schema.json +48 -0
- package/src/config/schema/training-metrics.schema.d.ts +1 -1
- package/src/config/schema/training-metrics.schema.js +4 -3
- package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
- package/src/config/source-packages/registry.json +25 -0
- package/src/config/supported-operations.json +5 -0
- package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
- package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
- package/src/config/transforms/execution-graph-transforms.js +227 -230
- package/src/converter/conversion-plan.d.ts +2 -0
- package/src/converter/conversion-plan.js +18 -4
- package/src/converter/core.d.ts +8 -0
- package/src/converter/core.js +574 -12
- package/src/converter/quantization-info.js +179 -2
- package/src/converter/quantizer.d.ts +5 -4
- package/src/converter/quantizer.js +16 -2
- package/src/experimental/adapters/lora-loader.js +109 -24
- package/src/experimental/diffusion/index.d.ts +24 -0
- package/src/experimental/diffusion/index.js +10 -0
- package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
- package/src/experimental/distribution/descriptor-transport.js +342 -0
- package/src/experimental/training/README.md +11 -0
- package/src/experimental/training/autograd.js +33 -8
- package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
- package/src/experimental/training/datasets/gepa-frontier.js +220 -0
- package/src/experimental/training/datasets/index.d.ts +13 -1
- package/src/experimental/training/datasets/index.js +8 -1
- package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
- package/src/experimental/training/datasets/teacher-traces.js +279 -0
- package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
- package/src/experimental/training/datasets/text-pairs.js +82 -5
- package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
- package/src/experimental/training/distillation/student-fixture.js +290 -32
- package/src/experimental/training/export.d.ts +9 -0
- package/src/experimental/training/export.js +113 -7
- package/src/experimental/training/index.d.ts +13 -1
- package/src/experimental/training/index.js +13 -1
- package/src/experimental/training/lora-pipeline.d.ts +143 -0
- package/src/experimental/training/lora-pipeline.js +1737 -49
- package/src/experimental/training/lora.js +23 -1
- package/src/experimental/training/loss.js +1 -1
- package/src/experimental/training/operator-agent-eval.d.ts +65 -0
- package/src/experimental/training/operator-agent-eval.js +326 -0
- package/src/experimental/training/operator-command.js +310 -51
- package/src/experimental/training/runner.d.ts +1 -0
- package/src/experimental/training/runner.js +4 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
- package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/registry.json +186 -6
- package/src/experimental/training/workloads.d.ts +37 -0
- package/src/experimental/training/workloads.js +110 -1
- package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
- package/src/formats/rdrr/functional-descriptor.js +230 -0
- package/src/formats/rdrr/index.d.ts +1 -0
- package/src/formats/rdrr/index.js +1 -0
- package/src/formats/rdrr/parsing.js +22 -6
- package/src/formats/rdrr/source-transform-contract.js +10 -1
- package/src/formats/rdrr/storage-descriptor.js +9 -2
- package/src/formats/rdrr/types.d.ts +5 -3
- package/src/formats/rdrr/validation.js +15 -1
- package/src/gpu/command-recorder.d.ts +43 -3
- package/src/gpu/command-recorder.js +122 -25
- package/src/gpu/device.js +111 -11
- package/src/gpu/kernel-tuner/cache.js +1 -5
- package/src/gpu/kernels/attention.js +17 -9
- package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
- package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
- package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
- package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
- package/src/gpu/kernels/cast.js +74 -67
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
- package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
- package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
- package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
- package/src/gpu/kernels/dispatch.js +13 -3
- package/src/gpu/kernels/fused_ffn.js +3 -4
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
- package/src/gpu/kernels/gather.d.ts +84 -1
- package/src/gpu/kernels/gather.js +353 -5
- package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
- package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
- package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
- package/src/gpu/kernels/gelu.wgsl +11 -7
- package/src/gpu/kernels/gelu_f16.wgsl +11 -7
- package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
- package/src/gpu/kernels/gemma4-route-expert.js +126 -0
- package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
- package/src/gpu/kernels/index.d.ts +71 -0
- package/src/gpu/kernels/index.js +52 -0
- package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
- package/src/gpu/kernels/kv-cache-write.js +126 -0
- package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
- package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
- package/src/gpu/kernels/lm-head-argmax.js +218 -0
- package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
- package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
- package/src/gpu/kernels/matmul-dispatch.js +37 -1
- package/src/gpu/kernels/matmul-selection.d.ts +5 -5
- package/src/gpu/kernels/matmul-selection.js +104 -9
- package/src/gpu/kernels/matmul.d.ts +1 -1
- package/src/gpu/kernels/matmul.js +151 -43
- package/src/gpu/kernels/moe.d.ts +1 -0
- package/src/gpu/kernels/moe.js +46 -18
- package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
- package/src/gpu/kernels/pipeline-cache.js +24 -0
- package/src/gpu/kernels/rep-penalty.d.ts +21 -0
- package/src/gpu/kernels/rep-penalty.js +17 -9
- package/src/gpu/kernels/residual.d.ts +1 -0
- package/src/gpu/kernels/residual.js +13 -1
- package/src/gpu/kernels/residual.wgsl +3 -3
- package/src/gpu/kernels/residual_f16.wgsl +5 -3
- package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
- package/src/gpu/kernels/residual_vec4.wgsl +6 -6
- package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
- package/src/gpu/kernels/rmsnorm-pair.js +152 -0
- package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
- package/src/gpu/kernels/rmsnorm-qk.js +128 -0
- package/src/gpu/kernels/rmsnorm.d.ts +31 -0
- package/src/gpu/kernels/rmsnorm.js +32 -7
- package/src/gpu/kernels/rmsnorm.wgsl +10 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
- package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
- package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
- package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
- package/src/gpu/kernels/rope-qk.d.ts +41 -0
- package/src/gpu/kernels/rope-qk.js +143 -0
- package/src/gpu/kernels/rope_qk.wgsl +95 -0
- package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
- package/src/gpu/kernels/rule-registry.js +50 -10
- package/src/gpu/kernels/sample.js +212 -187
- package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
- package/src/gpu/kernels/shader-cache.js +1 -21
- package/src/gpu/kernels/soft-embedding.d.ts +28 -0
- package/src/gpu/kernels/soft-embedding.js +447 -0
- package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
- package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
- package/src/gpu/kernels/softmax.wgsl +5 -80
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
- package/src/gpu/kernels/uniform-utils.js +2 -5
- package/src/gpu/kernels/utils.d.ts +16 -1
- package/src/gpu/kernels/utils.js +42 -13
- package/src/gpu/readback-utils.js +10 -2
- package/src/gpu/uniform-cache.d.ts +10 -1
- package/src/gpu/uniform-cache.js +25 -7
- package/src/gpu/weight-buffer.d.ts +76 -7
- package/src/gpu/weight-buffer.js +85 -1
- package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
- package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
- package/src/inference/browser-harness-suite-helpers.js +438 -2
- package/src/inference/browser-harness-text-helpers.d.ts +77 -0
- package/src/inference/browser-harness-text-helpers.js +236 -11
- package/src/inference/browser-harness.d.ts +8 -0
- package/src/inference/browser-harness.js +292 -8
- package/src/inference/decode-ring.d.ts +1 -0
- package/src/inference/decode-ring.js +7 -5
- package/src/inference/kv-cache/base.d.ts +25 -0
- package/src/inference/kv-cache/base.js +109 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
- package/src/inference/kv-cache/mixed-geometry.js +109 -2
- package/src/inference/kv-cache/sliding-window.d.ts +14 -0
- package/src/inference/kv-cache/sliding-window.js +88 -0
- package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
- package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
- package/src/inference/moe-router.d.ts +14 -1
- package/src/inference/moe-router.js +24 -1
- package/src/inference/pipelines/audio/gemma4.js +52 -1
- package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
- package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
- package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
- package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
- package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
- package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
- package/src/inference/pipelines/text/attention/projections.js +329 -49
- package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
- package/src/inference/pipelines/text/attention/record.js +446 -73
- package/src/inference/pipelines/text/attention/run.js +251 -49
- package/src/inference/pipelines/text/attention/types.d.ts +5 -1
- package/src/inference/pipelines/text/config.d.ts +22 -1
- package/src/inference/pipelines/text/config.js +199 -22
- package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
- package/src/inference/pipelines/text/dtype-contract.js +4 -1
- package/src/inference/pipelines/text/embed.d.ts +4 -3
- package/src/inference/pipelines/text/embed.js +16 -6
- package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
- package/src/inference/pipelines/text/execution-plan.js +6 -0
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
- package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
- package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
- package/src/inference/pipelines/text/execution-v1.js +373 -139
- package/src/inference/pipelines/text/ffn/dense.js +44 -13
- package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
- package/src/inference/pipelines/text/ffn/moe.js +9 -2
- package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
- package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/standard.js +44 -4
- package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
- package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
- package/src/inference/pipelines/text/generator-helpers.js +5 -2
- package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
- package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
- package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
- package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
- package/src/inference/pipelines/text/generator-steps.js +268 -77
- package/src/inference/pipelines/text/generator.d.ts +39 -0
- package/src/inference/pipelines/text/generator.js +1105 -18
- package/src/inference/pipelines/text/init.d.ts +8 -1
- package/src/inference/pipelines/text/init.js +30 -1
- package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
- package/src/inference/pipelines/text/layer.js +272 -25
- package/src/inference/pipelines/text/linear-attention.js +6 -4
- package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
- package/src/inference/pipelines/text/logits/gpu.js +417 -66
- package/src/inference/pipelines/text/logits/index.d.ts +2 -2
- package/src/inference/pipelines/text/logits/index.js +96 -42
- package/src/inference/pipelines/text/logits/types.d.ts +2 -2
- package/src/inference/pipelines/text/lora-apply.js +17 -0
- package/src/inference/pipelines/text/model-load.js +5 -1
- package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
- package/src/inference/pipelines/text/moe-gpu.js +445 -16
- package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
- package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
- package/src/inference/pipelines/text/ops.d.ts +29 -2
- package/src/inference/pipelines/text/ops.js +19 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
- package/src/inference/pipelines/text/probes.js +4 -5
- package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
- package/src/inference/pipelines/text/sampling-config.js +103 -64
- package/src/inference/pipelines/text/sampling.js +13 -9
- package/src/inference/pipelines/text/state.js +17 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
- package/src/inference/pipelines/text/types.d.ts +84 -3
- package/src/inference/pipelines/text/weights.d.ts +4 -2
- package/src/inference/pipelines/text/weights.js +4 -5
- package/src/inference/pipelines/text.d.ts +24 -0
- package/src/inference/pipelines/text.js +162 -38
- package/src/inference/pipelines/vision/gemma4.js +20 -9
- package/src/inference/tokenizer.d.ts +30 -0
- package/src/inference/tokenizer.js +340 -164
- package/src/inference/tokenizers/bundled.js +71 -49
- package/src/loader/doppler-loader.d.ts +11 -0
- package/src/loader/doppler-loader.js +290 -15
- package/src/loader/embedding-limit-preflight.d.ts +31 -0
- package/src/loader/embedding-limit-preflight.js +216 -0
- package/src/loader/embedding-loader.d.ts +8 -2
- package/src/loader/embedding-loader.js +263 -8
- package/src/loader/experts/expert-loader.d.ts +4 -1
- package/src/loader/experts/expert-loader.js +174 -5
- package/src/loader/final-weights-loader.d.ts +5 -0
- package/src/loader/final-weights-loader.js +167 -4
- package/src/loader/layer-loader.d.ts +2 -0
- package/src/loader/layer-loader.js +74 -6
- package/src/loader/loader-types.d.ts +59 -2
- package/src/loader/manifest-config.d.ts +8 -0
- package/src/loader/shard-resolver.js +37 -4
- package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
- package/src/loader/tensors/functional-descriptor-loader.js +377 -0
- package/src/loader/tensors/tensor-loader.d.ts +36 -1
- package/src/loader/tensors/tensor-loader.js +437 -21
- package/src/loader/tensors/tensor-reader.d.ts +4 -1
- package/src/loader/tensors/tensor-reader.js +11 -4
- package/src/loader/weights.d.ts +3 -1
- package/src/memory/buffer-pool.js +12 -9
- package/src/models/diffusiongemma.d.ts +7 -0
- package/src/models/diffusiongemma.js +25 -0
- package/src/models/gemma4.js +24 -0
- package/src/models/qwen3.js +2 -2
- package/src/rules/execution-rules-contract-check.js +93 -10
- package/src/rules/inference/capability-transforms.rules.json +26 -2
- package/src/rules/inference/config.rules.json +1 -0
- package/src/rules/inference/dtype.rules.json +7 -0
- package/src/rules/inference/execution.rules.json +108 -17
- package/src/rules/inference/ffn.rules.json +18 -0
- package/src/rules/kernels/gather.rules.json +81 -0
- package/src/rules/kernels/matmul.rules.json +19 -0
- package/src/rules/kernels/moe.rules.json +3 -0
- package/src/rules/kernels/rep-penalty.rules.json +6 -0
- package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
- package/src/rules/kernels/rmsnorm.rules.json +42 -7
- package/src/rules/kernels/rope-qk.rules.json +6 -0
- package/src/rules/loader/tensor-loader.rules.json +8 -0
- package/src/rules/rule-registry.js +6 -0
- package/src/storage/artifact-storage-context.d.ts +1 -0
- package/src/storage/artifact-storage-context.js +10 -0
- package/src/tooling/browser-command-runner.d.ts +0 -3
- package/src/tooling/browser-command-runner.js +2 -14
- package/src/tooling/command-api-family-normalizers.js +4 -2
- package/src/tooling/command-api-helpers.js +2 -1
- package/src/tooling/command-api.d.ts +2 -0
- package/src/tooling/command-api.js +1 -0
- package/src/tooling/command-runner-shared.d.ts +4 -0
- package/src/tooling/command-runner-shared.js +2 -1
- package/src/tooling/diagnose-runner.js +2 -0
- package/src/tooling/litert-package-runtime.js +553 -16
- package/src/tooling/node-browser-command-runner.js +10 -4
- package/src/tooling/node-command-runner.js +7 -2
- package/src/tooling/node-convert-worker-pool.js +3 -0
- package/src/tooling/node-convert-worker.js +1 -0
- package/src/tooling/node-converter.js +12 -0
- package/src/tooling/node-webgpu.d.ts +1 -0
- package/src/tooling/node-webgpu.js +23 -11
- package/src/tooling/program-bundle.d.ts +52 -1
- package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
- package/src/tooling/source-artifact-adapter.js +7 -2
- package/src/tooling/source-package-profiles.d.ts +18 -1
- package/src/tooling/source-package-profiles.js +109 -1
- package/src/tooling/source-runtime-bundle.d.ts +1 -0
- package/src/tooling/source-runtime-bundle.js +27 -0
- package/src/tooling/source-runtime-converter-config.js +12 -3
- package/src/tooling-experimental-exports.shared.d.ts +10 -0
- package/src/tooling-experimental-exports.shared.js +10 -0
- package/src/version.js +1 -1
package/src/gpu/kernels/moe.js
CHANGED
|
@@ -145,26 +145,46 @@ function getMoEGatherBindGroupLayout(device) {
|
|
|
145
145
|
// Required because auto layout can omit bindings in some driver/compiler paths.
|
|
146
146
|
let scatterAddDynamicBindGroupLayout = null;
|
|
147
147
|
let scatterAddDynamicBindGroupLayoutEpoch = -1;
|
|
148
|
+
let scatterAddDynamicScaledBindGroupLayout = null;
|
|
149
|
+
let scatterAddDynamicScaledBindGroupLayoutEpoch = -1;
|
|
148
150
|
|
|
149
|
-
function getScatterAddDynamicBindGroupLayout(device) {
|
|
151
|
+
function getScatterAddDynamicBindGroupLayout(device, hasExpertScale = false) {
|
|
150
152
|
const epoch = getDeviceEpoch();
|
|
151
|
-
if (scatterAddDynamicBindGroupLayout && scatterAddDynamicBindGroupLayoutEpoch === epoch) {
|
|
153
|
+
if (!hasExpertScale && scatterAddDynamicBindGroupLayout && scatterAddDynamicBindGroupLayoutEpoch === epoch) {
|
|
152
154
|
return scatterAddDynamicBindGroupLayout;
|
|
153
155
|
}
|
|
156
|
+
if (hasExpertScale && scatterAddDynamicScaledBindGroupLayout && scatterAddDynamicScaledBindGroupLayoutEpoch === epoch) {
|
|
157
|
+
return scatterAddDynamicScaledBindGroupLayout;
|
|
158
|
+
}
|
|
154
159
|
|
|
155
|
-
|
|
156
|
-
label:
|
|
160
|
+
const layout = device.createBindGroupLayout({
|
|
161
|
+
label: hasExpertScale
|
|
162
|
+
? 'scatter_add_dynamic_scaled_explicit_layout'
|
|
163
|
+
: 'scatter_add_dynamic_explicit_layout',
|
|
157
164
|
entries: [
|
|
158
165
|
{ binding: 0, visibility: GPUShaderStage.COMPUTE, buffer: { type: 'uniform' } },
|
|
159
166
|
{ binding: 1, visibility: GPUShaderStage.COMPUTE, buffer: { type: 'read-only-storage' } },
|
|
160
167
|
{ binding: 2, visibility: GPUShaderStage.COMPUTE, buffer: { type: 'read-only-storage' } },
|
|
161
168
|
{ binding: 3, visibility: GPUShaderStage.COMPUTE, buffer: { type: 'read-only-storage' } },
|
|
162
169
|
{ binding: 4, visibility: GPUShaderStage.COMPUTE, buffer: { type: 'read-only-storage' } },
|
|
163
|
-
|
|
170
|
+
...(hasExpertScale
|
|
171
|
+
? [
|
|
172
|
+
{ binding: 5, visibility: GPUShaderStage.COMPUTE, buffer: { type: 'read-only-storage' } },
|
|
173
|
+
{ binding: 6, visibility: GPUShaderStage.COMPUTE, buffer: { type: 'storage' } },
|
|
174
|
+
]
|
|
175
|
+
: [
|
|
176
|
+
{ binding: 5, visibility: GPUShaderStage.COMPUTE, buffer: { type: 'storage' } },
|
|
177
|
+
]),
|
|
164
178
|
],
|
|
165
179
|
});
|
|
166
|
-
|
|
167
|
-
|
|
180
|
+
if (hasExpertScale) {
|
|
181
|
+
scatterAddDynamicScaledBindGroupLayout = layout;
|
|
182
|
+
scatterAddDynamicScaledBindGroupLayoutEpoch = epoch;
|
|
183
|
+
} else {
|
|
184
|
+
scatterAddDynamicBindGroupLayout = layout;
|
|
185
|
+
scatterAddDynamicBindGroupLayoutEpoch = epoch;
|
|
186
|
+
}
|
|
187
|
+
return layout;
|
|
168
188
|
}
|
|
169
189
|
|
|
170
190
|
let moeOffsetsBindGroupLayout = null;
|
|
@@ -450,8 +470,9 @@ export async function recordMoEBuildTokenOffsets(recorder, tokenCounts, tokenMap
|
|
|
450
470
|
|
|
451
471
|
async function executeScatterAddDynamic(recorder, expertOutputs, indices, weights, tokenOffsets, numTokens, hiddenSize, topK, options = {}) {
|
|
452
472
|
const execution = resolveExecution(recorder);
|
|
453
|
-
const { outputBuffer = null, weightsDtype = 'f32' } = options;
|
|
473
|
+
const { outputBuffer = null, weightsDtype = 'f32', perExpertScale = null } = options;
|
|
454
474
|
const ownsOutput = outputBuffer == null;
|
|
475
|
+
const hasExpertScale = perExpertScale != null;
|
|
455
476
|
|
|
456
477
|
if (weightsDtype === 'f16' && expertOutputs.dtype !== 'f16') {
|
|
457
478
|
throw new Error('ScatterAddDynamic f16 weights require f16 expert outputs');
|
|
@@ -460,8 +481,9 @@ async function executeScatterAddDynamic(recorder, expertOutputs, indices, weight
|
|
|
460
481
|
const variant = selectKernelRuleValue('moe', 'scatterAddVariant', {
|
|
461
482
|
outputDtype: expertOutputs.dtype,
|
|
462
483
|
weightsDtype,
|
|
484
|
+
hasExpertScale,
|
|
463
485
|
});
|
|
464
|
-
const explicitLayout = getScatterAddDynamicBindGroupLayout(execution.device);
|
|
486
|
+
const explicitLayout = getScatterAddDynamicBindGroupLayout(execution.device, hasExpertScale);
|
|
465
487
|
const pipeline = await createPipeline('scatter_add', variant, explicitLayout);
|
|
466
488
|
|
|
467
489
|
const bytesPerElement = expertOutputs.dtype === 'f16' ? 2 : 4;
|
|
@@ -475,17 +497,24 @@ async function executeScatterAddDynamic(recorder, expertOutputs, indices, weight
|
|
|
475
497
|
});
|
|
476
498
|
|
|
477
499
|
try {
|
|
500
|
+
const entries = [
|
|
501
|
+
{ binding: 0, resource: { buffer: uniformBuffer } },
|
|
502
|
+
{ binding: 1, resource: { buffer: expertOutputs.buffer } },
|
|
503
|
+
{ binding: 2, resource: { buffer: indices } },
|
|
504
|
+
{ binding: 3, resource: { buffer: weights } },
|
|
505
|
+
{ binding: 4, resource: { buffer: tokenOffsets } },
|
|
506
|
+
];
|
|
507
|
+
if (hasExpertScale) {
|
|
508
|
+
entries.push({ binding: 5, resource: { buffer: perExpertScale } });
|
|
509
|
+
entries.push({ binding: 6, resource: { buffer: outputBuf } });
|
|
510
|
+
} else {
|
|
511
|
+
entries.push({ binding: 5, resource: { buffer: outputBuf } });
|
|
512
|
+
}
|
|
513
|
+
|
|
478
514
|
const bindGroup = await createBindGroupWithValidation(execution.device, {
|
|
479
515
|
label: 'scatter_add_dynamic_bind_group',
|
|
480
516
|
layout: pipeline.getBindGroupLayout(0),
|
|
481
|
-
entries
|
|
482
|
-
{ binding: 0, resource: { buffer: uniformBuffer } },
|
|
483
|
-
{ binding: 1, resource: { buffer: expertOutputs.buffer } },
|
|
484
|
-
{ binding: 2, resource: { buffer: indices } },
|
|
485
|
-
{ binding: 3, resource: { buffer: weights } },
|
|
486
|
-
{ binding: 4, resource: { buffer: tokenOffsets } },
|
|
487
|
-
{ binding: 5, resource: { buffer: outputBuf } },
|
|
488
|
-
],
|
|
517
|
+
entries,
|
|
489
518
|
}, `scatter_add_dynamic:${variant}`);
|
|
490
519
|
|
|
491
520
|
const workgroups = Math.ceil((numTokens * topK * hiddenSize) / WORKGROUP_SIZES.DEFAULT);
|
|
@@ -524,4 +553,3 @@ export async function runScatterAddDynamic(expertOutputs, indices, weights, toke
|
|
|
524
553
|
return executeScatterAddDynamic(null, expertOutputs, indices, weights, tokenOffsets, numTokens, hiddenSize, topK, options);
|
|
525
554
|
}
|
|
526
555
|
|
|
527
|
-
|
|
@@ -33,6 +33,14 @@ export declare function getOrCreatePipelineLayout(
|
|
|
33
33
|
deviceOverride?: GPUDevice | null
|
|
34
34
|
): GPUPipelineLayout;
|
|
35
35
|
|
|
36
|
+
/**
|
|
37
|
+
* Get a cached bind group layout from a compute pipeline.
|
|
38
|
+
*/
|
|
39
|
+
export declare function getPipelineBindGroupLayout(
|
|
40
|
+
pipeline: GPUComputePipeline,
|
|
41
|
+
index?: number
|
|
42
|
+
): GPUBindGroupLayout;
|
|
43
|
+
|
|
36
44
|
// ============================================================================
|
|
37
45
|
// Pipeline Creation
|
|
38
46
|
// ============================================================================
|
|
@@ -19,6 +19,8 @@ const bindGroupLayoutCache = new Map();
|
|
|
19
19
|
|
|
20
20
|
const pipelineLayoutCache = new Map();
|
|
21
21
|
|
|
22
|
+
let pipelineBindGroupLayoutCache = new WeakMap();
|
|
23
|
+
|
|
22
24
|
let pipelineCacheEpoch = -1;
|
|
23
25
|
const deviceIds = new WeakMap();
|
|
24
26
|
let nextDeviceId = 1;
|
|
@@ -38,6 +40,7 @@ function ensureCacheEpoch() {
|
|
|
38
40
|
pipelineCache.clear();
|
|
39
41
|
bindGroupLayoutCache.clear();
|
|
40
42
|
pipelineLayoutCache.clear();
|
|
43
|
+
pipelineBindGroupLayoutCache = new WeakMap();
|
|
41
44
|
pipelineCacheEpoch = epoch;
|
|
42
45
|
}
|
|
43
46
|
}
|
|
@@ -100,6 +103,26 @@ export function getOrCreatePipelineLayout(
|
|
|
100
103
|
return layout;
|
|
101
104
|
}
|
|
102
105
|
|
|
106
|
+
export function getPipelineBindGroupLayout(pipeline, index = 0) {
|
|
107
|
+
ensureCacheEpoch();
|
|
108
|
+
if (!pipeline || typeof pipeline.getBindGroupLayout !== 'function') {
|
|
109
|
+
throw new Error('getPipelineBindGroupLayout requires a GPUComputePipeline');
|
|
110
|
+
}
|
|
111
|
+
const normalizedIndex = Number.isInteger(index) && index >= 0 ? index : 0;
|
|
112
|
+
let layoutsByIndex = pipelineBindGroupLayoutCache.get(pipeline);
|
|
113
|
+
if (!layoutsByIndex) {
|
|
114
|
+
layoutsByIndex = new Map();
|
|
115
|
+
pipelineBindGroupLayoutCache.set(pipeline, layoutsByIndex);
|
|
116
|
+
}
|
|
117
|
+
const cached = layoutsByIndex.get(normalizedIndex);
|
|
118
|
+
if (cached) {
|
|
119
|
+
return cached;
|
|
120
|
+
}
|
|
121
|
+
const layout = pipeline.getBindGroupLayout(normalizedIndex);
|
|
122
|
+
layoutsByIndex.set(normalizedIndex, layout);
|
|
123
|
+
return layout;
|
|
124
|
+
}
|
|
125
|
+
|
|
103
126
|
// ============================================================================
|
|
104
127
|
// Pipeline Creation
|
|
105
128
|
// ============================================================================
|
|
@@ -292,6 +315,7 @@ export function clearPipelineCaches() {
|
|
|
292
315
|
pipelineCache.clear();
|
|
293
316
|
bindGroupLayoutCache.clear();
|
|
294
317
|
pipelineLayoutCache.clear();
|
|
318
|
+
pipelineBindGroupLayoutCache = new WeakMap();
|
|
295
319
|
pipelineCacheEpoch = getDeviceEpoch();
|
|
296
320
|
}
|
|
297
321
|
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
export interface RepPenaltyOptions {
|
|
2
|
+
vocabSize: number;
|
|
3
|
+
historyCount: number;
|
|
4
|
+
penalty: number;
|
|
5
|
+
batchCount: number;
|
|
6
|
+
batchOffset: number;
|
|
7
|
+
logitsDtype: 'f16' | 'f32';
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
export declare function selectRepPenaltyVariant(useF16: boolean): 'default' | 'default_f16';
|
|
11
|
+
|
|
12
|
+
export declare function recordRepPenalty(
|
|
13
|
+
recorder: {
|
|
14
|
+
device: GPUDevice;
|
|
15
|
+
beginComputePass(label: string): GPUComputePassEncoder;
|
|
16
|
+
},
|
|
17
|
+
logitsBuffer: GPUBuffer,
|
|
18
|
+
historyBuffer: GPUBuffer,
|
|
19
|
+
batchTokensBuffer: GPUBuffer,
|
|
20
|
+
options: RepPenaltyOptions
|
|
21
|
+
): Promise<void>;
|
|
@@ -1,10 +1,8 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
1
|
import { getDevice, getKernelCapabilities } from '../device.js';
|
|
5
2
|
import { createPipeline, getOrCreateBindGroupLayout } from './pipeline-cache.js';
|
|
6
3
|
import { createUniformBufferWithView } from './uniform-utils.js';
|
|
7
4
|
import { WORKGROUP_SIZES } from './constants.js';
|
|
5
|
+
import { selectRuleValue } from './rule-registry.js';
|
|
8
6
|
|
|
9
7
|
|
|
10
8
|
function getRepPenaltyBindGroupLayout(device) {
|
|
@@ -20,8 +18,8 @@ function getRepPenaltyBindGroupLayout(device) {
|
|
|
20
18
|
);
|
|
21
19
|
}
|
|
22
20
|
|
|
23
|
-
function
|
|
24
|
-
return
|
|
21
|
+
export function selectRepPenaltyVariant(useF16) {
|
|
22
|
+
return selectRuleValue('repPenalty', 'variant', { useF16 });
|
|
25
23
|
}
|
|
26
24
|
|
|
27
25
|
function createRepPenaltyUniformBuffer(device, recorder, options) {
|
|
@@ -54,11 +52,21 @@ export async function recordRepPenalty(
|
|
|
54
52
|
vocabSize,
|
|
55
53
|
historyCount,
|
|
56
54
|
penalty,
|
|
57
|
-
batchCount
|
|
58
|
-
batchOffset
|
|
59
|
-
logitsDtype
|
|
55
|
+
batchCount,
|
|
56
|
+
batchOffset,
|
|
57
|
+
logitsDtype,
|
|
60
58
|
} = options;
|
|
61
59
|
|
|
60
|
+
if (!Number.isFinite(batchCount) || batchCount < 0) {
|
|
61
|
+
throw new Error('[RepPenalty] batchCount is required and must be non-negative.');
|
|
62
|
+
}
|
|
63
|
+
if (!Number.isFinite(batchOffset) || batchOffset < 0) {
|
|
64
|
+
throw new Error('[RepPenalty] batchOffset is required and must be non-negative.');
|
|
65
|
+
}
|
|
66
|
+
if (logitsDtype !== 'f16' && logitsDtype !== 'f32') {
|
|
67
|
+
throw new Error('[RepPenalty] logitsDtype must be "f16" or "f32".');
|
|
68
|
+
}
|
|
69
|
+
|
|
62
70
|
if (penalty === 1.0 || (historyCount === 0 && batchCount === 0)) {
|
|
63
71
|
return;
|
|
64
72
|
}
|
|
@@ -69,7 +77,7 @@ export async function recordRepPenalty(
|
|
|
69
77
|
throw new Error('[RepPenalty] F16 logits requested but shader-f16 is unavailable.');
|
|
70
78
|
}
|
|
71
79
|
|
|
72
|
-
const variant =
|
|
80
|
+
const variant = selectRepPenaltyVariant(useF16);
|
|
73
81
|
const layout = getRepPenaltyBindGroupLayout(device);
|
|
74
82
|
const pipeline = await createPipeline('rep_penalty', variant, layout);
|
|
75
83
|
|
|
@@ -14,6 +14,7 @@ import type { ExecutionV1PoliciesSchema } from '../../config/schema/execution-v1
|
|
|
14
14
|
/** Residual kernel options */
|
|
15
15
|
export interface ResidualOptions extends OutputBufferOptions {
|
|
16
16
|
useVec4?: boolean;
|
|
17
|
+
outputScale?: number | null;
|
|
17
18
|
dataOffset?: number;
|
|
18
19
|
biasOffset?: number;
|
|
19
20
|
executionPolicies?: ExecutionV1PoliciesSchema | null;
|
|
@@ -108,9 +108,21 @@ function planResidualDispatch(target, size, elementsPerWorkgroup) {
|
|
|
108
108
|
};
|
|
109
109
|
}
|
|
110
110
|
|
|
111
|
+
function resolveResidualOutputScale(outputScale) {
|
|
112
|
+
if (outputScale == null) {
|
|
113
|
+
return 1;
|
|
114
|
+
}
|
|
115
|
+
const value = Number(outputScale);
|
|
116
|
+
if (!Number.isFinite(value)) {
|
|
117
|
+
throw new Error(`[Residual] outputScale must be finite; got "${String(outputScale)}".`);
|
|
118
|
+
}
|
|
119
|
+
return value;
|
|
120
|
+
}
|
|
121
|
+
|
|
111
122
|
async function _residualAdd(target, a, b, size, options = {}) {
|
|
112
123
|
const recorder = target && typeof target.beginComputePass === 'function' ? target : null;
|
|
113
124
|
const { useVec4 = true, outputBuffer = null } = options;
|
|
125
|
+
const outputScale = resolveResidualOutputScale(options.outputScale);
|
|
114
126
|
const ownsOutput = outputBuffer == null;
|
|
115
127
|
|
|
116
128
|
const { a: aAligned, b: bAligned, temps } = await alignResidualInputs(
|
|
@@ -136,7 +148,7 @@ async function _residualAdd(target, a, b, size, options = {}) {
|
|
|
136
148
|
await unifiedKernelWrapper(
|
|
137
149
|
'residual', target, variant,
|
|
138
150
|
[aAligned, bAligned, output],
|
|
139
|
-
{ size, scale:
|
|
151
|
+
{ size, scale: outputScale, _pad1: dispatchPlan.dispatchStride, _pad2: 0 },
|
|
140
152
|
dispatchPlan.workgroups
|
|
141
153
|
);
|
|
142
154
|
return createTensor(output, outputDtype, [size], 'residual_output');
|
|
@@ -4,12 +4,12 @@
|
|
|
4
4
|
* Residual Add Kernel
|
|
5
5
|
*
|
|
6
6
|
* Performs element-wise addition for residual connections.
|
|
7
|
-
* output = a + b
|
|
7
|
+
* output = (a + b) * scale
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
10
|
struct Uniforms {
|
|
11
11
|
size: u32, // Total number of elements
|
|
12
|
-
scale: f32, //
|
|
12
|
+
scale: f32, // Output scale for main/add_vec4; residual scale for add_scaled
|
|
13
13
|
_pad1: u32,
|
|
14
14
|
_pad2: u32,
|
|
15
15
|
}
|
|
@@ -28,7 +28,7 @@ fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
|
|
|
28
28
|
if (idx >= u.size) {
|
|
29
29
|
return;
|
|
30
30
|
}
|
|
31
|
-
output[idx] = a[idx] + b[idx];
|
|
31
|
+
output[idx] = (a[idx] + b[idx]) * u.scale;
|
|
32
32
|
}
|
|
33
33
|
|
|
34
34
|
// In-place version: output = output + b
|
|
@@ -1,19 +1,21 @@
|
|
|
1
1
|
// AUTO-GENERATED from src/gpu/kernels/residual.wgsl.
|
|
2
2
|
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
|
|
3
|
+
// AUTO-GENERATED from src/gpu/kernels/residual.wgsl.
|
|
4
|
+
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
|
|
3
5
|
// residual_f16.wgsl
|
|
4
6
|
|
|
5
7
|
/**
|
|
6
8
|
* Residual Add Kernel (F16)
|
|
7
9
|
*
|
|
8
10
|
* Performs element-wise addition for residual connections.
|
|
9
|
-
* output = a + b
|
|
11
|
+
* output = (a + b) * scale
|
|
10
12
|
*/
|
|
11
13
|
|
|
12
14
|
enable f16;
|
|
13
15
|
|
|
14
16
|
struct Uniforms {
|
|
15
17
|
size: u32, // Total number of elements
|
|
16
|
-
scale: f32, //
|
|
18
|
+
scale: f32, // Output scale for main/add_vec4; residual scale for add_scaled
|
|
17
19
|
_pad1: u32,
|
|
18
20
|
_pad2: u32,
|
|
19
21
|
}
|
|
@@ -32,5 +34,5 @@ fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
|
|
|
32
34
|
if (idx >= u.size) {
|
|
33
35
|
return;
|
|
34
36
|
}
|
|
35
|
-
output[idx] = a[idx] + b[idx];
|
|
37
|
+
output[idx] = f16(f32(a[idx] + b[idx]) * u.scale);
|
|
36
38
|
}
|
|
@@ -10,7 +10,7 @@ enable f16;
|
|
|
10
10
|
|
|
11
11
|
struct Uniforms {
|
|
12
12
|
size: u32, // Total number of elements
|
|
13
|
-
scale: f32, //
|
|
13
|
+
scale: f32, // Output scale
|
|
14
14
|
_pad1: u32,
|
|
15
15
|
_pad2: u32,
|
|
16
16
|
}
|
|
@@ -36,13 +36,13 @@ fn add_vec4(@builtin(global_invocation_id) gid: vec3<u32>) {
|
|
|
36
36
|
let remaining = min(4u, size - idx);
|
|
37
37
|
|
|
38
38
|
if (remaining >= 4u) {
|
|
39
|
-
output[idx] = a[idx] + b[idx];
|
|
40
|
-
output[idx + 1u] = a[idx + 1u] + b[idx + 1u];
|
|
41
|
-
output[idx + 2u] = a[idx + 2u] + b[idx + 2u];
|
|
42
|
-
output[idx + 3u] = a[idx + 3u] + b[idx + 3u];
|
|
39
|
+
output[idx] = f16(f32(a[idx] + b[idx]) * u.scale);
|
|
40
|
+
output[idx + 1u] = f16(f32(a[idx + 1u] + b[idx + 1u]) * u.scale);
|
|
41
|
+
output[idx + 2u] = f16(f32(a[idx + 2u] + b[idx + 2u]) * u.scale);
|
|
42
|
+
output[idx + 3u] = f16(f32(a[idx + 3u] + b[idx + 3u]) * u.scale);
|
|
43
43
|
} else {
|
|
44
44
|
for (var i = 0u; i < remaining; i = i + 1u) {
|
|
45
|
-
output[idx + i] = a[idx + i] + b[idx + i];
|
|
45
|
+
output[idx + i] = f16(f32(a[idx + i] + b[idx + i]) * u.scale);
|
|
46
46
|
}
|
|
47
47
|
}
|
|
48
48
|
}
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
|
|
9
9
|
struct Uniforms {
|
|
10
10
|
size: u32, // Total number of elements
|
|
11
|
-
scale: f32, //
|
|
11
|
+
scale: f32, // Output scale
|
|
12
12
|
_pad1: u32,
|
|
13
13
|
_pad2: u32,
|
|
14
14
|
}
|
|
@@ -35,13 +35,13 @@ fn add_vec4(@builtin(global_invocation_id) gid: vec3<u32>) {
|
|
|
35
35
|
let remaining = min(4u, size - idx);
|
|
36
36
|
|
|
37
37
|
if (remaining >= 4u) {
|
|
38
|
-
output[idx] = a[idx] + b[idx];
|
|
39
|
-
output[idx + 1u] = a[idx + 1u] + b[idx + 1u];
|
|
40
|
-
output[idx + 2u] = a[idx + 2u] + b[idx + 2u];
|
|
41
|
-
output[idx + 3u] = a[idx + 3u] + b[idx + 3u];
|
|
38
|
+
output[idx] = (a[idx] + b[idx]) * u.scale;
|
|
39
|
+
output[idx + 1u] = (a[idx + 1u] + b[idx + 1u]) * u.scale;
|
|
40
|
+
output[idx + 2u] = (a[idx + 2u] + b[idx + 2u]) * u.scale;
|
|
41
|
+
output[idx + 3u] = (a[idx + 3u] + b[idx + 3u]) * u.scale;
|
|
42
42
|
} else {
|
|
43
43
|
for (var i = 0u; i < remaining; i = i + 1u) {
|
|
44
|
-
output[idx + i] = a[idx + i] + b[idx + i];
|
|
44
|
+
output[idx + i] = (a[idx + i] + b[idx + i]) * u.scale;
|
|
45
45
|
}
|
|
46
46
|
}
|
|
47
47
|
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import type { Tensor } from '../tensor.js';
|
|
2
|
+
import type { CommandRecorder } from '../command-recorder.js';
|
|
3
|
+
import type { WeightBuffer, TensorLike } from '../weight-buffer.js';
|
|
4
|
+
|
|
5
|
+
export declare const RMSNORM_PAIR_CACHE_LIMIT: number;
|
|
6
|
+
|
|
7
|
+
export interface SandwichRMSNormPairOptions {
|
|
8
|
+
batchSize?: number;
|
|
9
|
+
hiddenSize?: number | null;
|
|
10
|
+
rmsNormWeightOffset?: boolean;
|
|
11
|
+
label?: string | null;
|
|
12
|
+
layerIdx?: number;
|
|
13
|
+
postOutputBuffer?: GPUBuffer | null;
|
|
14
|
+
preOutputBuffer?: GPUBuffer | null;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export interface SandwichRMSNormPairResult {
|
|
18
|
+
postAttn: Tensor;
|
|
19
|
+
ffnInput: Tensor;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export declare function runSandwichRMSNormPair(
|
|
23
|
+
input: Tensor,
|
|
24
|
+
residual: Tensor | null,
|
|
25
|
+
postWeight: GPUBuffer | WeightBuffer | TensorLike,
|
|
26
|
+
preWeight: GPUBuffer | WeightBuffer | TensorLike,
|
|
27
|
+
eps: number,
|
|
28
|
+
options?: SandwichRMSNormPairOptions
|
|
29
|
+
): Promise<SandwichRMSNormPairResult>;
|
|
30
|
+
|
|
31
|
+
export declare function recordSandwichRMSNormPair(
|
|
32
|
+
recorder: CommandRecorder,
|
|
33
|
+
input: Tensor,
|
|
34
|
+
residual: Tensor | null,
|
|
35
|
+
postWeight: GPUBuffer | WeightBuffer | TensorLike,
|
|
36
|
+
preWeight: GPUBuffer | WeightBuffer | TensorLike,
|
|
37
|
+
eps: number,
|
|
38
|
+
options?: SandwichRMSNormPairOptions
|
|
39
|
+
): Promise<SandwichRMSNormPairResult>;
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
import { acquireBuffer, releaseBuffer } from '../../memory/buffer-pool.js';
|
|
2
|
+
import { padToQ4KBlock } from '../../config/schema/index.js';
|
|
3
|
+
import { createTensor } from '../tensor.js';
|
|
4
|
+
import { getBuffer } from '../weight-buffer.js';
|
|
5
|
+
import { getKernelCapabilities } from '../device.js';
|
|
6
|
+
import { unifiedKernelWrapper } from './utils.js';
|
|
7
|
+
import {
|
|
8
|
+
assertRMSNormWeightBuffer,
|
|
9
|
+
planRMSNormDispatch,
|
|
10
|
+
resolveNormWeightDtype,
|
|
11
|
+
} from './rmsnorm.js';
|
|
12
|
+
|
|
13
|
+
export const RMSNORM_PAIR_CACHE_LIMIT = 4608;
|
|
14
|
+
|
|
15
|
+
function inferHiddenSize(input, hiddenSize) {
|
|
16
|
+
if (hiddenSize != null) return hiddenSize;
|
|
17
|
+
const shape = input?.shape;
|
|
18
|
+
if (Array.isArray(shape) && shape.length > 0) {
|
|
19
|
+
return shape[shape.length - 1];
|
|
20
|
+
}
|
|
21
|
+
return null;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function resolveVariant() {
|
|
25
|
+
const caps = getKernelCapabilities();
|
|
26
|
+
return caps?.hasSubgroups ? 'subgroup' : 'default';
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function resolveDispatchLabel(label) {
|
|
30
|
+
if (typeof label !== 'string' || label.length === 0) {
|
|
31
|
+
return 'rmsnorm_pair';
|
|
32
|
+
}
|
|
33
|
+
const normalized = label.replace(/^L\d+\./, '').replace(/\s+/g, '_');
|
|
34
|
+
return `rmsnorm_pair:${normalized}`;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
function assertRMSNormPairInputs(input, residual, hiddenSize) {
|
|
38
|
+
if (input?.dtype !== 'f32') {
|
|
39
|
+
throw new Error(`[rmsnorm_pair] input dtype must be f32; got "${String(input?.dtype)}".`);
|
|
40
|
+
}
|
|
41
|
+
if (residual && residual.dtype !== 'f32') {
|
|
42
|
+
throw new Error(`[rmsnorm_pair] residual dtype must be f32; got "${String(residual.dtype)}".`);
|
|
43
|
+
}
|
|
44
|
+
if (!Number.isInteger(hiddenSize) || hiddenSize <= 0) {
|
|
45
|
+
throw new Error(`[rmsnorm_pair] hiddenSize must be a positive integer; got "${String(hiddenSize)}".`);
|
|
46
|
+
}
|
|
47
|
+
if (hiddenSize > RMSNORM_PAIR_CACHE_LIMIT) {
|
|
48
|
+
throw new Error(
|
|
49
|
+
`[rmsnorm_pair] hiddenSize=${hiddenSize} exceeds cache limit ${RMSNORM_PAIR_CACHE_LIMIT}.`
|
|
50
|
+
);
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
async function runRMSNormPairImpl(
|
|
55
|
+
target,
|
|
56
|
+
input,
|
|
57
|
+
residual,
|
|
58
|
+
postWeight,
|
|
59
|
+
preWeight,
|
|
60
|
+
eps,
|
|
61
|
+
options = {}
|
|
62
|
+
) {
|
|
63
|
+
const {
|
|
64
|
+
batchSize = 1,
|
|
65
|
+
hiddenSize = null,
|
|
66
|
+
rmsNormWeightOffset = false,
|
|
67
|
+
label = null,
|
|
68
|
+
postOutputBuffer = null,
|
|
69
|
+
preOutputBuffer = null,
|
|
70
|
+
} = options;
|
|
71
|
+
const inferredHiddenSize = inferHiddenSize(input, hiddenSize);
|
|
72
|
+
assertRMSNormPairInputs(input, residual, inferredHiddenSize);
|
|
73
|
+
|
|
74
|
+
const postWeightBuffer = getBuffer(postWeight);
|
|
75
|
+
const preWeightBuffer = getBuffer(preWeight);
|
|
76
|
+
assertRMSNormWeightBuffer(postWeight, postWeightBuffer, inferredHiddenSize);
|
|
77
|
+
assertRMSNormWeightBuffer(preWeight, preWeightBuffer, inferredHiddenSize);
|
|
78
|
+
const postWeightDtype = resolveNormWeightDtype(postWeight, inferredHiddenSize);
|
|
79
|
+
const preWeightDtype = resolveNormWeightDtype(preWeight, inferredHiddenSize);
|
|
80
|
+
|
|
81
|
+
const paddedHiddenSize = padToQ4KBlock(inferredHiddenSize);
|
|
82
|
+
const outputSize = batchSize * paddedHiddenSize * 4;
|
|
83
|
+
const ownsPostOutput = postOutputBuffer == null;
|
|
84
|
+
const ownsPreOutput = preOutputBuffer == null;
|
|
85
|
+
const postOutput = postOutputBuffer ?? acquireBuffer(outputSize, undefined, 'rmsnorm_pair_post_output');
|
|
86
|
+
const preOutput = preOutputBuffer ?? acquireBuffer(outputSize, undefined, 'rmsnorm_pair_pre_output');
|
|
87
|
+
const residualBuffer = residual?.buffer || residual || input.buffer;
|
|
88
|
+
const dispatchPlan = planRMSNormDispatch(target, batchSize);
|
|
89
|
+
const variant = resolveVariant();
|
|
90
|
+
let completed = false;
|
|
91
|
+
|
|
92
|
+
try {
|
|
93
|
+
await unifiedKernelWrapper(
|
|
94
|
+
'rmsnorm_pair',
|
|
95
|
+
target,
|
|
96
|
+
variant,
|
|
97
|
+
[input, residualBuffer, postWeightBuffer, preWeightBuffer, postOutput, preOutput],
|
|
98
|
+
{
|
|
99
|
+
hidden_size: inferredHiddenSize,
|
|
100
|
+
num_tokens: batchSize,
|
|
101
|
+
eps,
|
|
102
|
+
has_residual: residual ? 1 : 0,
|
|
103
|
+
token_stride: dispatchPlan.tokenStride,
|
|
104
|
+
_pad0: 0,
|
|
105
|
+
_pad1: 0,
|
|
106
|
+
_pad2: 0,
|
|
107
|
+
},
|
|
108
|
+
dispatchPlan.workgroups,
|
|
109
|
+
{
|
|
110
|
+
RMS_NORM_OFFSET: rmsNormWeightOffset,
|
|
111
|
+
POST_WEIGHT_IS_F16: postWeightDtype === 'f16',
|
|
112
|
+
PRE_WEIGHT_IS_F16: preWeightDtype === 'f16',
|
|
113
|
+
},
|
|
114
|
+
null,
|
|
115
|
+
resolveDispatchLabel(label)
|
|
116
|
+
);
|
|
117
|
+
|
|
118
|
+
completed = true;
|
|
119
|
+
return {
|
|
120
|
+
postAttn: createTensor(postOutput, 'f32', [batchSize, inferredHiddenSize], 'rmsnorm_pair_post_output'),
|
|
121
|
+
ffnInput: createTensor(preOutput, 'f32', [batchSize, inferredHiddenSize], 'rmsnorm_pair_pre_output'),
|
|
122
|
+
};
|
|
123
|
+
} finally {
|
|
124
|
+
if (!completed) {
|
|
125
|
+
if (ownsPostOutput) releaseBuffer(postOutput);
|
|
126
|
+
if (ownsPreOutput) releaseBuffer(preOutput);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
export async function runSandwichRMSNormPair(
|
|
132
|
+
input,
|
|
133
|
+
residual,
|
|
134
|
+
postWeight,
|
|
135
|
+
preWeight,
|
|
136
|
+
eps,
|
|
137
|
+
options = {}
|
|
138
|
+
) {
|
|
139
|
+
return runRMSNormPairImpl(null, input, residual, postWeight, preWeight, eps, options);
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
export async function recordSandwichRMSNormPair(
|
|
143
|
+
recorder,
|
|
144
|
+
input,
|
|
145
|
+
residual,
|
|
146
|
+
postWeight,
|
|
147
|
+
preWeight,
|
|
148
|
+
eps,
|
|
149
|
+
options = {}
|
|
150
|
+
) {
|
|
151
|
+
return runRMSNormPairImpl(recorder, input, residual, postWeight, preWeight, eps, options);
|
|
152
|
+
}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import type { Tensor } from '../tensor.js';
|
|
2
|
+
import type { CommandRecorder } from '../command-recorder.js';
|
|
3
|
+
import type { WeightBuffer, TensorLike } from '../weight-buffer.js';
|
|
4
|
+
|
|
5
|
+
export interface RMSNormQKOptions {
|
|
6
|
+
numTokens: number;
|
|
7
|
+
numHeads: number;
|
|
8
|
+
numKVHeads: number;
|
|
9
|
+
headDim: number;
|
|
10
|
+
rmsNormWeightOffset?: boolean;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export interface RMSNormQKResult {
|
|
14
|
+
q: Tensor;
|
|
15
|
+
k: Tensor;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export function canUseRMSNormQK(
|
|
19
|
+
qTensor: Tensor | null | undefined,
|
|
20
|
+
kTensor: Tensor | null | undefined,
|
|
21
|
+
options?: { skipKNorm?: boolean }
|
|
22
|
+
): boolean;
|
|
23
|
+
|
|
24
|
+
export declare function runRMSNormQK(
|
|
25
|
+
qTensor: Tensor,
|
|
26
|
+
kTensor: Tensor,
|
|
27
|
+
qWeight: GPUBuffer | WeightBuffer | TensorLike,
|
|
28
|
+
kWeight: GPUBuffer | WeightBuffer | TensorLike,
|
|
29
|
+
eps: number,
|
|
30
|
+
options: RMSNormQKOptions
|
|
31
|
+
): Promise<RMSNormQKResult>;
|
|
32
|
+
|
|
33
|
+
export declare function recordRMSNormQK(
|
|
34
|
+
recorder: CommandRecorder,
|
|
35
|
+
qTensor: Tensor,
|
|
36
|
+
kTensor: Tensor,
|
|
37
|
+
qWeight: GPUBuffer | WeightBuffer | TensorLike,
|
|
38
|
+
kWeight: GPUBuffer | WeightBuffer | TensorLike,
|
|
39
|
+
eps: number,
|
|
40
|
+
options: RMSNormQKOptions
|
|
41
|
+
): Promise<RMSNormQKResult>;
|