doppler-gpu 0.2.2 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +182 -0
- package/README.md +20 -8
- package/assets/doppler.svg +1601 -0
- package/package.json +82 -33
- package/src/cli/cli-model-resolution.js +275 -0
- package/src/cli/cli-output.js +453 -0
- package/src/cli/doppler-cli.js +817 -709
- package/src/cli/doppler-quickstart.d.ts +7 -23
- package/src/cli/doppler-quickstart.js +20 -4
- package/src/cli/doppler-serve.js +1 -1
- package/src/cli/runtime-profiles.d.ts +40 -0
- package/src/cli/runtime-profiles.js +225 -0
- package/src/client/doppler-api.js +1 -6
- package/src/client/doppler-registry.d.ts +12 -0
- package/src/client/doppler-registry.js +27 -0
- package/src/client/doppler-registry.json +79 -5
- package/src/client/failure-taxonomy.js +139 -0
- package/src/client/fault-injection.js +30 -0
- package/src/client/provider.d.ts +151 -0
- package/src/client/provider.js +374 -0
- package/src/client/receipt.js +156 -0
- package/src/client/runtime/index.js +56 -4
- package/src/client/runtime/model-manager.d.ts +0 -14
- package/src/client/runtime/model-manager.js +2 -42
- package/src/client/runtime/model-session.js +33 -0
- package/src/client/runtime/model-source.d.ts +13 -1
- package/src/client/runtime/model-source.js +128 -2
- package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
- package/src/client/runtime/node-quickstart-cache.js +353 -0
- package/src/client/runtime/source-runtime.js +1 -10
- package/src/client/runtime/types.d.ts +0 -60
- package/src/client/wrap-pipeline-handle.d.ts +20 -0
- package/src/client/wrap-pipeline-handle.js +84 -0
- package/src/config/README.md +3 -1
- package/src/config/conversion/README.md +23 -157
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +368 -62
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
- package/src/config/execution-contract-check.js +6 -5
- package/src/config/kernel-path-loader.d.ts +8 -2
- package/src/config/kernels/kernel-ref-digests.js +125 -122
- package/src/config/kernels/registry.d.ts +0 -35
- package/src/config/kernels/registry.js +5 -9
- package/src/config/kernels/registry.json +615 -368
- package/src/config/merge-helpers.d.ts +1 -1
- package/src/config/merge-helpers.js +1 -1
- package/src/config/merge.js +35 -7
- package/src/config/param-categories.d.ts +0 -9
- package/src/config/param-categories.js +1 -5
- package/src/config/param-validator.d.ts +2 -0
- package/src/config/param-validator.js +16 -0
- package/src/config/platforms/loader.d.ts +0 -27
- package/src/config/platforms/loader.js +0 -30
- package/src/config/required-inference-fields-contract-check.js +1 -7
- package/src/config/runtime/README.md +28 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +32 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
- package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
- package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
- package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
- package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
- package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
- package/src/config/schema/benchmark.schema.d.ts +2 -0
- package/src/config/schema/benchmark.schema.js +1 -0
- package/src/config/schema/bridge.schema.d.ts +16 -0
- package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
- package/src/config/schema/browser-suite-metrics.schema.js +2 -0
- package/src/config/schema/converter.schema.d.ts +5 -0
- package/src/config/schema/converter.schema.js +3 -0
- package/src/config/schema/debug.schema.d.ts +22 -1
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/distill-training.schema.d.ts +0 -1
- package/src/config/schema/distill-training.schema.js +1 -1
- package/src/config/schema/doppler.schema.d.ts +21 -3
- package/src/config/schema/doppler.schema.js +116 -9
- package/src/config/schema/emulation.schema.d.ts +3 -0
- package/src/config/schema/emulation.schema.js +16 -16
- package/src/config/schema/execution-v1.schema.d.ts +12 -1
- package/src/config/schema/execution-v1.schema.js +7 -1
- package/src/config/schema/index.d.ts +54 -1
- package/src/config/schema/index.js +15 -0
- package/src/config/schema/inference.schema.d.ts +3 -3
- package/src/config/schema/kernel-path.schema.d.ts +6 -27
- package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
- package/src/config/schema/kernel-thresholds.schema.js +3 -7
- package/src/config/schema/kvcache.schema.js +0 -9
- package/src/config/schema/loading.schema.d.ts +12 -0
- package/src/config/schema/loading.schema.js +4 -0
- package/src/config/schema/manifest.schema.d.ts +73 -1
- package/src/config/schema/manifest.schema.js +48 -2
- package/src/config/schema/moe.schema.js +0 -8
- package/src/config/schema/program-bundle.schema.d.ts +233 -0
- package/src/config/schema/program-bundle.schema.js +393 -0
- package/src/config/schema/ul-training.schema.d.ts +0 -1
- package/src/config/schema/ul-training.schema.js +1 -1
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
- package/src/config/supported-operations.d.ts +14 -0
- package/src/config/supported-operations.js +34 -0
- package/src/config/supported-operations.json +178 -0
- package/src/config/training-defaults.js +1 -7
- package/src/config/transforms/capability-transform-resolver.js +3 -1
- package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
- package/src/config/transforms/execution-graph-transforms.js +693 -29
- package/src/converter/conversion-plan.js +13 -10
- package/src/converter/core.d.ts +57 -0
- package/src/converter/core.js +582 -13
- package/src/converter/parsers/diffusion.js +1 -33
- package/src/converter/quantization-info.d.ts +5 -4
- package/src/converter/quantization-info.js +39 -3
- package/src/converter/quantizer.d.ts +19 -0
- package/src/converter/quantizer.js +81 -4
- package/src/debug/config.d.ts +0 -4
- package/src/debug/config.js +0 -17
- package/src/debug/index.d.ts +28 -30
- package/src/debug/index.js +0 -18
- package/src/debug/stats.d.ts +0 -8
- package/src/debug/stats.js +3 -3
- package/src/debug/tensor.d.ts +9 -2
- package/src/debug/tensor.js +11 -4
- package/src/debug/trace.js +2 -15
- package/src/errors/doppler-error.js +20 -0
- package/src/experimental/browser/quantization.d.ts +5 -0
- package/src/experimental/distribution/shard-delivery.js +179 -23
- package/src/experimental/training/artifacts.js +1 -14
- package/src/experimental/training/checkpoint.js +1 -14
- package/src/experimental/training/datasets/token-batch.d.ts +1 -1
- package/src/experimental/training/datasets/token-batch.js +2 -2
- package/src/experimental/training/distillation/dataset.js +2 -15
- package/src/experimental/training/distillation/eval.js +3 -3
- package/src/experimental/training/distillation/runtime.js +3 -16
- package/src/experimental/training/distillation/student-fixture.js +6 -6
- package/src/experimental/training/lora-pipeline.js +1 -14
- package/src/experimental/training/objectives/base.d.ts +2 -2
- package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.js +2 -2
- package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
- package/src/experimental/training/objectives/distill_triplet.js +2 -2
- package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
- package/src/experimental/training/operator-artifacts.js +1 -14
- package/src/experimental/training/runner.js +1 -14
- package/src/experimental/training/workload-packs/registry.json +6 -6
- package/src/experimental/training/workloads.js +1 -1
- package/src/formats/gguf/types.d.ts +0 -63
- package/src/formats/gguf/types.js +58 -38
- package/src/formats/litert/types.d.ts +0 -13
- package/src/formats/litert/types.js +2 -2
- package/src/formats/rdrr/distributed/index.d.ts +34 -0
- package/src/formats/rdrr/distributed/index.js +3 -0
- package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
- package/src/formats/rdrr/distributed/parsing.js +23 -0
- package/src/formats/rdrr/distributed/validation.d.ts +23 -0
- package/src/formats/rdrr/distributed/validation.js +131 -0
- package/src/formats/rdrr/groups.d.ts +1 -13
- package/src/formats/rdrr/groups.js +0 -34
- package/src/formats/rdrr/index.d.ts +9 -0
- package/src/formats/rdrr/index.js +9 -0
- package/src/formats/rdrr/integrity.d.ts +51 -0
- package/src/formats/rdrr/integrity.js +200 -0
- package/src/formats/rdrr/lowerings.d.ts +28 -0
- package/src/formats/rdrr/lowerings.js +112 -0
- package/src/formats/rdrr/manifest.d.ts +0 -14
- package/src/formats/rdrr/manifest.js +5 -58
- package/src/formats/rdrr/merkle.d.ts +24 -0
- package/src/formats/rdrr/merkle.js +118 -0
- package/src/formats/rdrr/parsing.js +4 -0
- package/src/formats/rdrr/slice-compiler.d.ts +29 -0
- package/src/formats/rdrr/slice-compiler.js +188 -0
- package/src/formats/rdrr/source-transform-contract.js +390 -0
- package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
- package/src/formats/rdrr/storage-descriptor.js +186 -0
- package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
- package/src/formats/rdrr/tensor-config-validator.js +0 -30
- package/src/formats/rdrr/types.d.ts +115 -1
- package/src/formats/rdrr/validation.js +245 -0
- package/src/formats/tflite/types.d.ts +8 -31
- package/src/formats/tflite/types.js +50 -5
- package/src/gpu/command-recorder.d.ts +9 -21
- package/src/gpu/command-recorder.js +41 -2
- package/src/gpu/device.d.ts +17 -3
- package/src/gpu/device.js +20 -4
- package/src/gpu/kernel-runtime.d.ts +8 -15
- package/src/gpu/kernel-selection-log.d.ts +0 -8
- package/src/gpu/kernel-selection-log.js +0 -15
- package/src/gpu/kernel-tuner/cache.js +0 -3
- package/src/gpu/kernels/README.md +50 -18
- package/src/gpu/kernels/activation-static-qdq.d.ts +22 -0
- package/src/gpu/kernels/activation-static-qdq.js +69 -0
- package/src/gpu/kernels/activation_static_qdq.wgsl +25 -0
- package/src/gpu/kernels/attention.d.ts +28 -19
- package/src/gpu/kernels/attention.js +443 -9
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_f16.wgsl +88 -30
- package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
- package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
- package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
- package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
- package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
- package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
- package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
- package/src/gpu/kernels/backward/index.d.ts +6 -0
- package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
- package/src/gpu/kernels/backward/utils.d.ts +35 -0
- package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
- package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
- package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
- package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
- package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
- package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
- package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
- package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
- package/src/gpu/kernels/constants.d.ts +9 -48
- package/src/gpu/kernels/constants.js +0 -23
- package/src/gpu/kernels/conv2d.d.ts +0 -4
- package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant.d.ts +0 -16
- package/src/gpu/kernels/dequant.js +1 -71
- package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant_shared.wgsl +1 -55
- package/src/gpu/kernels/dispatch.d.ts +12 -97
- package/src/gpu/kernels/dispatch.js +0 -117
- package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
- package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
- package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
- package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
- package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
- package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
- package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
- package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
- package/src/gpu/kernels/gather.d.ts +0 -7
- package/src/gpu/kernels/gather_f16.wgsl +1 -1
- package/src/gpu/kernels/gelu_f16.wgsl +14 -15
- package/src/gpu/kernels/groupnorm.d.ts +0 -4
- package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
- package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
- package/src/gpu/kernels/index.d.ts +13 -5
- package/src/gpu/kernels/index.js +9 -4
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
- package/src/gpu/kernels/layernorm.d.ts +0 -4
- package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
- package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
- package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
- package/src/gpu/kernels/logit-merge.d.ts +0 -24
- package/src/gpu/kernels/logit-merge.js +1 -17
- package/src/gpu/kernels/matmul-dispatch.js +7 -1
- package/src/gpu/kernels/matmul-selection.js +68 -1
- package/src/gpu/kernels/matmul.d.ts +0 -28
- package/src/gpu/kernels/matmul.js +101 -7
- package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
- package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
- package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
- package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
- package/src/gpu/kernels/modulate_f16.wgsl +1 -1
- package/src/gpu/kernels/moe.d.ts +0 -56
- package/src/gpu/kernels/moe.js +0 -15
- package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
- package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
- package/src/gpu/kernels/residual.d.ts +0 -12
- package/src/gpu/kernels/residual_f16.wgsl +2 -2
- package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
- package/src/gpu/kernels/rmsnorm.d.ts +0 -8
- package/src/gpu/kernels/rmsnorm.js +17 -1
- package/src/gpu/kernels/rmsnorm.wgsl +5 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
- package/src/gpu/kernels/rope.d.ts +1 -5
- package/src/gpu/kernels/rope.js +12 -2
- package/src/gpu/kernels/rope.wgsl +33 -19
- package/src/gpu/kernels/rope_f16.wgsl +39 -25
- package/src/gpu/kernels/sample.d.ts +0 -5
- package/src/gpu/kernels/sample.wgsl +7 -6
- package/src/gpu/kernels/sample_f16.wgsl +28 -30
- package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
- package/src/gpu/kernels/shader-cache.d.ts +13 -0
- package/src/gpu/kernels/shader-cache.js +27 -0
- package/src/gpu/kernels/silu_f16.wgsl +1 -1
- package/src/gpu/kernels/softmax.wgsl +4 -144
- package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
- package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
- package/src/gpu/kernels/topk_f16.wgsl +1 -1
- package/src/gpu/kernels/turboquant-codebook.js +11 -40
- package/src/gpu/kernels/uniform-utils.d.ts +18 -0
- package/src/gpu/kernels/upsample2d.d.ts +0 -4
- package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
- package/src/gpu/kernels/utils.d.ts +28 -0
- package/src/gpu/kernels/utils.js +41 -0
- package/src/gpu/perf-guards.d.ts +0 -12
- package/src/gpu/perf-guards.js +0 -62
- package/src/gpu/profiler.d.ts +0 -21
- package/src/gpu/profiler.js +0 -34
- package/src/gpu/submit-tracker.d.ts +0 -57
- package/src/gpu/submit-tracker.js +3 -62
- package/src/gpu/tensor.d.ts +0 -23
- package/src/gpu/tensor.js +0 -41
- package/src/gpu/uniform-cache.d.ts +0 -11
- package/src/gpu/uniform-cache.js +0 -23
- package/src/index.d.ts +1 -0
- package/src/index.js +1 -0
- package/src/inference/browser-harness-model-helpers.d.ts +0 -9
- package/src/inference/browser-harness-model-helpers.js +22 -13
- package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
- package/src/inference/browser-harness-runtime-helpers.js +2 -2
- package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
- package/src/inference/browser-harness-suite-helpers.js +87 -1
- package/src/inference/browser-harness-text-helpers.d.ts +13 -0
- package/src/inference/browser-harness-text-helpers.js +267 -18
- package/src/inference/browser-harness.d.ts +0 -29
- package/src/inference/browser-harness.js +271 -71
- package/src/inference/kv-cache/base.d.ts +6 -0
- package/src/inference/kv-cache/base.js +16 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
- package/src/inference/kv-cache/mixed-geometry.js +12 -2
- package/src/inference/kv-cache/quantized.js +2 -0
- package/src/inference/kv-cache/tiered.js +7 -0
- package/src/inference/kv-cache/types.d.ts +2 -0
- package/src/inference/multi-model-network.d.ts +1 -1
- package/src/inference/multi-model-network.js +1 -2
- package/src/inference/network-evolution.d.ts +0 -12
- package/src/inference/network-evolution.js +2 -2
- package/src/inference/pipelines/context.js +21 -2
- package/src/inference/pipelines/diffusion/init.js +2 -2
- package/src/inference/pipelines/diffusion/pipeline.js +41 -96
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
- package/src/inference/pipelines/diffusion/vae.js +4 -4
- package/src/inference/pipelines/energy/quintel.d.ts +15 -0
- package/src/inference/pipelines/structured/index.d.ts +2 -0
- package/src/inference/pipelines/structured/index.js +2 -0
- package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
- package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
- package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
- package/src/inference/pipelines/text/attention/attn-config.js +1 -1
- package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
- package/src/inference/pipelines/text/attention/projections.js +70 -23
- package/src/inference/pipelines/text/attention/record.js +118 -5
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
- package/src/inference/pipelines/text/attention/run.js +97 -6
- package/src/inference/pipelines/text/attention/types.d.ts +2 -0
- package/src/inference/pipelines/text/attention/types.js +9 -4
- package/src/inference/pipelines/text/chat-format.d.ts +0 -10
- package/src/inference/pipelines/text/chat-format.js +1 -8
- package/src/inference/pipelines/text/config.d.ts +3 -25
- package/src/inference/pipelines/text/config.js +105 -3
- package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
- package/src/inference/pipelines/text/drift-policy.js +1 -1
- package/src/inference/pipelines/text/embed.d.ts +23 -27
- package/src/inference/pipelines/text/embed.js +10 -43
- package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
- package/src/inference/pipelines/text/execution-plan.js +15 -2
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
- package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
- package/src/inference/pipelines/text/execution-v1.js +94 -25
- package/src/inference/pipelines/text/ffn/dense.js +180 -3
- package/src/inference/pipelines/text/ffn/standard.js +40 -13
- package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
- package/src/inference/pipelines/text/generator-helpers.d.ts +6 -2
- package/src/inference/pipelines/text/generator-helpers.js +20 -2
- package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
- package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
- package/src/inference/pipelines/text/generator-runtime.js +8 -6
- package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
- package/src/inference/pipelines/text/generator-steps.js +23 -9
- package/src/inference/pipelines/text/generator.d.ts +0 -6
- package/src/inference/pipelines/text/generator.js +254 -530
- package/src/inference/pipelines/text/init-chat-templates.js +86 -0
- package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
- package/src/inference/pipelines/text/init-rope.js +408 -0
- package/src/inference/pipelines/text/init.d.ts +8 -2
- package/src/inference/pipelines/text/init.js +190 -710
- package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
- package/src/inference/pipelines/text/kernel-trace.js +0 -33
- package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
- package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
- package/src/inference/pipelines/text/layer-plan.js +1 -1
- package/src/inference/pipelines/text/layer.d.ts +70 -8
- package/src/inference/pipelines/text/layer.js +105 -537
- package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
- package/src/inference/pipelines/text/load-errors.d.ts +21 -0
- package/src/inference/pipelines/text/load-errors.js +59 -0
- package/src/inference/pipelines/text/logits/cpu.js +2 -20
- package/src/inference/pipelines/text/logits/gpu.d.ts +5 -4
- package/src/inference/pipelines/text/logits/gpu.js +148 -11
- package/src/inference/pipelines/text/logits/index.d.ts +1 -1
- package/src/inference/pipelines/text/model-load.d.ts +5 -4
- package/src/inference/pipelines/text/model-load.js +74 -17
- package/src/inference/pipelines/text/ops.d.ts +13 -2
- package/src/inference/pipelines/text/ops.js +1 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +347 -59
- package/src/inference/pipelines/text/probes.d.ts +3 -6
- package/src/inference/pipelines/text/probes.js +198 -31
- package/src/inference/pipelines/text/sampling.d.ts +0 -5
- package/src/inference/pipelines/text/sampling.js +140 -0
- package/src/inference/pipelines/text/stage-names.js +12 -0
- package/src/inference/pipelines/text/state.js +2 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
- package/src/inference/pipelines/text/types.d.ts +8 -78
- package/src/inference/pipelines/text/weights.d.ts +0 -50
- package/src/inference/pipelines/text/weights.js +2 -51
- package/src/inference/pipelines/text.d.ts +9 -1
- package/src/inference/pipelines/text.js +233 -29
- package/src/inference/pipelines/vision/index.js +0 -107
- package/src/inference/runtime-model.d.ts +0 -7
- package/src/inference/runtime-model.js +0 -23
- package/src/inference/test-harness.d.ts +6 -22
- package/src/inference/test-harness.js +97 -23
- package/src/inference/tokenizer.js +1 -10
- package/src/inference/tokenizers/sentencepiece.js +65 -15
- package/src/inference/tokenizers/types.d.ts +0 -21
- package/src/loader/doppler-loader.d.ts +3 -0
- package/src/loader/doppler-loader.js +90 -12
- package/src/loader/embedding-loader.js +30 -5
- package/src/loader/loader-types.d.ts +4 -0
- package/src/loader/manifest-config.d.ts +2 -7
- package/src/loader/manifest-config.js +25 -1
- package/src/loader/memory-monitor.d.ts +17 -5
- package/src/loader/per-layer-input-loader.js +114 -1
- package/src/loader/quantization-constants.d.ts +6 -0
- package/src/loader/shard-resolver.d.ts +1 -0
- package/src/loader/shard-resolver.js +15 -0
- package/src/loader/tensors/source-transform.d.ts +15 -0
- package/src/loader/tensors/source-transform.js +317 -34
- package/src/loader/tensors/tensor-loader.d.ts +2 -9
- package/src/loader/tensors/tensor-loader.js +84 -1
- package/src/loader/tensors/tensor-reader.js +28 -2
- package/src/memory/address-table.d.ts +1 -9
- package/src/memory/buffer-pool.d.ts +0 -16
- package/src/memory/buffer-pool.js +1 -33
- package/src/memory/heap-manager.d.ts +18 -43
- package/src/memory/unified-detect.d.ts +19 -34
- package/src/models/embeddinggemma.d.ts +7 -0
- package/src/models/embeddinggemma.js +24 -0
- package/src/models/family.d.ts +24 -0
- package/src/models/family.js +25 -0
- package/src/models/gemma3.d.ts +7 -0
- package/src/models/gemma3.js +33 -0
- package/src/models/gemma4.d.ts +7 -0
- package/src/models/gemma4.js +33 -0
- package/src/models/qwen3.d.ts +7 -0
- package/src/models/qwen3.js +36 -0
- package/src/rules/converter/tokenizer.rules.json +5 -0
- package/src/rules/execution-rules-contract-check.js +72 -3
- package/src/rules/inference/capability-transforms.rules.json +87 -0
- package/src/rules/inference/execution.rules.json +11 -1
- package/src/rules/inference/ffn.rules.json +6 -2
- package/src/rules/loader/tensor-loader.rules.json +9 -0
- package/src/rules/rule-registry.js +53 -75
- package/src/storage/artifact-storage-context.d.ts +12 -3
- package/src/storage/artifact-storage-context.js +389 -98
- package/src/storage/backends/opfs-store.js +138 -88
- package/src/storage/blake3.js +9 -1
- package/src/storage/export.js +1 -5
- package/src/storage/normalize-model-id.js +3 -0
- package/src/storage/preflight.js +22 -0
- package/src/storage/quickstart-downloader.js +44 -56
- package/src/storage/reports.js +35 -11
- package/src/storage/shard-manager.d.ts +8 -2
- package/src/storage/shard-manager.js +176 -15
- package/src/storage/source-artifact-store.d.ts +0 -2
- package/src/storage/source-artifact-store.js +3 -9
- package/src/sw.js +0 -3
- package/src/tooling/browser-command-runner.js +39 -0
- package/src/tooling/command-api-constants.js +1 -1
- package/src/tooling/command-api-family-normalizers.d.ts +1 -0
- package/src/tooling/command-api-family-normalizers.js +49 -7
- package/src/tooling/command-api-helpers.d.ts +0 -2
- package/src/tooling/command-api-helpers.js +6 -2
- package/src/tooling/command-api.d.ts +20 -1
- package/src/tooling/command-api.js +10 -1
- package/src/tooling/command-envelope.d.ts +0 -1
- package/src/tooling/command-envelope.js +1 -1
- package/src/tooling/command-runner-shared.d.ts +10 -0
- package/src/tooling/conversion-config-materializer.d.ts +0 -10
- package/src/tooling/conversion-config-materializer.js +1 -1
- package/src/tooling/diagnose-runner.js +1 -4
- package/src/tooling/hf-registry-utils.d.ts +2 -6
- package/src/tooling/hf-registry-utils.js +90 -19
- package/src/tooling/kernel-path-builder/index.js +1 -4
- package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
- package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
- package/src/tooling/lean-execution-contract-runner.js +2 -2
- package/src/tooling/litert-package-runtime.d.ts +1 -1
- package/src/tooling/litert-package-runtime.js +373 -21
- package/src/tooling/node-browser-command-runner.d.ts +15 -0
- package/src/tooling/node-browser-command-runner.js +97 -13
- package/src/tooling/node-command-runner.js +36 -0
- package/src/tooling/node-convert-worker-pool.js +12 -0
- package/src/tooling/node-convert-worker.js +10 -1
- package/src/tooling/node-converter.js +90 -1
- package/src/tooling/node-source-runtime.js +257 -150
- package/src/tooling/opfs-cache.js +1 -4
- package/src/tooling/precision-replay-math.js +174 -0
- package/src/tooling/program-bundle-host.d.ts +16 -0
- package/src/tooling/program-bundle-host.js +20 -0
- package/src/tooling/program-bundle-parity.d.ts +28 -0
- package/src/tooling/program-bundle-parity.js +210 -0
- package/src/tooling/program-bundle.d.ts +61 -0
- package/src/tooling/program-bundle.js +1119 -0
- package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
- package/src/tooling/rdrr-integrity-refresh.js +271 -0
- package/src/tooling/reference-verify.js +191 -0
- package/src/tooling/source-artifact-adapter.d.ts +0 -20
- package/src/tooling/source-artifact-adapter.js +4 -4
- package/src/tooling/source-package-profiles.js +1 -7
- package/src/tooling/source-runtime-bundle.d.ts +27 -0
- package/src/tooling/source-runtime-bundle.js +56 -23
- package/src/tooling/source-runtime-converter-config.d.ts +0 -5
- package/src/tooling/source-runtime-converter-config.js +13 -9
- package/src/tooling/source-runtime-materializer.js +4 -8
- package/src/tooling-exports/device.d.ts +11 -0
- package/src/tooling-exports/device.js +14 -0
- package/src/tooling-exports/manifest.d.ts +8 -0
- package/src/tooling-exports/manifest.js +11 -0
- package/src/tooling-exports/storage.d.ts +25 -0
- package/src/tooling-exports/storage.js +29 -0
- package/src/tooling-exports/structured.d.ts +1 -0
- package/src/tooling-exports/structured.js +5 -0
- package/src/tooling-exports.d.ts +23 -0
- package/src/tooling-exports.js +17 -0
- package/src/tooling-exports.shared.d.ts +27 -30
- package/src/tooling-exports.shared.js +28 -29
- package/src/types/index.d.ts +0 -3
- package/src/types/index.js +0 -3
- package/src/utils/array-buffer.d.ts +4 -0
- package/src/utils/array-buffer.js +9 -0
- package/src/utils/canonical-hash.d.ts +7 -0
- package/src/utils/canonical-hash.js +74 -0
- package/src/utils/clone-json.d.ts +1 -0
- package/src/utils/clone-json.js +7 -0
- package/src/utils/encode-utf8.js +5 -0
- package/src/utils/index.d.ts +4 -1
- package/src/utils/index.js +4 -1
- package/src/utils/runtime-env.js +7 -0
- package/src/utils/sha256.d.ts +1 -0
- package/src/utils/sha256.js +18 -4
- package/src/utils/stable-sort-object.d.ts +1 -0
- package/src/utils/stable-sort-object.js +13 -0
- package/src/version.js +1 -1
- package/tools/convert-safetensors-node.js +1 -2
- package/src/bootstrap.d.ts +0 -1
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
- package/src/config/kernels/kernel-ref.d.ts +0 -17
- package/src/config/kernels/kernel-ref.js +0 -75
- package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
- package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
- package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
- package/src/config/runtime/kernels/fused-q4k.json +0 -13
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
- package/src/config/runtime/kernels/safe-q4k.json +0 -13
- package/src/config/runtime/overlays/debug-f32a.json +0 -3
- package/src/config/runtime/overlays/trace-config.json +0 -28
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
- package/src/debug/perf.d.ts +0 -33
- package/src/debug/perf.js +0 -51
- package/src/errors/index.d.ts +0 -1
- package/src/errors/index.js +0 -1
- package/src/formats/gguf/index.d.ts +0 -8
- package/src/formats/gguf/index.js +0 -4
- package/src/formats/litert/index.d.ts +0 -1
- package/src/formats/litert/index.js +0 -1
- package/src/formats/safetensors/index.d.ts +0 -8
- package/src/formats/safetensors/index.js +0 -4
- package/src/formats/tflite/index.d.ts +0 -1
- package/src/formats/tflite/index.js +0 -1
- package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
- package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
- package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
- package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
- package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
- package/src/inference/functiongemma.d.ts +0 -15
- package/src/inference/functiongemma.js +0 -1
- package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
- package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
- package/src/rules/kernels/kernel-validator.d.ts +0 -24
- package/src/rules/kernels/kernel-validator.js +0 -160
- package/src/sw.d.ts +0 -1
- package/src/tooling/bench-runner.js +0 -64
- package/src/tooling/distill-runner.js +0 -56
- package/src/tooling/lora-runner.js +0 -37
- package/src/tooling/verify-runner.js +0 -63
- package/src/types/gpu.d.ts +0 -185
- package/src/types/gpu.js +0 -5
- package/src/types/inference.d.ts +0 -197
- package/src/types/inference.js +0 -5
- package/src/types/model.d.ts +0 -130
- package/src/types/model.js +0 -5
- package/src/utils/format-bytes.js +0 -20
- /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
- /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
|
@@ -29,6 +29,7 @@ import {
|
|
|
29
29
|
selectMatmulKernel,
|
|
30
30
|
} from './matmul-selection.js';
|
|
31
31
|
import { selectRuleValue as selectKernelRuleValue } from './rule-registry.js';
|
|
32
|
+
import { getRuntimeConfig } from '../../config/runtime.js';
|
|
32
33
|
import {
|
|
33
34
|
MatmulKernel,
|
|
34
35
|
calculateMatmulDispatch,
|
|
@@ -36,6 +37,7 @@ import {
|
|
|
36
37
|
createMatmulBindGroupLayout,
|
|
37
38
|
getMatmulPipeline,
|
|
38
39
|
} from './matmul-dispatch.js';
|
|
40
|
+
import { __dbgRecord } from './utils.js';
|
|
39
41
|
|
|
40
42
|
export { isFusedQ4KDisabled, selectMatmulKernel };
|
|
41
43
|
export { createMatmulBindGroupLayout };
|
|
@@ -96,12 +98,22 @@ function assertBindGroupBuffer(kernelName, variant, bindingIndex, bindingLabel,
|
|
|
96
98
|
);
|
|
97
99
|
}
|
|
98
100
|
|
|
99
|
-
function createMatmulBindGroupEntries(variant, uniformBuffer, matmulInput, bBuffer, outputBuffer, offsets, bindingSizes) {
|
|
101
|
+
function createMatmulBindGroupEntries(variant, uniformBuffer, matmulInput, bBuffer, outputBuffer, offsets, bindingSizes, residualBuffer = null, normWeightBuffer = null) {
|
|
100
102
|
const isQ4KF16 = variant === 'q4_fused_multicol_f16'
|
|
101
103
|
|| variant === 'q4_fused_f16a'
|
|
102
104
|
|| variant === 'q4_fused_batched_f16'
|
|
103
105
|
|| variant === 'q4_fused_multicol_f16a'
|
|
104
|
-
|| variant === '
|
|
106
|
+
|| variant === 'q4_fused_multicol_f16a_f32acc'
|
|
107
|
+
|| variant === 'q4_fused_batched_f16a'
|
|
108
|
+
|| variant === 'q4_fused_batched_f16acc_f16a'
|
|
109
|
+
|| variant === 'q4_fused_prefill_tiled_f16'
|
|
110
|
+
|| variant === 'q4_fused_widetile_f16'
|
|
111
|
+
|| variant === 'q4_fused_widetile_f16a';
|
|
112
|
+
// 5-entry WideTile epilogue/prologue variants: output at binding 3 + one
|
|
113
|
+
// extra read-only buffer at binding 4 (residual for _residual, norm weight
|
|
114
|
+
// for _rmsnorm). Distinct from isQ4KF16 (which puts output at binding 4).
|
|
115
|
+
const isWideTileResidual = variant === 'q4_fused_widetile_residual';
|
|
116
|
+
const isWideTileRmsnorm = variant === 'q4_fused_rmsnorm_widetile';
|
|
105
117
|
|
|
106
118
|
assertBindGroupBuffer('matmul', variant, 0, 'uniforms', uniformBuffer);
|
|
107
119
|
assertBindGroupBuffer('matmul', variant, 1, 'input', matmulInput?.buffer, [
|
|
@@ -110,6 +122,18 @@ function createMatmulBindGroupEntries(variant, uniformBuffer, matmulInput, bBuff
|
|
|
110
122
|
]);
|
|
111
123
|
assertBindGroupBuffer('matmul', variant, 2, 'weights', bBuffer);
|
|
112
124
|
assertBindGroupBuffer('matmul', variant, isQ4KF16 ? 4 : 3, 'output', outputBuffer);
|
|
125
|
+
if (isWideTileResidual) {
|
|
126
|
+
if (!residualBuffer) {
|
|
127
|
+
throw new Error(`[Matmul] variant "${variant}" requires a residual buffer but none was provided.`);
|
|
128
|
+
}
|
|
129
|
+
assertBindGroupBuffer('matmul', variant, 4, 'residual', residualBuffer);
|
|
130
|
+
}
|
|
131
|
+
if (isWideTileRmsnorm) {
|
|
132
|
+
if (!normWeightBuffer) {
|
|
133
|
+
throw new Error(`[Matmul] variant "${variant}" requires a norm weight buffer but none was provided.`);
|
|
134
|
+
}
|
|
135
|
+
assertBindGroupBuffer('matmul', variant, 4, 'norm_weight', normWeightBuffer);
|
|
136
|
+
}
|
|
113
137
|
|
|
114
138
|
const entries = [
|
|
115
139
|
{ binding: 0, resource: { buffer: uniformBuffer } },
|
|
@@ -127,6 +151,18 @@ function createMatmulBindGroupEntries(variant, uniformBuffer, matmulInput, bBuff
|
|
|
127
151
|
binding: 3,
|
|
128
152
|
resource: { buffer: outputBuffer, offset: offsets.cOffset, size: bindingSizes.cBindingSize },
|
|
129
153
|
});
|
|
154
|
+
if (isWideTileResidual) {
|
|
155
|
+
entries.push({
|
|
156
|
+
binding: 4,
|
|
157
|
+
resource: { buffer: residualBuffer },
|
|
158
|
+
});
|
|
159
|
+
}
|
|
160
|
+
if (isWideTileRmsnorm) {
|
|
161
|
+
entries.push({
|
|
162
|
+
binding: 4,
|
|
163
|
+
resource: { buffer: normWeightBuffer },
|
|
164
|
+
});
|
|
165
|
+
}
|
|
130
166
|
}
|
|
131
167
|
|
|
132
168
|
return entries;
|
|
@@ -214,7 +250,23 @@ async function executeMatmul(recorder, A, B, M, N, K, options = {}) {
|
|
|
214
250
|
|
|
215
251
|
validateMatmulOffsets(opLabel, aOffset, bOffset, cOffset);
|
|
216
252
|
|
|
217
|
-
const
|
|
253
|
+
const runtimeSession = getRuntimeConfig().inference?.session;
|
|
254
|
+
const effectiveOptions = (
|
|
255
|
+
options.useTiledQ4KPrefill == null
|
|
256
|
+
|| options.useWideTileQ4KPrefill == null
|
|
257
|
+
|| options.useWideTileResidualFusion == null
|
|
258
|
+
|| options.useFusedRmsnormWideTile == null
|
|
259
|
+
)
|
|
260
|
+
? {
|
|
261
|
+
...options,
|
|
262
|
+
useTiledQ4KPrefill: options.useTiledQ4KPrefill ?? (runtimeSession?.useTiledQ4KPrefill === true),
|
|
263
|
+
useWideTileQ4KPrefill: options.useWideTileQ4KPrefill ?? (runtimeSession?.useWideTileQ4KPrefill === true),
|
|
264
|
+
useWideTileResidualFusion: options.useWideTileResidualFusion ?? (runtimeSession?.useWideTileResidualFusion === true),
|
|
265
|
+
useFusedRmsnormWideTile: options.useFusedRmsnormWideTile ?? (runtimeSession?.useFusedRmsnormWideTile === true),
|
|
266
|
+
}
|
|
267
|
+
: options;
|
|
268
|
+
|
|
269
|
+
let { variant, useQ4KFused, useGemv } = selectMatmulVariantAndFlags(
|
|
218
270
|
mode,
|
|
219
271
|
M,
|
|
220
272
|
N,
|
|
@@ -223,10 +275,35 @@ async function executeMatmul(recorder, A, B, M, N, K, options = {}) {
|
|
|
223
275
|
bDtype,
|
|
224
276
|
transposeB,
|
|
225
277
|
requestedOutputDtype,
|
|
226
|
-
|
|
278
|
+
effectiveOptions
|
|
227
279
|
);
|
|
228
280
|
|
|
229
|
-
|
|
281
|
+
if (
|
|
282
|
+
runtimeSession?.useF32AccumF16ioMatmul === true
|
|
283
|
+
&& useQ4KFused
|
|
284
|
+
&& variant === 'q4_fused_multicol_f16a'
|
|
285
|
+
) {
|
|
286
|
+
variant = 'q4_fused_multicol_f16a_f32acc';
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
let constants = resolveMatmulConstants(options, phase);
|
|
290
|
+
if (variant === 'f32' && constants && options.constants == null) {
|
|
291
|
+
constants = null;
|
|
292
|
+
}
|
|
293
|
+
// For the rmsnorm-fused WideTile variant, forward the caller's
|
|
294
|
+
// rmsNormOffset flag as a pipeline override constant. Gemma-family norm
|
|
295
|
+
// weights encode `(w - 1.0)`; other models encode `w`.
|
|
296
|
+
// Also forward WEIGHT_IS_F16 based on the norm weight buffer dtype so the
|
|
297
|
+
// kernel correctly unpacks f16-packed weights (Gemma hidden weights) vs
|
|
298
|
+
// f32 weights.
|
|
299
|
+
if (variant === 'q4_fused_rmsnorm_widetile' && options.rmsNormOffset != null) {
|
|
300
|
+
const normWeightDtype = getWeightDtype(options.normWeight) ?? 'f32';
|
|
301
|
+
constants = {
|
|
302
|
+
...(constants ?? {}),
|
|
303
|
+
RMS_NORM_OFFSET: options.rmsNormOffset === true,
|
|
304
|
+
WEIGHT_IS_F16: normWeightDtype === 'f16',
|
|
305
|
+
};
|
|
306
|
+
}
|
|
230
307
|
|
|
231
308
|
let matmulInput = A;
|
|
232
309
|
let matmulADtype = aDtype;
|
|
@@ -325,9 +402,12 @@ async function executeMatmul(recorder, A, B, M, N, K, options = {}) {
|
|
|
325
402
|
);
|
|
326
403
|
}
|
|
327
404
|
|
|
405
|
+
const __dbg = (typeof process !== "undefined" ? process : { env: {} }).env.DOPPLER_DBG_RECORD === '1';
|
|
406
|
+
const __t0 = __dbg ? performance.now() : 0;
|
|
328
407
|
const config = getMatmulConfig(variant, constants);
|
|
329
408
|
const kernel = new MatmulKernel(device);
|
|
330
409
|
const pipeline = await getMatmulPipeline(variant, constants);
|
|
410
|
+
const __tPipeline = __dbg ? performance.now() : 0;
|
|
331
411
|
|
|
332
412
|
const { output: C, outputSize, cBindingSize, actualOutputDtype } = resolveMatmulOutput(
|
|
333
413
|
variant,
|
|
@@ -358,6 +438,9 @@ async function executeMatmul(recorder, A, B, M, N, K, options = {}) {
|
|
|
358
438
|
let uniformBuffer = null;
|
|
359
439
|
let completed = false;
|
|
360
440
|
try {
|
|
441
|
+
const uniformExtras = variant === 'q4_fused_rmsnorm_widetile' && Number.isFinite(options.rmsNormEps)
|
|
442
|
+
? { eps: options.rmsNormEps }
|
|
443
|
+
: null;
|
|
361
444
|
uniformBuffer = createMatmulUniformBuffer(
|
|
362
445
|
'matmul_uniforms',
|
|
363
446
|
M,
|
|
@@ -368,9 +451,12 @@ async function executeMatmul(recorder, A, B, M, N, K, options = {}) {
|
|
|
368
451
|
transposeB,
|
|
369
452
|
dispatchPlan.uniformWorkgroupsX,
|
|
370
453
|
recorder || null,
|
|
371
|
-
device
|
|
454
|
+
device,
|
|
455
|
+
uniformExtras
|
|
372
456
|
);
|
|
373
457
|
|
|
458
|
+
const residualBuffer = options.residualTensor?.buffer ?? null;
|
|
459
|
+
const normWeightBuffer = options.normWeight?.buffer ?? options.normWeight ?? null;
|
|
374
460
|
const entries = createMatmulBindGroupEntries(
|
|
375
461
|
variant,
|
|
376
462
|
uniformBuffer,
|
|
@@ -382,20 +468,28 @@ async function executeMatmul(recorder, A, B, M, N, K, options = {}) {
|
|
|
382
468
|
aBindingSize: bindingSizes.aBindingSize,
|
|
383
469
|
bBindingSize: bindingSizes.bBindingSize,
|
|
384
470
|
cBindingSize,
|
|
385
|
-
}
|
|
471
|
+
},
|
|
472
|
+
residualBuffer,
|
|
473
|
+
normWeightBuffer
|
|
386
474
|
);
|
|
387
475
|
|
|
476
|
+
const __tBgStart = __dbg ? performance.now() : 0;
|
|
388
477
|
const bindGroup = device.createBindGroup({
|
|
389
478
|
label: 'matmul_bind_group',
|
|
390
479
|
layout: pipeline.getBindGroupLayout(0),
|
|
391
480
|
entries,
|
|
392
481
|
});
|
|
482
|
+
const __tBg = __dbg ? performance.now() : 0;
|
|
393
483
|
|
|
394
484
|
if (isRecord) {
|
|
395
485
|
kernel.record(recorder, pipeline, bindGroup, dispatchPlan.workgroups, buildProfileLabel(options));
|
|
396
486
|
} else {
|
|
397
487
|
kernel.dispatch(pipeline, bindGroup, dispatchPlan.workgroups);
|
|
398
488
|
}
|
|
489
|
+
if (__dbg) {
|
|
490
|
+
const __tEnd = performance.now();
|
|
491
|
+
__dbgRecord('matmul', variant, __tPipeline - __t0, __tBgStart - __tPipeline, __tBg - __tBgStart, __tEnd - __tBg);
|
|
492
|
+
}
|
|
399
493
|
completed = true;
|
|
400
494
|
return createTensor(C, actualOutputDtype, [M, N], 'matmul_output');
|
|
401
495
|
} finally {
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
// C[M,N] = A[M,K] * B^T[N,K] (transpose_b=1, SafeTensors/RDRR row-major layout)
|
|
9
9
|
//
|
|
10
10
|
// Design:
|
|
11
|
-
// - f16 storage
|
|
11
|
+
// - f16 storage and f16 accumulators for the all-f16 execution lane
|
|
12
12
|
// - coalesced scalar loads into shared tiles
|
|
13
13
|
// - Shared memory tiles: 64 x TILE_K for A and B
|
|
14
14
|
// - B is always assumed row-major (transpose_b=1) for RDRR models
|
|
@@ -64,11 +64,10 @@ fn main(
|
|
|
64
64
|
let row_base = wg_id.x * TILE_M + tx * THREAD_M;
|
|
65
65
|
let col_base = wg_id.y * TILE_N + ty * THREAD_N;
|
|
66
66
|
|
|
67
|
-
|
|
68
|
-
var
|
|
69
|
-
var
|
|
70
|
-
var
|
|
71
|
-
var acc30: f32 = 0.0; var acc31: f32 = 0.0; var acc32: f32 = 0.0; var acc33: f32 = 0.0;
|
|
67
|
+
var acc00: f16 = f16(0.0); var acc01: f16 = f16(0.0); var acc02: f16 = f16(0.0); var acc03: f16 = f16(0.0);
|
|
68
|
+
var acc10: f16 = f16(0.0); var acc11: f16 = f16(0.0); var acc12: f16 = f16(0.0); var acc13: f16 = f16(0.0);
|
|
69
|
+
var acc20: f16 = f16(0.0); var acc21: f16 = f16(0.0); var acc22: f16 = f16(0.0); var acc23: f16 = f16(0.0);
|
|
70
|
+
var acc30: f16 = f16(0.0); var acc31: f16 = f16(0.0); var acc32: f16 = f16(0.0); var acc33: f16 = f16(0.0);
|
|
72
71
|
|
|
73
72
|
let num_tiles = (u.K + TILE_K - 1u) / TILE_K;
|
|
74
73
|
|
|
@@ -120,16 +119,16 @@ fn main(
|
|
|
120
119
|
// B tile: tileB[n_local * TILE_K + k] for cols ty*4..ty*4+3
|
|
121
120
|
for (var k: u32 = 0u; k < TILE_K; k = k + 1u) {
|
|
122
121
|
// Load 4 A values for this thread's rows
|
|
123
|
-
let a0 =
|
|
124
|
-
let a1 =
|
|
125
|
-
let a2 =
|
|
126
|
-
let a3 =
|
|
122
|
+
let a0 = tileA[(tx * THREAD_M + 0u) * TILE_K + k];
|
|
123
|
+
let a1 = tileA[(tx * THREAD_M + 1u) * TILE_K + k];
|
|
124
|
+
let a2 = tileA[(tx * THREAD_M + 2u) * TILE_K + k];
|
|
125
|
+
let a3 = tileA[(tx * THREAD_M + 3u) * TILE_K + k];
|
|
127
126
|
|
|
128
127
|
// Load 4 B values for this thread's columns
|
|
129
|
-
let b0 =
|
|
130
|
-
let b1 =
|
|
131
|
-
let b2 =
|
|
132
|
-
let b3 =
|
|
128
|
+
let b0 = tileB[(ty * THREAD_N + 0u) * TILE_K + k];
|
|
129
|
+
let b1 = tileB[(ty * THREAD_N + 1u) * TILE_K + k];
|
|
130
|
+
let b2 = tileB[(ty * THREAD_N + 2u) * TILE_K + k];
|
|
131
|
+
let b3 = tileB[(ty * THREAD_N + 3u) * TILE_K + k];
|
|
133
132
|
|
|
134
133
|
// Outer product: 4x4 accumulation
|
|
135
134
|
acc00 += a0 * b0; acc01 += a0 * b1; acc02 += a0 * b2; acc03 += a0 * b3;
|
|
@@ -142,24 +141,24 @@ fn main(
|
|
|
142
141
|
}
|
|
143
142
|
|
|
144
143
|
// Write 4x4 output tile
|
|
145
|
-
let alpha = u.alpha;
|
|
146
|
-
if (row_base + 0u < u.M && col_base + 0u < u.N) { C[(row_base + 0u) * u.N + col_base + 0u] =
|
|
147
|
-
if (row_base + 0u < u.M && col_base + 1u < u.N) { C[(row_base + 0u) * u.N + col_base + 1u] =
|
|
148
|
-
if (row_base + 0u < u.M && col_base + 2u < u.N) { C[(row_base + 0u) * u.N + col_base + 2u] =
|
|
149
|
-
if (row_base + 0u < u.M && col_base + 3u < u.N) { C[(row_base + 0u) * u.N + col_base + 3u] =
|
|
150
|
-
|
|
151
|
-
if (row_base + 1u < u.M && col_base + 0u < u.N) { C[(row_base + 1u) * u.N + col_base + 0u] =
|
|
152
|
-
if (row_base + 1u < u.M && col_base + 1u < u.N) { C[(row_base + 1u) * u.N + col_base + 1u] =
|
|
153
|
-
if (row_base + 1u < u.M && col_base + 2u < u.N) { C[(row_base + 1u) * u.N + col_base + 2u] =
|
|
154
|
-
if (row_base + 1u < u.M && col_base + 3u < u.N) { C[(row_base + 1u) * u.N + col_base + 3u] =
|
|
155
|
-
|
|
156
|
-
if (row_base + 2u < u.M && col_base + 0u < u.N) { C[(row_base + 2u) * u.N + col_base + 0u] =
|
|
157
|
-
if (row_base + 2u < u.M && col_base + 1u < u.N) { C[(row_base + 2u) * u.N + col_base + 1u] =
|
|
158
|
-
if (row_base + 2u < u.M && col_base + 2u < u.N) { C[(row_base + 2u) * u.N + col_base + 2u] =
|
|
159
|
-
if (row_base + 2u < u.M && col_base + 3u < u.N) { C[(row_base + 2u) * u.N + col_base + 3u] =
|
|
160
|
-
|
|
161
|
-
if (row_base + 3u < u.M && col_base + 0u < u.N) { C[(row_base + 3u) * u.N + col_base + 0u] =
|
|
162
|
-
if (row_base + 3u < u.M && col_base + 1u < u.N) { C[(row_base + 3u) * u.N + col_base + 1u] =
|
|
163
|
-
if (row_base + 3u < u.M && col_base + 2u < u.N) { C[(row_base + 3u) * u.N + col_base + 2u] =
|
|
164
|
-
if (row_base + 3u < u.M && col_base + 3u < u.N) { C[(row_base + 3u) * u.N + col_base + 3u] =
|
|
144
|
+
let alpha = f16(u.alpha);
|
|
145
|
+
if (row_base + 0u < u.M && col_base + 0u < u.N) { C[(row_base + 0u) * u.N + col_base + 0u] = acc00 * alpha; }
|
|
146
|
+
if (row_base + 0u < u.M && col_base + 1u < u.N) { C[(row_base + 0u) * u.N + col_base + 1u] = acc01 * alpha; }
|
|
147
|
+
if (row_base + 0u < u.M && col_base + 2u < u.N) { C[(row_base + 0u) * u.N + col_base + 2u] = acc02 * alpha; }
|
|
148
|
+
if (row_base + 0u < u.M && col_base + 3u < u.N) { C[(row_base + 0u) * u.N + col_base + 3u] = acc03 * alpha; }
|
|
149
|
+
|
|
150
|
+
if (row_base + 1u < u.M && col_base + 0u < u.N) { C[(row_base + 1u) * u.N + col_base + 0u] = acc10 * alpha; }
|
|
151
|
+
if (row_base + 1u < u.M && col_base + 1u < u.N) { C[(row_base + 1u) * u.N + col_base + 1u] = acc11 * alpha; }
|
|
152
|
+
if (row_base + 1u < u.M && col_base + 2u < u.N) { C[(row_base + 1u) * u.N + col_base + 2u] = acc12 * alpha; }
|
|
153
|
+
if (row_base + 1u < u.M && col_base + 3u < u.N) { C[(row_base + 1u) * u.N + col_base + 3u] = acc13 * alpha; }
|
|
154
|
+
|
|
155
|
+
if (row_base + 2u < u.M && col_base + 0u < u.N) { C[(row_base + 2u) * u.N + col_base + 0u] = acc20 * alpha; }
|
|
156
|
+
if (row_base + 2u < u.M && col_base + 1u < u.N) { C[(row_base + 2u) * u.N + col_base + 1u] = acc21 * alpha; }
|
|
157
|
+
if (row_base + 2u < u.M && col_base + 2u < u.N) { C[(row_base + 2u) * u.N + col_base + 2u] = acc22 * alpha; }
|
|
158
|
+
if (row_base + 2u < u.M && col_base + 3u < u.N) { C[(row_base + 2u) * u.N + col_base + 3u] = acc23 * alpha; }
|
|
159
|
+
|
|
160
|
+
if (row_base + 3u < u.M && col_base + 0u < u.N) { C[(row_base + 3u) * u.N + col_base + 0u] = acc30 * alpha; }
|
|
161
|
+
if (row_base + 3u < u.M && col_base + 1u < u.N) { C[(row_base + 3u) * u.N + col_base + 1u] = acc31 * alpha; }
|
|
162
|
+
if (row_base + 3u < u.M && col_base + 2u < u.N) { C[(row_base + 3u) * u.N + col_base + 2u] = acc32 * alpha; }
|
|
163
|
+
if (row_base + 3u < u.M && col_base + 3u < u.N) { C[(row_base + 3u) * u.N + col_base + 3u] = acc33 * alpha; }
|
|
165
164
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// AUTO-GENERATED from src/gpu/kernels/matmul_gemv_residual.wgsl.
|
|
2
|
-
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:
|
|
2
|
+
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
|
|
3
3
|
// Fused GEMV + Residual Kernel - Single dispatch for output projection + residual add
|
|
4
4
|
// For M=1 decode: C[N] = A[K] * B^T[K,N] + residual[N]
|
|
5
5
|
//
|
|
@@ -152,7 +152,6 @@ fn main(
|
|
|
152
152
|
// Tune-time overrides can retile this without changing entry points.
|
|
153
153
|
override MULTICOL_COLS_PER_WG: u32 = 32u;
|
|
154
154
|
override MULTICOL_THREADS_PER_COL: u32 = 8u;
|
|
155
|
-
const MULTICOL_MAX_SUBGROUPS: u32 = 8u; // Support sg_size >= 1 (unlikely but safe)
|
|
156
155
|
|
|
157
156
|
// Shared memory for reduction (one slot per thread)
|
|
158
157
|
var<workgroup> multicol_wg_sums: array<f32, MAX_WORKGROUP_SIZE>;
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
// 4. Loop unrolling for better ILP
|
|
9
9
|
//
|
|
10
10
|
// A is f16 (activations), B is f16 (weights), C is f16.
|
|
11
|
-
//
|
|
11
|
+
// The Gemma 4 31B all-f16 lane uses f16 accumulation on main_multicol.
|
|
12
12
|
// transpose_b=0: B is [K, N] (GGUF/column-major), access B[k * N + col]
|
|
13
13
|
// transpose_b=1: B is [N, K] (SafeTensors/row-major), access B[col * K + k]
|
|
14
14
|
//
|
|
@@ -20,15 +20,15 @@ enable subgroups;
|
|
|
20
20
|
|
|
21
21
|
override WORKGROUP_SIZE: u32 = 256u;
|
|
22
22
|
const MAX_WORKGROUP_SIZE: u32 = 256u;
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
23
|
+
override COLS_PER_WG: u32 = 4u; // Each workgroup computes 4 output columns
|
|
24
|
+
override THREADS_PER_COL: u32 = 64u; // 256 / 4 = 64 threads per column
|
|
25
|
+
override MAX_SUBGROUPS_PER_COL: u32 = 16u; // Support sg_size >= 4 (64/4 = 16)
|
|
26
26
|
|
|
27
27
|
// Alternative layout: more columns per workgroup, fewer threads per column.
|
|
28
28
|
// This reduces workgroup count and improves occupancy for mid-sized K.
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
29
|
+
override COLS_PER_WG_COLS8: u32 = 8u; // 256 / 8 = 32 threads per column
|
|
30
|
+
override THREADS_PER_COL_COLS8: u32 = 32u;
|
|
31
|
+
override MAX_SUBGROUPS_PER_COL_COLS8: u32 = 8u; // Support sg_size >= 4 (32/4 = 8)
|
|
32
32
|
var<workgroup> wg_sums_cols8: array<f32, 64>;
|
|
33
33
|
|
|
34
34
|
struct Uniforms {
|
|
@@ -166,10 +166,9 @@ fn main(
|
|
|
166
166
|
// Tune-time overrides can retile this without changing entry points.
|
|
167
167
|
override MULTICOL_COLS_PER_WG: u32 = 32u;
|
|
168
168
|
override MULTICOL_THREADS_PER_COL: u32 = 8u;
|
|
169
|
-
const MULTICOL_MAX_SUBGROUPS: u32 = 8u; // Support sg_size >= 1 (unlikely but safe)
|
|
170
169
|
|
|
171
170
|
// Shared memory for reduction (one slot per thread)
|
|
172
|
-
var<workgroup> multicol_wg_sums: array<
|
|
171
|
+
var<workgroup> multicol_wg_sums: array<f16, MAX_WORKGROUP_SIZE>;
|
|
173
172
|
|
|
174
173
|
@compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
|
|
175
174
|
fn main_multicol(
|
|
@@ -193,7 +192,7 @@ fn main_multicol(
|
|
|
193
192
|
// Track validity
|
|
194
193
|
let is_valid = col < u.N;
|
|
195
194
|
|
|
196
|
-
var partial_sum:
|
|
195
|
+
var partial_sum: f16 = f16(0.0);
|
|
197
196
|
|
|
198
197
|
if (is_valid) {
|
|
199
198
|
// Each of 8 threads splits K
|
|
@@ -218,30 +217,30 @@ fn main_multicol(
|
|
|
218
217
|
B[b_row_offset + k + 2u],
|
|
219
218
|
B[b_row_offset + k + 3u]
|
|
220
219
|
);
|
|
221
|
-
partial_sum +=
|
|
220
|
+
partial_sum += dot(a_vec, b_vec);
|
|
222
221
|
}
|
|
223
222
|
|
|
224
223
|
for (; k < k_end; k = k + 1u) {
|
|
225
|
-
partial_sum +=
|
|
224
|
+
partial_sum += A[k] * B[b_row_offset + k];
|
|
226
225
|
}
|
|
227
226
|
} else {
|
|
228
227
|
// B is [K, N] (GGUF/column-major): B[k, col] = B[k * N + col]
|
|
229
228
|
for (; k < k_aligned_end; k = k + 4u) {
|
|
230
|
-
let a0 =
|
|
231
|
-
let a1 =
|
|
232
|
-
let a2 =
|
|
233
|
-
let a3 =
|
|
229
|
+
let a0 = A[k];
|
|
230
|
+
let a1 = A[k + 1u];
|
|
231
|
+
let a2 = A[k + 2u];
|
|
232
|
+
let a3 = A[k + 3u];
|
|
234
233
|
|
|
235
|
-
let b0 =
|
|
236
|
-
let b1 =
|
|
237
|
-
let b2 =
|
|
238
|
-
let b3 =
|
|
234
|
+
let b0 = B[k * u.N + col];
|
|
235
|
+
let b1 = B[(k + 1u) * u.N + col];
|
|
236
|
+
let b2 = B[(k + 2u) * u.N + col];
|
|
237
|
+
let b3 = B[(k + 3u) * u.N + col];
|
|
239
238
|
|
|
240
239
|
partial_sum = partial_sum + a0 * b0 + a1 * b1 + a2 * b2 + a3 * b3;
|
|
241
240
|
}
|
|
242
241
|
|
|
243
242
|
for (; k < k_end; k = k + 1u) {
|
|
244
|
-
partial_sum = partial_sum +
|
|
243
|
+
partial_sum = partial_sum + A[k] * B[k * u.N + col];
|
|
245
244
|
}
|
|
246
245
|
}
|
|
247
246
|
}
|
|
@@ -252,12 +251,12 @@ fn main_multicol(
|
|
|
252
251
|
|
|
253
252
|
// Thread 0 of each column reduces its 8 values
|
|
254
253
|
if (thread_in_col == 0u && is_valid) {
|
|
255
|
-
var final_sum:
|
|
254
|
+
var final_sum: f16 = f16(0.0);
|
|
256
255
|
let base = col_in_wg * MULTICOL_THREADS_PER_COL;
|
|
257
256
|
for (var i: u32 = 0u; i < MULTICOL_THREADS_PER_COL; i = i + 1u) {
|
|
258
257
|
final_sum = final_sum + multicol_wg_sums[base + i];
|
|
259
258
|
}
|
|
260
|
-
C[col] =
|
|
259
|
+
C[col] = final_sum * f16(u.alpha);
|
|
261
260
|
}
|
|
262
261
|
}
|
|
263
262
|
|
|
@@ -442,8 +441,8 @@ fn main_cols8(
|
|
|
442
441
|
// giving large contiguous reads for better cache utilization.
|
|
443
442
|
// Ideal for standard projections (N=1536..8960) where K is moderate.
|
|
444
443
|
|
|
445
|
-
|
|
446
|
-
|
|
444
|
+
override COLS_PER_WG_4T: u32 = 64u;
|
|
445
|
+
override THREADS_PER_COL_4T: u32 = 4u;
|
|
447
446
|
var<workgroup> wg_sums_4t: array<f32, MAX_WORKGROUP_SIZE>;
|
|
448
447
|
|
|
449
448
|
@compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// AUTO-GENERATED from src/gpu/kernels/modulate.wgsl.
|
|
2
|
-
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:
|
|
2
|
+
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
|
|
3
3
|
// Modulate kernel (f16)
|
|
4
4
|
// Applies per-channel affine and optional gating.
|
|
5
5
|
|
package/src/gpu/kernels/moe.d.ts
CHANGED
|
@@ -37,18 +37,6 @@ export declare function runTopK(
|
|
|
37
37
|
options?: MoEOptions
|
|
38
38
|
): Promise<{ indices: GPUBuffer; weights: GPUBuffer }>;
|
|
39
39
|
|
|
40
|
-
/**
|
|
41
|
-
* Record top-K expert selection (batched, no submit)
|
|
42
|
-
*/
|
|
43
|
-
export declare function recordTopK(
|
|
44
|
-
recorder: CommandRecorder,
|
|
45
|
-
probs: GPUBuffer,
|
|
46
|
-
numTokens: number,
|
|
47
|
-
numExperts: number,
|
|
48
|
-
topK: number,
|
|
49
|
-
options?: MoEOptions
|
|
50
|
-
): Promise<{ indices: GPUBuffer; weights: GPUBuffer }>;
|
|
51
|
-
|
|
52
40
|
/**
|
|
53
41
|
* Run MoE gather (dispatch tokens to experts)
|
|
54
42
|
*/
|
|
@@ -62,20 +50,6 @@ export declare function runMoEGather(
|
|
|
62
50
|
options?: MoEOptions
|
|
63
51
|
): Promise<MoEGatherResult>;
|
|
64
52
|
|
|
65
|
-
/**
|
|
66
|
-
* Record MoE gather (batched, no submit)
|
|
67
|
-
*/
|
|
68
|
-
export declare function recordMoEGather(
|
|
69
|
-
recorder: CommandRecorder,
|
|
70
|
-
hiddenStates: Tensor,
|
|
71
|
-
expertIndices: GPUBuffer,
|
|
72
|
-
numTokens: number,
|
|
73
|
-
hiddenSize: number,
|
|
74
|
-
numExperts: number,
|
|
75
|
-
topK: number,
|
|
76
|
-
options?: MoEOptions
|
|
77
|
-
): Promise<MoEGatherResult>;
|
|
78
|
-
|
|
79
53
|
/**
|
|
80
54
|
* Build token offsets for dynamic scatter on GPU.
|
|
81
55
|
*
|
|
@@ -119,21 +93,6 @@ export declare function runScatterAdd(
|
|
|
119
93
|
options?: MoEOptions
|
|
120
94
|
): Promise<Tensor>;
|
|
121
95
|
|
|
122
|
-
/**
|
|
123
|
-
* Record scatter-add (batched, no submit)
|
|
124
|
-
*/
|
|
125
|
-
export declare function recordScatterAdd(
|
|
126
|
-
recorder: CommandRecorder,
|
|
127
|
-
expertOutputs: Tensor,
|
|
128
|
-
indices: GPUBuffer,
|
|
129
|
-
weights: GPUBuffer,
|
|
130
|
-
numTokens: number,
|
|
131
|
-
hiddenSize: number,
|
|
132
|
-
numExperts: number,
|
|
133
|
-
topK: number,
|
|
134
|
-
options?: MoEOptions
|
|
135
|
-
): Promise<Tensor>;
|
|
136
|
-
|
|
137
96
|
/**
|
|
138
97
|
* Run dynamic scatter-add with token offsets
|
|
139
98
|
*/
|
|
@@ -147,18 +106,3 @@ export declare function runScatterAddDynamic(
|
|
|
147
106
|
topK: number,
|
|
148
107
|
options?: MoEOptions
|
|
149
108
|
): Promise<Tensor>;
|
|
150
|
-
|
|
151
|
-
/**
|
|
152
|
-
* Record dynamic scatter-add (batched, no submit)
|
|
153
|
-
*/
|
|
154
|
-
export declare function recordScatterAddDynamic(
|
|
155
|
-
recorder: CommandRecorder,
|
|
156
|
-
expertOutputs: Tensor,
|
|
157
|
-
indices: GPUBuffer,
|
|
158
|
-
weights: GPUBuffer,
|
|
159
|
-
tokenOffsets: GPUBuffer,
|
|
160
|
-
numTokens: number,
|
|
161
|
-
hiddenSize: number,
|
|
162
|
-
topK: number,
|
|
163
|
-
options?: MoEOptions
|
|
164
|
-
): Promise<Tensor>;
|
package/src/gpu/kernels/moe.js
CHANGED
|
@@ -115,10 +115,6 @@ export async function runTopK(probs, numTokens, numExperts, topK, options = {})
|
|
|
115
115
|
}
|
|
116
116
|
|
|
117
117
|
|
|
118
|
-
export async function recordTopK(recorder, probs, numTokens, numExperts, topK, options = {}) {
|
|
119
|
-
return executeTopK(recorder, probs, numTokens, numExperts, topK, options);
|
|
120
|
-
}
|
|
121
|
-
|
|
122
118
|
// Cached explicit bind group layout for MoE gather (all 6 bindings).
|
|
123
119
|
// Internal postmortems cover why this explicit layout is required.
|
|
124
120
|
|
|
@@ -312,10 +308,6 @@ export async function runMoEGather(hiddenStates, expertIndices, numTokens, hidde
|
|
|
312
308
|
}
|
|
313
309
|
|
|
314
310
|
|
|
315
|
-
export async function recordMoEGather(recorder, hiddenStates, expertIndices, numTokens, hiddenSize, numExperts, topK, options = {}) {
|
|
316
|
-
return executeMoEGather(recorder, hiddenStates, expertIndices, numTokens, hiddenSize, numExperts, topK, options);
|
|
317
|
-
}
|
|
318
|
-
|
|
319
311
|
async function executeScatterAdd(recorder, expertOutputs, indices, weights, numTokens, hiddenSize, numExperts, topK, options = {}) {
|
|
320
312
|
const execution = resolveExecution(recorder);
|
|
321
313
|
const { outputBuffer = null } = options;
|
|
@@ -384,10 +376,6 @@ export async function runScatterAdd(expertOutputs, indices, weights, numTokens,
|
|
|
384
376
|
}
|
|
385
377
|
|
|
386
378
|
|
|
387
|
-
export async function recordScatterAdd(recorder, expertOutputs, indices, weights, numTokens, hiddenSize, numExperts, topK, options = {}) {
|
|
388
|
-
return executeScatterAdd(recorder, expertOutputs, indices, weights, numTokens, hiddenSize, numExperts, topK, options);
|
|
389
|
-
}
|
|
390
|
-
|
|
391
379
|
async function executeMoEBuildTokenOffsets(recorder, tokenCounts, tokenMap, numTokens, numExperts, topK, maxTokensPerExpert, options = {}) {
|
|
392
380
|
const execution = resolveExecution(recorder);
|
|
393
381
|
const { outputBuffer = null } = options;
|
|
@@ -537,6 +525,3 @@ export async function runScatterAddDynamic(expertOutputs, indices, weights, toke
|
|
|
537
525
|
}
|
|
538
526
|
|
|
539
527
|
|
|
540
|
-
export async function recordScatterAddDynamic(recorder, expertOutputs, indices, weights, tokenOffsets, numTokens, hiddenSize, topK, options = {}) {
|
|
541
|
-
return executeScatterAddDynamic(recorder, expertOutputs, indices, weights, tokenOffsets, numTokens, hiddenSize, topK, options);
|
|
542
|
-
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// AUTO-GENERATED from src/gpu/kernels/moe_gather.wgsl.
|
|
2
|
-
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:
|
|
2
|
+
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
|
|
3
3
|
// moe_gather_f16.wgsl
|
|
4
4
|
|
|
5
5
|
/**
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// AUTO-GENERATED from src/gpu/kernels/pixel_shuffle.wgsl.
|
|
2
|
-
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:
|
|
2
|
+
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
|
|
3
3
|
// Pixel shuffle (tokens -> CHW) f16
|
|
4
4
|
|
|
5
5
|
enable f16;
|
|
@@ -19,18 +19,6 @@ export interface ResidualOptions extends OutputBufferOptions {
|
|
|
19
19
|
executionPolicies?: ExecutionV1PoliciesSchema | null;
|
|
20
20
|
}
|
|
21
21
|
|
|
22
|
-
/** Context for residual variant selection rules. */
|
|
23
|
-
export interface ResidualSelectionContext {
|
|
24
|
-
outputDtype: 'f16' | 'f32';
|
|
25
|
-
useVec4: boolean;
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
/** Context for bias add variant selection rules. */
|
|
29
|
-
export interface BiasAddSelectionContext {
|
|
30
|
-
dataDtype: 'f16' | 'f32';
|
|
31
|
-
biasDtype: 'f16' | 'f32';
|
|
32
|
-
}
|
|
33
|
-
|
|
34
22
|
/**
|
|
35
23
|
* Run residual add (element-wise addition)
|
|
36
24
|
*/
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// AUTO-GENERATED from src/gpu/kernels/residual.wgsl.
|
|
2
|
-
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:
|
|
2
|
+
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
|
|
3
3
|
// residual_f16.wgsl
|
|
4
4
|
|
|
5
5
|
/**
|
|
@@ -32,5 +32,5 @@ fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
|
|
|
32
32
|
if (idx >= u.size) {
|
|
33
33
|
return;
|
|
34
34
|
}
|
|
35
|
-
output[idx] =
|
|
35
|
+
output[idx] = a[idx] + b[idx];
|
|
36
36
|
}
|
|
@@ -36,13 +36,13 @@ fn add_vec4(@builtin(global_invocation_id) gid: vec3<u32>) {
|
|
|
36
36
|
let remaining = min(4u, size - idx);
|
|
37
37
|
|
|
38
38
|
if (remaining >= 4u) {
|
|
39
|
-
output[idx] =
|
|
40
|
-
output[idx + 1u] =
|
|
41
|
-
output[idx + 2u] =
|
|
42
|
-
output[idx + 3u] =
|
|
39
|
+
output[idx] = a[idx] + b[idx];
|
|
40
|
+
output[idx + 1u] = a[idx + 1u] + b[idx + 1u];
|
|
41
|
+
output[idx + 2u] = a[idx + 2u] + b[idx + 2u];
|
|
42
|
+
output[idx + 3u] = a[idx + 3u] + b[idx + 3u];
|
|
43
43
|
} else {
|
|
44
44
|
for (var i = 0u; i < remaining; i = i + 1u) {
|
|
45
|
-
output[idx + i] =
|
|
45
|
+
output[idx + i] = a[idx + i] + b[idx + i];
|
|
46
46
|
}
|
|
47
47
|
}
|
|
48
48
|
}
|
|
@@ -18,14 +18,6 @@ export interface RMSNormOptions extends OutputBufferOptions {
|
|
|
18
18
|
rmsNormWeightOffset?: boolean;
|
|
19
19
|
}
|
|
20
20
|
|
|
21
|
-
/** Context for RMSNorm variant selection rules. */
|
|
22
|
-
export interface RMSNormSelectionContext {
|
|
23
|
-
isF16: boolean;
|
|
24
|
-
residual: boolean;
|
|
25
|
-
hasSubgroups: boolean;
|
|
26
|
-
isSmall: boolean;
|
|
27
|
-
}
|
|
28
|
-
|
|
29
21
|
/**
|
|
30
22
|
* Select RMSNorm kernel variant based on options, tensor dtypes, and GPU capabilities.
|
|
31
23
|
*/
|